Bumped version

Android split parsers

.github/workflows/python-package.yml

@@ -1,7 +1,7 @@
 # This workflow will install Python dependencies, run tests and lint with a variety of Python versions
 # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
 
-name: Python package
+name: CI
 
 on:
   push:
@@ -16,7 +16,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [3.7, 3.8, 3.9]
+        # python-version: [3.7, 3.8, 3.9]
+        python-version: [3.8, 3.9]
 
     steps:
     - uses: actions/checkout@v2
@@ -27,8 +28,9 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        python -m pip install flake8 pytest safety
+        python -m pip install flake8 pytest safety stix2
         if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+        python -m pip install .
     - name: Lint with flake8
       run: |
         # stop the build if there are Python syntax errors or undefined names
@@ -37,7 +39,5 @@ jobs:
         flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
     - name: Safety checks
       run: safety check
-
-    # - name: Test with pytest
-    #   run: |
-    #     pytest
+    - name: Test with pytest
+      run: pytest

.gitignore

@@ -131,3 +131,6 @@ dmypy.json
 
 # Temporal files
 *~
+
+# IDEA Dev Environment
+.idea

README.md

@@ -6,6 +6,8 @@
 
 [![](https://img.shields.io/pypi/v/mvt)](https://pypi.org/project/mvt/)
 [![Documentation Status](https://readthedocs.org/projects/mvt/badge/?version=latest)](https://docs.mvt.re/en/latest/?badge=latest)
+[![CI](https://github.com/mvt-project/mvt/actions/workflows/python-package.yml/badge.svg)](https://github.com/mvt-project/mvt/actions/workflows/python-package.yml)
+[![Downloads](https://pepy.tech/badge/mvt)](https://pepy.tech/project/mvt)
 
 Mobile Verification Toolkit (MVT) is a collection of utilities to simplify and automate the process of gathering forensic traces helpful to identify a potential compromise of Android and iOS devices.
 

dev/mvt-android

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021 The MVT Project Authors.
+# Copyright (c) 2021-2022 The MVT Project Authors.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 

dev/mvt-ios

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021 The MVT Project Authors.
+# Copyright (c) 2021-2022 The MVT Project Authors.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 

docs/docker.md

@@ -10,6 +10,11 @@ cd mvt
 docker build -t mvt .
 ```
 
+Optionally, you may need to specify your platform to Docker in order to build successfully (Apple M1)
+```bash
+docker build --platform amd64 -t mvt .
+```
+
 Test if the image was created successfully:
 
 ```bash

docs/iocs.md

@@ -41,4 +41,6 @@ export MVT_STIX2="/home/user/IOC1.stix2:/home/user/IOC2.stix2"
     - [Predator from Cytrox](https://citizenlab.ca/2021/12/pegasus-vs-predator-dissidents-doubly-infected-iphone-reveals-cytrox-mercenary-spyware/) ([STIX2](https://raw.githubusercontent.com/AmnestyTech/investigations/master/2021-12-16_cytrox/cytrox.stix2))
 - [This repository](https://github.com/Te-k/stalkerware-indicators) contains IOCs for Android stalkerware including [a STIX MVT-compatible file](https://raw.githubusercontent.com/Te-k/stalkerware-indicators/master/stalkerware.stix2).
 
+You can automaticallly download the latest public indicator files with the command `mvt-ios download-iocs` or `mvt-android download-iocs`. These commands download the list of indicators listed [here](https://github.com/mvt-project/mvt/blob/main/public_indicators.json) and store them in the [appdir](https://pypi.org/project/appdirs/) folder. They are then loaded automatically by mvt.
+
 Please [open an issue](https://github.com/mvt-project/mvt/issues/) to suggest new sources of STIX-formatted IOCs.

mvt/__init__.py

@@ -1,4 +1,4 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021 The MVT Project Authors.
+# Copyright (c) 2021-2022 The MVT Project Authors.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/

mvt/android/__init__.py

@@ -1,5 +1,5 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021 The MVT Project Authors.
+# Copyright (c) 2021-2022 The MVT Project Authors.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 

mvt/android/download_apks.py

@@ -1,5 +1,5 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021 The MVT Project Authors.
+# Copyright (c) 2021-2022 The MVT Project Authors.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
@@ -57,7 +57,7 @@ class DownloadAPKs(AndroidExtraction):
         :param json_path: Path to the apks.json file to parse.
 
         """
-        with open(json_path, "r") as handle:
+        with open(json_path, "r", encoding="utf-8") as handle:
             packages = json.load(handle)
             return cls(packages=packages)
 
@@ -173,7 +173,7 @@ class DownloadAPKs(AndroidExtraction):
     def save_json(self):
         """Save the results to the package.json file."""
         json_path = os.path.join(self.output_folder, "apks.json")
-        with open(json_path, "w") as handle:
+        with open(json_path, "w", encoding="utf-8") as handle:
             json.dump(self.packages, handle, indent=4)
 
     def run(self):

mvt/android/lookups/__init__.py

@@ -1,4 +1,4 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021 The MVT Project Authors.
+# Copyright (c) 2021-2022 The MVT Project Authors.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/

mvt/android/lookups/koodous.py

@@ -1,5 +1,5 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021 The MVT Project Authors.
+# Copyright (c) 2021-2022 The MVT Project Authors.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 

mvt/android/lookups/virustotal.py

@@ -1,5 +1,5 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021 The MVT Project Authors.
+# Copyright (c) 2021-2022 The MVT Project Authors.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
@@ -39,6 +39,10 @@ def get_virustotal_report(hashes):
 
 
 def virustotal_lookup(packages):
+    # NOTE: This is temporary, until we resolved the issue.
+    log.error("Unfortunately VirusTotal lookup is disabled until further notice, due to unresolved issues with the API service.")
+    return
+
     log.info("Looking up all extracted files on VirusTotal (www.virustotal.com)")
 
     unique_hashes = []

mvt/android/modules/__init__.py

@@ -1,4 +1,4 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021 The MVT Project Authors.
+# Copyright (c) 2021-2022 The MVT Project Authors.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/

mvt/android/modules/adb/__init__.py

@@ -1,23 +1,29 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021 The MVT Project Authors.
+# Copyright (c) 2021-2022 The MVT Project Authors.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
 from .chrome_history import ChromeHistory
-from .dumpsys_batterystats import DumpsysBatterystats
+from .dumpsys_accessibility import DumpsysAccessibility
+from .dumpsys_activities import DumpsysActivities
+from .dumpsys_battery_daily import DumpsysBatteryDaily
+from .dumpsys_battery_history import DumpsysBatteryHistory
+from .dumpsys_dbinfo import DumpsysDBInfo
 from .dumpsys_full import DumpsysFull
-from .dumpsys_packages import DumpsysPackages
-from .dumpsys_procstats import DumpsysProcstats
 from .dumpsys_receivers import DumpsysReceivers
 from .files import Files
+from .getprop import Getprop
 from .logcat import Logcat
 from .packages import Packages
 from .processes import Processes
-from .rootbinaries import RootBinaries
+from .root_binaries import RootBinaries
+from .selinux_status import SELinuxStatus
+from .settings import Settings
 from .sms import SMS
 from .whatsapp import Whatsapp
 
-ADB_MODULES = [ChromeHistory, SMS, Whatsapp, Processes,
-               DumpsysBatterystats, DumpsysProcstats,
-               DumpsysPackages, DumpsysReceivers, DumpsysFull,
-               Packages, RootBinaries, Logcat, Files]
+ADB_MODULES = [ChromeHistory, SMS, Whatsapp, Processes, Getprop, Settings,
+               SELinuxStatus, DumpsysBatteryHistory, DumpsysBatteryDaily,
+               DumpsysReceivers, DumpsysActivities, DumpsysAccessibility,
+               DumpsysDBInfo, DumpsysFull, Packages, Logcat, RootBinaries,
+               Files]

mvt/android/modules/adb/base.py

@@ -1,5 +1,5 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021 The MVT Project Authors.
+# Copyright (c) 2021-2022 The MVT Project Authors.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
@@ -15,7 +15,7 @@ from adb_shell.adb_device import AdbDeviceTcp, AdbDeviceUsb
 from adb_shell.auth.keygen import keygen, write_public_keyfile
 from adb_shell.auth.sign_pythonrsa import PythonRSASigner
 from adb_shell.exceptions import (AdbCommandFailureException, DeviceAuthError,
-                                  UsbReadFailedError)
+                                  UsbDeviceNotFoundError, UsbReadFailedError)
 from usb1 import USBErrorAccess, USBErrorBusy
 
 from mvt.common.module import InsufficientPrivileges, MVTModule
@@ -65,7 +65,11 @@ class AndroidExtraction(MVTModule):
         # If no serial was specified or if the serial does not seem to be
         # a HOST:PORT definition, we use the USB transport.
         if not self.serial or ":" not in self.serial:
-            self.device = AdbDeviceUsb(serial=self.serial)
+            try:
+                self.device = AdbDeviceUsb(serial=self.serial)
+            except UsbDeviceNotFoundError:
+                log.critical("No device found. Make sure it is connected and unlocked.")
+                sys.exit(-1)
         # Otherwise we try to use the TCP transport.
         else:
             addr = self.serial.split(":")
@@ -112,7 +116,7 @@ class AndroidExtraction(MVTModule):
         :returns: Output of command
 
         """
-        return self.device.shell(command)
+        return self.device.shell(command, read_timeout_s=200.0)
 
     def _adb_check_if_root(self):
         """Check if we have a `su` binary on the Android device.

mvt/android/modules/adb/chrome_history.py

@@ -1,5 +1,5 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021 The MVT Project Authors.
+# Copyright (c) 2021-2022 The MVT Project Authors.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
@@ -68,7 +68,7 @@ class ChromeHistory(AndroidExtraction):
                 "url": item[1],
                 "visit_id": item[2],
                 "timestamp": item[3],
-                "isodate": convert_timestamp_to_iso(convert_chrometime_to_unix[item[3]]),
+                "isodate": convert_timestamp_to_iso(convert_chrometime_to_unix(item[3])),
                 "redirect_source": item[4],
             })
 
@@ -78,5 +78,8 @@ class ChromeHistory(AndroidExtraction):
         log.info("Extracted a total of %d history items", len(self.results))
 
     def run(self):
-        self._adb_process_file(os.path.join("/", CHROME_HISTORY_PATH),
-                               self._parse_db)
+        try:
+            self._adb_process_file(os.path.join("/", CHROME_HISTORY_PATH),
+                                   self._parse_db)
+        except Exception as e:
+            self.log.error(e)

mvt/android/modules/adb/dumpsys_accessibility.py

@@ -0,0 +1,45 @@
+# Mobile Verification Toolkit (MVT)
+# Copyright (c) 2021-2022 The MVT Project Authors.
+# Use of this software is governed by the MVT License 1.1 that can be found at
+#   https://license.mvt.re/1.1/
+
+import logging
+
+from mvt.android.parsers import parse_dumpsys_accessibility
+
+from .base import AndroidExtraction
+
+log = logging.getLogger(__name__)
+
+
+class DumpsysAccessibility(AndroidExtraction):
+    """This module extracts stats on accessibility."""
+
+    def __init__(self, file_path=None, base_folder=None, output_folder=None,
+                 serial=None, fast_mode=False, log=None, results=[]):
+        super().__init__(file_path=file_path, base_folder=base_folder,
+                         output_folder=output_folder, fast_mode=fast_mode,
+                         log=log, results=results)
+
+    def check_indicators(self):
+        if not self.indicators:
+            return
+
+        for result in self.results:
+            ioc = self.indicators.check_app_id(result["package_name"])
+            if ioc:
+                result["matched_indicator"] = ioc
+                self.detected.append(result)
+                continue
+
+    def run(self):
+        self._adb_connect()
+        output = self._adb_command("dumpsys accessibility")
+        self._adb_disconnect()
+
+        self.results = parse_dumpsys_accessibility(output)
+
+        for result in self.results:
+            log.info("Found installed accessibility service \"%s\"", result.get("service"))
+
+        self.log.info("Identified a total of %d accessibility services", len(self.results))

mvt/android/modules/adb/dumpsys_activities.py

@@ -0,0 +1,45 @@
+# Mobile Verification Toolkit (MVT)
+# Copyright (c) 2021-2022 The MVT Project Authors.
+# Use of this software is governed by the MVT License 1.1 that can be found at
+#   https://license.mvt.re/1.1/
+
+import logging
+
+from mvt.android.parsers import parse_dumpsys_activity_resolver_table
+
+from .base import AndroidExtraction
+
+log = logging.getLogger(__name__)
+
+
+class DumpsysActivities(AndroidExtraction):
+    """This module extracts details on receivers for risky activities."""
+
+    def __init__(self, file_path=None, base_folder=None, output_folder=None,
+                 serial=None, fast_mode=False, log=None, results=[]):
+        super().__init__(file_path=file_path, base_folder=base_folder,
+                         output_folder=output_folder, fast_mode=fast_mode,
+                         log=log, results=results)
+
+        self.results = results if results else {}
+
+    def check_indicators(self):
+        if not self.indicators:
+            return
+
+        for intent, activities in self.results.items():
+            for activity in activities:
+                ioc = self.indicators.check_app_id(activity["package_name"])
+                if ioc:
+                    activity["matched_indicator"] = ioc
+                    self.detected.append({intent: activity})
+                    continue
+
+    def run(self):
+        self._adb_connect()
+        output = self._adb_command("dumpsys package")
+        self._adb_disconnect()
+
+        self.results = parse_dumpsys_activity_resolver_table(output)
+
+        self.log.info("Extracted activities for %d intents", len(self.results))

mvt/android/modules/adb/dumpsys_battery_daily.py

@@ -0,0 +1,50 @@
+# Mobile Verification Toolkit (MVT)
+# Copyright (c) 2021-2022 The MVT Project Authors.
+# Use of this software is governed by the MVT License 1.1 that can be found at
+#   https://license.mvt.re/1.1/
+
+import logging
+
+from mvt.android.parsers import parse_dumpsys_battery_daily
+
+from .base import AndroidExtraction
+
+log = logging.getLogger(__name__)
+
+
+class DumpsysBatteryDaily(AndroidExtraction):
+    """This module extracts records from battery daily updates."""
+
+    def __init__(self, file_path=None, base_folder=None, output_folder=None,
+                 serial=None, fast_mode=False, log=None, results=[]):
+        super().__init__(file_path=file_path, base_folder=base_folder,
+                         output_folder=output_folder, fast_mode=fast_mode,
+                         log=log, results=results)
+
+    def serialize(self, record):
+        return {
+            "timestamp": record["from"],
+            "module": self.__class__.__name__,
+            "event": "battery_daily",
+            "data": f"Recorded update of package {record['package_name']} with vers {record['vers']}"
+        }
+
+    def check_indicators(self):
+        if not self.indicators:
+            return
+
+        for result in self.results:
+            ioc = self.indicators.check_app_id(result["package_name"])
+            if ioc:
+                result["matched_indicator"] = ioc
+                self.detected.append(result)
+                continue
+
+    def run(self):
+        self._adb_connect()
+        output = self._adb_command("dumpsys batterystats --daily")
+        self._adb_disconnect()
+
+        self.results = parse_dumpsys_battery_daily(output)
+
+        self.log.info("Extracted %d records from battery daily stats", len(self.results))

mvt/android/modules/adb/dumpsys_battery_history.py

@@ -0,0 +1,42 @@
+# Mobile Verification Toolkit (MVT)
+# Copyright (c) 2021-2022 The MVT Project Authors.
+# Use of this software is governed by the MVT License 1.1 that can be found at
+#   https://license.mvt.re/1.1/
+
+import logging
+
+from mvt.android.parsers import parse_dumpsys_battery_history
+
+from .base import AndroidExtraction
+
+log = logging.getLogger(__name__)
+
+
+class DumpsysBatteryHistory(AndroidExtraction):
+    """This module extracts records from battery history events."""
+
+    def __init__(self, file_path=None, base_folder=None, output_folder=None,
+                 serial=None, fast_mode=False, log=None, results=[]):
+        super().__init__(file_path=file_path, base_folder=base_folder,
+                         output_folder=output_folder, fast_mode=fast_mode,
+                         log=log, results=results)
+
+    def check_indicators(self):
+        if not self.indicators:
+            return
+
+        for result in self.results:
+            ioc = self.indicators.check_app_id(result["package_name"])
+            if ioc:
+                result["matched_indicator"] = ioc
+                self.detected.append(result)
+                continue
+
+    def run(self):
+        self._adb_connect()
+        output = self._adb_command("dumpsys batterystats --history")
+        self._adb_disconnect()
+
+        self.results = parse_dumpsys_battery_history(output)
+
+        self.log.info("Extracted %d records from battery history", len(self.results))

mvt/android/modules/adb/dumpsys_batterystats.py

@@ -1,46 +0,0 @@
-# Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021 The MVT Project Authors.
-# Use of this software is governed by the MVT License 1.1 that can be found at
-#   https://license.mvt.re/1.1/
-
-import logging
-import os
-
-from .base import AndroidExtraction
-
-log = logging.getLogger(__name__)
-
-
-class DumpsysBatterystats(AndroidExtraction):
-    """This module extracts stats on battery consumption by processes."""
-
-    def __init__(self, file_path=None, base_folder=None, output_folder=None,
-                 serial=None, fast_mode=False, log=None, results=[]):
-        super().__init__(file_path=file_path, base_folder=base_folder,
-                         output_folder=output_folder, fast_mode=fast_mode,
-                         log=log, results=results)
-
-    def run(self):
-        self._adb_connect()
-
-        stats = self._adb_command("dumpsys batterystats")
-        if self.output_folder:
-            stats_path = os.path.join(self.output_folder,
-                                      "dumpsys_batterystats.txt")
-            with open(stats_path, "w") as handle:
-                handle.write(stats)
-
-            log.info("Records from dumpsys batterystats stored at %s",
-                     stats_path)
-
-        history = self._adb_command("dumpsys batterystats --history")
-        if self.output_folder:
-            history_path = os.path.join(self.output_folder,
-                                        "dumpsys_batterystats_history.txt")
-            with open(history_path, "w") as handle:
-                handle.write(history)
-
-            log.info("History records from dumpsys batterystats stored at %s",
-                     history_path)
-
-        self._adb_disconnect()

mvt/android/modules/adb/dumpsys_dbinfo.py

@@ -0,0 +1,48 @@
+# Mobile Verification Toolkit (MVT)
+# Copyright (c) 2021-2022 The MVT Project Authors.
+# Use of this software is governed by the MVT License 1.1 that can be found at
+#   https://license.mvt.re/1.1/
+
+import logging
+import re
+
+from mvt.android.parsers import parse_dumpsys_dbinfo
+
+from .base import AndroidExtraction
+
+log = logging.getLogger(__name__)
+
+
+class DumpsysDBInfo(AndroidExtraction):
+    """This module extracts records from battery daily updates."""
+
+    slug = "dumpsys_dbinfo"
+
+    def __init__(self, file_path=None, base_folder=None, output_folder=None,
+                 serial=None, fast_mode=False, log=None, results=[]):
+        super().__init__(file_path=file_path, base_folder=base_folder,
+                         output_folder=output_folder, fast_mode=fast_mode,
+                         log=log, results=results)
+
+    def check_indicators(self):
+        if not self.indicators:
+            return
+
+        for result in self.results:
+            path = result.get("path", "")
+            for part in path.split("/"):
+                ioc = self.indicators.check_app_id(part)
+                if ioc:
+                    result["matched_indicator"] = ioc
+                    self.detected.append(result)
+                    continue
+
+    def run(self):
+        self._adb_connect()
+        output = self._adb_command("dumpsys dbinfo")
+        self._adb_disconnect()
+
+        self.results = parse_dumpsys_dbinfo(output)
+
+        self.log.info("Extracted a total of %d records from database information",
+                      len(self.results))

mvt/android/modules/adb/dumpsys_full.py

@@ -1,5 +1,5 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021 The MVT Project Authors.
+# Copyright (c) 2021-2022 The MVT Project Authors.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
@@ -23,14 +23,12 @@ class DumpsysFull(AndroidExtraction):
     def run(self):
         self._adb_connect()
 
-        stats = self._adb_command("dumpsys")
+        output = self._adb_command("dumpsys")
         if self.output_folder:
-            stats_path = os.path.join(self.output_folder,
-                                      "dumpsys.txt")
-            with open(stats_path, "w") as handle:
-                handle.write(stats)
+            output_path = os.path.join(self.output_folder, "dumpsys.txt")
+            with open(output_path, "w", encoding="utf-8") as handle:
+                handle.write(output)
 
-            log.info("Full dumpsys output stored at %s",
-                     stats_path)
+            log.info("Full dumpsys output stored at %s", output_path)
 
         self._adb_disconnect()

mvt/android/modules/adb/dumpsys_receivers.py

@@ -1,18 +1,21 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021 The MVT Project Authors.
+# Copyright (c) 2021-2022 The MVT Project Authors.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
 import logging
 
+from mvt.android.parsers import parse_dumpsys_receiver_resolver_table
+
 from .base import AndroidExtraction
 
 log = logging.getLogger(__name__)
 
-ACTION_NEW_OUTGOING_SMS = "android.provider.Telephony.NEW_OUTGOING_SMS"
-ACTION_SMS_RECEIVED = "android.provider.Telephony.SMS_RECEIVED"
-ACTION_DATA_SMS_RECEIVED = "android.intent.action.DATA_SMS_RECEIVED"
-ACTION_PHONE_STATE = "android.intent.action.PHONE_STATE"
+INTENT_NEW_OUTGOING_SMS = "android.provider.Telephony.NEW_OUTGOING_SMS"
+INTENT_SMS_RECEIVED = "android.provider.Telephony.SMS_RECEIVED"
+INTENT_DATA_SMS_RECEIVED = "android.intent.action.DATA_SMS_RECEIVED"
+INTENT_PHONE_STATE = "android.intent.action.PHONE_STATE"
+INTENT_NEW_OUTGOING_CALL = "android.intent.action.NEW_OUTGOING_CALL"
 
 
 class DumpsysReceivers(AndroidExtraction):
@@ -24,64 +27,40 @@ class DumpsysReceivers(AndroidExtraction):
                          output_folder=output_folder, fast_mode=fast_mode,
                          log=log, results=results)
 
+        self.results = results if results else {}
+
+    def check_indicators(self):
+        if not self.indicators:
+            return
+
+        for intent, receivers in self.results.items():
+            for receiver in receivers:
+                if intent == INTENT_NEW_OUTGOING_SMS:
+                    self.log.info("Found a receiver to intercept outgoing SMS messages: \"%s\"",
+                                  receiver["receiver"])
+                elif intent == INTENT_SMS_RECEIVED:
+                    self.log.info("Found a receiver to intercept incoming SMS messages: \"%s\"",
+                                  receiver["receiver"])
+                elif intent == INTENT_DATA_SMS_RECEIVED:
+                    self.log.info("Found a receiver to intercept incoming data SMS message: \"%s\"",
+                                  receiver["receiver"])
+                elif intent == INTENT_PHONE_STATE:
+                    self.log.info("Found a receiver monitoring telephony state/incoming calls: \"%s\"",
+                                  receiver["receiver"])
+                elif intent == INTENT_NEW_OUTGOING_CALL:
+                    self.log.info("Found a receiver monitoring outgoing calls: \"%s\"",
+                                  receiver["receiver"])
+
+            ioc = self.indicators.check_app_id(receiver["package_name"])
+            if ioc:
+                receiver["matched_indicator"] = ioc
+                self.detected.append({intent: receiver})
+                continue
+
     def run(self):
         self._adb_connect()
 
         output = self._adb_command("dumpsys package")
-        if not output:
-            return
-
-        activity = None
-        for line in output.split("\n"):
-            # Find activity block markers.
-            if line.strip().startswith(ACTION_NEW_OUTGOING_SMS):
-                activity = ACTION_NEW_OUTGOING_SMS
-                continue
-            elif line.strip().startswith(ACTION_SMS_RECEIVED):
-                activity = ACTION_SMS_RECEIVED
-                continue
-            elif line.strip().startswith(ACTION_PHONE_STATE):
-                activity = ACTION_PHONE_STATE
-                continue
-            elif line.strip().startswith(ACTION_DATA_SMS_RECEIVED):
-                activity = ACTION_DATA_SMS_RECEIVED
-                continue
-
-            # If we are not in an activity block yet, skip.
-            if not activity:
-                continue
-
-            # If we are in a block but the line does not start with 8 spaces
-            # it means the block ended a new one started, so we reset and
-            # continue.
-            if not line.startswith(" " * 8):
-                activity = None
-                continue
-
-            # If we got this far, we are processing receivers for the
-            # activities we are interested in.
-            receiver = line.strip().split(" ")[1]
-            package_name = receiver.split("/")[0]
-            if package_name == "com.google.android.gms":
-                continue
-
-            if activity == ACTION_NEW_OUTGOING_SMS:
-                self.log.info("Found a receiver to intercept outgoing SMS messages: \"%s\"",
-                              receiver)
-            elif activity == ACTION_SMS_RECEIVED:
-                self.log.info("Found a receiver to intercept incoming SMS messages: \"%s\"",
-                              receiver)
-            elif activity == ACTION_DATA_SMS_RECEIVED:
-                self.log.info("Found a receiver to intercept incoming data SMS message: \"%s\"",
-                              receiver)
-            elif activity == ACTION_PHONE_STATE:
-                self.log.info("Found a receiver monitoring telephony state: \"%s\"",
-                              receiver)
-
-            self.results.append({
-                "activity": activity,
-                "package_name": package_name,
-                "receiver": receiver,
-            })
+        self.results = parse_dumpsys_receiver_resolver_table(output)
 
         self._adb_disconnect()

mvt/android/modules/adb/files.py

@@ -1,14 +1,13 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021 The MVT Project Authors.
+# Copyright (c) 2021-2022 The MVT Project Authors.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
-import logging
-import os
-import stat
 import datetime
+import logging
+import stat
 
-from mvt.common.utils import check_for_links, convert_timestamp_to_iso
+from mvt.common.utils import convert_timestamp_to_iso
 
 from .base import AndroidExtraction
 
@@ -23,30 +22,16 @@ class Files(AndroidExtraction):
         super().__init__(file_path=file_path, base_folder=base_folder,
                          output_folder=output_folder, fast_mode=fast_mode,
                          log=log, results=results)
-        self.full_find = None
+        self.full_find = False
 
-    def find_path(self, file_path):
-        """Checks if Android system supports full find command output"""
-        # Check find command params on first run
-        # Run find command with correct args and parse results.
+    def find_files(self, folder):
+        if self.full_find:
+            output = self._adb_command(f"find '{folder}' -printf '%T@ %m %s %u %g %p\n' 2> /dev/null")
 
-        # Check that full file printf options are suppported on first run.
-        if self.full_find == None:
-            output = self._adb_command(f"find '/' -maxdepth 1 -printf '%T@ %m %s %u %g %p\n' 2> /dev/null")
-            if not (output or output.strip().splitlines()):
-                # Full  find command failed to generate output, fallback to basic file arguments
-                self.full_find = False
-            else:
-                self.full_find = True
-
-        found_files = []
-        if self.full_find == True:
-            # Run full file command and collect additonal file information.
-            output = self._adb_command(f"find '{file_path}' -printf '%T@ %m %s %u %g %p\n' 2> /dev/null")
             for file_line in output.splitlines():
                 [unix_timestamp, mode, size, owner, group, full_path] = file_line.rstrip().split(" ", 5)
                 mod_time = convert_timestamp_to_iso(datetime.datetime.utcfromtimestamp(int(float(unix_timestamp))))
-                found_files.append({
+                self.results.append({
                     "path": full_path,
                     "modified_time": mod_time,
                     "mode": mode,
@@ -57,14 +42,9 @@ class Files(AndroidExtraction):
                     "group": group,
                 })
         else:
-            # Run a basic listing of file paths.
-            output = self._adb_command(f"find '{file_path}' 2> /dev/null")
+            output = self._adb_command(f"find '{folder}' 2> /dev/null")
             for file_line in output.splitlines():
-                found_files.append({
-                    "path": file_line.rstrip()
-                })
-
-        return found_files
+                self.results.append({"path": file_line.rstrip()})
 
     def serialize(self, record):
         if "modified_time" in record:
@@ -86,39 +66,32 @@ class Files(AndroidExtraction):
     def check_indicators(self):
         """Check file list for known suspicious files or suspicious properties"""
         self.check_suspicious()
+
         if not self.indicators:
             return
 
         for result in self.results:
-            if self.indicators.check_filename(result["path"]):
-                self.log.warning("Found a known suspicous filename at path: \"%s\"", result["path"])
-                self.detected.append(result)
-
             if self.indicators.check_file_path(result["path"]):
                 self.log.warning("Found a known suspicous file at path: \"%s\"", result["path"])
                 self.detected.append(result)
 
     def run(self):
         self._adb_connect()
-        found_file_paths = []
 
-        DATA_PATHS = ["/data/local/tmp/", "/sdcard/", "/tmp/"]
-        for path in DATA_PATHS:
-            file_info = self.find_path(path)
-            found_file_paths.extend(file_info)
+        output = self._adb_command("find '/' -maxdepth 1 -printf '%T@ %m %s %u %g %p\n' 2> /dev/null")
+        if output or output.strip().splitlines():
+            self.full_find = True
 
-        # Store results
-        self.results.extend(found_file_paths)
-        self.log.info("Found %s files in primary Android data directories.", len(found_file_paths))
+        for data_path in ["/data/local/tmp/", "/sdcard/", "/tmp/"]:
+            self.find_files(data_path)
+
+        self.log.info("Found %s files in primary Android data directories", len(self.results))
 
         if self.fast_mode:
             self.log.info("Flag --fast was enabled: skipping full file listing")
         else:
-            self.log.info("Flag --fast was not enabled: processing full file listing. "
-                          "This may take a while...")
-            output = self.find_path("/")
-            if output and self.output_folder:
-                self.results.extend(output)
-                log.info("List of visible files stored in files.json")
+            self.log.info("Processing full file listing. This may take a while...")
+            self.find_files("/")
+            self.log.info("Found %s total files", len(self.results))
 
         self._adb_disconnect()

mvt/android/modules/adb/getprop.py

@@ -1,18 +1,20 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021 The MVT Project Authors.
+# Copyright (c) 2021-2022 The MVT Project Authors.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
 import logging
-import os
+import re
+
+from mvt.android.parsers import parse_getprop
 
 from .base import AndroidExtraction
 
 log = logging.getLogger(__name__)
 
 
-class DumpsysProcstats(AndroidExtraction):
-    """This module extracts stats on memory consumption by processes."""
+class Getprop(AndroidExtraction):
+    """This module extracts device properties from getprop command."""
 
     def __init__(self, file_path=None, base_folder=None, output_folder=None,
                  serial=None, fast_mode=False, log=None, results=[]):
@@ -20,17 +22,13 @@ class DumpsysProcstats(AndroidExtraction):
                          output_folder=output_folder, fast_mode=fast_mode,
                          log=log, results=results)
 
+        self.results = {} if not results else results
+
     def run(self):
         self._adb_connect()
-
-        output = self._adb_command("dumpsys procstats")
-        if self.output_folder:
-            procstats_path = os.path.join(self.output_folder,
-                                          "dumpsys_procstats.txt")
-            with open(procstats_path, "w") as handle:
-                handle.write(output)
-
-            log.info("Records from dumpsys procstats stored at %s",
-                     procstats_path)
-
+        output = self._adb_command("getprop")
         self._adb_disconnect()
+
+        self.results = parse_getprop(output)
+
+        self.log.info("Extracted %d Android system properties", len(self.results))

mvt/android/modules/adb/logcat.py

@@ -1,5 +1,5 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021 The MVT Project Authors.
+# Copyright (c) 2021-2022 The MVT Project Authors.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
@@ -31,7 +31,7 @@ class Logcat(AndroidExtraction):
         if self.output_folder:
             logcat_path = os.path.join(self.output_folder,
                                        "logcat.txt")
-            with open(logcat_path, "w") as handle:
+            with open(logcat_path, "w", encoding="utf-8") as handle:
                 handle.write(output)
 
             log.info("Current logcat logs stored at %s",
@@ -39,7 +39,7 @@ class Logcat(AndroidExtraction):
 
             logcat_last_path = os.path.join(self.output_folder,
                                             "logcat_last.txt")
-            with open(logcat_last_path, "w") as handle:
+            with open(logcat_last_path, "w", encoding="utf-8") as handle:
                 handle.write(last_output)
 
             log.info("Logcat logs prior to last reboot stored at %s",

mvt/android/modules/adb/packages.py

@@ -1,5 +1,5 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021 The MVT Project Authors.
+# Copyright (c) 2021-2022 The MVT Project Authors.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
@@ -8,10 +8,37 @@ import os
 
 import pkg_resources
 
+from mvt.android.lookups.koodous import koodous_lookup
+from mvt.android.lookups.virustotal import virustotal_lookup
+
 from .base import AndroidExtraction
 
 log = logging.getLogger(__name__)
 
+DANGEROUS_PERMISSIONS_THRESHOLD = 10
+DANGEROUS_PERMISSIONS = [
+    "android.permission.ACCESS_COARSE_LOCATION",
+    "android.permission.ACCESS_FINE_LOCATION",
+    "android.permission.AUTHENTICATE_ACCOUNTS",
+    "android.permission.CAMERA",
+    "android.permission.DISABLE_KEYGUARD",
+    "android.permission.PROCESS_OUTGOING_CALLS",
+    "android.permission.READ_CALENDAR",
+    "android.permission.READ_CALL_LOG",
+    "android.permission.READ_CONTACTS",
+    "android.permission.READ_PHONE_STATE",
+    "android.permission.READ_SMS",
+    "android.permission.RECEIVE_MMS",
+    "android.permission.RECEIVE_SMS",
+    "android.permission.RECEIVE_WAP_PUSH",
+    "android.permission.RECORD_AUDIO",
+    "android.permission.SEND_SMS",
+    "android.permission.SYSTEM_ALERT_WINDOW",
+    "android.permission.USE_CREDENTIALS",
+    "android.permission.USE_SIP",
+    "com.android.browser.permission.READ_HISTORY_BOOKMARKS",
+]
+
 
 class Packages(AndroidExtraction):
     """This module extracts the list of installed packages."""
@@ -42,12 +69,9 @@ class Packages(AndroidExtraction):
         return records
 
     def check_indicators(self):
-        if not self.indicators:
-            return
-
         root_packages_path = os.path.join("..", "..", "data", "root_packages.txt")
         root_packages_string = pkg_resources.resource_string(__name__, root_packages_path)
-        root_packages = root_packages_string.decode("utf-8").split("\n")
+        root_packages = root_packages_string.decode("utf-8").splitlines()
         root_packages = [rp.strip() for rp in root_packages]
 
         for result in self.results:
@@ -55,16 +79,62 @@ class Packages(AndroidExtraction):
                 self.log.warning("Found an installed package related to rooting/jailbreaking: \"%s\"",
                                  result["package_name"])
                 self.detected.append(result)
-            if result["package_name"] in self.indicators.ioc_app_ids:
-                self.log.warning("Found a malicious package name: \"%s\"",
-                                 result["package_name"])
+                continue
+
+            if not self.indicators:
+                continue
+
+            ioc = self.indicators.check_app_id(result.get("package_name"))
+            if ioc:
+                result["matched_indicator"] = ioc
                 self.detected.append(result)
-                for file in result["files"]:
-                    if file["sha256"] in self.indicators.ioc_files_sha256:
-                        self.log.warning("Found a malicious APK: \"%s\" %s",
-                                         result["package_name"],
-                                         file["sha256"])
-                        self.detected.append(result)
+                continue
+
+            for package_file in result["files"]:
+                ioc = self.indicators.check_file_hash(package_file["sha256"])
+                if ioc:
+                    result["matched_indicator"] = ioc
+                    self.detected.append(result)
+
+    @staticmethod
+    def parse_package_for_details(output):
+        details = {
+            "uid": "",
+            "version_name": "",
+            "version_code": "",
+            "timestamp": "",
+            "first_install_time": "",
+            "last_update_time": "",
+            "requested_permissions": [],
+        }
+
+        in_permissions = False
+        for line in output.splitlines():
+            if in_permissions:
+                if line.startswith(" " * 4) and not line.startswith(" " * 6):
+                    in_permissions = False
+                    continue
+
+                permission = line.strip().split(":")[0]
+                details["requested_permissions"].append(permission)
+
+            if line.strip().startswith("userId="):
+                details["uid"] = line.split("=")[1].strip()
+            elif line.strip().startswith("versionName="):
+                details["version_name"] = line.split("=")[1].strip()
+            elif line.strip().startswith("versionCode="):
+                details["version_code"] = line.split("=", 1)[1].strip()
+            elif line.strip().startswith("timeStamp="):
+                details["timestamp"] = line.split("=")[1].strip()
+            elif line.strip().startswith("firstInstallTime="):
+                details["first_install_time"] = line.split("=")[1].strip()
+            elif line.strip().startswith("lastUpdateTime="):
+                details["last_update_time"] = line.split("=")[1].strip()
+            elif line.strip() == "requested permissions:":
+                in_permissions = True
+                continue
+
+        return details
 
     def _get_files_for_package(self, package_name):
         output = self._adb_command(f"pm path {package_name}")
@@ -73,7 +143,7 @@ class Packages(AndroidExtraction):
             return []
 
         package_files = []
-        for file_path in output.split("\n"):
+        for file_path in output.splitlines():
             file_path = file_path.strip()
 
             md5 = self._adb_command(f"md5sum {file_path}").split(" ")[0]
@@ -94,8 +164,9 @@ class Packages(AndroidExtraction):
     def run(self):
         self._adb_connect()
 
-        packages = self._adb_command("pm list packages -U -u -i -f")
-        for line in packages.split("\n"):
+        packages = self._adb_command("pm list packages -u -i -f")
+
+        for line in packages.splitlines():
             line = line.strip()
             if not line.startswith("package:"):
                 continue
@@ -111,31 +182,22 @@ class Packages(AndroidExtraction):
                 if installer == "null":
                     installer = None
 
-            try:
-                uid = fields[2].split(":")[1].strip()
-            except IndexError:
-                uid = None
-
-            dumpsys = self._adb_command(f"dumpsys package {package_name} | grep -A2 timeStamp").split("\n")
-            timestamp = dumpsys[0].split("=")[1].strip()
-            first_install = dumpsys[1].split("=")[1].strip()
-            last_update = dumpsys[2].split("=")[1].strip()
-
             package_files = self._get_files_for_package(package_name)
-
-            self.results.append({
+            new_package = {
                 "package_name": package_name,
                 "file_name": file_name,
                 "installer": installer,
-                "timestamp": timestamp,
-                "first_install_time": first_install,
-                "last_update_time": last_update,
-                "uid": uid,
                 "disabled": False,
                 "system": False,
                 "third_party": False,
                 "files": package_files,
-            })
+            }
+
+            dumpsys_package = self._adb_command(f"dumpsys package {package_name}")
+            package_details = self.parse_package_for_details(dumpsys_package)
+            new_package.update(package_details)
+
+            self.results.append(new_package)
 
         cmds = [
             {"field": "disabled", "arg": "-d"},
@@ -144,7 +206,7 @@ class Packages(AndroidExtraction):
         ]
         for cmd in cmds:
             output = self._adb_command(f"pm list packages {cmd['arg']}")
-            for line in output.split("\n"):
+            for line in output.splitlines():
                 line = line.strip()
                 if not line.startswith("package:"):
                     continue
@@ -154,13 +216,32 @@ class Packages(AndroidExtraction):
                     if result["package_name"] == package_name:
                         self.results[i][cmd["field"]] = True
 
+        for result in self.results:
+            if not result["third_party"]:
+                continue
+
+            dangerous_permissions_count = 0
+            for perm in result["requested_permissions"]:
+                if perm in DANGEROUS_PERMISSIONS:
+                    dangerous_permissions_count += 1
+
+            if dangerous_permissions_count >= DANGEROUS_PERMISSIONS_THRESHOLD:
+                self.log.info("Third-party package \"%s\" requested %d potentially dangerous permissions",
+                              result["package_name"], dangerous_permissions_count)
+
+        packages_to_lookup = []
         for result in self.results:
             if result["system"]:
                 continue
 
+            packages_to_lookup.append(result)
             self.log.info("Found non-system package with name \"%s\" installed by \"%s\" on %s",
                           result["package_name"], result["installer"], result["timestamp"])
 
+        if not self.fast_mode:
+            virustotal_lookup(packages_to_lookup)
+            koodous_lookup(packages_to_lookup)
+
         self.log.info("Extracted at total of %d installed package names",
                       len(self.results))
 

mvt/android/modules/adb/processes.py

@@ -1,5 +1,5 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021 The MVT Project Authors.
+# Copyright (c) 2021-2022 The MVT Project Authors.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
@@ -19,12 +19,22 @@ class Processes(AndroidExtraction):
                          output_folder=output_folder, fast_mode=fast_mode,
                          log=log, results=results)
 
+    def check_indicators(self):
+        if not self.indicators:
+            return
+
+        for result in self.results:
+            ioc = self.indicators.check_app_id(result.get("name", ""))
+            if ioc:
+                result["matched_indicator"] = ioc
+                self.detected.append(result)
+
     def run(self):
         self._adb_connect()
 
         output = self._adb_command("ps -e")
 
-        for line in output.split("\n")[1:]:
+        for line in output.splitlines()[1:]:
             line = line.strip()
             if line == "":
                 continue

mvt/android/modules/adb/root_binaries.py

@@ -1,5 +1,5 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021 The MVT Project Authors.
+# Copyright (c) 2021-2022 The MVT Project Authors.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
@@ -25,7 +25,7 @@ class RootBinaries(AndroidExtraction):
     def run(self):
         root_binaries_path = os.path.join("..", "..", "data", "root_binaries.txt")
         root_binaries_string = pkg_resources.resource_string(__name__, root_binaries_path)
-        root_binaries = root_binaries_string.decode("utf-8").split("\n")
+        root_binaries = root_binaries_string.decode("utf-8").splitlines()
 
         self._adb_connect()