Bumped version

Android split parsers

.github/workflows/python-package.yml

@@ -1,7 +1,7 @@
 # This workflow will install Python dependencies, run tests and lint with a variety of Python versions
 # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
 
-name: Python package
+name: CI
 
 on:
   push:
@@ -16,7 +16,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [3.7, 3.8, 3.9]
+        # python-version: [3.7, 3.8, 3.9]
+        python-version: [3.8, 3.9]
 
     steps:
     - uses: actions/checkout@v2
@@ -27,8 +28,9 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        python -m pip install flake8 pytest safety
+        python -m pip install flake8 pytest safety stix2
         if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+        python -m pip install .
     - name: Lint with flake8
       run: |
         # stop the build if there are Python syntax errors or undefined names
@@ -37,7 +39,5 @@ jobs:
         flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
     - name: Safety checks
       run: safety check
-
-    # - name: Test with pytest
-    #   run: |
-    #     pytest
+    - name: Test with pytest
+      run: pytest

.gitignore

@@ -131,3 +131,6 @@ dmypy.json
 
 # Temporal files
 *~
+
+# IDEA Dev Environment
+.idea

Dockerfile

@@ -38,12 +38,15 @@ RUN apt update \
 # Build libimobiledevice
 # ----------------------
 RUN git clone https://github.com/libimobiledevice/libplist \
+  && git clone https://github.com/libimobiledevice/libimobiledevice-glue \
   && git clone https://github.com/libimobiledevice/libusbmuxd \
   && git clone https://github.com/libimobiledevice/libimobiledevice \
   && git clone https://github.com/libimobiledevice/usbmuxd \
 
   && cd libplist && ./autogen.sh && make && make install && ldconfig \
 
+  && cd ../libimobiledevice-glue && PKG_CONFIG_PATH=/usr/local/lib/pkgconfig ./autogen.sh --prefix=/usr && make && make install && ldconfig \
+
   && cd ../libusbmuxd && PKG_CONFIG_PATH=/usr/local/lib/pkgconfig ./autogen.sh && make && make install && ldconfig \
 
   && cd ../libimobiledevice && PKG_CONFIG_PATH=/usr/local/lib/pkgconfig ./autogen.sh --enable-debug && make && make install && ldconfig \
@@ -51,7 +54,7 @@ RUN git clone https://github.com/libimobiledevice/libplist \
   && cd ../usbmuxd && PKG_CONFIG_PATH=/usr/local/lib/pkgconfig ./autogen.sh --prefix=/usr --sysconfdir=/etc --localstatedir=/var --runstatedir=/run && make && make install \
 
   # Clean up.
-  && cd .. && rm -rf libplist libusbmuxd libimobiledevice usbmuxd
+  && cd .. && rm -rf libplist libimobiledevice-glue libusbmuxd libimobiledevice usbmuxd
 
 # Installing MVT
 # --------------

README.md

@@ -6,6 +6,8 @@
 
 [![](https://img.shields.io/pypi/v/mvt)](https://pypi.org/project/mvt/)
 [![Documentation Status](https://readthedocs.org/projects/mvt/badge/?version=latest)](https://docs.mvt.re/en/latest/?badge=latest)
+[![CI](https://github.com/mvt-project/mvt/actions/workflows/python-package.yml/badge.svg)](https://github.com/mvt-project/mvt/actions/workflows/python-package.yml)
+[![Downloads](https://pepy.tech/badge/mvt)](https://pepy.tech/project/mvt)
 
 Mobile Verification Toolkit (MVT) is a collection of utilities to simplify and automate the process of gathering forensic traces helpful to identify a potential compromise of Android and iOS devices.
 
@@ -15,15 +17,15 @@ It has been developed and released by the [Amnesty International Security Lab](h
 
 ## Installation
 
-MVT can be installed from sources or from [PyPi](https://pypi.org/project/mvt/) (you will need some dependencies, check the [documentation](https://docs.mvt.re/en/latest/install.html)):
+MVT can be installed from sources or from [PyPi](https://pypi.org/project/mvt/) (you will need some dependencies, check the [documentation](https://docs.mvt.re/en/latest/install/)):
 
 ```
 pip3 install mvt
 ```
 
-Alternatively, you can decide to run MVT and all relevant tools through a [Docker container](https://docs.mvt.re/en/latest/docker.html).
+Alternatively, you can decide to run MVT and all relevant tools through a [Docker container](https://docs.mvt.re/en/latest/docker/).
 
-**Please note:** MVT is best run on Linux or Mac systems. [It does not currently support running natively on Windows.](https://docs.mvt.re/en/latest/install.html#mvt-on-windows)
+**Please note:** MVT is best run on Linux or Mac systems. [It does not currently support running natively on Windows.](https://docs.mvt.re/en/latest/install/#mvt-on-windows)
 
 ## Usage
 
@@ -31,4 +33,4 @@ MVT provides two commands `mvt-ios` and `mvt-android`. [Check out the documentat
 
 ## License
 
-The purpose of MVT is to facilitate the ***consensual forensic analysis*** of devices of those who might be targets of sophisticated mobile spyware attacks, especially members of civil society and marginalized communities. We do not want MVT to enable privacy violations of non-consenting individuals.  In order to achieve this, MVT is released under its own license. [Read more here.](https://docs.mvt.re/en/latest/license.html)
+The purpose of MVT is to facilitate the ***consensual forensic analysis*** of devices of those who might be targets of sophisticated mobile spyware attacks, especially members of civil society and marginalized communities. We do not want MVT to enable privacy violations of non-consenting individuals.  In order to achieve this, MVT is released under its own license. [Read more here.](https://docs.mvt.re/en/latest/license/)

dev/mvt-android

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021 The MVT Project Authors.
+# Copyright (c) 2021-2022 The MVT Project Authors.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 

dev/mvt-ios

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021 The MVT Project Authors.
+# Copyright (c) 2021-2022 The MVT Project Authors.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 

docs/android/backup.md

@@ -22,7 +22,7 @@ adb backup -all
 
 ## Unpack the backup
 
-In order to reliable unpack th [Android Backup Extractor (ABE)](https://github.com/nelenkov/android-backup-extractor) to convert it to a readable file format. Make sure that java is installed on your system and use the following command:
+In order to unpack the backup, use [Android Backup Extractor (ABE)](https://github.com/nelenkov/android-backup-extractor) to convert it to a readable file format. Make sure that java is installed on your system and use the following command:
 
 ```bash
 java -jar ~/path/to/abe.jar unpack backup.ab backup.tar
@@ -31,6 +31,8 @@ tar xvf backup.tar
 
 If the backup is encrypted, ABE will prompt you to enter the password.
 
+Alternatively, [ab-decrypt](https://github.com/joernheissler/ab-decrypt) can be used for that purpose.
+
 ## Check the backup
 
 You can then extract SMSs containing links with MVT:

docs/docker.md

@@ -10,6 +10,11 @@ cd mvt
 docker build -t mvt .
 ```
 
+Optionally, you may need to specify your platform to Docker in order to build successfully (Apple M1)
+```bash
+docker build --platform amd64 -t mvt .
+```
+
 Test if the image was created successfully:
 
 ```bash

docs/iocs.md

@@ -28,9 +28,19 @@ The `--iocs` option can be invoked multiple times to let MVT import multiple STI
 mvt-ios check-backup --iocs ~/iocs/malware1.stix --iocs ~/iocs/malware2.stix2 /path/to/backup
 ```
 
+It is also possible to load STIX2 files automatically from the environment variable `MVT_STIX2`:
+
+```bash
+export MVT_STIX2="/home/user/IOC1.stix2:/home/user/IOC2.stix2"
+```
+
 ## Known repositories of STIX2 IOCs
 
 - The [Amnesty International investigations repository](https://github.com/AmnestyTech/investigations) contains STIX-formatted IOCs for:
     - [Pegasus](https://en.wikipedia.org/wiki/Pegasus_(spyware)) ([STIX2](https://raw.githubusercontent.com/AmnestyTech/investigations/master/2021-07-18_nso/pegasus.stix2))
+    - [Predator from Cytrox](https://citizenlab.ca/2021/12/pegasus-vs-predator-dissidents-doubly-infected-iphone-reveals-cytrox-mercenary-spyware/) ([STIX2](https://raw.githubusercontent.com/AmnestyTech/investigations/master/2021-12-16_cytrox/cytrox.stix2))
+- [This repository](https://github.com/Te-k/stalkerware-indicators) contains IOCs for Android stalkerware including [a STIX MVT-compatible file](https://raw.githubusercontent.com/Te-k/stalkerware-indicators/master/stalkerware.stix2).
+
+You can automaticallly download the latest public indicator files with the command `mvt-ios download-iocs` or `mvt-android download-iocs`. These commands download the list of indicators listed [here](https://github.com/mvt-project/mvt/blob/main/public_indicators.json) and store them in the [appdir](https://pypi.org/project/appdirs/) folder. They are then loaded automatically by mvt.
 
 Please [open an issue](https://github.com/mvt-project/mvt/issues/) to suggest new sources of STIX-formatted IOCs.

docs/requirements.txt

@@ -1,4 +1,4 @@
-mkdocs==1.2.1
+mkdocs==1.2.3
 mkdocs-autorefs
 mkdocs-material
 mkdocs-material-extensions

mvt/__init__.py

@@ -1,4 +1,4 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021 The MVT Project Authors.
+# Copyright (c) 2021-2022 The MVT Project Authors.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/

mvt/android/__init__.py

@@ -1,5 +1,5 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021 The MVT Project Authors.
+# Copyright (c) 2021-2022 The MVT Project Authors.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 

mvt/android/download_apks.py

@@ -1,5 +1,5 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021 The MVT Project Authors.
+# Copyright (c) 2021-2022 The MVT Project Authors.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
@@ -7,17 +7,16 @@ import json
 import logging
 import os
 
-import pkg_resources
 from tqdm import tqdm
 
 from mvt.common.module import InsufficientPrivileges
-from mvt.common.utils import get_sha256_from_file_path
 
 from .modules.adb.base import AndroidExtraction
 from .modules.adb.packages import Packages
 
 log = logging.getLogger(__name__)
 
+
 # TODO: Would be better to replace tqdm with rich.progress to reduce
 #       the number of dependencies. Need to investigate whether
 #       it's possible to have a similar callback system.
@@ -58,7 +57,7 @@ class DownloadAPKs(AndroidExtraction):
         :param json_path: Path to the apks.json file to parse.
 
         """
-        with open(json_path, "r") as handle:
+        with open(json_path, "r", encoding="utf-8") as handle:
             packages = json.load(handle)
             return cls(packages=packages)
 
@@ -139,7 +138,7 @@ class DownloadAPKs(AndroidExtraction):
                     packages_selection.append(package)
 
             log.info("Selected only %d packages which are not marked as system",
-                len(packages_selection))
+                     len(packages_selection))
 
         if len(packages_selection) == 0:
             log.info("No packages were selected for download")
@@ -158,44 +157,23 @@ class DownloadAPKs(AndroidExtraction):
             log.info("[%d/%d] Package: %s", counter, len(packages_selection),
                      package["package_name"])
 
-            # Get the file path for the specific package.
-            try:
-                output = self._adb_command(f"pm path {package['package_name']}")
-                output = output.strip().replace("package:", "")
-                if not output:
-                    continue
-            except Exception as e:
-                log.exception("Failed to get path of package %s: %s",
-                              package["package_name"], e)
-                self._adb_reconnect()
-                continue
-
             # Sometimes the package path contains multiple lines for multiple apks.
             # We loop through each line and download each file.
-            for path in output.split("\n"):
-                device_path = path.strip()
-                file_path = self.pull_package_file(package["package_name"],
-                                                   device_path)
-                if not file_path:
+            for package_file in package["files"]:
+                device_path = package_file["path"]
+                local_path = self.pull_package_file(package["package_name"],
+                                                    device_path)
+                if not local_path:
                     continue
 
-                file_info = {
-                    "path": device_path,
-                    "local_name": file_path,
-                    "sha256": get_sha256_from_file_path(file_path),
-                }
-
-                if "files" not in package:
-                    package["files"] = [file_info,]
-                else:
-                    package["files"].append(file_info)
+                package_file["local_path"] = local_path
 
         log.info("Download of selected packages completed")
 
     def save_json(self):
         """Save the results to the package.json file."""
         json_path = os.path.join(self.output_folder, "apks.json")
-        with open(json_path, "w") as handle:
+        with open(json_path, "w", encoding="utf-8") as handle:
             json.dump(self.packages, handle, indent=4)
 
     def run(self):

mvt/android/lookups/__init__.py

@@ -1,4 +1,4 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021 The MVT Project Authors.
+# Copyright (c) 2021-2022 The MVT Project Authors.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/

mvt/android/lookups/koodous.py

@@ -1,5 +1,5 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021 The MVT Project Authors.
+# Copyright (c) 2021-2022 The MVT Project Authors.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
@@ -13,6 +13,7 @@ from rich.text import Text
 
 log = logging.getLogger(__name__)
 
+
 def koodous_lookup(packages):
     log.info("Looking up all extracted files on Koodous (www.koodous.com)")
     log.info("This might take a while...")
@@ -32,7 +33,7 @@ def koodous_lookup(packages):
             res = requests.get(url)
             report = res.json()
 
-            row = [package["package_name"], file["local_name"]]
+            row = [package["package_name"], file["path"]]
 
             if "package_name" in report:
                 trusted = "no"

mvt/android/lookups/virustotal.py

@@ -1,5 +1,5 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021 The MVT Project Authors.
+# Copyright (c) 2021-2022 The MVT Project Authors.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
@@ -13,6 +13,7 @@ from rich.text import Text
 
 log = logging.getLogger(__name__)
 
+
 def get_virustotal_report(hashes):
     apikey = "233f22e200ca5822bd91103043ccac138b910db79f29af5616a9afe8b6f215ad"
     url = f"https://www.virustotal.com/partners/sysinternals/file-reports?apikey={apikey}"
@@ -36,7 +37,12 @@ def get_virustotal_report(hashes):
         log.error("Unexpected response from VirusTotal: %s", res.status_code)
         return None
 
+
 def virustotal_lookup(packages):
+    # NOTE: This is temporary, until we resolved the issue.
+    log.error("Unfortunately VirusTotal lookup is disabled until further notice, due to unresolved issues with the API service.")
+    return
+
     log.info("Looking up all extracted files on VirusTotal (www.virustotal.com)")
 
     unique_hashes = []
@@ -48,6 +54,7 @@ def virustotal_lookup(packages):
     total_unique_hashes = len(unique_hashes)
 
     detections = {}
+
     def virustotal_query(batch):
         report = get_virustotal_report(batch)
         if not report:
@@ -75,7 +82,7 @@ def virustotal_lookup(packages):
 
     for package in packages:
         for file in package.get("files", []):
-            row = [package["package_name"], file["local_name"]]
+            row = [package["package_name"], file["path"]]
 
             if file["sha256"] in detections:
                 detection = detections[file["sha256"]]

mvt/android/modules/__init__.py

@@ -1,4 +1,4 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021 The MVT Project Authors.
+# Copyright (c) 2021-2022 The MVT Project Authors.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/

mvt/android/modules/adb/__init__.py

@@ -1,23 +1,29 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021 The MVT Project Authors.
+# Copyright (c) 2021-2022 The MVT Project Authors.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
 from .chrome_history import ChromeHistory
-from .dumpsys_batterystats import DumpsysBatterystats
+from .dumpsys_accessibility import DumpsysAccessibility
+from .dumpsys_activities import DumpsysActivities
+from .dumpsys_battery_daily import DumpsysBatteryDaily
+from .dumpsys_battery_history import DumpsysBatteryHistory
+from .dumpsys_dbinfo import DumpsysDBInfo
 from .dumpsys_full import DumpsysFull
-from .dumpsys_packages import DumpsysPackages
-from .dumpsys_procstats import DumpsysProcstats
 from .dumpsys_receivers import DumpsysReceivers
 from .files import Files
+from .getprop import Getprop
 from .logcat import Logcat
 from .packages import Packages
 from .processes import Processes
-from .rootbinaries import RootBinaries
+from .root_binaries import RootBinaries
+from .selinux_status import SELinuxStatus
+from .settings import Settings
 from .sms import SMS
 from .whatsapp import Whatsapp
 
-ADB_MODULES = [ChromeHistory, SMS, Whatsapp, Processes,
-               DumpsysBatterystats, DumpsysProcstats,
-               DumpsysPackages, DumpsysReceivers, DumpsysFull,
-               Packages, RootBinaries, Logcat, Files]
+ADB_MODULES = [ChromeHistory, SMS, Whatsapp, Processes, Getprop, Settings,
+               SELinuxStatus, DumpsysBatteryHistory, DumpsysBatteryDaily,
+               DumpsysReceivers, DumpsysActivities, DumpsysAccessibility,
+               DumpsysDBInfo, DumpsysFull, Packages, Logcat, RootBinaries,
+               Files]

mvt/android/modules/adb/base.py

@@ -1,5 +1,5 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021 The MVT Project Authors.
+# Copyright (c) 2021-2022 The MVT Project Authors.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
@@ -15,7 +15,7 @@ from adb_shell.adb_device import AdbDeviceTcp, AdbDeviceUsb
 from adb_shell.auth.keygen import keygen, write_public_keyfile
 from adb_shell.auth.sign_pythonrsa import PythonRSASigner
 from adb_shell.exceptions import (AdbCommandFailureException, DeviceAuthError,
-                                  UsbReadFailedError)
+                                  UsbDeviceNotFoundError, UsbReadFailedError)
 from usb1 import USBErrorAccess, USBErrorBusy
 
 from mvt.common.module import InsufficientPrivileges, MVTModule
@@ -25,6 +25,7 @@ log = logging.getLogger(__name__)
 ADB_KEY_PATH = os.path.expanduser("~/.android/adbkey")
 ADB_PUB_KEY_PATH = os.path.expanduser("~/.android/adbkey.pub")
 
+
 class AndroidExtraction(MVTModule):
     """This class provides a base for all Android extraction modules."""
 
@@ -41,7 +42,7 @@ class AndroidExtraction(MVTModule):
     def _adb_check_keys():
         """Make sure Android adb keys exist."""
         if not os.path.isdir(os.path.dirname(ADB_KEY_PATH)):
-            os.path.makedirs(os.path.dirname(ADB_KEY_PATH))
+            os.makedirs(os.path.dirname(ADB_KEY_PATH))
 
         if not os.path.exists(ADB_KEY_PATH):
             keygen(ADB_KEY_PATH)
@@ -56,12 +57,19 @@ class AndroidExtraction(MVTModule):
         with open(ADB_KEY_PATH, "rb") as handle:
             priv_key = handle.read()
 
-        signer = PythonRSASigner("", priv_key)
+        with open(ADB_PUB_KEY_PATH, "rb") as handle:
+            pub_key = handle.read()
+
+        signer = PythonRSASigner(pub_key, priv_key)
 
         # If no serial was specified or if the serial does not seem to be
         # a HOST:PORT definition, we use the USB transport.
         if not self.serial or ":" not in self.serial:
-            self.device = AdbDeviceUsb(serial=self.serial)
+            try:
+                self.device = AdbDeviceUsb(serial=self.serial)
+            except UsbDeviceNotFoundError:
+                log.critical("No device found. Make sure it is connected and unlocked.")
+                sys.exit(-1)
         # Otherwise we try to use the TCP transport.
         else:
             addr = self.serial.split(":")
@@ -86,7 +94,7 @@ class AndroidExtraction(MVTModule):
             except OSError as e:
                 if e.errno == 113 and self.serial:
                     log.critical("Unable to connect to the device %s: did you specify the correct IP addres?",
-                              self.serial)
+                                 self.serial)
                     sys.exit(-1)
             else:
                 break
@@ -108,7 +116,7 @@ class AndroidExtraction(MVTModule):
         :returns: Output of command
 
         """
-        return self.device.shell(command)
+        return self.device.shell(command, read_timeout_s=200.0)
 
     def _adb_check_if_root(self):
         """Check if we have a `su` binary on the Android device.
@@ -132,7 +140,7 @@ class AndroidExtraction(MVTModule):
 
         """
         return self._adb_command(f"su -c {command}")
-    
+
     def _adb_check_file_exists(self, file):
         """Verify that a file exists.
 
@@ -166,7 +174,7 @@ class AndroidExtraction(MVTModule):
                 self._adb_download_root(remote_path, local_path, progress_callback)
             else:
                 raise Exception(f"Unable to download file {remote_path}: {e}")
-    
+
     def _adb_download_root(self, remote_path, local_path, progress_callback=None):
         try:
             # Check if we have root, if not raise an Exception.
@@ -191,7 +199,7 @@ class AndroidExtraction(MVTModule):
 
             # Delete the copy on /sdcard/.
             self._adb_command(f"rm -rf {new_remote_path}")
-            
+
         except AdbCommandFailureException as e:
             raise Exception(f"Unable to download file {remote_path}: {e}")
 

mvt/android/modules/adb/chrome_history.py

@@ -1,5 +1,5 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021 The MVT Project Authors.
+# Copyright (c) 2021-2022 The MVT Project Authors.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
@@ -16,6 +16,7 @@ log = logging.getLogger(__name__)
 
 CHROME_HISTORY_PATH = "data/data/com.android.chrome/app_chrome/Default/History"
 
+
 class ChromeHistory(AndroidExtraction):
     """This module extracts records from Android's Chrome browsing history."""
 
@@ -33,6 +34,14 @@ class ChromeHistory(AndroidExtraction):
             "data": f"{record['id']} - {record['url']} (visit ID: {record['visit_id']}, redirect source: {record['redirect_source']})"
         }
 
+    def check_indicators(self):
+        if not self.indicators:
+            return
+
+        for result in self.results:
+            if self.indicators.check_domain(result["url"]):
+                self.detected.append(result)
+
     def _parse_db(self, db_path):
         """Parse a Chrome History database file.
 
@@ -59,7 +68,7 @@ class ChromeHistory(AndroidExtraction):
                 "url": item[1],
                 "visit_id": item[2],
                 "timestamp": item[3],
-                "isodate": convert_timestamp_to_iso(convert_chrometime_to_unix[item[3]]),
+                "isodate": convert_timestamp_to_iso(convert_chrometime_to_unix(item[3])),
                 "redirect_source": item[4],
             })
 
@@ -69,5 +78,8 @@ class ChromeHistory(AndroidExtraction):
         log.info("Extracted a total of %d history items", len(self.results))
 
     def run(self):
-        self._adb_process_file(os.path.join("/", CHROME_HISTORY_PATH),
-                               self._parse_db)
+        try:
+            self._adb_process_file(os.path.join("/", CHROME_HISTORY_PATH),
+                                   self._parse_db)
+        except Exception as e:
+            self.log.error(e)

mvt/android/modules/adb/dumpsys_accessibility.py

@@ -0,0 +1,45 @@
+# Mobile Verification Toolkit (MVT)
+# Copyright (c) 2021-2022 The MVT Project Authors.
+# Use of this software is governed by the MVT License 1.1 that can be found at
+#   https://license.mvt.re/1.1/
+
+import logging
+
+from mvt.android.parsers import parse_dumpsys_accessibility
+
+from .base import AndroidExtraction
+
+log = logging.getLogger(__name__)
+
+
+class DumpsysAccessibility(AndroidExtraction):
+    """This module extracts stats on accessibility."""
+
+    def __init__(self, file_path=None, base_folder=None, output_folder=None,
+                 serial=None, fast_mode=False, log=None, results=[]):
+        super().__init__(file_path=file_path, base_folder=base_folder,
+                         output_folder=output_folder, fast_mode=fast_mode,
+                         log=log, results=results)
+
+    def check_indicators(self):
+        if not self.indicators:
+            return
+
+        for result in self.results:
+            ioc = self.indicators.check_app_id(result["package_name"])
+            if ioc:
+                result["matched_indicator"] = ioc
+                self.detected.append(result)
+                continue
+
+    def run(self):
+        self._adb_connect()
+        output = self._adb_command("dumpsys accessibility")
+        self._adb_disconnect()
+
+        self.results = parse_dumpsys_accessibility(output)
+
+        for result in self.results:
+            log.info("Found installed accessibility service \"%s\"", result.get("service"))
+
+        self.log.info("Identified a total of %d accessibility services", len(self.results))

mvt/android/modules/adb/dumpsys_activities.py

@@ -0,0 +1,45 @@
+# Mobile Verification Toolkit (MVT)
+# Copyright (c) 2021-2022 The MVT Project Authors.
+# Use of this software is governed by the MVT License 1.1 that can be found at
+#   https://license.mvt.re/1.1/
+
+import logging
+
+from mvt.android.parsers import parse_dumpsys_activity_resolver_table
+
+from .base import AndroidExtraction
+
+log = logging.getLogger(__name__)
+
+
+class DumpsysActivities(AndroidExtraction):
+    """This module extracts details on receivers for risky activities."""
+
+    def __init__(self, file_path=None, base_folder=None, output_folder=None,
+                 serial=None, fast_mode=False, log=None, results=[]):
+        super().__init__(file_path=file_path, base_folder=base_folder,
+                         output_folder=output_folder, fast_mode=fast_mode,
+                         log=log, results=results)
+
+        self.results = results if results else {}
+
+    def check_indicators(self):
+        if not self.indicators:
+            return
+
+        for intent, activities in self.results.items():
+            for activity in activities:
+                ioc = self.indicators.check_app_id(activity["package_name"])
+                if ioc:
+                    activity["matched_indicator"] = ioc
+                    self.detected.append({intent: activity})
+                    continue
+
+    def run(self):
+        self._adb_connect()
+        output = self._adb_command("dumpsys package")
+        self._adb_disconnect()
+
+        self.results = parse_dumpsys_activity_resolver_table(output)
+
+        self.log.info("Extracted activities for %d intents", len(self.results))

mvt/android/modules/adb/dumpsys_battery_daily.py

@@ -0,0 +1,50 @@
+# Mobile Verification Toolkit (MVT)
+# Copyright (c) 2021-2022 The MVT Project Authors.
+# Use of this software is governed by the MVT License 1.1 that can be found at
+#   https://license.mvt.re/1.1/
+
+import logging
+
+from mvt.android.parsers import parse_dumpsys_battery_daily
+
+from .base import AndroidExtraction
+
+log = logging.getLogger(__name__)
+
+
+class DumpsysBatteryDaily(AndroidExtraction):
+    """This module extracts records from battery daily updates."""
+
+    def __init__(self, file_path=None, base_folder=None, output_folder=None,
+                 serial=None, fast_mode=False, log=None, results=[]):
+        super().__init__(file_path=file_path, base_folder=base_folder,
+                         output_folder=output_folder, fast_mode=fast_mode,
+                         log=log, results=results)
+
+    def serialize(self, record):
+        return {
+            "timestamp": record["from"],
+            "module": self.__class__.__name__,
+            "event": "battery_daily",
+            "data": f"Recorded update of package {record['package_name']} with vers {record['vers']}"
+        }
+
+    def check_indicators(self):
+        if not self.indicators:
+            return
+
+        for result in self.results:
+            ioc = self.indicators.check_app_id(result["package_name"])
+            if ioc:
+                result["matched_indicator"] = ioc
+                self.detected.append(result)
+                continue
+
+    def run(self):
+        self._adb_connect()
+        output = self._adb_command("dumpsys batterystats --daily")
+        self._adb_disconnect()
+
+        self.results = parse_dumpsys_battery_daily(output)
+
+        self.log.info("Extracted %d records from battery daily stats", len(self.results))

mvt/android/modules/adb/dumpsys_battery_history.py

@@ -0,0 +1,42 @@
+# Mobile Verification Toolkit (MVT)
+# Copyright (c) 2021-2022 The MVT Project Authors.
+# Use of this software is governed by the MVT License 1.1 that can be found at
+#   https://license.mvt.re/1.1/
+
+import logging
+
+from mvt.android.parsers import parse_dumpsys_battery_history
+
+from .base import AndroidExtraction
+
+log = logging.getLogger(__name__)
+
+
+class DumpsysBatteryHistory(AndroidExtraction):
+    """This module extracts records from battery history events."""
+
+    def __init__(self, file_path=None, base_folder=None, output_folder=None,
+                 serial=None, fast_mode=False, log=None, results=[]):
+        super().__init__(file_path=file_path, base_folder=base_folder,
+                         output_folder=output_folder, fast_mode=fast_mode,
+                         log=log, results=results)
+
+    def check_indicators(self):
+        if not self.indicators:
+            return
+
+        for result in self.results:
+            ioc = self.indicators.check_app_id(result["package_name"])
+            if ioc:
+                result["matched_indicator"] = ioc
+                self.detected.append(result)
+                continue
+
+    def run(self):
+        self._adb_connect()
+        output = self._adb_command("dumpsys batterystats --history")
+        self._adb_disconnect()
+
+        self.results = parse_dumpsys_battery_history(output)
+
+        self.log.info("Extracted %d records from battery history", len(self.results))

mvt/android/modules/adb/dumpsys_batterystats.py

@@ -1,45 +0,0 @@
-# Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021 The MVT Project Authors.
-# Use of this software is governed by the MVT License 1.1 that can be found at
-#   https://license.mvt.re/1.1/
-
-import logging
-import os
-
-from .base import AndroidExtraction
-
-log = logging.getLogger(__name__)
-
-class DumpsysBatterystats(AndroidExtraction):
-    """This module extracts stats on battery consumption by processes."""
-
-    def __init__(self, file_path=None, base_folder=None, output_folder=None,
-                 serial=None, fast_mode=False, log=None, results=[]):
-        super().__init__(file_path=file_path, base_folder=base_folder,
-                         output_folder=output_folder, fast_mode=fast_mode,
-                         log=log, results=results)
-
-    def run(self):
-        self._adb_connect()
-
-        stats = self._adb_command("dumpsys batterystats")
-        if self.output_folder:
-            stats_path = os.path.join(self.output_folder,
-                                      "dumpsys_batterystats.txt")
-            with open(stats_path, "w") as handle:
-                handle.write(stats)
-
-            log.info("Records from dumpsys batterystats stored at %s",
-                      stats_path)
-
-        history = self._adb_command("dumpsys batterystats --history")
-        if self.output_folder:
-            history_path = os.path.join(self.output_folder,
-                                        "dumpsys_batterystats_history.txt")
-            with open(history_path, "w") as handle:
-                handle.write(history)
-
-            log.info("History records from dumpsys batterystats stored at %s",
-                     history_path)
-
-        self._adb_disconnect()

mvt/android/modules/adb/dumpsys_dbinfo.py

@@ -0,0 +1,48 @@
+# Mobile Verification Toolkit (MVT)
+# Copyright (c) 2021-2022 The MVT Project Authors.
+# Use of this software is governed by the MVT License 1.1 that can be found at
+#   https://license.mvt.re/1.1/
+
+import logging
+import re
+
+from mvt.android.parsers import parse_dumpsys_dbinfo
+
+from .base import AndroidExtraction
+
+log = logging.getLogger(__name__)
+
+
+class DumpsysDBInfo(AndroidExtraction):
+    """This module extracts records from battery daily updates."""
+
+    slug = "dumpsys_dbinfo"
+
+    def __init__(self, file_path=None, base_folder=None, output_folder=None,
+                 serial=None, fast_mode=False, log=None, results=[]):
+        super().__init__(file_path=file_path, base_folder=base_folder,
+                         output_folder=output_folder, fast_mode=fast_mode,
+                         log=log, results=results)
+
+    def check_indicators(self):
+        if not self.indicators:
+            return
+
+        for result in self.results:
+            path = result.get("path", "")
+            for part in path.split("/"):
+                ioc = self.indicators.check_app_id(part)
+                if ioc:
+                    result["matched_indicator"] = ioc
+                    self.detected.append(result)
+                    continue
+
+    def run(self):
+        self._adb_connect()
+        output = self._adb_command("dumpsys dbinfo")
+        self._adb_disconnect()
+
+        self.results = parse_dumpsys_dbinfo(output)
+
+        self.log.info("Extracted a total of %d records from database information",
+                      len(self.results))

mvt/android/modules/adb/dumpsys_full.py

@@ -1,5 +1,5 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021 The MVT Project Authors.
+# Copyright (c) 2021-2022 The MVT Project Authors.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
@@ -10,6 +10,7 @@ from .base import AndroidExtraction
 
 log = logging.getLogger(__name__)
 
+
 class DumpsysFull(AndroidExtraction):
     """This module extracts stats on battery consumption by processes."""
 
@@ -22,14 +23,12 @@ class DumpsysFull(AndroidExtraction):
     def run(self):
         self._adb_connect()
 
-        stats = self._adb_command("dumpsys")
+        output = self._adb_command("dumpsys")
         if self.output_folder:
-            stats_path = os.path.join(self.output_folder,
-                                      "dumpsys.txt")
-            with open(stats_path, "w") as handle:
-                handle.write(stats)
+            output_path = os.path.join(self.output_folder, "dumpsys.txt")
+            with open(output_path, "w", encoding="utf-8") as handle:
+                handle.write(output)
 
-            log.info("Full dumpsys output stored at %s",
-                      stats_path)
+            log.info("Full dumpsys output stored at %s", output_path)
 
         self._adb_disconnect()

mvt/android/modules/adb/dumpsys_receivers.py

@@ -1,19 +1,22 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021 The MVT Project Authors.
+# Copyright (c) 2021-2022 The MVT Project Authors.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
 import logging
-import os
+
+from mvt.android.parsers import parse_dumpsys_receiver_resolver_table
 
 from .base import AndroidExtraction
 
 log = logging.getLogger(__name__)
 
-ACTION_NEW_OUTGOING_SMS = "android.provider.Telephony.NEW_OUTGOING_SMS"
-ACTION_SMS_RECEIVED = "android.provider.Telephony.SMS_RECEIVED"
-ACTION_DATA_SMS_RECEIVED = "android.intent.action.DATA_SMS_RECEIVED"
-ACTION_PHONE_STATE = "android.intent.action.PHONE_STATE"
+INTENT_NEW_OUTGOING_SMS = "android.provider.Telephony.NEW_OUTGOING_SMS"
+INTENT_SMS_RECEIVED = "android.provider.Telephony.SMS_RECEIVED"
+INTENT_DATA_SMS_RECEIVED = "android.intent.action.DATA_SMS_RECEIVED"
+INTENT_PHONE_STATE = "android.intent.action.PHONE_STATE"
+INTENT_NEW_OUTGOING_CALL = "android.intent.action.NEW_OUTGOING_CALL"
+
 
 class DumpsysReceivers(AndroidExtraction):
     """This module extracts details on receivers for risky activities."""
@@ -24,64 +27,40 @@ class DumpsysReceivers(AndroidExtraction):
                          output_folder=output_folder, fast_mode=fast_mode,
                          log=log, results=results)
 
+        self.results = results if results else {}
+
+    def check_indicators(self):
+        if not self.indicators:
+            return
+
+        for intent, receivers in self.results.items():
+            for receiver in receivers:
+                if intent == INTENT_NEW_OUTGOING_SMS:
+                    self.log.info("Found a receiver to intercept outgoing SMS messages: \"%s\"",
+                                  receiver["receiver"])
+                elif intent == INTENT_SMS_RECEIVED:
+                    self.log.info("Found a receiver to intercept incoming SMS messages: \"%s\"",
+                                  receiver["receiver"])
+                elif intent == INTENT_DATA_SMS_RECEIVED:
+                    self.log.info("Found a receiver to intercept incoming data SMS message: \"%s\"",
+                                  receiver["receiver"])
+                elif intent == INTENT_PHONE_STATE:
+                    self.log.info("Found a receiver monitoring telephony state/incoming calls: \"%s\"",
+                                  receiver["receiver"])
+                elif intent == INTENT_NEW_OUTGOING_CALL:
+                    self.log.info("Found a receiver monitoring outgoing calls: \"%s\"",
+                                  receiver["receiver"])
+
+            ioc = self.indicators.check_app_id(receiver["package_name"])
+            if ioc:
+                receiver["matched_indicator"] = ioc
+                self.detected.append({intent: receiver})
+                continue
+
     def run(self):
         self._adb_connect()
 
         output = self._adb_command("dumpsys package")
-        if not output:
-            return
-
-        activity = None
-        for line in output.split("\n"):
-            # Find activity block markers.
-            if line.strip().startswith(ACTION_NEW_OUTGOING_SMS):
-                activity = ACTION_NEW_OUTGOING_SMS
-                continue
-            elif line.strip().startswith(ACTION_SMS_RECEIVED):
-                activity = ACTION_SMS_RECEIVED
-                continue
-            elif line.strip().startswith(ACTION_PHONE_STATE):
-                activity = ACTION_PHONE_STATE
-                continue
-            elif line.strip().startswith(ACTION_DATA_SMS_RECEIVED):
-                activity = ACTION_DATA_SMS_RECEIVED
-                continue
-
-            # If we are not in an activity block yet, skip.
-            if not activity:
-                continue
-
-            # If we are in a block but the line does not start with 8 spaces
-            # it means the block ended a new one started, so we reset and
-            # continue.
-            if not line.startswith(" " * 8):
-                activity = None
-                continue
-
-            # If we got this far, we are processing receivers for the
-            # activities we are interested in.
-            receiver = line.strip().split(" ")[1]
-            package_name = receiver.split("/")[0]
-            if package_name == "com.google.android.gms":
-                continue
-
-            if activity == ACTION_NEW_OUTGOING_SMS:
-                self.log.info("Found a receiver to intercept outgoing SMS messages: \"%s\"",
-                    receiver)
-            elif activity == ACTION_SMS_RECEIVED:
-                self.log.info("Found a receiver to intercept incoming SMS messages: \"%s\"",
-                    receiver)
-            elif activity == ACTION_DATA_SMS_RECEIVED:
-                self.log.info("Found a receiver to intercept incoming data SMS message: \"%s\"",
-                    receiver)
-            elif activity == ACTION_PHONE_STATE:
-                self.log.info("Found a receiver monitoring telephony state: \"%s\"",
-                    receiver)
-
-            self.results.append({
-                "activity": activity,
-                "package_name": package_name,
-                "receiver": receiver,
-            })
+        self.results = parse_dumpsys_receiver_resolver_table(output)
 
         self._adb_disconnect()

mvt/android/modules/adb/files.py

@@ -1,33 +1,97 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021 The MVT Project Authors.
+# Copyright (c) 2021-2022 The MVT Project Authors.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
+import datetime
 import logging
-import os
+import stat
+
+from mvt.common.utils import convert_timestamp_to_iso
 
 from .base import AndroidExtraction
 
 log = logging.getLogger(__name__)
 
+
 class Files(AndroidExtraction):
-    """This module extracts the list of installed packages."""
+    """This module extracts the list of files on the device."""
 
     def __init__(self, file_path=None, base_folder=None, output_folder=None,
                  serial=None, fast_mode=False, log=None, results=[]):
         super().__init__(file_path=file_path, base_folder=base_folder,
                          output_folder=output_folder, fast_mode=fast_mode,
                          log=log, results=results)
+        self.full_find = False
+
+    def find_files(self, folder):
+        if self.full_find:
+            output = self._adb_command(f"find '{folder}' -printf '%T@ %m %s %u %g %p\n' 2> /dev/null")
+
+            for file_line in output.splitlines():
+                [unix_timestamp, mode, size, owner, group, full_path] = file_line.rstrip().split(" ", 5)
+                mod_time = convert_timestamp_to_iso(datetime.datetime.utcfromtimestamp(int(float(unix_timestamp))))
+                self.results.append({
+                    "path": full_path,
+                    "modified_time": mod_time,
+                    "mode": mode,
+                    "is_suid": (int(mode, 8) & stat.S_ISUID) == 2048,
+                    "is_sgid": (int(mode, 8) & stat.S_ISGID) == 1024,
+                    "size": size,
+                    "owner": owner,
+                    "group": group,
+                })
+        else:
+            output = self._adb_command(f"find '{folder}' 2> /dev/null")
+            for file_line in output.splitlines():
+                self.results.append({"path": file_line.rstrip()})
+
+    def serialize(self, record):
+        if "modified_time" in record:
+            return {
+                "timestamp": record["modified_time"],
+                "module": self.__class__.__name__,
+                "event": "file_modified",
+                "data": record["path"],
+            }
+
+    def check_suspicious(self):
+        """Check for files with suspicious permissions"""
+        for result in sorted(self.results, key=lambda item: item["path"]):
+            if result.get("is_suid"):
+                self.log.warning("Found an SUID file in a non-standard directory \"%s\".",
+                                 result["path"])
+                self.detected.append(result)
+
+    def check_indicators(self):
+        """Check file list for known suspicious files or suspicious properties"""
+        self.check_suspicious()
+
+        if not self.indicators:
+            return
+
+        for result in self.results:
+            if self.indicators.check_file_path(result["path"]):
+                self.log.warning("Found a known suspicous file at path: \"%s\"", result["path"])
+                self.detected.append(result)
 
     def run(self):
         self._adb_connect()
 
-        output = self._adb_command("find / -type f 2> /dev/null")
-        if output and self.output_folder:
-            files_txt_path = os.path.join(self.output_folder, "files.txt")
-            with open(files_txt_path, "w") as handle:
-                handle.write(output)
+        output = self._adb_command("find '/' -maxdepth 1 -printf '%T@ %m %s %u %g %p\n' 2> /dev/null")
+        if output or output.strip().splitlines():
+            self.full_find = True
 
-            log.info("List of visible files stored at %s", files_txt_path)
+        for data_path in ["/data/local/tmp/", "/sdcard/", "/tmp/"]:
+            self.find_files(data_path)
+
+        self.log.info("Found %s files in primary Android data directories", len(self.results))
+
+        if self.fast_mode:
+            self.log.info("Flag --fast was enabled: skipping full file listing")
+        else:
+            self.log.info("Processing full file listing. This may take a while...")
+            self.find_files("/")
+            self.log.info("Found %s total files", len(self.results))
 
         self._adb_disconnect()

mvt/android/modules/adb/getprop.py

@@ -1,17 +1,20 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021 The MVT Project Authors.
+# Copyright (c) 2021-2022 The MVT Project Authors.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
 import logging
-import os
+import re
+
+from mvt.android.parsers import parse_getprop
 
 from .base import AndroidExtraction
 
 log = logging.getLogger(__name__)
 
-class DumpsysProcstats(AndroidExtraction):
-    """This module extracts stats on memory consumption by processes."""
+
+class Getprop(AndroidExtraction):
+    """This module extracts device properties from getprop command."""
 
     def __init__(self, file_path=None, base_folder=None, output_folder=None,
                  serial=None, fast_mode=False, log=None, results=[]):
@@ -19,17 +22,13 @@ class DumpsysProcstats(AndroidExtraction):
                          output_folder=output_folder, fast_mode=fast_mode,
                          log=log, results=results)
 
+        self.results = {} if not results else results
+
     def run(self):
         self._adb_connect()
-
-        output = self._adb_command("dumpsys procstats")
-        if self.output_folder:
-            procstats_path = os.path.join(self.output_folder,
-                                          "dumpsys_procstats.txt")
-            with open(procstats_path, "w") as handle:
-                handle.write(output)
-
-            log.info("Records from dumpsys procstats stored at %s",
-                     procstats_path)
-
+        output = self._adb_command("getprop")
         self._adb_disconnect()
+
+        self.results = parse_getprop(output)
+
+        self.log.info("Extracted %d Android system properties", len(self.results))