Autofix for ruff

Add a module to parse uninstalled apps from dumpsys data

docs/command_completion.md

@@ -0,0 +1,37 @@
+# Command Completion 
+
+MVT utilizes the [Click](https://click.palletsprojects.com/en/stable/) library for creating its command line interface. 
+
+Click provides tab completion support for Bash (version 4.4 and up), Zsh, and Fish.
+
+To enable it, you need to manually register a special function with your shell, which varies depending on the shell you are using.
+
+The following describes how to generate the command completion scripts and add them to your shell configuration. 
+
+`You will need to start a new shell for the changes to take effect.`
+
+### For Bash
+
+```bash
+# Generates bash completion scripts
+echo "$(_MVT_IOS_COMPLETE=bash_source mvt-ios)" > ~/.mvt-ios-complete.bash &&
+echo "$(_MVT_ANDROID_COMPLETE=bash_source mvt-android)" > ~/.mvt-android-complete.bash
+
+# Sources the scripts in ~/.bashrc.
+. ~/.mvt-ios-complete.bash && .  ~/.mvt-android-complete.bash
+```
+
+### For Zsh
+
+```bash
+# Generates zsh completion scripts
+echo "$(_MVT_IOS_COMPLETE=zsh_source mvt-ios)" >  ~/.mvt-ios-complete.zsh &&
+echo "$(_MVT_ANDROID_COMPLETE=zsh_source mvt-android)" > ~/.mvt-android-complete.zsh
+
+# Sources the scripts in ~/.zshrc.
+.  ~/.mvt-ios-complete.zsh  && .  ~/.mvt-android-complete.zsh
+```
+
+For more information, visit the official [Click Docs](https://click.palletsprojects.com/en/stable/shell-completion/#enabling-completion).
+
+

src/mvt/android/artifacts/dumpsys_appops.py

@@ -51,7 +51,7 @@ class DumpsysAppopsArtifact(AndroidArtifact):
                     and perm["access"] == "allow"
                 ):
                     self.log.info(
-                        "Package %s with REQUEST_INSTALL_PACKAGES " "permission",
+                        "Package %s with REQUEST_INSTALL_PACKAGES permission",
                         result["package_name"],
                     )
 

src/mvt/android/artifacts/dumpsys_packages.py

@@ -16,8 +16,7 @@ class DumpsysPackagesArtifact(AndroidArtifact):
         for result in self.results:
             if result["package_name"] in ROOT_PACKAGES:
                 self.log.warning(
-                    "Found an installed package related to "
-                    'rooting/jailbreaking: "%s"',
+                    'Found an installed package related to rooting/jailbreaking: "%s"',
                     result["package_name"],
                 )
                 self.detected.append(result)

src/mvt/android/artifacts/dumpsys_platform_compat.py

@@ -0,0 +1,42 @@
+# Mobile Verification Toolkit (MVT)
+# Copyright (c) 2021-2023 The MVT Authors.
+# Use of this software is governed by the MVT License 1.1 that can be found at
+#   https://license.mvt.re/1.1/
+
+from .artifact import AndroidArtifact
+
+
+class DumpsysPlatformCompatArtifact(AndroidArtifact):
+    """
+    Parser for uninstalled apps listed in platform_compat section.
+    """
+
+    def check_indicators(self) -> None:
+        if not self.indicators:
+            return
+
+        for result in self.results:
+            ioc = self.indicators.check_app_id(result["package_name"])
+            if ioc:
+                result["matched_indicator"] = ioc
+                self.detected.append(result)
+                continue
+
+    def parse(self, data: str) -> None:
+        for line in data.splitlines():
+            if not line.startswith("ChangeId(168419799; name=DOWNSCALED;"):
+                continue
+
+            if line.strip() == "":
+                break
+
+            # Look for rawOverrides field
+            if "rawOverrides={" in line:
+                # Extract the content inside the braces for rawOverrides
+                overrides_field = line.split("rawOverrides={", 1)[1].split("};", 1)[0]
+
+                for entry in overrides_field.split(", "):
+                    # Extract app name
+                    uninstall_app = entry.split("=")[0].strip()
+
+                    self.results.append({"package_name": uninstall_app})

src/mvt/android/modules/adb/base.py

@@ -326,8 +326,7 @@ class AndroidExtraction(MVTModule):
 
         if not header["backup"]:
             self.log.error(
-                "Extracting SMS via Android backup failed. "
-                "No valid backup data found."
+                "Extracting SMS via Android backup failed. No valid backup data found."
             )
             return None
 

src/mvt/android/modules/adb/packages.py

@@ -75,8 +75,7 @@ class Packages(AndroidExtraction):
         for result in self.results:
             if result["package_name"] in ROOT_PACKAGES:
                 self.log.warning(
-                    "Found an installed package related to "
-                    'rooting/jailbreaking: "%s"',
+                    'Found an installed package related to rooting/jailbreaking: "%s"',
                     result["package_name"],
                 )
                 self.detected.append(result)

src/mvt/android/modules/adb/sms.py

@@ -70,7 +70,7 @@ class SMS(AndroidExtraction):
             "timestamp": record["isodate"],
             "module": self.__class__.__name__,
             "event": f"sms_{record['direction']}",
-            "data": f"{record.get('address', 'unknown source')}: \"{body}\"",
+            "data": f'{record.get("address", "unknown source")}: "{body}"',
         }
 
     def check_indicators(self) -> None:

src/mvt/android/modules/androidqf/__init__.py

@@ -14,6 +14,7 @@ from .dumpsys_receivers import DumpsysReceivers
 from .dumpsys_adb import DumpsysADBState
 from .getprop import Getprop
 from .packages import Packages
+from .dumpsys_platform_compat import DumpsysPlatformCompat
 from .processes import Processes
 from .settings import Settings
 from .sms import SMS
@@ -29,6 +30,7 @@ ANDROIDQF_MODULES = [
     DumpsysBatteryHistory,
     DumpsysADBState,
     Packages,
+    DumpsysPlatformCompat,
     Processes,
     Getprop,
     Settings,

src/mvt/android/modules/androidqf/dumpsys_platform_compat.py

@@ -0,0 +1,44 @@
+# Mobile Verification Toolkit (MVT)
+# Copyright (c) 2021-2023 The MVT Authors.
+# Use of this software is governed by the MVT License 1.1 that can be found at
+#   https://license.mvt.re/1.1/
+
+import logging
+from typing import Optional
+
+from mvt.android.artifacts.dumpsys_platform_compat import DumpsysPlatformCompatArtifact
+
+from .base import AndroidQFModule
+
+
+class DumpsysPlatformCompat(DumpsysPlatformCompatArtifact, AndroidQFModule):
+    """This module extracts details on uninstalled apps."""
+
+    def __init__(
+        self,
+        file_path: Optional[str] = None,
+        target_path: Optional[str] = None,
+        results_path: Optional[str] = None,
+        module_options: Optional[dict] = None,
+        log: logging.Logger = logging.getLogger(__name__),
+        results: Optional[list] = None,
+    ) -> None:
+        super().__init__(
+            file_path=file_path,
+            target_path=target_path,
+            results_path=results_path,
+            module_options=module_options,
+            log=log,
+            results=results,
+        )
+
+    def run(self) -> None:
+        dumpsys_file = self._get_files_by_pattern("*/dumpsys.txt")
+        if not dumpsys_file:
+            return
+
+        data = self._get_file_content(dumpsys_file[0]).decode("utf-8", errors="replace")
+        content = self.extract_dumpsys_section(data, "DUMP OF SERVICE platform_compat:")
+        self.parse(content)
+
+        self.log.info("Found %d uninstalled apps", len(self.results))

src/mvt/android/modules/androidqf/packages.py

@@ -44,8 +44,7 @@ class Packages(AndroidQFModule):
         for result in self.results:
             if result["name"] in ROOT_PACKAGES:
                 self.log.warning(
-                    "Found an installed package related to "
-                    'rooting/jailbreaking: "%s"',
+                    'Found an installed package related to rooting/jailbreaking: "%s"',
                     result["name"],
                 )
                 self.detected.append(result)

src/mvt/android/modules/bugreport/__init__.py

@@ -11,6 +11,7 @@ from .battery_history import BatteryHistory
 from .dbinfo import DBInfo
 from .getprop import Getprop
 from .packages import Packages
+from .platform_compat import PlatformCompat
 from .receivers import Receivers
 from .adb_state import DumpsysADBState
 
@@ -23,6 +24,7 @@ BUGREPORT_MODULES = [
     DBInfo,
     Getprop,
     Packages,
+    PlatformCompat,
     Receivers,
     DumpsysADBState,
 ]

src/mvt/android/modules/bugreport/platform_compat.py

@@ -0,0 +1,48 @@
+# Mobile Verification Toolkit (MVT)
+# Copyright (c) 2021-2023 The MVT Authors.
+# Use of this software is governed by the MVT License 1.1 that can be found at
+#   https://license.mvt.re/1.1/
+
+import logging
+from typing import Optional
+
+from mvt.android.artifacts.dumpsys_platform_compat import DumpsysPlatformCompatArtifact
+
+from mvt.android.modules.bugreport.base import BugReportModule
+
+
+class PlatformCompat(DumpsysPlatformCompatArtifact, BugReportModule):
+    """This module extracts details on uninstalled apps."""
+
+    def __init__(
+        self,
+        file_path: Optional[str] = None,
+        target_path: Optional[str] = None,
+        results_path: Optional[str] = None,
+        module_options: Optional[dict] = None,
+        log: logging.Logger = logging.getLogger(__name__),
+        results: Optional[list] = None,
+    ) -> None:
+        super().__init__(
+            file_path=file_path,
+            target_path=target_path,
+            results_path=results_path,
+            module_options=module_options,
+            log=log,
+            results=results,
+        )
+
+    def run(self) -> None:
+        data = self._get_dumpstate_file()
+        if not data:
+            self.log.error(
+                "Unable to find dumpstate file. "
+                "Did you provide a valid bug report archive?"
+            )
+            return
+
+        data = data.decode("utf-8", errors="replace")
+        content = self.extract_dumpsys_section(data, "DUMP OF SERVICE platform_compat:")
+        self.parse(content)
+
+        self.log.info("Found %d uninstalled apps", len(self.results))

src/mvt/common/command.py

@@ -81,7 +81,7 @@ class Command:
             os.path.join(self.results_path, "command.log")
         )
         formatter = logging.Formatter(
-            "%(asctime)s - %(name)s - " "%(levelname)s - %(message)s"
+            "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
         )
         file_handler.setLevel(logging.DEBUG)
         file_handler.setFormatter(formatter)

src/mvt/common/indicators.py

@@ -383,8 +383,7 @@ class Indicators:
         for ioc in self.get_iocs("urls"):
             if ioc["value"] == url:
                 self.log.warning(
-                    "Found a known suspicious URL %s "
-                    'matching indicator "%s" from "%s"',
+                    'Found a known suspicious URL %s matching indicator "%s" from "%s"',
                     url,
                     ioc["value"],
                     ioc["name"],

src/mvt/ios/cli.py

@@ -100,7 +100,7 @@ def decrypt_backup(ctx, destination, password, key_file, hashes, backup_path):
     if key_file:
         if MVT_IOS_BACKUP_PASSWORD in os.environ:
             log.info(
-                "Ignoring %s environment variable, using --key-file" "'%s' instead",
+                "Ignoring %s environment variable, using --key-file'%s' instead",
                 MVT_IOS_BACKUP_PASSWORD,
                 key_file,
             )
@@ -114,7 +114,7 @@ def decrypt_backup(ctx, destination, password, key_file, hashes, backup_path):
 
         if MVT_IOS_BACKUP_PASSWORD in os.environ:
             log.info(
-                "Ignoring %s environment variable, using --password" "argument instead",
+                "Ignoring %s environment variable, using --passwordargument instead",
                 MVT_IOS_BACKUP_PASSWORD,
             )
 
@@ -168,8 +168,7 @@ def extract_key(password, key_file, backup_path):
 
         if MVT_IOS_BACKUP_PASSWORD in os.environ:
             log.info(
-                "Ignoring %s environment variable, using --password "
-                "argument instead",
+                "Ignoring %s environment variable, using --password argument instead",
                 MVT_IOS_BACKUP_PASSWORD,
             )
     elif MVT_IOS_BACKUP_PASSWORD in os.environ:

src/mvt/ios/modules/backup/backup_info.py

@@ -41,7 +41,7 @@ class BackupInfo(IOSExtraction):
         info_path = os.path.join(self.target_path, "Info.plist")
         if not os.path.exists(info_path):
             raise DatabaseNotFoundError(
-                "No Info.plist at backup path, unable to extract device " "information"
+                "No Info.plist at backup path, unable to extract device information"
             )
 
         with open(info_path, "rb") as handle:

src/mvt/ios/modules/backup/manifest.py

@@ -110,8 +110,7 @@ class Manifest(IOSExtraction):
                 ioc = self.indicators.check_url(part)
                 if ioc:
                     self.log.warning(
-                        'Found mention of domain "%s" in a backup file with '
-                        "path: %s",
+                        'Found mention of domain "%s" in a backup file with path: %s',
                         ioc["value"],
                         rel_path,
                     )

src/mvt/ios/modules/base.py

@@ -74,7 +74,7 @@ class IOSExtraction(MVTModule):
 
         if not shutil.which("sqlite3"):
             raise DatabaseCorruptedError(
-                "failed to recover without sqlite3 binary: please install " "sqlite3!"
+                "failed to recover without sqlite3 binary: please install sqlite3!"
             )
         if '"' in file_path:
             raise DatabaseCorruptedError(

src/mvt/ios/modules/mixed/sms.py

@@ -43,7 +43,7 @@ class SMS(IOSExtraction):
 
     def serialize(self, record: dict) -> Union[dict, list]:
         text = record["text"].replace("\n", "\\n")
-        sms_data = f"{record['service']}: {record['guid']} \"{text}\" from {record['phone_number']} ({record['account']})"
+        sms_data = f'{record["service"]}: {record["guid"]} "{text}" from {record["phone_number"]} ({record["account"]})'
         records = [
             {
                 "timestamp": record["isodate"],

src/mvt/ios/modules/mixed/webkit_session_resource_log.py

@@ -100,7 +100,7 @@ class WebkitSessionResourceLog(IOSExtraction):
                         redirect_path += ", ".join(source_domains)
                         redirect_path += " -> "
 
-                    redirect_path += f"ORIGIN: \"{entry['origin']}\""
+                    redirect_path += f'ORIGIN: "{entry["origin"]}"'
 
                     if len(destination_domains) > 0:
                         redirect_path += " -> "

tests/android/test_artifact_dumpsys_platform_compat.py

@@ -0,0 +1,40 @@
+# Mobile Verification Toolkit (MVT)
+# Copyright (c) 2021-2023 The MVT Authors.
+# Use of this software is governed by the MVT License 1.1 that can be found at
+#   https://license.mvt.re/1.1/
+import logging
+
+from mvt.android.artifacts.dumpsys_platform_compat import DumpsysPlatformCompatArtifact
+from mvt.common.indicators import Indicators
+
+from ..utils import get_artifact
+
+
+class TestDumpsysPlatformCompatArtifact:
+    def test_parsing(self):
+        dbi = DumpsysPlatformCompatArtifact()
+        file = get_artifact("android_data/dumpsys_platform_compat.txt")
+        with open(file) as f:
+            data = f.read()
+
+        assert len(dbi.results) == 0
+        dbi.parse(data)
+        assert len(dbi.results) == 2
+        assert dbi.results[0]["package_name"] == "org.torproject.torbrowser"
+        assert dbi.results[1]["package_name"] == "org.article19.circulo.next"
+
+    def test_ioc_check(self, indicator_file):
+        dbi = DumpsysPlatformCompatArtifact()
+        file = get_artifact("android_data/dumpsys_platform_compat.txt")
+        with open(file) as f:
+            data = f.read()
+        dbi.parse(data)
+
+        ind = Indicators(log=logging.getLogger())
+        ind.parse_stix2(indicator_file)
+        ind.ioc_collections[0]["app_ids"].append("org.torproject.torbrowser")
+        ind.ioc_collections[0]["app_ids"].append("org.article19.circulo.next")
+        dbi.indicators = ind
+        assert len(dbi.detected) == 0
+        dbi.check_indicators()
+        assert len(dbi.detected) == 2

tests/android_androidqf/test_dumpsys_platform_compat.py

@@ -0,0 +1,23 @@
+# Mobile Verification Toolkit (MVT)
+# Copyright (c) 2021-2023 The MVT Authors.
+# Use of this software is governed by the MVT License 1.1 that can be found at
+#   https://license.mvt.re/1.1/
+
+from pathlib import Path
+
+from mvt.android.modules.androidqf.dumpsys_platform_compat import DumpsysPlatformCompat
+from mvt.common.module import run_module
+
+from ..utils import get_android_androidqf, list_files
+
+
+class TestDumpsysPlatformCompatModule:
+    def test_parsing(self):
+        data_path = get_android_androidqf()
+        m = DumpsysPlatformCompat(target_path=data_path)
+        files = list_files(data_path)
+        parent_path = Path(data_path).absolute().parent.as_posix()
+        m.from_folder(parent_path, files)
+        run_module(m)
+        assert len(m.results) == 2
+        assert len(m.detected) == 0

tests/artifacts/android_data/bugreport/dumpstate.txt

@@ -246,6 +246,23 @@ Packages:
         com.instagram.direct.share.handler.DirectMultipleExternalMediaShareActivity
         com.instagram.share.handleractivity.ClipsShareHandlerActivity
         com.instagram.direct.share.handler.DirectMultipleExternalMediaShareActivityInterop
-
+--------- 0.053s was the duration of dumpsys appops, ending at: 2022-03-29 23:14:27
+-------------------------------------------------------------------------------
+DUMP OF SERVICE platform_compat:
+ChangeId(180326845; name=OVERRIDE_MIN_ASPECT_RATIO_MEDIUM; disabled; overridable)
+ChangeId(189969744; name=DOWNSCALE_65; disabled; overridable)
+ChangeId(183372781; name=ENABLE_RAW_SYSTEM_GALLERY_ACCESS; enableSinceTargetSdk=30)
+ChangeId(150939131; name=ADD_CONTENT_OBSERVER_FLAGS; enableSinceTargetSdk=30)
+ChangeId(226439802; name=SCHEDULE_EXACT_ALARM_DENIED_BY_DEFAULT; disabled)
+ChangeId(270674727; name=ENABLE_STRICT_FORMATTER_VALIDATION; enableSinceTargetSdk=35)
+ChangeId(183155436; name=ALWAYS_USE_CONTEXT_USER; enableSinceTargetSdk=33)
+ChangeId(303742236; name=ROLE_MANAGER_USER_HANDLE_AWARE; enableSinceTargetSdk=35)
+ChangeId(203800354; name=MEDIA_CONTROL_SESSION_ACTIONS; enableSinceTargetSdk=33)
+ChangeId(144027538; name=BLOCK_GPS_STATUS_USAGE; enableSinceTargetSdk=31)
+ChangeId(189969749; name=DOWNSCALE_35; disabled; overridable)
+ChangeId(143539591; name=SELINUX_LATEST_CHANGES; disabled)
+ChangeId(247079863; name=DISALLOW_INVALID_GROUP_REFERENCE; enableSinceTargetSdk=34)
+ChangeId(174227820; name=FORCE_DISABLE_HEVC_SUPPORT; disabled)
+ChangeId(168419799; name=DOWNSCALED; disabled; packageOverrides={com.google.android.apps.tachyon=false, org.torproject.torbrowser=false}; rawOverrides={org.torproject.torbrowser=false, org.article19.circulo.next=false}; overridable)
 
 

tests/artifacts/android_data/dumpsys_platform_compat.txt

@@ -0,0 +1,16 @@
+DUMP OF SERVICE platform_compat:
+ChangeId(180326845; name=OVERRIDE_MIN_ASPECT_RATIO_MEDIUM; disabled; overridable)
+ChangeId(189969744; name=DOWNSCALE_65; disabled; overridable)
+ChangeId(183372781; name=ENABLE_RAW_SYSTEM_GALLERY_ACCESS; enableSinceTargetSdk=30)
+ChangeId(150939131; name=ADD_CONTENT_OBSERVER_FLAGS; enableSinceTargetSdk=30)
+ChangeId(226439802; name=SCHEDULE_EXACT_ALARM_DENIED_BY_DEFAULT; disabled)
+ChangeId(270674727; name=ENABLE_STRICT_FORMATTER_VALIDATION; enableSinceTargetSdk=35)
+ChangeId(183155436; name=ALWAYS_USE_CONTEXT_USER; enableSinceTargetSdk=33)
+ChangeId(303742236; name=ROLE_MANAGER_USER_HANDLE_AWARE; enableSinceTargetSdk=35)
+ChangeId(203800354; name=MEDIA_CONTROL_SESSION_ACTIONS; enableSinceTargetSdk=33)
+ChangeId(144027538; name=BLOCK_GPS_STATUS_USAGE; enableSinceTargetSdk=31)
+ChangeId(189969749; name=DOWNSCALE_35; disabled; overridable)
+ChangeId(143539591; name=SELINUX_LATEST_CHANGES; disabled)
+ChangeId(247079863; name=DISALLOW_INVALID_GROUP_REFERENCE; enableSinceTargetSdk=34)
+ChangeId(174227820; name=FORCE_DISABLE_HEVC_SUPPORT; disabled)
+ChangeId(168419799; name=DOWNSCALED; disabled; packageOverrides={com.google.android.apps.tachyon=false, org.torproject.torbrowser=false}; rawOverrides={org.torproject.torbrowser=false, org.article19.circulo.next=false}; overridable)

tests/artifacts/androidqf/dumpsys.txt

@@ -379,4 +379,22 @@ Daily stats:
       Update com.google.android.projection.gearhead vers=99632623
       Update com.google.android.projection.gearhead vers=99632623
       Update com.google.android.projection.gearhead vers=99632623
+--------- 0.053s was the duration of dumpsys batterystats, ending at: 2024-03-21 11:07:22
+-------------------------------------------------------------------------------
+DUMP OF SERVICE platform_compat:
+ChangeId(180326845; name=OVERRIDE_MIN_ASPECT_RATIO_MEDIUM; disabled; overridable)
+ChangeId(189969744; name=DOWNSCALE_65; disabled; overridable)
+ChangeId(183372781; name=ENABLE_RAW_SYSTEM_GALLERY_ACCESS; enableSinceTargetSdk=30)
+ChangeId(150939131; name=ADD_CONTENT_OBSERVER_FLAGS; enableSinceTargetSdk=30)
+ChangeId(226439802; name=SCHEDULE_EXACT_ALARM_DENIED_BY_DEFAULT; disabled)
+ChangeId(270674727; name=ENABLE_STRICT_FORMATTER_VALIDATION; enableSinceTargetSdk=35)
+ChangeId(183155436; name=ALWAYS_USE_CONTEXT_USER; enableSinceTargetSdk=33)
+ChangeId(303742236; name=ROLE_MANAGER_USER_HANDLE_AWARE; enableSinceTargetSdk=35)
+ChangeId(203800354; name=MEDIA_CONTROL_SESSION_ACTIONS; enableSinceTargetSdk=33)
+ChangeId(144027538; name=BLOCK_GPS_STATUS_USAGE; enableSinceTargetSdk=31)
+ChangeId(189969749; name=DOWNSCALE_35; disabled; overridable)
+ChangeId(143539591; name=SELINUX_LATEST_CHANGES; disabled)
+ChangeId(247079863; name=DISALLOW_INVALID_GROUP_REFERENCE; enableSinceTargetSdk=34)
+ChangeId(174227820; name=FORCE_DISABLE_HEVC_SUPPORT; disabled)
+ChangeId(168419799; name=DOWNSCALED; disabled; packageOverrides={com.google.android.apps.tachyon=false, org.torproject.torbrowser=false}; rawOverrides={org.torproject.torbrowser=false, org.article19.circulo.next=false}; overridable)
 

tests/common/test_utils.py

@@ -75,7 +75,7 @@ class TestHashes:
         # This needs to be updated when we add or edit files in AndroidQF folder
         assert (
             hashes[1]["sha256"]
-            == "1bd255f656a7f9d5647a730f0f0cc47053115576f11532d41bf28c16635b193d"
+            == "9fb6396b64cfff30e2a459a64496d3c1386926d09edd68be2d878de45fa7b3a9"
         )