Merge branch 'main' into feature/android-sub-module-loading

.github/workflows/tests.yml

@@ -12,7 +12,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.10', '3.11', '3.12', '3.13']
+        python-version: ['3.8', '3.9', '3.10']  # , '3.11']
 
     steps:
       - uses: actions/checkout@v4
@@ -35,4 +35,4 @@ jobs:
         if: github.event_name == 'pull_request'
         with:
           pytest-coverage-path: ./pytest-coverage.txt
-          junitxml-path: ./pytest.xml
+          junitxml-path: ./pytest.xml

.github/workflows/update-ios-data.yml

@@ -21,7 +21,6 @@ jobs:
           title: '[auto] Update iOS releases and versions'
           commit-message: Add new iOS versions and build numbers
           branch: auto/add-new-ios-releases
-          draft: true
           body: |
             This is an automated pull request to update the iOS releases and version numbers.
           add-paths: |

Dockerfile

@@ -103,7 +103,7 @@ RUN git clone https://github.com/libimobiledevice/usbmuxd && cd usbmuxd \
 
 
 # Create main image
-FROM ubuntu:24.04 as main
+FROM ubuntu:22.04 as main
 
 LABEL org.opencontainers.image.url="https://mvt.re"
 LABEL org.opencontainers.image.documentation="https://docs.mvt.re"
@@ -135,7 +135,8 @@ COPY --from=build-usbmuxd /build /
 COPY . mvt/
 RUN apt-get update \
    && apt-get install -y git python3-pip \
-   && PIP_NO_CACHE_DIR=1 pip3 install --break-system-packages ./mvt \
+   && PIP_NO_CACHE_DIR=1 pip3 install --upgrade pip \
+   && PIP_NO_CACHE_DIR=1 pip3 install ./mvt \
    && apt-get remove -y python3-pip git && apt-get autoremove -y \
    && rm -rf /var/lib/apt/lists/* \
    && rm -rf mvt

pyproject.toml

@@ -19,26 +19,25 @@ classifiers = [
     "Programming Language :: Python"
 ]
 dependencies = [
-    "click==8.2.1",
-    "rich==14.0.0",
-    "tld==0.13.1",
-    "requests==2.32.2",
-    "simplejson==3.20.1",
-    "packaging==25.0",
-    "appdirs==1.4.4",
-    "iOSbackup==0.9.925",
-    "adb-shell[usb]==0.4.4",
-    "libusb1==3.3.1",
-    "cryptography==45.0.3",
-    "PyYAML>=6.0.2",
-    "pyahocorasick==2.1.0",
-    "betterproto==1.2.5",
-    "pydantic==2.11.5",
-    "pydantic-settings==2.9.1",
-    "NSKeyedUnArchiver==1.5.2",
-    "python-dateutil==2.9.0.post0",
+    "click >=8.1.3",
+    "rich >=12.6.0",
+    "tld >=0.12.6",
+    "requests >=2.28.1",
+    "simplejson >=3.17.6",
+    "packaging >=21.3",
+    "appdirs >=1.4.4",
+    "iOSbackup >=0.9.923",
+    "adb-shell[usb] >=0.4.3",
+    "libusb1 >=3.0.0",
+    "cryptography >=42.0.5",
+    "pyyaml >=6.0",
+    "pyahocorasick >= 2.0.0",
+    "betterproto >=1.2.0",
+    "pydantic >= 2.10.0",
+    "pydantic-settings >= 2.7.0",
+    'backports.zoneinfo; python_version < "3.9"',
 ]
-requires-python = ">= 3.10"
+requires-python = ">= 3.8"
 
 [project.urls]
 homepage = "https://docs.mvt.re/en/latest/"
@@ -104,4 +103,4 @@ where = ["src"]
 mvt = ["ios/data/*.json"]
 
 [tool.setuptools.dynamic]
-version = {attr = "mvt.common.version.MVT_VERSION"}
+version = {attr = "mvt.common.version.MVT_VERSION"}

src/mvt/android/artifacts/dumpsys_adb.py

@@ -4,14 +4,13 @@
 #   https://license.mvt.re/1.1/
 
 import base64
-import binascii
 import hashlib
 
 from .artifact import AndroidArtifact
 
 
 class DumpsysADBArtifact(AndroidArtifact):
-    multiline_fields = ["user_keys", "keystore"]
+    multiline_fields = ["user_keys"]
 
     def indented_dump_parser(self, dump_data):
         """
@@ -68,38 +67,14 @@ class DumpsysADBArtifact(AndroidArtifact):
 
         return res
 
-    def parse_xml(self, xml_data):
-        """
-        Parse XML data from dumpsys ADB output
-        """
-        import xml.etree.ElementTree as ET
-
-        keystore = []
-        keystore_root = ET.fromstring(xml_data)
-        for adb_key in keystore_root.findall("adbKey"):
-            key_info = self.calculate_key_info(adb_key.get("key").encode("utf-8"))
-            key_info["last_connected"] = adb_key.get("lastConnection")
-            keystore.append(key_info)
-
-        return keystore
-
     @staticmethod
     def calculate_key_info(user_key: bytes) -> str:
-        if b" " in user_key:
-            key_base64, user = user_key.split(b" ", 1)
-        else:
-            key_base64, user = user_key, b""
-
-        try:
-            key_raw = base64.b64decode(key_base64)
-            key_fingerprint = hashlib.md5(key_raw).hexdigest().upper()
-            key_fingerprint_colon = ":".join(
-                [key_fingerprint[i : i + 2] for i in range(0, len(key_fingerprint), 2)]
-            )
-        except binascii.Error:
-            # Impossible to parse base64
-            key_fingerprint_colon = ""
-
+        key_base64, user = user_key.split(b" ", 1)
+        key_raw = base64.b64decode(key_base64)
+        key_fingerprint = hashlib.md5(key_raw).hexdigest().upper()
+        key_fingerprint_colon = ":".join(
+            [key_fingerprint[i : i + 2] for i in range(0, len(key_fingerprint), 2)]
+        )
         return {
             "user": user.decode("utf-8"),
             "fingerprint": key_fingerprint_colon,
@@ -140,24 +115,8 @@ class DumpsysADBArtifact(AndroidArtifact):
         if parsed.get("debugging_manager") is None:
             self.log.error("Unable to find expected ADB entries in dumpsys output")  # noqa
             return
-
-        # Keystore can be in different levels, as the basic parser
-        # is not always consistent due to different dumpsys formats.
-        if parsed.get("keystore"):
-            keystore_data = b"\n".join(parsed["keystore"])
-        elif parsed["debugging_manager"].get("keystore"):
-            keystore_data = b"\n".join(parsed["debugging_manager"]["keystore"])
         else:
-            keystore_data = None
-
-        # Keystore is in XML format on some devices and we need to parse it
-        if keystore_data and keystore_data.startswith(b"<?xml"):
-            parsed["debugging_manager"]["keystore"] = self.parse_xml(keystore_data)
-        else:
-            # Keystore is not XML format
-            parsed["debugging_manager"]["keystore"] = keystore_data
-
-        parsed = parsed["debugging_manager"]
+            parsed = parsed["debugging_manager"]
 
         # Calculate key fingerprints for better readability
         key_info = []

src/mvt/android/artifacts/tombstone_crashes.py

@@ -8,7 +8,6 @@ from typing import List, Optional, Union
 
 import pydantic
 import betterproto
-from dateutil import parser
 
 from mvt.common.utils import convert_datetime_to_iso
 from mvt.android.parsers.proto.tombstone import Tombstone
@@ -63,7 +62,7 @@ class TombstoneCrashResult(pydantic.BaseModel):
     process_name: Optional[str] = None
     binary_path: Optional[str] = None
     selinux_label: Optional[str] = None
-    uid: int
+    uid: Optional[int] = None
     signal_info: SignalInfo
     cause: Optional[str] = None
     extra: Optional[str] = None
@@ -125,9 +124,7 @@ class TombstoneCrashArtifact(AndroidArtifact):
         Parse Android tombstone crash files from a protobuf object.
         """
         tombstone_pb = Tombstone().parse(data)
-        tombstone_dict = tombstone_pb.to_dict(
-            betterproto.Casing.SNAKE, include_default_values=True
-        )
+        tombstone_dict = tombstone_pb.to_dict(betterproto.Casing.SNAKE)
 
         # Add some extra metadata
         tombstone_dict["timestamp"] = self._parse_timestamp_string(
@@ -255,7 +252,12 @@ class TombstoneCrashArtifact(AndroidArtifact):
 
     @staticmethod
     def _parse_timestamp_string(timestamp: str) -> str:
-        timestamp_parsed = parser.parse(timestamp)
+        timestamp_date, timezone = timestamp.split("+")
+        # Truncate microseconds before parsing
+        timestamp_without_micro = timestamp_date.split(".")[0] + "+" + timezone
+        timestamp_parsed = datetime.datetime.strptime(
+            timestamp_without_micro, "%Y-%m-%d %H:%M:%S%z"
+        )
 
         # HACK: Swap the local timestamp to UTC, so keep the original time and avoid timezone conversion.
         local_timestamp = timestamp_parsed.replace(tzinfo=datetime.timezone.utc)

src/mvt/android/cmd_check_androidqf.py

@@ -12,6 +12,8 @@ from typing import List, Optional
 from mvt.common.command import Command
 
 from .modules.androidqf import ANDROIDQF_MODULES
+from .modules.bugreport import BUGREPORT_MODULES
+from .modules.bugreport.base import BugReportModule
 
 log = logging.getLogger(__name__)
 
@@ -39,7 +41,11 @@ class CmdAndroidCheckAndroidQF(Command):
         )
 
         self.name = "check-androidqf"
-        self.modules = ANDROIDQF_MODULES
+
+        # We can load AndroidQF and bugreport modules here, as
+        # AndroidQF dump will contain a bugreport.
+        self.modules = ANDROIDQF_MODULES + BUGREPORT_MODULES
+        # TODO: Check how to namespace and deduplicate modules.
 
         self.format: Optional[str] = None
         self.archive: Optional[zipfile.ZipFile] = None
@@ -54,12 +60,44 @@ class CmdAndroidCheckAndroidQF(Command):
                 for fname in subfiles:
                     file_path = os.path.relpath(os.path.join(root, fname), parent_path)
                     self.files.append(file_path)
+
         elif os.path.isfile(self.target_path):
             self.format = "zip"
             self.archive = zipfile.ZipFile(self.target_path)
             self.files = self.archive.namelist()
 
+    def load_bugreport(self):
+        # Refactor this file list loading
+        # First we need to find the bugreport file location
+        bugreport_zip_path = None
+        for file_name in self.files:
+            if file_name.endswith("bugreport.zip"):
+                bugreport_zip_path = file_name
+                break
+        else:
+            self.log.warning("No bugreport.zip found in the AndroidQF dump")
+            return None
+
+        if self.format == "zip":
+            # Create handle to the bugreport.zip file inside the AndroidQF dump
+            handle = self.archive.open(bugreport_zip_path)
+            bugreport_zip = zipfile.ZipFile(handle)
+        else:
+            # Load the bugreport.zip file from the extracted AndroidQF dump on disk.
+            parent_path = Path(self.target_path).absolute().parent.as_posix()
+            bug_report_path = os.path.join(parent_path, bugreport_zip_path)
+            bugreport_zip = zipfile.ZipFile(bug_report_path)
+
+        return bugreport_zip
+
     def module_init(self, module):
+        if isinstance(module, BugReportModule):
+            bugreport_archive = self.load_bugreport()
+            if not bugreport_archive:
+                return
+            module.from_zip(bugreport_archive, bugreport_archive.namelist())
+            return
+
         if self.format == "zip":
             module.from_zip_file(self.archive, self.files)
         else:

src/mvt/common/cmd_check_iocs.py

@@ -65,10 +65,6 @@ class CmdCheckIOCS(Command):
                 m = iocs_module.from_json(
                     file_path, log=logging.getLogger(iocs_module.__module__)
                 )
-                if not m:
-                    log.warning("No result from this module, skipping it")
-                    continue
-
                 if self.iocs.total_ioc_count > 0:
                     m.indicators = self.iocs
                     m.indicators.log = m.log

src/mvt/common/logo.py

@@ -29,7 +29,7 @@ def check_updates() -> None:
         if latest_version:
             rich_print(
                 f"\t\t[bold]Version {latest_version} is available! "
-                "Upgrade mvt with `pip3 install -U mvt` or with `pipx upgrade mvt`[/bold]"
+                "Upgrade mvt with `pip3 install -U mvt`[/bold]"
             )
 
     # Then we check for indicators files updates.

src/mvt/common/module.py

@@ -69,14 +69,10 @@ class MVTModule:
     @classmethod
     def from_json(cls, json_path: str, log: logging.Logger):
         with open(json_path, "r", encoding="utf-8") as handle:
-            try:
-                results = json.load(handle)
-                if log:
-                    log.info('Loaded %d results from "%s"', len(results), json_path)
-                return cls(results=results, log=log)
-            except json.decoder.JSONDecodeError as err:
-                log.error('Error to decode the json "%s" file: "%s"', json_path, err)
-                return None
+            results = json.load(handle)
+            if log:
+                log.info('Loaded %d results from "%s"', len(results), json_path)
+            return cls(results=results, log=log)
 
     @classmethod
     def get_slug(cls) -> str:

src/mvt/common/version.py

@@ -3,4 +3,4 @@
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
-MVT_VERSION = "2.6.1"
+MVT_VERSION = "2.6.0"

src/mvt/ios/data/ios_versions.json

@@ -891,10 +891,6 @@
         "version": "15.8.2",
         "build": "19H384"
     },
-    {
-        "version": "15.8.4",
-        "build": "19H390"
-    },
     {
         "build": "20A362",
         "version": "16.0"
@@ -996,10 +992,6 @@
         "version": "16.7.8",
         "build": "20H343"
     },
-    {
-        "version": "16.7.11",
-        "build": "20H360"
-    },
     {
         "version": "17.0",
         "build": "21A327"
@@ -1084,10 +1076,6 @@
         "version": "17.6.1",
         "build": "21G101"
     },
-    {
-        "version": "17.7.7",
-        "build": "21H433"
-    },
     {
         "version": "18",
         "build": "22A3354"
@@ -1115,21 +1103,5 @@
     {
         "version": "18.3",
         "build": "22D63"
-    },
-    {
-        "version": "18.3.1",
-        "build": "22D72"
-    },
-    {
-        "version": "18.4",
-        "build": "22E240"
-    },
-    {
-        "version": "18.4.1",
-        "build": "22E252"
-    },
-    {
-        "version": "18.5",
-        "build": "22F76"
     }
 ]

src/mvt/ios/modules/mixed/global_preferences.py

@@ -43,8 +43,6 @@ class GlobalPreferences(IOSExtraction):
                     self.log.warning("Lockdown mode enabled")
                 else:
                     self.log.warning("Lockdown mode disabled")
-                return
-        self.log.warning("Lockdown mode disabled")
 
     def process_file(self, file_path: str) -> None:
         with open(file_path, "rb") as handle:

tests/android/test_artifact_dumpsys_adb.py

@@ -29,28 +29,3 @@ class TestDumpsysADBArtifact:
             user_key["fingerprint"] == "F0:A1:3D:8C:B3:F4:7B:09:9F:EE:8B:D8:38:2E:BD:C6"
         )
         assert user_key["user"] == "user@linux"
-
-    def test_parsing_adb_xml(self):
-        da_adb = DumpsysADBArtifact()
-        file = get_artifact("android_data/dumpsys_adb_xml.txt")
-        with open(file, "rb") as f:
-            data = f.read()
-
-        da_adb.parse(data)
-
-        assert len(da_adb.results) == 1
-
-        adb_data = da_adb.results[0]
-        assert "user_keys" in adb_data
-        assert len(adb_data["user_keys"]) == 1
-
-        # Check key and fingerprint parsed successfully.
-        expected_fingerprint = "F0:0B:27:08:E3:68:7B:FA:4C:79:A2:B4:BF:0E:CF:70"
-        user_key = adb_data["user_keys"][0]
-        user_key["fingerprint"] == expected_fingerprint
-        assert user_key["user"] == "user@laptop"
-
-        key_store_entry = adb_data["keystore"][0]
-        assert key_store_entry["user"] == "user@laptop"
-        assert key_store_entry["fingerprint"] == expected_fingerprint
-        assert key_store_entry["last_connected"] == "1628501829898"

tests/android/test_artifact_tombstones.py

@@ -64,4 +64,4 @@ class TestTombstoneCrashArtifact:
         # We often don't know the time offset for a log entry and so can't convert everything to UTC.
         # MVT should output the local time only:
         # So original 2023-04-12 12:32:40.518290770+0200 -> 2023-04-12 12:32:40.000000
-        assert tombstone_result.get("timestamp") == "2023-04-12 12:32:40.518290"
+        assert tombstone_result.get("timestamp") == "2023-04-12 12:32:40.000000"

tests/android_bugreport/test_bugreport.py

@@ -9,7 +9,6 @@ from pathlib import Path
 from mvt.android.modules.bugreport.appops import Appops
 from mvt.android.modules.bugreport.getprop import Getprop
 from mvt.android.modules.bugreport.packages import Packages
-from mvt.android.modules.bugreport.tombstones import Tombstones
 from mvt.common.module import run_module
 
 from ..utils import get_artifact_folder
@@ -55,8 +54,3 @@ class TestBugreportAnalysis:
     def test_getprop_module(self):
         m = self.launch_bug_report_module(Getprop)
         assert len(m.results) == 0
-
-    def test_tombstones_modules(self):
-        m = self.launch_bug_report_module(Tombstones)
-        assert len(m.results) == 2
-        assert m.results[1]["pid"] == 3559

tests/artifacts/android_data/bugreport/FS/data/tombstones/tombstone_00

@@ -1,27 +0,0 @@
-*** *** *** *** *** *** *** *** *** *** *** *** *** *** *** ***
-Build fingerprint: 'samsung/a10eea/a10:10/.190711.020/A105:user/release-keys'
-Revision: '5'
-ABI: 'arm'
-Timestamp: 2021-09-29 17:43:49+0200
-pid: 9850, tid: 9893, name: UsbFfs-worker  >>> /system/bin/adbd <<<
-uid: 2000
-signal 6 (SIGABRT), code -1 (SI_QUEUE), fault addr --------
-Abort message: 'Check failed: payload.size() <= bytes_left (payload.size()=99, bytes_left=51) '
-    r0  00000000  r1  000026a5  r2  00000006  r3  f11fad98
-    r4  f11fadac  r5  f11fad90  r6  0000267a  r7  0000016b
-    r8  f11fada8  r9  f11fad98  r10 f11fadc8  r11 f11fadb8
-    ip  000026a5  sp  f11fad68  lr  f20c23b7  pc  f20c23ca
-
-backtrace:
-      #00 pc 000603ca  /apex/com.android.runtime/lib/bionic/libc.so (abort+166) (BuildId: 320fbdc2a1289fadd7dacae7f2eb77a3)
-      #01 pc 00007e23  /system/lib/libbase.so (android::base::DefaultAborter(char const*)+6) (BuildId: a28585ee446ea17e3e6fcf9c907fff2a)
-      #02 pc 0000855f  /system/lib/libbase.so (android::base::LogMessage::~LogMessage()+406) (BuildId: a28585ee446ea17e3e6fcf9c907fff2a)
-      #03 pc 000309cf  /system/lib/libadbd.so (UsbFfsConnection::ProcessRead(IoBlock*)+814) (BuildId: 3645b175977ae210c156a57b25dfa599)
-      #04 pc 00030459  /system/lib/libadbd.so (UsbFfsConnection::HandleRead(TransferId, long long)+84) (BuildId: 3645b175977ae210c156a57b25dfa599)
-      #05 pc 00030349  /system/lib/libadbd.so (UsbFfsConnection::ReadEvents()+92) (BuildId: 3645b175977ae210c156a57b25dfa599)
-      #06 pc 00030169  /system/lib/libadbd.so (_ZZN16UsbFfsConnection11StartWorkerEvENKUlvE_clEv+504) (BuildId: 3645b175977ae210c156a57b25dfa599)
-      #07 pc 0002ff53  /system/lib/libadbd.so (_ZNSt3__114__thread_proxyINS_5tupleIJNS_10unique_ptrINS_15__thread_structENS_14default_deleteIS3_EEEEZN16UsbFfsConnection11StartWorkerEvEUlvE_EEEEEPvSA_+26) (BuildId: 3645b175977ae210c156a57b25dfa599)
-      #08 pc 000a75b3  /apex/com.android.runtime/lib/bionic/libc.so (__pthread_start(void*)+20) (BuildId: 320fbdc2a1289fadd7dacae7f2eb77a3)
-      #09 pc 00061b33  /apex/com.android.runtime/lib/bionic/libc.so (__start_thread+30) (BuildId: 320fbdc2a1289fadd7dacae7f2eb77a3)
-
-

tests/artifacts/android_data/bugreport/FS/data/tombstones/tombstone_01

@@ -1,38 +0,0 @@
-*** *** *** *** *** *** *** *** *** *** *** *** *** *** *** ***
-Build fingerprint: 'samsung/a10eea/a10:11/RP1A.200720.012/A105:user/release-keys'
-Revision: '5'
-ABI: 'arm'
-Timestamp: 2023-08-21 23:28:59-0400
-pid: 3559, tid: 3568, name: tzts_daemon  >>> /vendor/bin/tzts_daemon <<<
-uid: 1000
-signal 11 (SIGSEGV), code 1 (SEGV_MAPERR), fault addr 0xe8b4d14c
-    r0  e8b4d14c  r1  e8b4d14c  r2  0000002b  r3  00000004
-    r4  00000000  r5  e8b4d14c  r6  00000000  r7  00000000
-    r8  e7ef78b0  r9  0000002b  r10 e7ef7dad  r11 e7ef7400
-    ip  00000000  sp  e7ef7208  lr  e89f4b01  pc  e89c273a
-
-backtrace:
-      #00 pc 0005f73a  /apex/com.android.runtime/lib/bionic/libc.so (strlen_a15+54) (BuildId: fef5b751123147ea65bf3f4f798c9518)
-      #01 pc 00091afd  /apex/com.android.runtime/lib/bionic/libc.so (__vfprintf+3364) (BuildId: fef5b751123147ea65bf3f4f798c9518)
-      #02 pc 000a68e5  /apex/com.android.runtime/lib/bionic/libc.so (vsnprintf+152) (BuildId: fef5b751123147ea65bf3f4f798c9518)
-      #03 pc 000051cf  /system/lib/liblog.so (__android_log_vprint+74) (BuildId: 3fcead474cd0ecbdafb529ff176b0d13)
-      #04 pc 000012e8  /vendor/bin/tzts_daemon
-
-memory near r0:
-    e8b4d12c -------- -------- -------- --------  ................
-    e8b4d13c -------- -------- -------- --------  ................
-    e8b4d14c -------- -------- -------- --------  ................
-    e8b4d15c -------- -------- -------- --------  ................
-    e8b4d16c -------- -------- -------- --------  ................
-    e8b4d17c -------- -------- -------- --------  ................
-    e8b4d18c -------- -------- -------- --------  ................
-    e8b4d19c -------- -------- -------- --------  ................
-    e8b4d1ac -------- -------- -------- --------  ................
-    e8b4d1bc -------- -------- -------- --------  ................
-    e8b4d1cc -------- -------- -------- --------  ................
-    e8b4d1dc -------- -------- -------- --------  ................
-    e8b4d1ec -------- -------- -------- --------  ................
-    e8b4d1fc -------- -------- -------- --------  ................
-    e8b4d20c -------- -------- -------- --------  ................
-    e8b4d21c -------- -------- -------- --------  ................
-

tests/artifacts/android_data/dumpsys_adb_xml.txt

@@ -1,16 +0,0 @@
--------------------------------------------------------------------------------
-DUMP OF SERVICE adb:
-ADB MANAGER STATE (dumpsys adb):
-{
-  debugging_manager={
-    connected_to_adb=true
-    user_keys=QAAAAAcgbytJst31DsaSP7hn8QcBXKR9NPVPK9MZssFVSNIP user@laptop
-
-    keystore=<?xml version='1.0' encoding='utf-8' standalone='yes' ?>
-    <keyStore version="1">
-    <adbKey key="QAAAAAcgbytJst31DsaSP7hn8QcBXKR9NPVPK9MZssFVSNIP user@laptop" lastConnection="1628501829898" />
-    </keyStore>
-
-  }
-}
---------- 0.012s was the duration of dumpsys adb, ending at: 2025-02-04 20:25:58