Fix bug where default values were dropped when parsing protobuf tombstones

.github/dependabot.yml

@@ -1,11 +0,0 @@
-# To get started with Dependabot version updates, you'll need to specify which
-# package ecosystems to update and where the package manifests are located.
-# Please see the documentation for all configuration options:
-# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file
-
-version: 2
-updates:
-  - package-ecosystem: "pip" # See documentation for possible values
-    directory: "/" # Location of package manifests
-    schedule:
-      interval: "weekly"

.github/workflows/tests.yml

@@ -12,7 +12,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.10', '3.11', '3.12', '3.13']
+        python-version: ['3.8', '3.9', '3.10']  # , '3.11']
 
     steps:
       - uses: actions/checkout@v4
@@ -35,4 +35,4 @@ jobs:
         if: github.event_name == 'pull_request'
         with:
           pytest-coverage-path: ./pytest-coverage.txt
-          junitxml-path: ./pytest.xml
+          junitxml-path: ./pytest.xml

Dockerfile

@@ -103,7 +103,7 @@ RUN git clone https://github.com/libimobiledevice/usbmuxd && cd usbmuxd \
 
 
 # Create main image
-FROM ubuntu:24.04 as main
+FROM ubuntu:22.04 as main
 
 LABEL org.opencontainers.image.url="https://mvt.re"
 LABEL org.opencontainers.image.documentation="https://docs.mvt.re"
@@ -135,7 +135,8 @@ COPY --from=build-usbmuxd /build /
 COPY . mvt/
 RUN apt-get update \
    && apt-get install -y git python3-pip \
-   && PIP_NO_CACHE_DIR=1 pip3 install --break-system-packages ./mvt \
+   && PIP_NO_CACHE_DIR=1 pip3 install --upgrade pip \
+   && PIP_NO_CACHE_DIR=1 pip3 install ./mvt \
    && apt-get remove -y python3-pip git && apt-get autoremove -y \
    && rm -rf /var/lib/apt/lists/* \
    && rm -rf mvt

docs/requirements.txt

@@ -1,5 +1,5 @@
 mkdocs==1.6.1
-mkdocs-autorefs==1.4.2
-mkdocs-material==9.6.16
+mkdocs-autorefs==1.2.0
+mkdocs-material==9.5.42
 mkdocs-material-extensions==1.3.1
-mkdocstrings==0.30.0
+mkdocstrings==0.23.0

pyproject.toml

@@ -19,26 +19,25 @@ classifiers = [
     "Programming Language :: Python"
 ]
 dependencies = [
-    "click==8.2.1",
-    "rich==14.1.0",
-    "tld==0.13.1",
-    "requests==2.32.4",
-    "simplejson==3.20.1",
-    "packaging==25.0",
-    "appdirs==1.4.4",
-    "iOSbackup==0.9.925",
-    "adb-shell[usb]==0.4.4",
-    "libusb1==3.3.1",
-    "cryptography==45.0.6",
-    "PyYAML>=6.0.2",
-    "pyahocorasick==2.2.0",
-    "betterproto==1.2.5",
-    "pydantic==2.11.7",
-    "pydantic-settings==2.10.1",
-    "NSKeyedUnArchiver==1.5.2",
-    "python-dateutil==2.9.0.post0",
+    "click >=8.1.3",
+    "rich >=12.6.0",
+    "tld >=0.12.6",
+    "requests >=2.28.1",
+    "simplejson >=3.17.6",
+    "packaging >=21.3",
+    "appdirs >=1.4.4",
+    "iOSbackup >=0.9.923",
+    "adb-shell[usb] >=0.4.3",
+    "libusb1 >=3.0.0",
+    "cryptography >=42.0.5",
+    "pyyaml >=6.0",
+    "pyahocorasick >= 2.0.0",
+    "betterproto >=1.2.0",
+    "pydantic >= 2.10.0",
+    "pydantic-settings >= 2.7.0",
+    'backports.zoneinfo; python_version < "3.9"',
 ]
-requires-python = ">= 3.10"
+requires-python = ">= 3.8"
 
 [project.urls]
 homepage = "https://docs.mvt.re/en/latest/"
@@ -104,4 +103,4 @@ where = ["src"]
 mvt = ["ios/data/*.json"]
 
 [tool.setuptools.dynamic]
-version = {attr = "mvt.common.version.MVT_VERSION"}
+version = {attr = "mvt.common.version.MVT_VERSION"}

src/mvt/android/artifacts/dumpsys_adb.py

@@ -4,7 +4,6 @@
 #   https://license.mvt.re/1.1/
 
 import base64
-import binascii
 import hashlib
 
 from .artifact import AndroidArtifact
@@ -21,22 +20,12 @@ class DumpsysADBArtifact(AndroidArtifact):
         stack = [res]
         cur_indent = 0
         in_multiline = False
-        # Normalize line endings to handle both Unix (\n) and Windows (\r\n)
-        normalized_data = dump_data.replace(b"\r\n", b"\n").replace(b"\r", b"\n")
-        for line in normalized_data.strip(b"\n").split(b"\n"):
-            # Skip completely empty lines
-            if not line.strip():
-                continue
-
+        for line in dump_data.strip(b"\n").split(b"\n"):
             # Track the level of indentation
             indent = len(line) - len(line.lstrip())
             if indent < cur_indent:
                 # If the current line is less indented than the previous one, back out
-                while len(stack) > 1 and indent < cur_indent:
-                    stack.pop()
-                    # Check if we were in multiline mode and need to exit it
-                    if in_multiline and not isinstance(stack[-1], list):
-                        in_multiline = False
+                stack.pop()
                 cur_indent = indent
             else:
                 cur_indent = indent
@@ -48,30 +37,12 @@ class DumpsysADBArtifact(AndroidArtifact):
 
             # Annoyingly, some values are multiline and don't have a key on each line
             if in_multiline:
-                if key == "" and len(vals) < 2:
+                if key == "":
                     # If the line is empty, it's the terminator for the multiline value
                     in_multiline = False
                     stack.pop()
-                    current_dict = stack[-1]
-                elif len(vals) >= 2 and (key in self.multiline_fields or key == "}" or vals[1] == b"{"):
-                    # If we encounter a new field while in multiline mode, exit multiline mode
-                    # and process this line as a new field
-                    in_multiline = False
-                    stack.pop()
-                    current_dict = stack[-1]
-                    # Don't continue here - let the line be processed as a new field
                 else:
-                    # When in multiline mode, the top of stack should be a list
-                    if isinstance(stack[-1], list):
-                        stack[-1].append(line.lstrip())
-                    else:
-                        # Something went wrong with the stack, exit multiline mode
-                        in_multiline = False
-                        current_dict = stack[-1]
-                    continue
-
-            # Skip lines that don't have a value after '='
-            if len(vals) < 2:
+                    current_dict.append(line.lstrip())
                 continue
 
             if key == "}":
@@ -118,16 +89,11 @@ class DumpsysADBArtifact(AndroidArtifact):
         else:
             key_base64, user = user_key, b""
 
-        try:
-            key_raw = base64.b64decode(key_base64)
-            key_fingerprint = hashlib.md5(key_raw).hexdigest().upper()
-            key_fingerprint_colon = ":".join(
-                [key_fingerprint[i : i + 2] for i in range(0, len(key_fingerprint), 2)]
-            )
-        except binascii.Error:
-            # Impossible to parse base64
-            key_fingerprint_colon = ""
-
+        key_raw = base64.b64decode(key_base64)
+        key_fingerprint = hashlib.md5(key_raw).hexdigest().upper()
+        key_fingerprint_colon = ":".join(
+            [key_fingerprint[i : i + 2] for i in range(0, len(key_fingerprint), 2)]
+        )
         return {
             "user": user.decode("utf-8"),
             "fingerprint": key_fingerprint_colon,
@@ -161,16 +127,7 @@ class DumpsysADBArtifact(AndroidArtifact):
 
         # TODO: Parse AdbDebuggingManager line in output.
         start_of_json = content.find(b"\n{") + 2
-
-        # Handle both Unix (\n) and Windows (\r\n) line endings
-        end_of_json = content.rfind(b"}\n")
-        if end_of_json == -1:
-            end_of_json = content.rfind(b"}\r\n")
-            if end_of_json == -1:
-                self.log.error("Unable to find end of JSON block in dumpsys output")
-                return
-
-        end_of_json -= 2
+        end_of_json = content.rfind(b"}\n") - 2
         json_content = content[start_of_json:end_of_json].rstrip()
 
         parsed = self.indented_dump_parser(json_content)

src/mvt/android/artifacts/settings.py

@@ -51,6 +51,11 @@ ANDROID_DANGEROUS_SETTINGS = [
         "key": "send_action_app_error",
         "safe_value": "1",
     },
+    {
+        "description": "enabled installation of non Google Play apps",
+        "key": "install_non_market_apps",
+        "safe_value": "0",
+    },
     {
         "description": "enabled accessibility services",
         "key": "accessibility_enabled",

src/mvt/android/artifacts/tombstone_crashes.py

@@ -8,7 +8,6 @@ from typing import List, Optional, Union
 
 import pydantic
 import betterproto
-from dateutil import parser
 
 from mvt.common.utils import convert_datetime_to_iso
 from mvt.android.parsers.proto.tombstone import Tombstone
@@ -255,7 +254,12 @@ class TombstoneCrashArtifact(AndroidArtifact):
 
     @staticmethod
     def _parse_timestamp_string(timestamp: str) -> str:
-        timestamp_parsed = parser.parse(timestamp)
+        timestamp_date, timezone = timestamp.split("+")
+        # Truncate microseconds before parsing
+        timestamp_without_micro = timestamp_date.split(".")[0] + "+" + timezone
+        timestamp_parsed = datetime.datetime.strptime(
+            timestamp_without_micro, "%Y-%m-%d %H:%M:%S%z"
+        )
 
         # HACK: Swap the local timestamp to UTC, so keep the original time and avoid timezone conversion.
         local_timestamp = timestamp_parsed.replace(tzinfo=datetime.timezone.utc)

src/mvt/android/modules/androidqf/files.py

@@ -112,18 +112,10 @@ class Files(AndroidQFModule):
 
     def run(self) -> None:
         if timezone := self._get_device_timezone():
-            try:
-                device_timezone = zoneinfo.ZoneInfo(timezone)
-            except zoneinfo.ZoneInfoNotFoundError:
-                self.log.warning("Device timezone '%s' not found, using UTC", timezone)
-                device_timezone = datetime.timezone.utc
+            device_timezone = zoneinfo.ZoneInfo(timezone)
         else:
             self.log.warning("Unable to determine device timezone, using UTC")
-            try:
-                device_timezone = zoneinfo.ZoneInfo("UTC")
-            except zoneinfo.ZoneInfoNotFoundError:
-                # Fallback for Windows systems where zoneinfo might not have UTC
-                device_timezone = datetime.timezone.utc
+            device_timezone = zoneinfo.ZoneInfo("UTC")
 
         for file in self._get_files_by_pattern("*/files.json"):
             rawdata = self._get_file_content(file).decode("utf-8", errors="ignore")

src/mvt/android/parsers/backup.py

@@ -231,7 +231,6 @@ def parse_sms_file(data):
             entry.pop("mms_body")
 
         body = entry.get("body", None)
-        message_links = None
         if body:
             message_links = check_for_links(entry["body"])
 

src/mvt/common/cmd_check_iocs.py

@@ -65,10 +65,6 @@ class CmdCheckIOCS(Command):
                 m = iocs_module.from_json(
                     file_path, log=logging.getLogger(iocs_module.__module__)
                 )
-                if not m:
-                    log.warning("No result from this module, skipping it")
-                    continue
-
                 if self.iocs.total_ioc_count > 0:
                     m.indicators = self.iocs
                     m.indicators.log = m.log

src/mvt/common/indicators.py

@@ -654,8 +654,7 @@ class Indicators:
             return None
 
         for ioc in self.get_iocs("processes"):
-            # Use os-agnostic path splitting to handle both Windows (\) and Unix (/) separators
-            parts = file_path.replace("\\", "/").split("/")
+            parts = file_path.split("/")
             if ioc["value"] in parts:
                 self.log.warning(
                     "Found known suspicious process name mentioned in file at "

src/mvt/common/module.py

@@ -69,14 +69,10 @@ class MVTModule:
     @classmethod
     def from_json(cls, json_path: str, log: logging.Logger):
         with open(json_path, "r", encoding="utf-8") as handle:
-            try:
-                results = json.load(handle)
-                if log:
-                    log.info('Loaded %d results from "%s"', len(results), json_path)
-                return cls(results=results, log=log)
-            except json.decoder.JSONDecodeError as err:
-                log.error('Error to decode the json "%s" file: "%s"', json_path, err)
-                return None
+            results = json.load(handle)
+            if log:
+                log.info('Loaded %d results from "%s"', len(results), json_path)
+            return cls(results=results, log=log)
 
     @classmethod
     def get_slug(cls) -> str:

src/mvt/common/version.py

@@ -3,4 +3,4 @@
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
-MVT_VERSION = "2.6.1"
+MVT_VERSION = "2.6.0"

src/mvt/ios/data/ios_versions.json

@@ -891,10 +891,6 @@
         "version": "15.8.2",
         "build": "19H384"
     },
-    {
-        "version": "15.8.4",
-        "build": "19H390"
-    },
     {
         "build": "20A362",
         "version": "16.0"
@@ -996,10 +992,6 @@
         "version": "16.7.8",
         "build": "20H343"
     },
-    {
-        "version": "16.7.11",
-        "build": "20H360"
-    },
     {
         "version": "17.0",
         "build": "21A327"
@@ -1084,10 +1076,6 @@
         "version": "17.6.1",
         "build": "21G101"
     },
-    {
-        "version": "17.7.7",
-        "build": "21H433"
-    },
     {
         "version": "18",
         "build": "22A3354"
@@ -1119,21 +1107,5 @@
     {
         "version": "18.3.1",
         "build": "22D72"
-    },
-    {
-        "version": "18.4",
-        "build": "22E240"
-    },
-    {
-        "version": "18.4.1",
-        "build": "22E252"
-    },
-    {
-        "version": "18.5",
-        "build": "22F76"
-    },
-    {
-        "version": "18.6",
-        "build": "22G86"
     }
 ]

src/mvt/ios/modules/mixed/global_preferences.py

@@ -43,8 +43,6 @@ class GlobalPreferences(IOSExtraction):
                     self.log.warning("Lockdown mode enabled")
                 else:
                     self.log.warning("Lockdown mode disabled")
-                return
-        self.log.warning("Lockdown mode disabled")
 
     def process_file(self, file_path: str) -> None:
         with open(file_path, "rb") as handle:

src/mvt/ios/modules/mixed/safari_browserstate.py

@@ -95,17 +95,14 @@ class SafariBrowserState(IOSExtraction):
             )
         except sqlite3.OperationalError:
             # Old version iOS <12 likely
-            try:
-                cur.execute(
-                    """
-                    SELECT
-                        title, url, user_visible_url, last_viewed_time, session_data
-                    FROM tabs
-                    ORDER BY last_viewed_time;
+            cur.execute(
                 """
-                )
-            except sqlite3.OperationalError as e:
-                self.log.error(f"Error executing query: {e}")
+                SELECT
+                    title, url, user_visible_url, last_viewed_time, session_data
+                FROM tabs
+                ORDER BY last_viewed_time;
+            """
+            )
 
         for row in cur:
             session_entries = []

src/mvt/ios/modules/mixed/tcc.py

@@ -116,16 +116,13 @@ class TCC(IOSExtraction):
                 )
                 db_version = "v2"
             except sqlite3.OperationalError:
-                try:
-                    cur.execute(
-                        """SELECT
-                        service, client, client_type, allowed,
-                        prompt_count
-                        FROM access;"""
-                    )
-                    db_version = "v1"
-                except sqlite3.OperationalError as e:
-                    self.log.error(f"Error parsing TCC database: {e}")
+                cur.execute(
+                    """SELECT
+                    service, client, client_type, allowed,
+                    prompt_count
+                    FROM access;"""
+                )
+                db_version = "v1"
 
         for row in cur:
             service = row[0]

tests/android/test_artifact_tombstones.py

@@ -64,4 +64,4 @@ class TestTombstoneCrashArtifact:
         # We often don't know the time offset for a log entry and so can't convert everything to UTC.
         # MVT should output the local time only:
         # So original 2023-04-12 12:32:40.518290770+0200 -> 2023-04-12 12:32:40.000000
-        assert tombstone_result.get("timestamp") == "2023-04-12 12:32:40.518290"
+        assert tombstone_result.get("timestamp") == "2023-04-12 12:32:40.000000"

tests/android_bugreport/test_bugreport.py

@@ -9,7 +9,6 @@ from pathlib import Path
 from mvt.android.modules.bugreport.appops import Appops
 from mvt.android.modules.bugreport.getprop import Getprop
 from mvt.android.modules.bugreport.packages import Packages
-from mvt.android.modules.bugreport.tombstones import Tombstones
 from mvt.common.module import run_module
 
 from ..utils import get_artifact_folder
@@ -55,8 +54,3 @@ class TestBugreportAnalysis:
     def test_getprop_module(self):
         m = self.launch_bug_report_module(Getprop)
         assert len(m.results) == 0
-
-    def test_tombstones_modules(self):
-        m = self.launch_bug_report_module(Tombstones)
-        assert len(m.results) == 2
-        assert m.results[1]["pid"] == 3559

tests/artifacts/android_data/bugreport/FS/data/tombstones/tombstone_00

@@ -1,27 +0,0 @@
-*** *** *** *** *** *** *** *** *** *** *** *** *** *** *** ***
-Build fingerprint: 'samsung/a10eea/a10:10/.190711.020/A105:user/release-keys'
-Revision: '5'
-ABI: 'arm'
-Timestamp: 2021-09-29 17:43:49+0200
-pid: 9850, tid: 9893, name: UsbFfs-worker  >>> /system/bin/adbd <<<
-uid: 2000
-signal 6 (SIGABRT), code -1 (SI_QUEUE), fault addr --------
-Abort message: 'Check failed: payload.size() <= bytes_left (payload.size()=99, bytes_left=51) '
-    r0  00000000  r1  000026a5  r2  00000006  r3  f11fad98
-    r4  f11fadac  r5  f11fad90  r6  0000267a  r7  0000016b
-    r8  f11fada8  r9  f11fad98  r10 f11fadc8  r11 f11fadb8
-    ip  000026a5  sp  f11fad68  lr  f20c23b7  pc  f20c23ca
-
-backtrace:
-      #00 pc 000603ca  /apex/com.android.runtime/lib/bionic/libc.so (abort+166) (BuildId: 320fbdc2a1289fadd7dacae7f2eb77a3)
-      #01 pc 00007e23  /system/lib/libbase.so (android::base::DefaultAborter(char const*)+6) (BuildId: a28585ee446ea17e3e6fcf9c907fff2a)
-      #02 pc 0000855f  /system/lib/libbase.so (android::base::LogMessage::~LogMessage()+406) (BuildId: a28585ee446ea17e3e6fcf9c907fff2a)
-      #03 pc 000309cf  /system/lib/libadbd.so (UsbFfsConnection::ProcessRead(IoBlock*)+814) (BuildId: 3645b175977ae210c156a57b25dfa599)
-      #04 pc 00030459  /system/lib/libadbd.so (UsbFfsConnection::HandleRead(TransferId, long long)+84) (BuildId: 3645b175977ae210c156a57b25dfa599)
-      #05 pc 00030349  /system/lib/libadbd.so (UsbFfsConnection::ReadEvents()+92) (BuildId: 3645b175977ae210c156a57b25dfa599)
-      #06 pc 00030169  /system/lib/libadbd.so (_ZZN16UsbFfsConnection11StartWorkerEvENKUlvE_clEv+504) (BuildId: 3645b175977ae210c156a57b25dfa599)
-      #07 pc 0002ff53  /system/lib/libadbd.so (_ZNSt3__114__thread_proxyINS_5tupleIJNS_10unique_ptrINS_15__thread_structENS_14default_deleteIS3_EEEEZN16UsbFfsConnection11StartWorkerEvEUlvE_EEEEEPvSA_+26) (BuildId: 3645b175977ae210c156a57b25dfa599)
-      #08 pc 000a75b3  /apex/com.android.runtime/lib/bionic/libc.so (__pthread_start(void*)+20) (BuildId: 320fbdc2a1289fadd7dacae7f2eb77a3)
-      #09 pc 00061b33  /apex/com.android.runtime/lib/bionic/libc.so (__start_thread+30) (BuildId: 320fbdc2a1289fadd7dacae7f2eb77a3)
-
-

tests/artifacts/android_data/bugreport/FS/data/tombstones/tombstone_01

@@ -1,38 +0,0 @@
-*** *** *** *** *** *** *** *** *** *** *** *** *** *** *** ***
-Build fingerprint: 'samsung/a10eea/a10:11/RP1A.200720.012/A105:user/release-keys'
-Revision: '5'
-ABI: 'arm'
-Timestamp: 2023-08-21 23:28:59-0400
-pid: 3559, tid: 3568, name: tzts_daemon  >>> /vendor/bin/tzts_daemon <<<
-uid: 1000
-signal 11 (SIGSEGV), code 1 (SEGV_MAPERR), fault addr 0xe8b4d14c
-    r0  e8b4d14c  r1  e8b4d14c  r2  0000002b  r3  00000004
-    r4  00000000  r5  e8b4d14c  r6  00000000  r7  00000000
-    r8  e7ef78b0  r9  0000002b  r10 e7ef7dad  r11 e7ef7400
-    ip  00000000  sp  e7ef7208  lr  e89f4b01  pc  e89c273a
-
-backtrace:
-      #00 pc 0005f73a  /apex/com.android.runtime/lib/bionic/libc.so (strlen_a15+54) (BuildId: fef5b751123147ea65bf3f4f798c9518)
-      #01 pc 00091afd  /apex/com.android.runtime/lib/bionic/libc.so (__vfprintf+3364) (BuildId: fef5b751123147ea65bf3f4f798c9518)
-      #02 pc 000a68e5  /apex/com.android.runtime/lib/bionic/libc.so (vsnprintf+152) (BuildId: fef5b751123147ea65bf3f4f798c9518)
-      #03 pc 000051cf  /system/lib/liblog.so (__android_log_vprint+74) (BuildId: 3fcead474cd0ecbdafb529ff176b0d13)
-      #04 pc 000012e8  /vendor/bin/tzts_daemon
-
-memory near r0:
-    e8b4d12c -------- -------- -------- --------  ................
-    e8b4d13c -------- -------- -------- --------  ................
-    e8b4d14c -------- -------- -------- --------  ................
-    e8b4d15c -------- -------- -------- --------  ................
-    e8b4d16c -------- -------- -------- --------  ................
-    e8b4d17c -------- -------- -------- --------  ................
-    e8b4d18c -------- -------- -------- --------  ................
-    e8b4d19c -------- -------- -------- --------  ................
-    e8b4d1ac -------- -------- -------- --------  ................
-    e8b4d1bc -------- -------- -------- --------  ................
-    e8b4d1cc -------- -------- -------- --------  ................
-    e8b4d1dc -------- -------- -------- --------  ................
-    e8b4d1ec -------- -------- -------- --------  ................
-    e8b4d1fc -------- -------- -------- --------  ................
-    e8b4d20c -------- -------- -------- --------  ................
-    e8b4d21c -------- -------- -------- --------  ................
-