Merge branch 'feature/android-sub-module-loading' into local-timeline-fixes

Most other Android timestamps appear to be local time. The
results timeline is more useful if all the timestamps
are consistent. I would prefer to use UTC, but that would
mean converting all the other timestamps to UTC as well. We probably
do not have sufficient information to do that accurately,
especially if the device is moving between timezones..

Makefile

@@ -25,6 +25,12 @@ install:
 test-requirements:
 	python3 -m pip install --upgrade -r test-requirements.txt
 
+generate-proto-parsers:
+	# Generate python parsers for protobuf files
+	PROTO_DIR="src/mvt/android/parsers/proto/"; \
+	PROTO_FILES=$$(find $(PROTO_DIR) -iname "*.proto"); \
+	protoc -I$(PROTO_DIR) --python_betterproto_out=$(PROTO_DIR) $$PROTO_FILES
+
 clean:
 	rm -rf $(PWD)/build $(PWD)/dist $(PWD)/src/mvt.egg-info
 

pyproject.toml

@@ -32,6 +32,7 @@ dependencies = [
     "cryptography >=42.0.5",
     "pyyaml >=6.0",
     "pyahocorasick >= 2.0.0",
+    "betterproto >=1.2.0",
 ]
 requires-python = ">= 3.8"
 

src/mvt/android/artifacts/file_timestamps.py

@@ -0,0 +1,43 @@
+# Mobile Verification Toolkit (MVT)
+# Copyright (c) 2021-2023 The MVT Authors.
+# Use of this software is governed by the MVT License 1.1 that can be found at
+#   https://license.mvt.re/1.1/
+from typing import Union
+
+from .artifact import AndroidArtifact
+
+
+class FileTimestampsArtifact(AndroidArtifact):
+    def serialize(self, record: dict) -> Union[dict, list]:
+        records = []
+
+        for ts in set(
+            [
+                record.get("access_time"),
+                record.get("changed_time"),
+                record.get("modified_time"),
+            ]
+        ):
+            if not ts:
+                continue
+
+            macb = ""
+            macb += "M" if ts == record.get("modified_time") else "-"
+            macb += "A" if ts == record.get("access_time") else "-"
+            macb += "C" if ts == record.get("changed_time") else "-"
+            macb += "-"
+
+            msg = record["path"]
+            if record.get("context"):
+                msg += f" ({record['context']})"
+
+            records.append(
+                {
+                    "timestamp": ts,
+                    "module": self.__class__.__name__,
+                    "event": macb,
+                    "data": msg,
+                }
+            )
+
+        return records

src/mvt/android/artifacts/getprop.py

@@ -42,6 +42,17 @@ class GetProp(AndroidArtifact):
             entry = {"name": matches[0][0], "value": matches[0][1]}
             self.results.append(entry)
 
+    def get_device_timezone(self) -> str:
+        """
+        Get the device timezone from the getprop results
+
+        Used in other moduels to calculate the timezone offset
+        """
+        for entry in self.results:
+            if entry["name"] == "persist.sys.timezone":
+                return entry["value"]
+        return None
+
     def check_indicators(self) -> None:
         for entry in self.results:
             if entry["name"] in INTERESTING_PROPERTIES:

src/mvt/android/artifacts/tombstone_crashes.py

@@ -0,0 +1,13 @@
+# Mobile Verification Toolkit (MVT)
+# Copyright (c) 2021-2023 The MVT Authors.
+# Use of this software is governed by the MVT License 1.1 that can be found at
+#   https://license.mvt.re/1.1/
+
+
+from .artifact import AndroidArtifact
+
+
+class TombstoneCrashArtifact(AndroidArtifact):
+    def parse(self, content: bytes) -> None:
+        """
+        Parse Android tombstone crash files."""

src/mvt/android/cmd_check_androidqf.py

@@ -12,6 +12,8 @@ from typing import List, Optional
 from mvt.common.command import Command
 
 from .modules.androidqf import ANDROIDQF_MODULES
+from .modules.bugreport import BUGREPORT_MODULES
+from .modules.bugreport.base import BugReportModule
 
 log = logging.getLogger(__name__)
 
@@ -39,7 +41,11 @@ class CmdAndroidCheckAndroidQF(Command):
         )
 
         self.name = "check-androidqf"
-        self.modules = ANDROIDQF_MODULES
+
+        # We can load AndroidQF and bugreport modules here, as
+        # AndroidQF dump will contain a bugreport.
+        self.modules = ANDROIDQF_MODULES + BUGREPORT_MODULES
+        # TODO: Check how to namespace and deduplicate modules.
 
         self.format: Optional[str] = None
         self.archive: Optional[zipfile.ZipFile] = None
@@ -54,12 +60,44 @@ class CmdAndroidCheckAndroidQF(Command):
                 for fname in subfiles:
                     file_path = os.path.relpath(os.path.join(root, fname), parent_path)
                     self.files.append(file_path)
+
         elif os.path.isfile(self.target_path):
             self.format = "zip"
             self.archive = zipfile.ZipFile(self.target_path)
             self.files = self.archive.namelist()
 
+    def load_bugreport(self):
+        # Refactor this file list loading
+        # First we need to find the bugreport file location
+        bugreport_zip_path = None
+        for file_name in self.files:
+            if file_name.endswith("bugreport.zip"):
+                bugreport_zip_path = file_name
+                break
+        else:
+            self.log.warning("No bugreport.zip found in the AndroidQF dump")
+            return None
+
+        if self.format == "zip":
+            # Create handle to the bugreport.zip file inside the AndroidQF dump
+            handle = self.archive.open(bugreport_zip_path)
+            bugreport_zip = zipfile.ZipFile(handle)
+        else:
+            # Load the bugreport.zip file from the extracted AndroidQF dump on disk.
+            parent_path = Path(self.target_path).absolute().parent.as_posix()
+            bug_report_path = os.path.join(parent_path, bugreport_zip_path)
+            bugreport_zip = zipfile.ZipFile(bug_report_path)
+
+        return bugreport_zip
+
     def module_init(self, module):
+        if isinstance(module, BugReportModule):
+            bugreport_archive = self.load_bugreport()
+            if not bugreport_archive:
+                return
+            module.from_zip(bugreport_archive, bugreport_archive.namelist())
+            return
+
         if self.format == "zip":
             module.from_zip_file(self.archive, self.files)
         else:

src/mvt/android/modules/androidqf/base.py

@@ -48,6 +48,31 @@ class AndroidQFModule(MVTModule):
     def _get_files_by_pattern(self, pattern: str):
         return fnmatch.filter(self.files, pattern)
 
+    def _get_device_timezone(self):
+        """
+        Get the device timezone from the getprop.txt file.
+
+        This is needed to map local timestamps stored in some
+        Android log files to UTC/timezone-aware timestamps.
+        """
+        get_prop_files = self._get_files_by_pattern("*/getprop.txt")
+        prop_data = self._get_file_content(get_prop_files[0]).decode("utf-8")
+
+        from mvt.android.artifacts.getprop import GetProp
+
+        properties_artifact = GetProp()
+        properties_artifact.parse(prop_data)
+        timezone = properties_artifact.get_device_timezone()
+        if timezone:
+            self.log.debug("Identified local phone timezone: %s", timezone)
+            return timezone
+
+        self.log.warning(
+            "Could not find or determine local device timezone. "
+            "Some timestamps and timeline data may be incorrect."
+        )
+        return None
+
     def _get_file_content(self, file_path):
         if self.archive:
             handle = self.archive.open(file_path)

src/mvt/android/modules/androidqf/files.py

@@ -6,6 +6,7 @@
 import datetime
 import json
 import logging
+from zoneinfo import ZoneInfo
 from typing import Optional, Union
 
 from mvt.android.modules.androidqf.base import AndroidQFModule
@@ -106,6 +107,12 @@ class Files(AndroidQFModule):
             # TODO: adds SHA1 and MD5 when available in MVT
 
     def run(self) -> None:
+        if timezone := self._get_device_timezone():
+            device_timezone = ZoneInfo(timezone)
+        else:
+            self.log.warning("Unable to determine device timezone, using UTC")
+            device_timezone = ZoneInfo("UTC")
+
         for file in self._get_files_by_pattern("*/files.json"):
             rawdata = self._get_file_content(file).decode("utf-8", errors="ignore")
             try:
@@ -120,11 +127,18 @@ class Files(AndroidQFModule):
             for file_data in data:
                 for ts in ["access_time", "changed_time", "modified_time"]:
                     if ts in file_data:
-                        file_data[ts] = convert_datetime_to_iso(
-                            datetime.datetime.fromtimestamp(
-                                file_data[ts], tz=datetime.timezone.utc
-                            )
+                        utc_timestamp = datetime.datetime.fromtimestamp(
+                            file_data[ts], tz=datetime.timezone.utc
                         )
+                        # Convert the UTC timestamp to local tiem on Android device's local timezone
+                        local_timestamp = utc_timestamp.astimezone(device_timezone)
+
+                        # HACK: We only output the UTC timestamp in convert_datetime_to_iso, we
+                        # set the timestamp timezone to UTC, to avoid the timezone conversion again.
+                        local_timestamp = local_timestamp.replace(
+                            tzinfo=datetime.timezone.utc
+                        )
+                        file_data[ts] = convert_datetime_to_iso(local_timestamp)
 
                 self.results.append(file_data)
 

src/mvt/android/modules/androidqf/logfile_timestamps.py

@@ -0,0 +1,65 @@
+# Mobile Verification Toolkit (MVT)
+# Copyright (c) 2021-2023 The MVT Authors.
+# Use of this software is governed by the MVT License 1.1 that can be found at
+#   https://license.mvt.re/1.1/
+
+import os
+import datetime
+import logging
+from typing import Optional
+
+from mvt.common.utils import convert_datetime_to_iso
+from .base import AndroidQFModule
+from mvt.android.artifacts.file_timestamps import FileTimestampsArtifact
+
+
+class LogsFileTimestamps(FileTimestampsArtifact, AndroidQFModule):
+    """This module extracts records from battery daily updates."""
+
+    slug = "logfile_timestamps"
+
+    def __init__(
+        self,
+        file_path: Optional[str] = None,
+        target_path: Optional[str] = None,
+        results_path: Optional[str] = None,
+        module_options: Optional[dict] = None,
+        log: logging.Logger = logging.getLogger(__name__),
+        results: Optional[list] = None,
+    ) -> None:
+        super().__init__(
+            file_path=file_path,
+            target_path=target_path,
+            results_path=results_path,
+            module_options=module_options,
+            log=log,
+            results=results,
+        )
+
+    def _get_file_modification_time(self, file_path: str) -> dict:
+        if self.archive:
+            file_timetuple = self.archive.getinfo(file_path).date_time
+            return datetime.datetime(*file_timetuple)
+        else:
+            file_stat = os.stat(os.path.join(self.parent_path, file_path))
+            return datetime.datetime.fromtimestamp(file_stat.st_mtime)
+
+    def run(self) -> None:
+        filesystem_files = self._get_files_by_pattern("*/logs/*")
+
+        self.results = []
+        for file in filesystem_files:
+            # Only the modification time is available in the zip file metadata.
+            # The timezone is the local timezone of the machine the phone.
+            modification_time = self._get_file_modification_time(file)
+            self.results.append(
+                {
+                    "path": file,
+                    "modified_time": convert_datetime_to_iso(modification_time),
+                }
+            )
+
+        self.log.info(
+            "Extracted a total of %d filesystem timestamps from AndroidQF logs directory.",
+            len(self.results),
+        )

src/mvt/android/modules/bugreport/__init__.py

@@ -13,6 +13,7 @@ from .getprop import Getprop
 from .packages import Packages
 from .receivers import Receivers
 from .adb_state import DumpsysADBState
+from .fs_timestamps import BugReportTimestamps
 
 BUGREPORT_MODULES = [
     Accessibility,
@@ -25,4 +26,5 @@ BUGREPORT_MODULES = [
     Packages,
     Receivers,
     DumpsysADBState,
+    BugReportTimestamps,
 ]

src/mvt/android/modules/bugreport/base.py

@@ -6,6 +6,7 @@
 import fnmatch
 import logging
 import os
+
 from typing import List, Optional
 from zipfile import ZipFile
 

src/mvt/android/modules/bugreport/fs_timestamps.py

@@ -0,0 +1,65 @@
+# Mobile Verification Toolkit (MVT)
+# Copyright (c) 2021-2023 The MVT Authors.
+# Use of this software is governed by the MVT License 1.1 that can be found at
+#   https://license.mvt.re/1.1/
+
+import os
+import logging
+import datetime
+from typing import Optional
+
+from mvt.common.utils import convert_datetime_to_iso
+from .base import BugReportModule
+from mvt.android.artifacts.file_timestamps import FileTimestampsArtifact
+
+
+class BugReportTimestamps(FileTimestampsArtifact, BugReportModule):
+    """This module extracts records from battery daily updates."""
+
+    slug = "bugreport_timestamps"
+
+    def __init__(
+        self,
+        file_path: Optional[str] = None,
+        target_path: Optional[str] = None,
+        results_path: Optional[str] = None,
+        module_options: Optional[dict] = None,
+        log: logging.Logger = logging.getLogger(__name__),
+        results: Optional[list] = None,
+    ) -> None:
+        super().__init__(
+            file_path=file_path,
+            target_path=target_path,
+            results_path=results_path,
+            module_options=module_options,
+            log=log,
+            results=results,
+        )
+
+    def _get_file_modification_time(self, file_path: str) -> dict:
+        if self.zip_archive:
+            file_timetuple = self.zip_archive.getinfo(file_path).date_time
+            return datetime.datetime(*file_timetuple)
+        else:
+            file_stat = os.stat(os.path.join(self.extract_path, file_path))
+            return datetime.datetime.fromtimestamp(file_stat.st_mtime)
+
+    def run(self) -> None:
+        filesystem_files = self._get_files_by_pattern("FS/*")
+
+        self.results = []
+        for file in filesystem_files:
+            # Only the modification time is available in the zip file metadata.
+            # The timezone is the local timezone of the machine the phone.
+            modification_time = self._get_file_modification_time(file)
+            self.results.append(
+                {
+                    "path": file,
+                    "modified_time": convert_datetime_to_iso(modification_time),
+                }
+            )
+
+        self.log.info(
+            "Extracted a total of %d filesystem timestamps from bugreport.",
+            len(self.results),
+        )

src/mvt/android/modules/bugreport/tombstones.py

@@ -0,0 +1,59 @@
+# Mobile Verification Toolkit (MVT)
+# Copyright (c) 2021-2023 The MVT Authors.
+# Use of this software is governed by the MVT License 1.1 that can be found at
+#   https://license.mvt.re/1.1/
+
+import logging
+from typing import Optional
+
+from mvt.android.artifacts.tombstone_crashes import TombstoneCrashArtifact
+from .base import BugReportModule
+
+
+class Tombstones(TombstoneCrashArtifact, BugReportModule):
+    """This module extracts records from battery daily updates."""
+
+    slug = "tombstones"
+
+    def __init__(
+        self,
+        file_path: Optional[str] = None,
+        target_path: Optional[str] = None,
+        results_path: Optional[str] = None,
+        module_options: Optional[dict] = None,
+        log: logging.Logger = logging.getLogger(__name__),
+        results: Optional[list] = None,
+    ) -> None:
+        super().__init__(
+            file_path=file_path,
+            target_path=target_path,
+            results_path=results_path,
+            module_options=module_options,
+            log=log,
+            results=results,
+        )
+
+    def run(self) -> None:
+        tombstone_files = self._get_files_by_pattern("*/tombstone_*")
+        if not tombstone_files:
+            self.log.error(
+                "Unable to find any tombstone files. "
+                "Did you provide a valid bugreport archive?"
+            )
+            return
+
+        for tombstone_file in tombstone_files:
+            if tombstone_file.endswith("*.pb"):
+                self.log.info("Skipping protobuf tombstone file: %s", tombstone_file)
+                continue
+
+            print(tombstone_file)
+            tombstone_data = self._get_file_content(tombstone_file)
+            tombstone = self.parse_tombstone(tombstone_data)
+            print(tombstone)
+            break
+
+        # self.log.info(
+        #     "Extracted a total of %d database connection pool records",
+        #     len(self.results),
+        # )

src/mvt/android/parsers/proto/tombstone.proto

@@ -0,0 +1,195 @@
+// tombstone.proto file from Android source
+// Src: https://android.googlesource.com/platform/system/core/+/refs/heads/main/debuggerd/proto/tombstone.proto
+//
+// Protobuf definition for Android tombstones.
+//
+// An app can get hold of these for any `REASON_CRASH_NATIVE` instance of
+// `android.app.ApplicationExitInfo`.
+//
+// https://developer.android.com/reference/android/app/ApplicationExitInfo#getTraceInputStream()
+//
+syntax = "proto3";
+option java_package = "com.android.server.os";
+option java_outer_classname = "TombstoneProtos";
+// NOTE TO OEMS:
+// If you add custom fields to this proto, do not use numbers in the reserved range.
+message CrashDetail {
+  bytes name = 1;
+  bytes data = 2;
+  reserved 3 to 999;
+}
+message StackHistoryBufferEntry {
+  BacktraceFrame addr = 1;
+  uint64 fp = 2;
+  uint64 tag = 3;
+  reserved 4 to 999;
+}
+message StackHistoryBuffer {
+  uint64 tid = 1;
+  repeated StackHistoryBufferEntry entries = 2;
+  reserved 3 to 999;
+}
+message Tombstone {
+  Architecture arch = 1;
+  Architecture guest_arch = 24;
+  string build_fingerprint = 2;
+  string revision = 3;
+  string timestamp = 4;
+  uint32 pid = 5;
+  uint32 tid = 6;
+  uint32 uid = 7;
+  string selinux_label = 8;
+  repeated string command_line = 9;
+  // Process uptime in seconds.
+  uint32 process_uptime = 20;
+  Signal signal_info = 10;
+  string abort_message = 14;
+  repeated CrashDetail crash_details = 21;
+  repeated Cause causes = 15;
+  map<uint32, Thread> threads = 16;
+  map<uint32, Thread> guest_threads = 25;
+  repeated MemoryMapping memory_mappings = 17;
+  repeated LogBuffer log_buffers = 18;
+  repeated FD open_fds = 19;
+  uint32 page_size = 22;
+  bool has_been_16kb_mode = 23;
+  StackHistoryBuffer stack_history_buffer = 26;
+  reserved 27 to 999;
+}
+enum Architecture {
+  ARM32 = 0;
+  ARM64 = 1;
+  X86 = 2;
+  X86_64 = 3;
+  RISCV64 = 4;
+  NONE = 5;
+  reserved 6 to 999;
+}
+message Signal {
+  int32 number = 1;
+  string name = 2;
+  int32 code = 3;
+  string code_name = 4;
+  bool has_sender = 5;
+  int32 sender_uid = 6;
+  int32 sender_pid = 7;
+  bool has_fault_address = 8;
+  uint64 fault_address = 9;
+  // Note, may or may not contain the dump of the actual memory contents. Currently, on arm64, we
+  // only include metadata, and not the contents.
+  MemoryDump fault_adjacent_metadata = 10;
+  reserved 11 to 999;
+}
+message HeapObject {
+  uint64 address = 1;
+  uint64 size = 2;
+  uint64 allocation_tid = 3;
+  repeated BacktraceFrame allocation_backtrace = 4;
+  uint64 deallocation_tid = 5;
+  repeated BacktraceFrame deallocation_backtrace = 6;
+}
+message MemoryError {
+  enum Tool {
+    GWP_ASAN = 0;
+    SCUDO = 1;
+    reserved 2 to 999;
+  }
+  Tool tool = 1;
+  enum Type {
+    UNKNOWN = 0;
+    USE_AFTER_FREE = 1;
+    DOUBLE_FREE = 2;
+    INVALID_FREE = 3;
+    BUFFER_OVERFLOW = 4;
+    BUFFER_UNDERFLOW = 5;
+    reserved 6 to 999;
+  }
+  Type type = 2;
+  oneof location {
+    HeapObject heap = 3;
+  }
+  reserved 4 to 999;
+}
+message Cause {
+  string human_readable = 1;
+  oneof details {
+    MemoryError memory_error = 2;
+  }
+  reserved 3 to 999;
+}
+message Register {
+  string name = 1;
+  uint64 u64 = 2;
+  reserved 3 to 999;
+}
+message Thread {
+  int32 id = 1;
+  string name = 2;
+  repeated Register registers = 3;
+  repeated string backtrace_note = 7;
+  repeated string unreadable_elf_files = 9;
+  repeated BacktraceFrame current_backtrace = 4;
+  repeated MemoryDump memory_dump = 5;
+  int64 tagged_addr_ctrl = 6;
+  int64 pac_enabled_keys = 8;
+  reserved 10 to 999;
+}
+message BacktraceFrame {
+  uint64 rel_pc = 1;
+  uint64 pc = 2;
+  uint64 sp = 3;
+  string function_name = 4;
+  uint64 function_offset = 5;
+  string file_name = 6;
+  uint64 file_map_offset = 7;
+  string build_id = 8;
+  reserved 9 to 999;
+}
+message ArmMTEMetadata {
+  // One memory tag per granule (e.g. every 16 bytes) of regular memory.
+  bytes memory_tags = 1;
+  reserved 2 to 999;
+}
+message MemoryDump {
+  string register_name = 1;
+  string mapping_name = 2;
+  uint64 begin_address = 3;
+  bytes memory = 4;
+  oneof metadata {
+    ArmMTEMetadata arm_mte_metadata = 6;
+  }
+  reserved 5, 7 to 999;
+}
+message MemoryMapping {
+  uint64 begin_address = 1;
+  uint64 end_address = 2;
+  uint64 offset = 3;
+  bool read = 4;
+  bool write = 5;
+  bool execute = 6;
+  string mapping_name = 7;
+  string build_id = 8;
+  uint64 load_bias = 9;
+  reserved 10 to 999;
+}
+message FD {
+  int32 fd = 1;
+  string path = 2;
+  string owner = 3;
+  uint64 tag = 4;
+  reserved 5 to 999;
+}
+message LogBuffer {
+  string name = 1;
+  repeated LogMessage logs = 2;
+  reserved 3 to 999;
+}
+message LogMessage {
+  string timestamp = 1;
+  uint32 pid = 2;
+  uint32 tid = 3;
+  uint32 priority = 4;
+  string tag = 5;
+  string message = 6;
+  reserved 7 to 999;
+}

src/mvt/android/parsers/proto/tombstone.py

@@ -0,0 +1,208 @@
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# sources: tombstone.proto
+# plugin: python-betterproto
+from dataclasses import dataclass
+from typing import Dict, List
+
+import betterproto
+
+
+class Architecture(betterproto.Enum):
+    ARM32 = 0
+    ARM64 = 1
+    X86 = 2
+    X86_64 = 3
+    RISCV64 = 4
+    NONE = 5
+
+
+class MemoryErrorTool(betterproto.Enum):
+    GWP_ASAN = 0
+    SCUDO = 1
+
+
+class MemoryErrorType(betterproto.Enum):
+    UNKNOWN = 0
+    USE_AFTER_FREE = 1
+    DOUBLE_FREE = 2
+    INVALID_FREE = 3
+    BUFFER_OVERFLOW = 4
+    BUFFER_UNDERFLOW = 5
+
+
+@dataclass
+class CrashDetail(betterproto.Message):
+    """
+    NOTE TO OEMS: If you add custom fields to this proto, do not use numbers in
+    the reserved range.
+    """
+
+    name: bytes = betterproto.bytes_field(1)
+    data: bytes = betterproto.bytes_field(2)
+
+
+@dataclass
+class StackHistoryBufferEntry(betterproto.Message):
+    addr: "BacktraceFrame" = betterproto.message_field(1)
+    fp: int = betterproto.uint64_field(2)
+    tag: int = betterproto.uint64_field(3)
+
+
+@dataclass
+class StackHistoryBuffer(betterproto.Message):
+    tid: int = betterproto.uint64_field(1)
+    entries: List["StackHistoryBufferEntry"] = betterproto.message_field(2)
+
+
+@dataclass
+class Tombstone(betterproto.Message):
+    arch: "Architecture" = betterproto.enum_field(1)
+    guest_arch: "Architecture" = betterproto.enum_field(24)
+    build_fingerprint: str = betterproto.string_field(2)
+    revision: str = betterproto.string_field(3)
+    timestamp: str = betterproto.string_field(4)
+    pid: int = betterproto.uint32_field(5)
+    tid: int = betterproto.uint32_field(6)
+    uid: int = betterproto.uint32_field(7)
+    selinux_label: str = betterproto.string_field(8)
+    command_line: List[str] = betterproto.string_field(9)
+    # Process uptime in seconds.
+    process_uptime: int = betterproto.uint32_field(20)
+    signal_info: "Signal" = betterproto.message_field(10)
+    abort_message: str = betterproto.string_field(14)
+    crash_details: List["CrashDetail"] = betterproto.message_field(21)
+    causes: List["Cause"] = betterproto.message_field(15)
+    threads: Dict[int, "Thread"] = betterproto.map_field(
+        16, betterproto.TYPE_UINT32, betterproto.TYPE_MESSAGE
+    )
+    guest_threads: Dict[int, "Thread"] = betterproto.map_field(
+        25, betterproto.TYPE_UINT32, betterproto.TYPE_MESSAGE
+    )
+    memory_mappings: List["MemoryMapping"] = betterproto.message_field(17)
+    log_buffers: List["LogBuffer"] = betterproto.message_field(18)
+    open_fds: List["FD"] = betterproto.message_field(19)
+    page_size: int = betterproto.uint32_field(22)
+    has_been_16kb_mode: bool = betterproto.bool_field(23)
+    stack_history_buffer: "StackHistoryBuffer" = betterproto.message_field(26)
+
+
+@dataclass
+class Signal(betterproto.Message):
+    number: int = betterproto.int32_field(1)
+    name: str = betterproto.string_field(2)
+    code: int = betterproto.int32_field(3)
+    code_name: str = betterproto.string_field(4)
+    has_sender: bool = betterproto.bool_field(5)
+    sender_uid: int = betterproto.int32_field(6)
+    sender_pid: int = betterproto.int32_field(7)
+    has_fault_address: bool = betterproto.bool_field(8)
+    fault_address: int = betterproto.uint64_field(9)
+    # Note, may or may not contain the dump of the actual memory contents.
+    # Currently, on arm64, we only include metadata, and not the contents.
+    fault_adjacent_metadata: "MemoryDump" = betterproto.message_field(10)
+
+
+@dataclass
+class HeapObject(betterproto.Message):
+    address: int = betterproto.uint64_field(1)
+    size: int = betterproto.uint64_field(2)
+    allocation_tid: int = betterproto.uint64_field(3)
+    allocation_backtrace: List["BacktraceFrame"] = betterproto.message_field(4)
+    deallocation_tid: int = betterproto.uint64_field(5)
+    deallocation_backtrace: List["BacktraceFrame"] = betterproto.message_field(6)
+
+
+@dataclass
+class MemoryError(betterproto.Message):
+    tool: "MemoryErrorTool" = betterproto.enum_field(1)
+    type: "MemoryErrorType" = betterproto.enum_field(2)
+    heap: "HeapObject" = betterproto.message_field(3, group="location")
+
+
+@dataclass
+class Cause(betterproto.Message):
+    human_readable: str = betterproto.string_field(1)
+    memory_error: "MemoryError" = betterproto.message_field(2, group="details")
+
+
+@dataclass
+class Register(betterproto.Message):
+    name: str = betterproto.string_field(1)
+    u64: int = betterproto.uint64_field(2)
+
+
+@dataclass
+class Thread(betterproto.Message):
+    id: int = betterproto.int32_field(1)
+    name: str = betterproto.string_field(2)
+    registers: List["Register"] = betterproto.message_field(3)
+    backtrace_note: List[str] = betterproto.string_field(7)
+    unreadable_elf_files: List[str] = betterproto.string_field(9)
+    current_backtrace: List["BacktraceFrame"] = betterproto.message_field(4)
+    memory_dump: List["MemoryDump"] = betterproto.message_field(5)
+    tagged_addr_ctrl: int = betterproto.int64_field(6)
+    pac_enabled_keys: int = betterproto.int64_field(8)
+
+
+@dataclass
+class BacktraceFrame(betterproto.Message):
+    rel_pc: int = betterproto.uint64_field(1)
+    pc: int = betterproto.uint64_field(2)
+    sp: int = betterproto.uint64_field(3)
+    function_name: str = betterproto.string_field(4)
+    function_offset: int = betterproto.uint64_field(5)
+    file_name: str = betterproto.string_field(6)
+    file_map_offset: int = betterproto.uint64_field(7)
+    build_id: str = betterproto.string_field(8)
+
+
+@dataclass
+class ArmMTEMetadata(betterproto.Message):
+    # One memory tag per granule (e.g. every 16 bytes) of regular memory.
+    memory_tags: bytes = betterproto.bytes_field(1)
+
+
+@dataclass
+class MemoryDump(betterproto.Message):
+    register_name: str = betterproto.string_field(1)
+    mapping_name: str = betterproto.string_field(2)
+    begin_address: int = betterproto.uint64_field(3)
+    memory: bytes = betterproto.bytes_field(4)
+    arm_mte_metadata: "ArmMTEMetadata" = betterproto.message_field(6, group="metadata")
+
+
+@dataclass
+class MemoryMapping(betterproto.Message):
+    begin_address: int = betterproto.uint64_field(1)
+    end_address: int = betterproto.uint64_field(2)
+    offset: int = betterproto.uint64_field(3)
+    read: bool = betterproto.bool_field(4)
+    write: bool = betterproto.bool_field(5)
+    execute: bool = betterproto.bool_field(6)
+    mapping_name: str = betterproto.string_field(7)
+    build_id: str = betterproto.string_field(8)
+    load_bias: int = betterproto.uint64_field(9)
+
+
+@dataclass
+class FD(betterproto.Message):
+    fd: int = betterproto.int32_field(1)
+    path: str = betterproto.string_field(2)
+    owner: str = betterproto.string_field(3)
+    tag: int = betterproto.uint64_field(4)
+
+
+@dataclass
+class LogBuffer(betterproto.Message):
+    name: str = betterproto.string_field(1)
+    logs: List["LogMessage"] = betterproto.message_field(2)
+
+
+@dataclass
+class LogMessage(betterproto.Message):
+    timestamp: str = betterproto.string_field(1)
+    pid: int = betterproto.uint32_field(2)
+    tid: int = betterproto.uint32_field(3)
+    priority: int = betterproto.uint32_field(4)
+    tag: str = betterproto.string_field(5)
+    message: str = betterproto.string_field(6)

test-requirements.txt

@@ -6,3 +6,4 @@ pytest-mock>=3.14.0
 stix2>=3.0.1
 ruff>=0.1.6
 mypy>=1.7.1
+betterproto[compiler]

tests/android/test_artifact_tombstones.py

@@ -0,0 +1,40 @@
+# Mobile Verification Toolkit (MVT)
+# Copyright (c) 2021-2023 The MVT Authors.
+# Use of this software is governed by the MVT License 1.1 that can be found at
+#   https://license.mvt.re/1.1/
+
+from mvt.android.artifacts.tombstone_crashes import TombstoneCrashArtifact
+from mvt.android.parsers.proto.tombstone import Tombstone
+
+from ..utils import get_artifact
+
+
+class TestTombstoneCrashArtifact:
+    # def test_tombtone_process_parsing(self):
+    #     tombstone_artifact = TombstoneCrashArtifact()
+    #     file = get_artifact("android_data/tombstone_process.txt")
+    #     with open(file, "rb") as f:
+    #         data = f.read()
+
+    #     tombstone_artifact.parse_text(data)
+    #     assert len(tombstone_artifact.results) == 1
+
+    # def test_tombtone_kernel_parsing(self):
+    #     tombstone_artifact = TombstoneCrashArtifact()
+    #     file = get_artifact("android_data/tombstone_kernel.txt")
+    #     with open(file, "rb") as f:
+    #         data = f.read()
+
+    #     tombstone_artifact.parse_text(data)
+    #     assert len(tombstone_artifact.results) == 1
+
+    def test_tombstone_pb_process_parsing(self):
+        file = get_artifact("android_data/tombstone_process.pb")
+        with open(file, "rb") as f:
+            data = f.read()
+
+        parsed_tombstone = Tombstone().parse(data)
+        assert parsed_tombstone
+        assert parsed_tombstone.command_line == ["/vendor/bin/hw/android.hardware.media.c2@1.2-mediatek"]
+        assert parsed_tombstone.uid == 1046
+        assert parsed_tombstone.timestamp == "2023-04-12 12:32:40.518290770+0200"