mirror of
https://github.com/mvt-project/mvt
synced 2025-10-21 22:42:15 +02:00
Compare commits
45 Commits
v2.6.0
...
Te-k-patch
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
7a1ed628ef | ||
|
|
c9ac12f336 | ||
|
|
486e3e7e9b | ||
|
|
be1fc3bd8b | ||
|
|
4757cff262 | ||
|
|
61f51caf31 | ||
|
|
511063fd0e | ||
|
|
88bc5672cb | ||
|
|
0fce0acf7a | ||
|
|
61f95d07d3 | ||
|
|
3dedd169c4 | ||
|
|
e34e03d3a3 | ||
|
|
34374699ce | ||
|
|
cf5aa7c89f | ||
|
|
2766739512 | ||
|
|
9c84afb4b0 | ||
|
|
80fc8bd879 | ||
|
|
ca41f7f106 | ||
|
|
55ddd86ad5 | ||
|
|
b184eeedf4 | ||
|
|
4e97e85350 | ||
|
|
e5865b166e | ||
|
|
a2dabb4267 | ||
|
|
b7595b62eb | ||
|
|
02c02ca15c | ||
|
|
6da33394fe | ||
|
|
086871e21d | ||
|
|
f32830c649 | ||
|
|
edcad488ab | ||
|
|
43901c96a0 | ||
|
|
0962383b46 | ||
|
|
34cd08fd9a | ||
|
|
579b53f7ec | ||
|
|
0fbf24e82a | ||
|
|
458195a0ab | ||
|
|
52e854b8b7 | ||
|
|
0f1eec3971 | ||
|
|
f4425865c0 | ||
|
|
28c0c86c4e | ||
|
|
154e6dab15 | ||
|
|
8e895d3d07 | ||
|
|
bc09e2a394 | ||
|
|
2d0de088dd | ||
|
|
8694e7a047 | ||
|
|
9b41ba99aa |
11
.github/dependabot.yml
vendored
Normal file
11
.github/dependabot.yml
vendored
Normal file
@@ -0,0 +1,11 @@
|
||||
# To get started with Dependabot version updates, you'll need to specify which
|
||||
# package ecosystems to update and where the package manifests are located.
|
||||
# Please see the documentation for all configuration options:
|
||||
# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file
|
||||
|
||||
version: 2
|
||||
updates:
|
||||
- package-ecosystem: "pip" # See documentation for possible values
|
||||
directory: "/" # Location of package manifests
|
||||
schedule:
|
||||
interval: "weekly"
|
||||
4
.github/workflows/tests.yml
vendored
4
.github/workflows/tests.yml
vendored
@@ -12,7 +12,7 @@ jobs:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
python-version: ['3.8', '3.9', '3.10'] # , '3.11']
|
||||
python-version: ['3.10', '3.11', '3.12', '3.13']
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
@@ -35,4 +35,4 @@ jobs:
|
||||
if: github.event_name == 'pull_request'
|
||||
with:
|
||||
pytest-coverage-path: ./pytest-coverage.txt
|
||||
junitxml-path: ./pytest.xml
|
||||
junitxml-path: ./pytest.xml
|
||||
|
||||
1
.github/workflows/update-ios-data.yml
vendored
1
.github/workflows/update-ios-data.yml
vendored
@@ -21,6 +21,7 @@ jobs:
|
||||
title: '[auto] Update iOS releases and versions'
|
||||
commit-message: Add new iOS versions and build numbers
|
||||
branch: auto/add-new-ios-releases
|
||||
draft: true
|
||||
body: |
|
||||
This is an automated pull request to update the iOS releases and version numbers.
|
||||
add-paths: |
|
||||
|
||||
@@ -103,7 +103,7 @@ RUN git clone https://github.com/libimobiledevice/usbmuxd && cd usbmuxd \
|
||||
|
||||
|
||||
# Create main image
|
||||
FROM ubuntu:22.04 as main
|
||||
FROM ubuntu:24.04 as main
|
||||
|
||||
LABEL org.opencontainers.image.url="https://mvt.re"
|
||||
LABEL org.opencontainers.image.documentation="https://docs.mvt.re"
|
||||
@@ -135,8 +135,7 @@ COPY --from=build-usbmuxd /build /
|
||||
COPY . mvt/
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y git python3-pip \
|
||||
&& PIP_NO_CACHE_DIR=1 pip3 install --upgrade pip \
|
||||
&& PIP_NO_CACHE_DIR=1 pip3 install ./mvt \
|
||||
&& PIP_NO_CACHE_DIR=1 pip3 install --break-system-packages ./mvt \
|
||||
&& apt-get remove -y python3-pip git && apt-get autoremove -y \
|
||||
&& rm -rf /var/lib/apt/lists/* \
|
||||
&& rm -rf mvt
|
||||
|
||||
5
Makefile
5
Makefile
@@ -25,6 +25,11 @@ install:
|
||||
test-requirements:
|
||||
python3 -m pip install --upgrade -r test-requirements.txt
|
||||
|
||||
generate-proto-parsers:
|
||||
# Generate python parsers for protobuf files
|
||||
PROTO_FILES=$$(find src/mvt/android/parsers/proto/ -iname "*.proto"); \
|
||||
protoc -Isrc/mvt/android/parsers/proto/ --python_betterproto_out=src/mvt/android/parsers/proto/ $$PROTO_FILES
|
||||
|
||||
clean:
|
||||
rm -rf $(PWD)/build $(PWD)/dist $(PWD)/src/mvt.egg-info
|
||||
|
||||
|
||||
@@ -19,21 +19,26 @@ classifiers = [
|
||||
"Programming Language :: Python"
|
||||
]
|
||||
dependencies = [
|
||||
"click >=8.1.3",
|
||||
"rich >=12.6.0",
|
||||
"tld >=0.12.6",
|
||||
"requests >=2.28.1",
|
||||
"simplejson >=3.17.6",
|
||||
"packaging >=21.3",
|
||||
"appdirs >=1.4.4",
|
||||
"iOSbackup >=0.9.923",
|
||||
"adb-shell[usb] >=0.4.3",
|
||||
"libusb1 >=3.0.0",
|
||||
"cryptography >=42.0.5",
|
||||
"pyyaml >=6.0",
|
||||
"pyahocorasick >= 2.0.0",
|
||||
"click==8.2.1",
|
||||
"rich==14.0.0",
|
||||
"tld==0.13.1",
|
||||
"requests==2.32.4",
|
||||
"simplejson==3.20.1",
|
||||
"packaging==25.0",
|
||||
"appdirs==1.4.4",
|
||||
"iOSbackup==0.9.925",
|
||||
"adb-shell[usb]==0.4.4",
|
||||
"libusb1==3.3.1",
|
||||
"cryptography==45.0.3",
|
||||
"PyYAML>=6.0.2",
|
||||
"pyahocorasick==2.1.0",
|
||||
"betterproto==1.2.5",
|
||||
"pydantic==2.11.5",
|
||||
"pydantic-settings==2.9.1",
|
||||
"NSKeyedUnArchiver==1.5.2",
|
||||
"python-dateutil==2.9.0.post0",
|
||||
]
|
||||
requires-python = ">= 3.8"
|
||||
requires-python = ">= 3.10"
|
||||
|
||||
[project.urls]
|
||||
homepage = "https://docs.mvt.re/en/latest/"
|
||||
@@ -99,4 +104,4 @@ where = ["src"]
|
||||
mvt = ["ios/data/*.json"]
|
||||
|
||||
[tool.setuptools.dynamic]
|
||||
version = {attr = "mvt.common.version.MVT_VERSION"}
|
||||
version = {attr = "mvt.common.version.MVT_VERSION"}
|
||||
|
||||
@@ -4,13 +4,14 @@
|
||||
# https://license.mvt.re/1.1/
|
||||
|
||||
import base64
|
||||
import binascii
|
||||
import hashlib
|
||||
|
||||
from .artifact import AndroidArtifact
|
||||
|
||||
|
||||
class DumpsysADBArtifact(AndroidArtifact):
|
||||
multiline_fields = ["user_keys"]
|
||||
multiline_fields = ["user_keys", "keystore"]
|
||||
|
||||
def indented_dump_parser(self, dump_data):
|
||||
"""
|
||||
@@ -67,14 +68,38 @@ class DumpsysADBArtifact(AndroidArtifact):
|
||||
|
||||
return res
|
||||
|
||||
def parse_xml(self, xml_data):
|
||||
"""
|
||||
Parse XML data from dumpsys ADB output
|
||||
"""
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
keystore = []
|
||||
keystore_root = ET.fromstring(xml_data)
|
||||
for adb_key in keystore_root.findall("adbKey"):
|
||||
key_info = self.calculate_key_info(adb_key.get("key").encode("utf-8"))
|
||||
key_info["last_connected"] = adb_key.get("lastConnection")
|
||||
keystore.append(key_info)
|
||||
|
||||
return keystore
|
||||
|
||||
@staticmethod
|
||||
def calculate_key_info(user_key: bytes) -> str:
|
||||
key_base64, user = user_key.split(b" ", 1)
|
||||
key_raw = base64.b64decode(key_base64)
|
||||
key_fingerprint = hashlib.md5(key_raw).hexdigest().upper()
|
||||
key_fingerprint_colon = ":".join(
|
||||
[key_fingerprint[i : i + 2] for i in range(0, len(key_fingerprint), 2)]
|
||||
)
|
||||
if b" " in user_key:
|
||||
key_base64, user = user_key.split(b" ", 1)
|
||||
else:
|
||||
key_base64, user = user_key, b""
|
||||
|
||||
try:
|
||||
key_raw = base64.b64decode(key_base64)
|
||||
key_fingerprint = hashlib.md5(key_raw).hexdigest().upper()
|
||||
key_fingerprint_colon = ":".join(
|
||||
[key_fingerprint[i : i + 2] for i in range(0, len(key_fingerprint), 2)]
|
||||
)
|
||||
except binascii.Error:
|
||||
# Impossible to parse base64
|
||||
key_fingerprint_colon = ""
|
||||
|
||||
return {
|
||||
"user": user.decode("utf-8"),
|
||||
"fingerprint": key_fingerprint_colon,
|
||||
@@ -115,8 +140,24 @@ class DumpsysADBArtifact(AndroidArtifact):
|
||||
if parsed.get("debugging_manager") is None:
|
||||
self.log.error("Unable to find expected ADB entries in dumpsys output") # noqa
|
||||
return
|
||||
|
||||
# Keystore can be in different levels, as the basic parser
|
||||
# is not always consistent due to different dumpsys formats.
|
||||
if parsed.get("keystore"):
|
||||
keystore_data = b"\n".join(parsed["keystore"])
|
||||
elif parsed["debugging_manager"].get("keystore"):
|
||||
keystore_data = b"\n".join(parsed["debugging_manager"]["keystore"])
|
||||
else:
|
||||
parsed = parsed["debugging_manager"]
|
||||
keystore_data = None
|
||||
|
||||
# Keystore is in XML format on some devices and we need to parse it
|
||||
if keystore_data and keystore_data.startswith(b"<?xml"):
|
||||
parsed["debugging_manager"]["keystore"] = self.parse_xml(keystore_data)
|
||||
else:
|
||||
# Keystore is not XML format
|
||||
parsed["debugging_manager"]["keystore"] = keystore_data
|
||||
|
||||
parsed = parsed["debugging_manager"]
|
||||
|
||||
# Calculate key fingerprints for better readability
|
||||
key_info = []
|
||||
|
||||
@@ -11,6 +11,10 @@ from mvt.common.utils import convert_datetime_to_iso
|
||||
from .artifact import AndroidArtifact
|
||||
|
||||
|
||||
RISKY_PERMISSIONS = ["REQUEST_INSTALL_PACKAGES"]
|
||||
RISKY_PACKAGES = ["com.android.shell"]
|
||||
|
||||
|
||||
class DumpsysAppopsArtifact(AndroidArtifact):
|
||||
"""
|
||||
Parser for dumpsys app ops info
|
||||
@@ -45,15 +49,39 @@ class DumpsysAppopsArtifact(AndroidArtifact):
|
||||
self.detected.append(result)
|
||||
continue
|
||||
|
||||
detected_permissions = []
|
||||
for perm in result["permissions"]:
|
||||
if (
|
||||
perm["name"] == "REQUEST_INSTALL_PACKAGES"
|
||||
and perm["access"] == "allow"
|
||||
perm["name"] in RISKY_PERMISSIONS
|
||||
# and perm["access"] == "allow"
|
||||
):
|
||||
self.log.info(
|
||||
"Package %s with REQUEST_INSTALL_PACKAGES permission",
|
||||
result["package_name"],
|
||||
)
|
||||
detected_permissions.append(perm)
|
||||
for entry in sorted(perm["entries"], key=lambda x: x["timestamp"]):
|
||||
self.log.warning(
|
||||
"Package '%s' had risky permission '%s' set to '%s' at %s",
|
||||
result["package_name"],
|
||||
perm["name"],
|
||||
entry["access"],
|
||||
entry["timestamp"],
|
||||
)
|
||||
|
||||
elif result["package_name"] in RISKY_PACKAGES:
|
||||
detected_permissions.append(perm)
|
||||
for entry in sorted(perm["entries"], key=lambda x: x["timestamp"]):
|
||||
self.log.warning(
|
||||
"Risky package '%s' had '%s' permission set to '%s' at %s",
|
||||
result["package_name"],
|
||||
perm["name"],
|
||||
entry["access"],
|
||||
entry["timestamp"],
|
||||
)
|
||||
|
||||
if detected_permissions:
|
||||
# We clean the result to only include the risky permission, otherwise the timeline
|
||||
# will be polluted with all the other irrelevant permissions
|
||||
cleaned_result = result.copy()
|
||||
cleaned_result["permissions"] = detected_permissions
|
||||
self.detected.append(cleaned_result)
|
||||
|
||||
def parse(self, output: str) -> None:
|
||||
self.results: List[Dict[str, Any]] = []
|
||||
@@ -121,11 +149,16 @@ class DumpsysAppopsArtifact(AndroidArtifact):
|
||||
if line.startswith(" "):
|
||||
# Permission entry like:
|
||||
# Reject: [fg-s]2021-05-19 22:02:52.054 (-314d1h25m2s33ms)
|
||||
access_type = line.split(":")[0].strip()
|
||||
if access_type not in ["Access", "Reject"]:
|
||||
# Skipping invalid access type. Some entries are not in the format we expect
|
||||
continue
|
||||
|
||||
if entry:
|
||||
perm["entries"].append(entry)
|
||||
entry = {}
|
||||
|
||||
entry["access"] = line.split(":")[0].strip()
|
||||
entry["access"] = access_type
|
||||
entry["type"] = line[line.find("[") + 1 : line.find("]")]
|
||||
|
||||
try:
|
||||
|
||||
43
src/mvt/android/artifacts/file_timestamps.py
Normal file
43
src/mvt/android/artifacts/file_timestamps.py
Normal file
@@ -0,0 +1,43 @@
|
||||
# Mobile Verification Toolkit (MVT)
|
||||
# Copyright (c) 2021-2023 The MVT Authors.
|
||||
# Use of this software is governed by the MVT License 1.1 that can be found at
|
||||
# https://license.mvt.re/1.1/
|
||||
from typing import Union
|
||||
|
||||
from .artifact import AndroidArtifact
|
||||
|
||||
|
||||
class FileTimestampsArtifact(AndroidArtifact):
|
||||
def serialize(self, record: dict) -> Union[dict, list]:
|
||||
records = []
|
||||
|
||||
for ts in set(
|
||||
[
|
||||
record.get("access_time"),
|
||||
record.get("changed_time"),
|
||||
record.get("modified_time"),
|
||||
]
|
||||
):
|
||||
if not ts:
|
||||
continue
|
||||
|
||||
macb = ""
|
||||
macb += "M" if ts == record.get("modified_time") else "-"
|
||||
macb += "A" if ts == record.get("access_time") else "-"
|
||||
macb += "C" if ts == record.get("changed_time") else "-"
|
||||
macb += "-"
|
||||
|
||||
msg = record["path"]
|
||||
if record.get("context"):
|
||||
msg += f" ({record['context']})"
|
||||
|
||||
records.append(
|
||||
{
|
||||
"timestamp": ts,
|
||||
"module": self.__class__.__name__,
|
||||
"event": macb,
|
||||
"data": msg,
|
||||
}
|
||||
)
|
||||
|
||||
return records
|
||||
@@ -42,6 +42,17 @@ class GetProp(AndroidArtifact):
|
||||
entry = {"name": matches[0][0], "value": matches[0][1]}
|
||||
self.results.append(entry)
|
||||
|
||||
def get_device_timezone(self) -> str:
|
||||
"""
|
||||
Get the device timezone from the getprop results
|
||||
|
||||
Used in other moduels to calculate the timezone offset
|
||||
"""
|
||||
for entry in self.results:
|
||||
if entry["name"] == "persist.sys.timezone":
|
||||
return entry["value"]
|
||||
return None
|
||||
|
||||
def check_indicators(self) -> None:
|
||||
for entry in self.results:
|
||||
if entry["name"] in INTERESTING_PROPERTIES:
|
||||
|
||||
@@ -16,6 +16,11 @@ ANDROID_DANGEROUS_SETTINGS = [
|
||||
"key": "package_verifier_enable",
|
||||
"safe_value": "1",
|
||||
},
|
||||
{
|
||||
"description": "disabled APK package verification",
|
||||
"key": "package_verifier_state",
|
||||
"safe_value": "1",
|
||||
},
|
||||
{
|
||||
"description": "disabled Google Play Protect",
|
||||
"key": "package_verifier_user_consent",
|
||||
|
||||
270
src/mvt/android/artifacts/tombstone_crashes.py
Normal file
270
src/mvt/android/artifacts/tombstone_crashes.py
Normal file
File diff suppressed because it is too large
Load Diff
@@ -48,6 +48,37 @@ class AndroidQFModule(MVTModule):
|
||||
def _get_files_by_pattern(self, pattern: str):
|
||||
return fnmatch.filter(self.files, pattern)
|
||||
|
||||
def _get_device_timezone(self):
|
||||
"""
|
||||
Get the device timezone from the getprop.txt file.
|
||||
|
||||
This is needed to map local timestamps stored in some
|
||||
Android log files to UTC/timezone-aware timestamps.
|
||||
"""
|
||||
get_prop_files = self._get_files_by_pattern("*/getprop.txt")
|
||||
if not get_prop_files:
|
||||
self.log.warning(
|
||||
"Could not find getprop.txt file. "
|
||||
"Some timestamps and timeline data may be incorrect."
|
||||
)
|
||||
return None
|
||||
|
||||
from mvt.android.artifacts.getprop import GetProp
|
||||
|
||||
properties_artifact = GetProp()
|
||||
prop_data = self._get_file_content(get_prop_files[0]).decode("utf-8")
|
||||
properties_artifact.parse(prop_data)
|
||||
timezone = properties_artifact.get_device_timezone()
|
||||
if timezone:
|
||||
self.log.debug("Identified local phone timezone: %s", timezone)
|
||||
return timezone
|
||||
|
||||
self.log.warning(
|
||||
"Could not find or determine local device timezone. "
|
||||
"Some timestamps and timeline data may be incorrect."
|
||||
)
|
||||
return None
|
||||
|
||||
def _get_file_content(self, file_path):
|
||||
if self.archive:
|
||||
handle = self.archive.open(file_path)
|
||||
|
||||
@@ -6,6 +6,11 @@
|
||||
import datetime
|
||||
import json
|
||||
import logging
|
||||
|
||||
try:
|
||||
import zoneinfo
|
||||
except ImportError:
|
||||
from backports import zoneinfo
|
||||
from typing import Optional, Union
|
||||
|
||||
from mvt.android.modules.androidqf.base import AndroidQFModule
|
||||
@@ -106,6 +111,12 @@ class Files(AndroidQFModule):
|
||||
# TODO: adds SHA1 and MD5 when available in MVT
|
||||
|
||||
def run(self) -> None:
|
||||
if timezone := self._get_device_timezone():
|
||||
device_timezone = zoneinfo.ZoneInfo(timezone)
|
||||
else:
|
||||
self.log.warning("Unable to determine device timezone, using UTC")
|
||||
device_timezone = zoneinfo.ZoneInfo("UTC")
|
||||
|
||||
for file in self._get_files_by_pattern("*/files.json"):
|
||||
rawdata = self._get_file_content(file).decode("utf-8", errors="ignore")
|
||||
try:
|
||||
@@ -120,11 +131,18 @@ class Files(AndroidQFModule):
|
||||
for file_data in data:
|
||||
for ts in ["access_time", "changed_time", "modified_time"]:
|
||||
if ts in file_data:
|
||||
file_data[ts] = convert_datetime_to_iso(
|
||||
datetime.datetime.fromtimestamp(
|
||||
file_data[ts], tz=datetime.timezone.utc
|
||||
)
|
||||
utc_timestamp = datetime.datetime.fromtimestamp(
|
||||
file_data[ts], tz=datetime.timezone.utc
|
||||
)
|
||||
# Convert the UTC timestamp to local tiem on Android device's local timezone
|
||||
local_timestamp = utc_timestamp.astimezone(device_timezone)
|
||||
|
||||
# HACK: We only output the UTC timestamp in convert_datetime_to_iso, we
|
||||
# set the timestamp timezone to UTC, to avoid the timezone conversion again.
|
||||
local_timestamp = local_timestamp.replace(
|
||||
tzinfo=datetime.timezone.utc
|
||||
)
|
||||
file_data[ts] = convert_datetime_to_iso(local_timestamp)
|
||||
|
||||
self.results.append(file_data)
|
||||
|
||||
|
||||
65
src/mvt/android/modules/androidqf/logfile_timestamps.py
Normal file
65
src/mvt/android/modules/androidqf/logfile_timestamps.py
Normal file
@@ -0,0 +1,65 @@
|
||||
# Mobile Verification Toolkit (MVT)
|
||||
# Copyright (c) 2021-2023 The MVT Authors.
|
||||
# Use of this software is governed by the MVT License 1.1 that can be found at
|
||||
# https://license.mvt.re/1.1/
|
||||
|
||||
import os
|
||||
import datetime
|
||||
import logging
|
||||
from typing import Optional
|
||||
|
||||
from mvt.common.utils import convert_datetime_to_iso
|
||||
from .base import AndroidQFModule
|
||||
from mvt.android.artifacts.file_timestamps import FileTimestampsArtifact
|
||||
|
||||
|
||||
class LogsFileTimestamps(FileTimestampsArtifact, AndroidQFModule):
|
||||
"""This module extracts records from battery daily updates."""
|
||||
|
||||
slug = "logfile_timestamps"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
file_path: Optional[str] = None,
|
||||
target_path: Optional[str] = None,
|
||||
results_path: Optional[str] = None,
|
||||
module_options: Optional[dict] = None,
|
||||
log: logging.Logger = logging.getLogger(__name__),
|
||||
results: Optional[list] = None,
|
||||
) -> None:
|
||||
super().__init__(
|
||||
file_path=file_path,
|
||||
target_path=target_path,
|
||||
results_path=results_path,
|
||||
module_options=module_options,
|
||||
log=log,
|
||||
results=results,
|
||||
)
|
||||
|
||||
def _get_file_modification_time(self, file_path: str) -> dict:
|
||||
if self.archive:
|
||||
file_timetuple = self.archive.getinfo(file_path).date_time
|
||||
return datetime.datetime(*file_timetuple)
|
||||
else:
|
||||
file_stat = os.stat(os.path.join(self.parent_path, file_path))
|
||||
return datetime.datetime.fromtimestamp(file_stat.st_mtime)
|
||||
|
||||
def run(self) -> None:
|
||||
filesystem_files = self._get_files_by_pattern("*/logs/*")
|
||||
|
||||
self.results = []
|
||||
for file in filesystem_files:
|
||||
# Only the modification time is available in the zip file metadata.
|
||||
# The timezone is the local timezone of the machine the phone.
|
||||
modification_time = self._get_file_modification_time(file)
|
||||
self.results.append(
|
||||
{
|
||||
"path": file,
|
||||
"modified_time": convert_datetime_to_iso(modification_time),
|
||||
}
|
||||
)
|
||||
|
||||
self.log.info(
|
||||
"Extracted a total of %d filesystem timestamps from AndroidQF logs directory.",
|
||||
len(self.results),
|
||||
)
|
||||
@@ -3,10 +3,11 @@
|
||||
# Use of this software is governed by the MVT License 1.1 that can be found at
|
||||
# https://license.mvt.re/1.1/
|
||||
|
||||
import os
|
||||
|
||||
from rich.prompt import Prompt
|
||||
|
||||
from mvt.common.config import settings
|
||||
|
||||
MVT_ANDROID_BACKUP_PASSWORD = "MVT_ANDROID_BACKUP_PASSWORD"
|
||||
|
||||
|
||||
@@ -16,24 +17,24 @@ def cli_load_android_backup_password(log, backup_password):
|
||||
|
||||
Used in MVT CLI command parsers.
|
||||
"""
|
||||
password_from_env = os.environ.get(MVT_ANDROID_BACKUP_PASSWORD, None)
|
||||
password_from_env_or_config = settings.ANDROID_BACKUP_PASSWORD
|
||||
if backup_password:
|
||||
log.info(
|
||||
"Your password may be visible in the process table because it "
|
||||
"was supplied on the command line!"
|
||||
)
|
||||
if password_from_env:
|
||||
if password_from_env_or_config:
|
||||
log.info(
|
||||
"Ignoring %s environment variable, using --backup-password argument instead",
|
||||
MVT_ANDROID_BACKUP_PASSWORD,
|
||||
"MVT_ANDROID_BACKUP_PASSWORD",
|
||||
)
|
||||
return backup_password
|
||||
elif password_from_env:
|
||||
elif password_from_env_or_config:
|
||||
log.info(
|
||||
"Using backup password from %s environment variable",
|
||||
MVT_ANDROID_BACKUP_PASSWORD,
|
||||
"Using backup password from %s environment variable or config file",
|
||||
"MVT_ANDROID_BACKUP_PASSWORD",
|
||||
)
|
||||
return password_from_env
|
||||
return password_from_env_or_config
|
||||
|
||||
|
||||
def prompt_or_load_android_backup_password(log, module_options):
|
||||
|
||||
@@ -14,6 +14,8 @@ from .packages import Packages
|
||||
from .platform_compat import PlatformCompat
|
||||
from .receivers import Receivers
|
||||
from .adb_state import DumpsysADBState
|
||||
from .fs_timestamps import BugReportTimestamps
|
||||
from .tombstones import Tombstones
|
||||
|
||||
BUGREPORT_MODULES = [
|
||||
Accessibility,
|
||||
@@ -27,4 +29,6 @@ BUGREPORT_MODULES = [
|
||||
PlatformCompat,
|
||||
Receivers,
|
||||
DumpsysADBState,
|
||||
BugReportTimestamps,
|
||||
Tombstones,
|
||||
]
|
||||
|
||||
@@ -2,10 +2,11 @@
|
||||
# Copyright (c) 2021-2023 The MVT Authors.
|
||||
# See the file 'LICENSE' for usage and copying permissions, or find a copy at
|
||||
# https://github.com/mvt-project/mvt/blob/main/LICENSE
|
||||
|
||||
import datetime
|
||||
import fnmatch
|
||||
import logging
|
||||
import os
|
||||
|
||||
from typing import List, Optional
|
||||
from zipfile import ZipFile
|
||||
|
||||
@@ -91,3 +92,11 @@ class BugReportModule(MVTModule):
|
||||
return None
|
||||
|
||||
return self._get_file_content(dumpstate_logs[0])
|
||||
|
||||
def _get_file_modification_time(self, file_path: str) -> dict:
|
||||
if self.zip_archive:
|
||||
file_timetuple = self.zip_archive.getinfo(file_path).date_time
|
||||
return datetime.datetime(*file_timetuple)
|
||||
else:
|
||||
file_stat = os.stat(os.path.join(self.extract_path, file_path))
|
||||
return datetime.datetime.fromtimestamp(file_stat.st_mtime)
|
||||
|
||||
55
src/mvt/android/modules/bugreport/fs_timestamps.py
Normal file
55
src/mvt/android/modules/bugreport/fs_timestamps.py
Normal file
@@ -0,0 +1,55 @@
|
||||
# Mobile Verification Toolkit (MVT)
|
||||
# Copyright (c) 2021-2023 The MVT Authors.
|
||||
# Use of this software is governed by the MVT License 1.1 that can be found at
|
||||
# https://license.mvt.re/1.1/
|
||||
|
||||
import logging
|
||||
from typing import Optional
|
||||
|
||||
from mvt.common.utils import convert_datetime_to_iso
|
||||
from .base import BugReportModule
|
||||
from mvt.android.artifacts.file_timestamps import FileTimestampsArtifact
|
||||
|
||||
|
||||
class BugReportTimestamps(FileTimestampsArtifact, BugReportModule):
|
||||
"""This module extracts records from battery daily updates."""
|
||||
|
||||
slug = "bugreport_timestamps"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
file_path: Optional[str] = None,
|
||||
target_path: Optional[str] = None,
|
||||
results_path: Optional[str] = None,
|
||||
module_options: Optional[dict] = None,
|
||||
log: logging.Logger = logging.getLogger(__name__),
|
||||
results: Optional[list] = None,
|
||||
) -> None:
|
||||
super().__init__(
|
||||
file_path=file_path,
|
||||
target_path=target_path,
|
||||
results_path=results_path,
|
||||
module_options=module_options,
|
||||
log=log,
|
||||
results=results,
|
||||
)
|
||||
|
||||
def run(self) -> None:
|
||||
filesystem_files = self._get_files_by_pattern("FS/*")
|
||||
|
||||
self.results = []
|
||||
for file in filesystem_files:
|
||||
# Only the modification time is available in the zip file metadata.
|
||||
# The timezone is the local timezone of the machine the phone.
|
||||
modification_time = self._get_file_modification_time(file)
|
||||
self.results.append(
|
||||
{
|
||||
"path": file,
|
||||
"modified_time": convert_datetime_to_iso(modification_time),
|
||||
}
|
||||
)
|
||||
|
||||
self.log.info(
|
||||
"Extracted a total of %d filesystem timestamps from bugreport.",
|
||||
len(self.results),
|
||||
)
|
||||
64
src/mvt/android/modules/bugreport/tombstones.py
Normal file
64
src/mvt/android/modules/bugreport/tombstones.py
Normal file
@@ -0,0 +1,64 @@
|
||||
# Mobile Verification Toolkit (MVT)
|
||||
# Copyright (c) 2021-2023 The MVT Authors.
|
||||
# Use of this software is governed by the MVT License 1.1 that can be found at
|
||||
# https://license.mvt.re/1.1/
|
||||
|
||||
import logging
|
||||
from typing import Optional
|
||||
|
||||
from mvt.android.artifacts.tombstone_crashes import TombstoneCrashArtifact
|
||||
from .base import BugReportModule
|
||||
|
||||
|
||||
class Tombstones(TombstoneCrashArtifact, BugReportModule):
|
||||
"""This module extracts records from battery daily updates."""
|
||||
|
||||
slug = "tombstones"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
file_path: Optional[str] = None,
|
||||
target_path: Optional[str] = None,
|
||||
results_path: Optional[str] = None,
|
||||
module_options: Optional[dict] = None,
|
||||
log: logging.Logger = logging.getLogger(__name__),
|
||||
results: Optional[list] = None,
|
||||
) -> None:
|
||||
super().__init__(
|
||||
file_path=file_path,
|
||||
target_path=target_path,
|
||||
results_path=results_path,
|
||||
module_options=module_options,
|
||||
log=log,
|
||||
results=results,
|
||||
)
|
||||
|
||||
def run(self) -> None:
|
||||
tombstone_files = self._get_files_by_pattern("*/tombstone_*")
|
||||
if not tombstone_files:
|
||||
self.log.error(
|
||||
"Unable to find any tombstone files. "
|
||||
"Did you provide a valid bugreport archive?"
|
||||
)
|
||||
return
|
||||
|
||||
for tombstone_file in sorted(tombstone_files):
|
||||
tombstone_filename = tombstone_file.split("/")[-1]
|
||||
modification_time = self._get_file_modification_time(tombstone_file)
|
||||
tombstone_data = self._get_file_content(tombstone_file)
|
||||
|
||||
try:
|
||||
if tombstone_file.endswith(".pb"):
|
||||
self.parse_protobuf(
|
||||
tombstone_filename, modification_time, tombstone_data
|
||||
)
|
||||
else:
|
||||
self.parse(tombstone_filename, modification_time, tombstone_data)
|
||||
except ValueError as e:
|
||||
# Catch any exceptions raised during parsing or validation.
|
||||
self.log.error(f"Error parsing tombstone file {tombstone_file}: {e}")
|
||||
|
||||
self.log.info(
|
||||
"Extracted a total of %d tombstone files",
|
||||
len(self.results),
|
||||
)
|
||||
0
src/mvt/android/parsers/proto/__init__.py
Normal file
0
src/mvt/android/parsers/proto/__init__.py
Normal file
195
src/mvt/android/parsers/proto/tombstone.proto
Normal file
195
src/mvt/android/parsers/proto/tombstone.proto
Normal file
@@ -0,0 +1,195 @@
|
||||
// tombstone.proto file from Android source
|
||||
// Src: https://android.googlesource.com/platform/system/core/+/refs/heads/main/debuggerd/proto/tombstone.proto
|
||||
//
|
||||
// Protobuf definition for Android tombstones.
|
||||
//
|
||||
// An app can get hold of these for any `REASON_CRASH_NATIVE` instance of
|
||||
// `android.app.ApplicationExitInfo`.
|
||||
//
|
||||
// https://developer.android.com/reference/android/app/ApplicationExitInfo#getTraceInputStream()
|
||||
//
|
||||
syntax = "proto3";
|
||||
option java_package = "com.android.server.os";
|
||||
option java_outer_classname = "TombstoneProtos";
|
||||
// NOTE TO OEMS:
|
||||
// If you add custom fields to this proto, do not use numbers in the reserved range.
|
||||
message CrashDetail {
|
||||
bytes name = 1;
|
||||
bytes data = 2;
|
||||
reserved 3 to 999;
|
||||
}
|
||||
message StackHistoryBufferEntry {
|
||||
BacktraceFrame addr = 1;
|
||||
uint64 fp = 2;
|
||||
uint64 tag = 3;
|
||||
reserved 4 to 999;
|
||||
}
|
||||
message StackHistoryBuffer {
|
||||
uint64 tid = 1;
|
||||
repeated StackHistoryBufferEntry entries = 2;
|
||||
reserved 3 to 999;
|
||||
}
|
||||
message Tombstone {
|
||||
Architecture arch = 1;
|
||||
Architecture guest_arch = 24;
|
||||
string build_fingerprint = 2;
|
||||
string revision = 3;
|
||||
string timestamp = 4;
|
||||
uint32 pid = 5;
|
||||
uint32 tid = 6;
|
||||
uint32 uid = 7;
|
||||
string selinux_label = 8;
|
||||
repeated string command_line = 9;
|
||||
// Process uptime in seconds.
|
||||
uint32 process_uptime = 20;
|
||||
Signal signal_info = 10;
|
||||
string abort_message = 14;
|
||||
repeated CrashDetail crash_details = 21;
|
||||
repeated Cause causes = 15;
|
||||
map<uint32, Thread> threads = 16;
|
||||
map<uint32, Thread> guest_threads = 25;
|
||||
repeated MemoryMapping memory_mappings = 17;
|
||||
repeated LogBuffer log_buffers = 18;
|
||||
repeated FD open_fds = 19;
|
||||
uint32 page_size = 22;
|
||||
bool has_been_16kb_mode = 23;
|
||||
StackHistoryBuffer stack_history_buffer = 26;
|
||||
reserved 27 to 999;
|
||||
}
|
||||
enum Architecture {
|
||||
ARM32 = 0;
|
||||
ARM64 = 1;
|
||||
X86 = 2;
|
||||
X86_64 = 3;
|
||||
RISCV64 = 4;
|
||||
NONE = 5;
|
||||
reserved 6 to 999;
|
||||
}
|
||||
message Signal {
|
||||
int32 number = 1;
|
||||
string name = 2;
|
||||
int32 code = 3;
|
||||
string code_name = 4;
|
||||
bool has_sender = 5;
|
||||
int32 sender_uid = 6;
|
||||
int32 sender_pid = 7;
|
||||
bool has_fault_address = 8;
|
||||
uint64 fault_address = 9;
|
||||
// Note, may or may not contain the dump of the actual memory contents. Currently, on arm64, we
|
||||
// only include metadata, and not the contents.
|
||||
MemoryDump fault_adjacent_metadata = 10;
|
||||
reserved 11 to 999;
|
||||
}
|
||||
message HeapObject {
|
||||
uint64 address = 1;
|
||||
uint64 size = 2;
|
||||
uint64 allocation_tid = 3;
|
||||
repeated BacktraceFrame allocation_backtrace = 4;
|
||||
uint64 deallocation_tid = 5;
|
||||
repeated BacktraceFrame deallocation_backtrace = 6;
|
||||
}
|
||||
message MemoryError {
|
||||
enum Tool {
|
||||
GWP_ASAN = 0;
|
||||
SCUDO = 1;
|
||||
reserved 2 to 999;
|
||||
}
|
||||
Tool tool = 1;
|
||||
enum Type {
|
||||
UNKNOWN = 0;
|
||||
USE_AFTER_FREE = 1;
|
||||
DOUBLE_FREE = 2;
|
||||
INVALID_FREE = 3;
|
||||
BUFFER_OVERFLOW = 4;
|
||||
BUFFER_UNDERFLOW = 5;
|
||||
reserved 6 to 999;
|
||||
}
|
||||
Type type = 2;
|
||||
oneof location {
|
||||
HeapObject heap = 3;
|
||||
}
|
||||
reserved 4 to 999;
|
||||
}
|
||||
message Cause {
|
||||
string human_readable = 1;
|
||||
oneof details {
|
||||
MemoryError memory_error = 2;
|
||||
}
|
||||
reserved 3 to 999;
|
||||
}
|
||||
message Register {
|
||||
string name = 1;
|
||||
uint64 u64 = 2;
|
||||
reserved 3 to 999;
|
||||
}
|
||||
message Thread {
|
||||
int32 id = 1;
|
||||
string name = 2;
|
||||
repeated Register registers = 3;
|
||||
repeated string backtrace_note = 7;
|
||||
repeated string unreadable_elf_files = 9;
|
||||
repeated BacktraceFrame current_backtrace = 4;
|
||||
repeated MemoryDump memory_dump = 5;
|
||||
int64 tagged_addr_ctrl = 6;
|
||||
int64 pac_enabled_keys = 8;
|
||||
reserved 10 to 999;
|
||||
}
|
||||
message BacktraceFrame {
|
||||
uint64 rel_pc = 1;
|
||||
uint64 pc = 2;
|
||||
uint64 sp = 3;
|
||||
string function_name = 4;
|
||||
uint64 function_offset = 5;
|
||||
string file_name = 6;
|
||||
uint64 file_map_offset = 7;
|
||||
string build_id = 8;
|
||||
reserved 9 to 999;
|
||||
}
|
||||
message ArmMTEMetadata {
|
||||
// One memory tag per granule (e.g. every 16 bytes) of regular memory.
|
||||
bytes memory_tags = 1;
|
||||
reserved 2 to 999;
|
||||
}
|
||||
message MemoryDump {
|
||||
string register_name = 1;
|
||||
string mapping_name = 2;
|
||||
uint64 begin_address = 3;
|
||||
bytes memory = 4;
|
||||
oneof metadata {
|
||||
ArmMTEMetadata arm_mte_metadata = 6;
|
||||
}
|
||||
reserved 5, 7 to 999;
|
||||
}
|
||||
message MemoryMapping {
|
||||
uint64 begin_address = 1;
|
||||
uint64 end_address = 2;
|
||||
uint64 offset = 3;
|
||||
bool read = 4;
|
||||
bool write = 5;
|
||||
bool execute = 6;
|
||||
string mapping_name = 7;
|
||||
string build_id = 8;
|
||||
uint64 load_bias = 9;
|
||||
reserved 10 to 999;
|
||||
}
|
||||
message FD {
|
||||
int32 fd = 1;
|
||||
string path = 2;
|
||||
string owner = 3;
|
||||
uint64 tag = 4;
|
||||
reserved 5 to 999;
|
||||
}
|
||||
message LogBuffer {
|
||||
string name = 1;
|
||||
repeated LogMessage logs = 2;
|
||||
reserved 3 to 999;
|
||||
}
|
||||
message LogMessage {
|
||||
string timestamp = 1;
|
||||
uint32 pid = 2;
|
||||
uint32 tid = 3;
|
||||
uint32 priority = 4;
|
||||
string tag = 5;
|
||||
string message = 6;
|
||||
reserved 7 to 999;
|
||||
}
|
||||
208
src/mvt/android/parsers/proto/tombstone.py
Normal file
208
src/mvt/android/parsers/proto/tombstone.py
Normal file
@@ -0,0 +1,208 @@
|
||||
# Generated by the protocol buffer compiler. DO NOT EDIT!
|
||||
# sources: tombstone.proto
|
||||
# plugin: python-betterproto
|
||||
from dataclasses import dataclass
|
||||
from typing import Dict, List
|
||||
|
||||
import betterproto
|
||||
|
||||
|
||||
class Architecture(betterproto.Enum):
|
||||
ARM32 = 0
|
||||
ARM64 = 1
|
||||
X86 = 2
|
||||
X86_64 = 3
|
||||
RISCV64 = 4
|
||||
NONE = 5
|
||||
|
||||
|
||||
class MemoryErrorTool(betterproto.Enum):
|
||||
GWP_ASAN = 0
|
||||
SCUDO = 1
|
||||
|
||||
|
||||
class MemoryErrorType(betterproto.Enum):
|
||||
UNKNOWN = 0
|
||||
USE_AFTER_FREE = 1
|
||||
DOUBLE_FREE = 2
|
||||
INVALID_FREE = 3
|
||||
BUFFER_OVERFLOW = 4
|
||||
BUFFER_UNDERFLOW = 5
|
||||
|
||||
|
||||
@dataclass
|
||||
class CrashDetail(betterproto.Message):
|
||||
"""
|
||||
NOTE TO OEMS: If you add custom fields to this proto, do not use numbers in
|
||||
the reserved range.
|
||||
"""
|
||||
|
||||
name: bytes = betterproto.bytes_field(1)
|
||||
data: bytes = betterproto.bytes_field(2)
|
||||
|
||||
|
||||
@dataclass
|
||||
class StackHistoryBufferEntry(betterproto.Message):
|
||||
addr: "BacktraceFrame" = betterproto.message_field(1)
|
||||
fp: int = betterproto.uint64_field(2)
|
||||
tag: int = betterproto.uint64_field(3)
|
||||
|
||||
|
||||
@dataclass
|
||||
class StackHistoryBuffer(betterproto.Message):
|
||||
tid: int = betterproto.uint64_field(1)
|
||||
entries: List["StackHistoryBufferEntry"] = betterproto.message_field(2)
|
||||
|
||||
|
||||
@dataclass
|
||||
class Tombstone(betterproto.Message):
|
||||
arch: "Architecture" = betterproto.enum_field(1)
|
||||
guest_arch: "Architecture" = betterproto.enum_field(24)
|
||||
build_fingerprint: str = betterproto.string_field(2)
|
||||
revision: str = betterproto.string_field(3)
|
||||
timestamp: str = betterproto.string_field(4)
|
||||
pid: int = betterproto.uint32_field(5)
|
||||
tid: int = betterproto.uint32_field(6)
|
||||
uid: int = betterproto.uint32_field(7)
|
||||
selinux_label: str = betterproto.string_field(8)
|
||||
command_line: List[str] = betterproto.string_field(9)
|
||||
# Process uptime in seconds.
|
||||
process_uptime: int = betterproto.uint32_field(20)
|
||||
signal_info: "Signal" = betterproto.message_field(10)
|
||||
abort_message: str = betterproto.string_field(14)
|
||||
crash_details: List["CrashDetail"] = betterproto.message_field(21)
|
||||
causes: List["Cause"] = betterproto.message_field(15)
|
||||
threads: Dict[int, "Thread"] = betterproto.map_field(
|
||||
16, betterproto.TYPE_UINT32, betterproto.TYPE_MESSAGE
|
||||
)
|
||||
guest_threads: Dict[int, "Thread"] = betterproto.map_field(
|
||||
25, betterproto.TYPE_UINT32, betterproto.TYPE_MESSAGE
|
||||
)
|
||||
memory_mappings: List["MemoryMapping"] = betterproto.message_field(17)
|
||||
log_buffers: List["LogBuffer"] = betterproto.message_field(18)
|
||||
open_fds: List["FD"] = betterproto.message_field(19)
|
||||
page_size: int = betterproto.uint32_field(22)
|
||||
has_been_16kb_mode: bool = betterproto.bool_field(23)
|
||||
stack_history_buffer: "StackHistoryBuffer" = betterproto.message_field(26)
|
||||
|
||||
|
||||
@dataclass
|
||||
class Signal(betterproto.Message):
|
||||
number: int = betterproto.int32_field(1)
|
||||
name: str = betterproto.string_field(2)
|
||||
code: int = betterproto.int32_field(3)
|
||||
code_name: str = betterproto.string_field(4)
|
||||
has_sender: bool = betterproto.bool_field(5)
|
||||
sender_uid: int = betterproto.int32_field(6)
|
||||
sender_pid: int = betterproto.int32_field(7)
|
||||
has_fault_address: bool = betterproto.bool_field(8)
|
||||
fault_address: int = betterproto.uint64_field(9)
|
||||
# Note, may or may not contain the dump of the actual memory contents.
|
||||
# Currently, on arm64, we only include metadata, and not the contents.
|
||||
fault_adjacent_metadata: "MemoryDump" = betterproto.message_field(10)
|
||||
|
||||
|
||||
@dataclass
|
||||
class HeapObject(betterproto.Message):
|
||||
address: int = betterproto.uint64_field(1)
|
||||
size: int = betterproto.uint64_field(2)
|
||||
allocation_tid: int = betterproto.uint64_field(3)
|
||||
allocation_backtrace: List["BacktraceFrame"] = betterproto.message_field(4)
|
||||
deallocation_tid: int = betterproto.uint64_field(5)
|
||||
deallocation_backtrace: List["BacktraceFrame"] = betterproto.message_field(6)
|
||||
|
||||
|
||||
@dataclass
|
||||
class MemoryError(betterproto.Message):
|
||||
tool: "MemoryErrorTool" = betterproto.enum_field(1)
|
||||
type: "MemoryErrorType" = betterproto.enum_field(2)
|
||||
heap: "HeapObject" = betterproto.message_field(3, group="location")
|
||||
|
||||
|
||||
@dataclass
|
||||
class Cause(betterproto.Message):
|
||||
human_readable: str = betterproto.string_field(1)
|
||||
memory_error: "MemoryError" = betterproto.message_field(2, group="details")
|
||||
|
||||
|
||||
@dataclass
|
||||
class Register(betterproto.Message):
|
||||
name: str = betterproto.string_field(1)
|
||||
u64: int = betterproto.uint64_field(2)
|
||||
|
||||
|
||||
@dataclass
|
||||
class Thread(betterproto.Message):
|
||||
id: int = betterproto.int32_field(1)
|
||||
name: str = betterproto.string_field(2)
|
||||
registers: List["Register"] = betterproto.message_field(3)
|
||||
backtrace_note: List[str] = betterproto.string_field(7)
|
||||
unreadable_elf_files: List[str] = betterproto.string_field(9)
|
||||
current_backtrace: List["BacktraceFrame"] = betterproto.message_field(4)
|
||||
memory_dump: List["MemoryDump"] = betterproto.message_field(5)
|
||||
tagged_addr_ctrl: int = betterproto.int64_field(6)
|
||||
pac_enabled_keys: int = betterproto.int64_field(8)
|
||||
|
||||
|
||||
@dataclass
|
||||
class BacktraceFrame(betterproto.Message):
|
||||
rel_pc: int = betterproto.uint64_field(1)
|
||||
pc: int = betterproto.uint64_field(2)
|
||||
sp: int = betterproto.uint64_field(3)
|
||||
function_name: str = betterproto.string_field(4)
|
||||
function_offset: int = betterproto.uint64_field(5)
|
||||
file_name: str = betterproto.string_field(6)
|
||||
file_map_offset: int = betterproto.uint64_field(7)
|
||||
build_id: str = betterproto.string_field(8)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ArmMTEMetadata(betterproto.Message):
|
||||
# One memory tag per granule (e.g. every 16 bytes) of regular memory.
|
||||
memory_tags: bytes = betterproto.bytes_field(1)
|
||||
|
||||
|
||||
@dataclass
|
||||
class MemoryDump(betterproto.Message):
|
||||
register_name: str = betterproto.string_field(1)
|
||||
mapping_name: str = betterproto.string_field(2)
|
||||
begin_address: int = betterproto.uint64_field(3)
|
||||
memory: bytes = betterproto.bytes_field(4)
|
||||
arm_mte_metadata: "ArmMTEMetadata" = betterproto.message_field(6, group="metadata")
|
||||
|
||||
|
||||
@dataclass
|
||||
class MemoryMapping(betterproto.Message):
|
||||
begin_address: int = betterproto.uint64_field(1)
|
||||
end_address: int = betterproto.uint64_field(2)
|
||||
offset: int = betterproto.uint64_field(3)
|
||||
read: bool = betterproto.bool_field(4)
|
||||
write: bool = betterproto.bool_field(5)
|
||||
execute: bool = betterproto.bool_field(6)
|
||||
mapping_name: str = betterproto.string_field(7)
|
||||
build_id: str = betterproto.string_field(8)
|
||||
load_bias: int = betterproto.uint64_field(9)
|
||||
|
||||
|
||||
@dataclass
|
||||
class FD(betterproto.Message):
|
||||
fd: int = betterproto.int32_field(1)
|
||||
path: str = betterproto.string_field(2)
|
||||
owner: str = betterproto.string_field(3)
|
||||
tag: int = betterproto.uint64_field(4)
|
||||
|
||||
|
||||
@dataclass
|
||||
class LogBuffer(betterproto.Message):
|
||||
name: str = betterproto.string_field(1)
|
||||
logs: List["LogMessage"] = betterproto.message_field(2)
|
||||
|
||||
|
||||
@dataclass
|
||||
class LogMessage(betterproto.Message):
|
||||
timestamp: str = betterproto.string_field(1)
|
||||
pid: int = betterproto.uint32_field(2)
|
||||
tid: int = betterproto.uint32_field(3)
|
||||
priority: int = betterproto.uint32_field(4)
|
||||
tag: str = betterproto.string_field(5)
|
||||
message: str = betterproto.string_field(6)
|
||||
@@ -65,6 +65,10 @@ class CmdCheckIOCS(Command):
|
||||
m = iocs_module.from_json(
|
||||
file_path, log=logging.getLogger(iocs_module.__module__)
|
||||
)
|
||||
if not m:
|
||||
log.warning("No result from this module, skipping it")
|
||||
continue
|
||||
|
||||
if self.iocs.total_ioc_count > 0:
|
||||
m.indicators = self.iocs
|
||||
m.indicators.log = m.log
|
||||
|
||||
@@ -17,6 +17,7 @@ from mvt.common.utils import (
|
||||
generate_hashes_from_path,
|
||||
get_sha256_from_file_path,
|
||||
)
|
||||
from mvt.common.config import settings
|
||||
from mvt.common.version import MVT_VERSION
|
||||
|
||||
|
||||
@@ -100,15 +101,25 @@ class Command:
|
||||
if not self.results_path:
|
||||
return
|
||||
|
||||
# We use local timestamps in the timeline on Android as many
|
||||
# logs do not contain timezone information.
|
||||
if type(self).__name__.startswith("CmdAndroid"):
|
||||
is_utc = False
|
||||
else:
|
||||
is_utc = True
|
||||
|
||||
if len(self.timeline) > 0:
|
||||
save_timeline(
|
||||
self.timeline, os.path.join(self.results_path, "timeline.csv")
|
||||
self.timeline,
|
||||
os.path.join(self.results_path, "timeline.csv"),
|
||||
is_utc=is_utc,
|
||||
)
|
||||
|
||||
if len(self.timeline_detected) > 0:
|
||||
save_timeline(
|
||||
self.timeline_detected,
|
||||
os.path.join(self.results_path, "timeline_detected.csv"),
|
||||
is_utc=is_utc,
|
||||
)
|
||||
|
||||
def _store_info(self) -> None:
|
||||
@@ -132,7 +143,7 @@ class Command:
|
||||
if ioc_file_path and ioc_file_path not in info["ioc_files"]:
|
||||
info["ioc_files"].append(ioc_file_path)
|
||||
|
||||
if self.target_path and (os.environ.get("MVT_HASH_FILES") or self.hashes):
|
||||
if self.target_path and (settings.HASH_FILES or self.hashes):
|
||||
self.generate_hashes()
|
||||
|
||||
info["hashes"] = self.hash_values
|
||||
@@ -141,7 +152,7 @@ class Command:
|
||||
with open(info_path, "w+", encoding="utf-8") as handle:
|
||||
json.dump(info, handle, indent=4)
|
||||
|
||||
if self.target_path and (os.environ.get("MVT_HASH_FILES") or self.hashes):
|
||||
if self.target_path and (settings.HASH_FILES or self.hashes):
|
||||
info_hash = get_sha256_from_file_path(info_path)
|
||||
self.log.info('Reference hash of the info.json file: "%s"', info_hash)
|
||||
|
||||
|
||||
105
src/mvt/common/config.py
Normal file
105
src/mvt/common/config.py
Normal file
@@ -0,0 +1,105 @@
|
||||
import os
|
||||
import yaml
|
||||
import json
|
||||
|
||||
from typing import Tuple, Type, Optional
|
||||
from appdirs import user_config_dir
|
||||
from pydantic import AnyHttpUrl, Field
|
||||
from pydantic_settings import (
|
||||
BaseSettings,
|
||||
InitSettingsSource,
|
||||
PydanticBaseSettingsSource,
|
||||
SettingsConfigDict,
|
||||
YamlConfigSettingsSource,
|
||||
)
|
||||
|
||||
MVT_CONFIG_FOLDER = user_config_dir("mvt")
|
||||
MVT_CONFIG_PATH = os.path.join(MVT_CONFIG_FOLDER, "config.yaml")
|
||||
|
||||
|
||||
class MVTSettings(BaseSettings):
|
||||
model_config = SettingsConfigDict(
|
||||
env_prefix="MVT_",
|
||||
env_nested_delimiter="_",
|
||||
extra="ignore",
|
||||
nested_model_default_partial_updates=True,
|
||||
)
|
||||
# Allow to decided if want to load environment variables
|
||||
load_env: bool = Field(True, exclude=True)
|
||||
|
||||
# General settings
|
||||
PYPI_UPDATE_URL: AnyHttpUrl = Field(
|
||||
"https://pypi.org/pypi/mvt/json",
|
||||
validate_default=False,
|
||||
)
|
||||
NETWORK_ACCESS_ALLOWED: bool = True
|
||||
NETWORK_TIMEOUT: int = 15
|
||||
|
||||
# Command default settings, all can be specified by MVT_ prefixed environment variables too.
|
||||
IOS_BACKUP_PASSWORD: Optional[str] = Field(
|
||||
None, description="Default password to use to decrypt iOS backups"
|
||||
)
|
||||
ANDROID_BACKUP_PASSWORD: Optional[str] = Field(
|
||||
None, description="Default password to use to decrypt Android backups"
|
||||
)
|
||||
STIX2: Optional[str] = Field(
|
||||
None, description="List of directories where STIX2 files are stored"
|
||||
)
|
||||
VT_API_KEY: Optional[str] = Field(
|
||||
None, description="API key to use for VirusTotal lookups"
|
||||
)
|
||||
PROFILE: bool = Field(False, description="Profile the execution of MVT modules")
|
||||
HASH_FILES: bool = Field(False, description="Should MVT hash output files")
|
||||
|
||||
@classmethod
|
||||
def settings_customise_sources(
|
||||
cls,
|
||||
settings_cls: Type[BaseSettings],
|
||||
init_settings: InitSettingsSource,
|
||||
env_settings: PydanticBaseSettingsSource,
|
||||
dotenv_settings: PydanticBaseSettingsSource,
|
||||
file_secret_settings: PydanticBaseSettingsSource,
|
||||
) -> Tuple[PydanticBaseSettingsSource, ...]:
|
||||
sources = (
|
||||
YamlConfigSettingsSource(settings_cls, MVT_CONFIG_PATH),
|
||||
init_settings,
|
||||
)
|
||||
# Load env variables if enabled
|
||||
if init_settings.init_kwargs.get("load_env", True):
|
||||
sources = (env_settings,) + sources
|
||||
return sources
|
||||
|
||||
def save_settings(
|
||||
self,
|
||||
) -> None:
|
||||
"""
|
||||
Save the current settings to a file.
|
||||
"""
|
||||
if not os.path.isdir(MVT_CONFIG_FOLDER):
|
||||
os.makedirs(MVT_CONFIG_FOLDER)
|
||||
|
||||
# Dump the settings to the YAML file
|
||||
model_serializable = json.loads(self.model_dump_json(exclude_defaults=True))
|
||||
with open(MVT_CONFIG_PATH, "w") as config_file:
|
||||
config_file.write(yaml.dump(model_serializable, default_flow_style=False))
|
||||
|
||||
@classmethod
|
||||
def initialise(cls) -> "MVTSettings":
|
||||
"""
|
||||
Initialise the settings file.
|
||||
|
||||
We first initialise the settings (without env variable) and then persist
|
||||
them to file. This way we can update the config file with the default values.
|
||||
|
||||
Afterwards we load the settings again, this time including the env variables.
|
||||
"""
|
||||
# Set invalid env prefix to avoid loading env variables.
|
||||
settings = MVTSettings(load_env=False)
|
||||
settings.save_settings()
|
||||
|
||||
# Load the settings again with any ENV variables.
|
||||
settings = MVTSettings(load_env=True)
|
||||
return settings
|
||||
|
||||
|
||||
settings = MVTSettings.initialise()
|
||||
@@ -14,6 +14,7 @@ import ahocorasick
|
||||
from appdirs import user_data_dir
|
||||
|
||||
from .url import URL
|
||||
from .config import settings
|
||||
|
||||
MVT_DATA_FOLDER = user_data_dir("mvt")
|
||||
MVT_INDICATORS_FOLDER = os.path.join(MVT_DATA_FOLDER, "indicators")
|
||||
@@ -41,12 +42,12 @@ class Indicators:
|
||||
|
||||
def _check_stix2_env_variable(self) -> None:
|
||||
"""
|
||||
Checks if a variable MVT_STIX2 contains path to a STIX file. Also recursively searches through dirs in MVT_STIX2
|
||||
Checks if MVT_STIX2 setting or environment variable contains path to a STIX file. Also recursively searches through dirs in MVT_STIX2
|
||||
"""
|
||||
if "MVT_STIX2" not in os.environ:
|
||||
if not settings.STIX2:
|
||||
return
|
||||
|
||||
paths = os.environ["MVT_STIX2"].split(":")
|
||||
paths = settings.STIX2.split(":")
|
||||
for path in paths:
|
||||
if os.path.isfile(path) and path.lower().endswith(".stix2"):
|
||||
self.parse_stix2(path)
|
||||
|
||||
@@ -29,7 +29,7 @@ def check_updates() -> None:
|
||||
if latest_version:
|
||||
rich_print(
|
||||
f"\t\t[bold]Version {latest_version} is available! "
|
||||
"Upgrade mvt with `pip3 install -U mvt`[/bold]"
|
||||
"Upgrade mvt with `pip3 install -U mvt` or with `pipx upgrade mvt`[/bold]"
|
||||
)
|
||||
|
||||
# Then we check for indicators files updates.
|
||||
|
||||
@@ -69,10 +69,14 @@ class MVTModule:
|
||||
@classmethod
|
||||
def from_json(cls, json_path: str, log: logging.Logger):
|
||||
with open(json_path, "r", encoding="utf-8") as handle:
|
||||
results = json.load(handle)
|
||||
if log:
|
||||
log.info('Loaded %d results from "%s"', len(results), json_path)
|
||||
return cls(results=results, log=log)
|
||||
try:
|
||||
results = json.load(handle)
|
||||
if log:
|
||||
log.info('Loaded %d results from "%s"', len(results), json_path)
|
||||
return cls(results=results, log=log)
|
||||
except json.decoder.JSONDecodeError as err:
|
||||
log.error('Error to decode the json "%s" file: "%s"', json_path, err)
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
def get_slug(cls) -> str:
|
||||
@@ -227,7 +231,7 @@ def run_module(module: MVTModule) -> None:
|
||||
module.save_to_json()
|
||||
|
||||
|
||||
def save_timeline(timeline: list, timeline_path: str) -> None:
|
||||
def save_timeline(timeline: list, timeline_path: str, is_utc: bool = True) -> None:
|
||||
"""Save the timeline in a csv file.
|
||||
|
||||
:param timeline: List of records to order and store
|
||||
@@ -238,7 +242,12 @@ def save_timeline(timeline: list, timeline_path: str) -> None:
|
||||
csvoutput = csv.writer(
|
||||
handle, delimiter=",", quotechar='"', quoting=csv.QUOTE_ALL, escapechar="\\"
|
||||
)
|
||||
csvoutput.writerow(["UTC Timestamp", "Plugin", "Event", "Description"])
|
||||
|
||||
if is_utc:
|
||||
timestamp_header = "UTC Timestamp"
|
||||
else:
|
||||
timestamp_header = "Device Local Timestamp"
|
||||
csvoutput.writerow([timestamp_header, "Plugin", "Event", "Description"])
|
||||
|
||||
for event in sorted(
|
||||
timeline, key=lambda x: x["timestamp"] if x["timestamp"] is not None else ""
|
||||
|
||||
@@ -14,6 +14,7 @@ from packaging import version
|
||||
|
||||
from .indicators import MVT_DATA_FOLDER, MVT_INDICATORS_FOLDER
|
||||
from .version import MVT_VERSION
|
||||
from .config import settings
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -23,7 +24,7 @@ INDICATORS_CHECK_FREQUENCY = 12
|
||||
|
||||
class MVTUpdates:
|
||||
def check(self) -> str:
|
||||
res = requests.get("https://pypi.org/pypi/mvt/json", timeout=15)
|
||||
res = requests.get(settings.PYPI_UPDATE_URL, timeout=15)
|
||||
data = res.json()
|
||||
latest_version = data.get("info", {}).get("version", "")
|
||||
|
||||
|
||||
@@ -13,6 +13,7 @@ import re
|
||||
from typing import Any, Iterator, Union
|
||||
|
||||
from rich.logging import RichHandler
|
||||
from mvt.common.config import settings
|
||||
|
||||
|
||||
class CustomJSONEncoder(json.JSONEncoder):
|
||||
@@ -256,7 +257,7 @@ def set_verbose_logging(verbose: bool = False):
|
||||
|
||||
def exec_or_profile(module, globals, locals):
|
||||
"""Hook for profiling MVT modules"""
|
||||
if int(os.environ.get("MVT_PROFILE", False)):
|
||||
if settings.PROFILE:
|
||||
cProfile.runctx(module, globals, locals)
|
||||
else:
|
||||
exec(module, globals, locals)
|
||||
|
||||
@@ -3,4 +3,4 @@
|
||||
# Use of this software is governed by the MVT License 1.1 that can be found at
|
||||
# https://license.mvt.re/1.1/
|
||||
|
||||
MVT_VERSION = "2.6.0"
|
||||
MVT_VERSION = "2.6.1"
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user