1
mirror of https://github.com/mvt-project/mvt synced 2025-10-21 22:42:15 +02:00

Compare commits

..

45 Commits

Author SHA1 Message Date
Tek
7a1ed628ef Create dependabot.yml 2025-06-20 11:16:08 +02:00
dependabot[bot]
c9ac12f336 Bump requests from 2.32.2 to 2.32.4 (#642)
Bumps [requests](https://github.com/psf/requests) from 2.32.2 to 2.32.4.
- [Release notes](https://github.com/psf/requests/releases)
- [Changelog](https://github.com/psf/requests/blob/main/HISTORY.md)
- [Commits](https://github.com/psf/requests/compare/v2.32.2...v2.32.4)

---
updated-dependencies:
- dependency-name: requests
  dependency-version: 2.32.4
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2025-06-19 23:55:42 +02:00
Tek
486e3e7e9b Update global_preferences.py (#641)
Added a warning for lockdown mode when the property is not present
---------

Co-authored-by: makitos666 <makitos666@gmail.com>
2025-06-19 23:51:59 +02:00
besendorf
be1fc3bd8b Update NSKeyedUnarchiver (#636) 2025-06-12 22:42:02 +02:00
Tek
4757cff262 Fixes date parsing issue in tombstones (#635) 2025-06-12 20:49:31 +02:00
Tek
61f51caf31 Freeze versions and bump version (#632)
* Freeze versions and bump version
* Drops support for python below 3.10
2025-06-12 16:33:15 +02:00
besendorf
511063fd0e Update pyproject.toml (#630) 2025-06-04 13:00:04 +02:00
scribblemaniac
88bc5672cb Upgrade main dockerfile runtime to ubuntu:24.04 (#619)
Co-authored-by: Tek <tek@randhome.io>
2025-05-14 11:34:40 +02:00
github-actions[bot]
0fce0acf7a Add new iOS versions and build numbers (#626)
Co-authored-by: DonnchaC <DonnchaC@users.noreply.github.com>
2025-05-14 11:12:13 +02:00
github-actions[bot]
61f95d07d3 Add new iOS versions and build numbers (#625)
Co-authored-by: DonnchaC <DonnchaC@users.noreply.github.com>
2025-05-12 22:37:46 +02:00
ping2A
3dedd169c4 Fix issue #574 for a module without IOCs output (#620)
* Fix issue #574 for a module without IOCs output
2025-04-30 10:30:39 +02:00
Tek
e34e03d3a3 Fixes Android Dumpsys ADB parsing issue 2025-04-18 17:43:08 +02:00
github-actions[bot]
34374699ce Add new iOS versions and build numbers (#622)
Co-authored-by: DonnchaC <DonnchaC@users.noreply.github.com>
2025-04-17 09:46:17 +02:00
github-actions[bot]
cf5aa7c89f Add new iOS versions and build numbers (#618)
Co-authored-by: DonnchaC <DonnchaC@users.noreply.github.com>
2025-04-01 16:04:06 +02:00
Donncha Ó Cearbhaill
2766739512 Fix bug where default values were dropped when parsing protobuf tombstones (#617) 2025-03-11 14:10:34 +01:00
cacu
9c84afb4b0 Update logo.py (#615)
add instructions to update mvt via pipx
2025-03-11 13:46:59 +01:00
Donncha Ó Cearbhaill
80fc8bd879 Fix YAML format (#611) 2025-02-21 15:48:00 +01:00
Donncha Ó Cearbhaill
ca41f7f106 Always open automatic PRs as drafts (#609) 2025-02-21 15:35:06 +01:00
github-actions[bot]
55ddd86ad5 Add new iOS versions and build numbers (#607)
Co-authored-by: DonnchaC <DonnchaC@users.noreply.github.com>
2025-02-21 15:24:27 +01:00
Donncha Ó Cearbhaill
b184eeedf4 Handle XML encoded ADB keystore and fix parsing bugs (#605) 2025-02-07 02:00:24 +01:00
Donncha Ó Cearbhaill
4e97e85350 Load Android device timezone info and add additional file modification logs (#567)
* Use local timestamp for Files module timeline.

Most other Android timestamps appear to be local time. The
results timeline is more useful if all the timestamps
are consistent. I would prefer to use UTC, but that would
mean converting all the other timestamps to UTC as well. We probably
do not have sufficient information to do that accurately,
especially if the device is moving between timezones..

* Add file timestamp modules to add logs into timeline

* Handle case were we cannot load device timezone

* Fix crash if prop file does not exist

* Move _get_file_modification_time to BugReportModule

* Add backport for timezone and fix Tombstone module to use local time.

* Fix import for backported Zoneinfo

* Fix ruff error
2025-02-06 20:51:15 +01:00
Donncha Ó Cearbhaill
e5865b166e Merge pull request #568 from mvt-project/feature/tombstone-parser
Add parser for Android tombstone files
2025-02-06 20:15:21 +01:00
Donncha Ó Cearbhaill
a2dabb4267 Fix generate-proto-parsers Makefile command 2025-02-06 20:11:54 +01:00
Donncha Ó Cearbhaill
b7595b62eb Add initial tombstone parser
This supports parsing tombstone files from Android bugreports. The parser
can load both the legacy text format and the new binary protobuf format.
2025-02-06 20:07:05 +01:00
Donncha Ó Cearbhaill
02c02ca15c Merge branch 'main' into feature/tombstone-parser 2025-02-03 18:44:00 +01:00
Donncha Ó Cearbhaill
6da33394fe Merge pull request #592 from mvt-project/feature/config-file
Reworking handling of config options
2025-01-30 13:32:53 +01:00
Donncha Ó Cearbhaill
086871e21d Merge branch 'main' into feature/config-file 2025-01-30 13:15:28 +01:00
Donncha Ó Cearbhaill
f32830c649 Merge pull request #603 from mvt-project/feature/add-suspicious-android-setting
Add additional Android security warnings
2025-01-30 13:12:14 +01:00
Donncha Ó Cearbhaill
edcad488ab Merge branch 'main' into feature/add-suspicious-android-setting 2025-01-30 13:10:00 +01:00
Donncha Ó Cearbhaill
43901c96a0 Add improved heuristic detections to AppOps module 2025-01-30 13:02:26 +01:00
Donncha Ó Cearbhaill
0962383b46 Alert on potentially suspicious permissions from ADB 2025-01-30 11:48:19 +01:00
Donncha Ó Cearbhaill
34cd08fd9a Add additional Android security setting to warn on 2025-01-30 11:35:18 +01:00
github-actions[bot]
579b53f7ec Add new iOS versions and build numbers (#602)
Co-authored-by: DonnchaC <DonnchaC@users.noreply.github.com>
2025-01-28 01:27:17 +01:00
Donncha Ó Cearbhaill
0fbf24e82a Merge branch 'main' into feature/config-file 2025-01-14 14:33:40 +01:00
Donncha Ó Cearbhaill
458195a0ab Fix optional typing syntax for Python 3.8 2024-12-25 00:28:02 +00:00
Donncha Ó Cearbhaill
52e854b8b7 Add missing import 2024-12-25 00:23:36 +00:00
Donncha Ó Cearbhaill
0f1eec3971 Add Pydantic dependencies 2024-12-25 00:21:42 +00:00
Donncha Ó Cearbhaill
f4425865c0 Add missed modules using updated settings module 2024-12-25 00:14:14 +00:00
Donncha Ó Cearbhaill
28c0c86c4e Update MVT code to use config file rather than raw env variables 2024-12-25 00:09:29 +00:00
Donncha Ó Cearbhaill
154e6dab15 Add config file parser for MVT 2024-12-24 23:30:18 +00:00
Donncha Ó Cearbhaill
8e895d3d07 Remove protobuf compiler dependency, only needed for dev 2024-10-28 13:10:37 +01:00
Donncha Ó Cearbhaill
bc09e2a394 Initial tests for tombstone parsing 2024-10-28 10:51:58 +01:00
Donncha Ó Cearbhaill
2d0de088dd Add generated protobuf parser 2024-10-28 10:38:19 +01:00
Donncha Ó Cearbhaill
8694e7a047 Add protobuf parser generation 2024-10-28 10:37:30 +01:00
Donncha Ó Cearbhaill
9b41ba99aa WIP: initial tombstone modules 2024-10-28 10:34:53 +01:00
48 changed files with 2512 additions and 67 deletions

11
.github/dependabot.yml vendored Normal file
View File

@@ -0,0 +1,11 @@
# To get started with Dependabot version updates, you'll need to specify which
# package ecosystems to update and where the package manifests are located.
# Please see the documentation for all configuration options:
# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file
version: 2
updates:
- package-ecosystem: "pip" # See documentation for possible values
directory: "/" # Location of package manifests
schedule:
interval: "weekly"

View File

@@ -12,7 +12,7 @@ jobs:
strategy:
fail-fast: false
matrix:
python-version: ['3.8', '3.9', '3.10'] # , '3.11']
python-version: ['3.10', '3.11', '3.12', '3.13']
steps:
- uses: actions/checkout@v4
@@ -35,4 +35,4 @@ jobs:
if: github.event_name == 'pull_request'
with:
pytest-coverage-path: ./pytest-coverage.txt
junitxml-path: ./pytest.xml
junitxml-path: ./pytest.xml

View File

@@ -21,6 +21,7 @@ jobs:
title: '[auto] Update iOS releases and versions'
commit-message: Add new iOS versions and build numbers
branch: auto/add-new-ios-releases
draft: true
body: |
This is an automated pull request to update the iOS releases and version numbers.
add-paths: |

View File

@@ -103,7 +103,7 @@ RUN git clone https://github.com/libimobiledevice/usbmuxd && cd usbmuxd \
# Create main image
FROM ubuntu:22.04 as main
FROM ubuntu:24.04 as main
LABEL org.opencontainers.image.url="https://mvt.re"
LABEL org.opencontainers.image.documentation="https://docs.mvt.re"
@@ -135,8 +135,7 @@ COPY --from=build-usbmuxd /build /
COPY . mvt/
RUN apt-get update \
&& apt-get install -y git python3-pip \
&& PIP_NO_CACHE_DIR=1 pip3 install --upgrade pip \
&& PIP_NO_CACHE_DIR=1 pip3 install ./mvt \
&& PIP_NO_CACHE_DIR=1 pip3 install --break-system-packages ./mvt \
&& apt-get remove -y python3-pip git && apt-get autoremove -y \
&& rm -rf /var/lib/apt/lists/* \
&& rm -rf mvt

View File

@@ -25,6 +25,11 @@ install:
test-requirements:
python3 -m pip install --upgrade -r test-requirements.txt
generate-proto-parsers:
# Generate python parsers for protobuf files
PROTO_FILES=$$(find src/mvt/android/parsers/proto/ -iname "*.proto"); \
protoc -Isrc/mvt/android/parsers/proto/ --python_betterproto_out=src/mvt/android/parsers/proto/ $$PROTO_FILES
clean:
rm -rf $(PWD)/build $(PWD)/dist $(PWD)/src/mvt.egg-info

View File

@@ -19,21 +19,26 @@ classifiers = [
"Programming Language :: Python"
]
dependencies = [
"click >=8.1.3",
"rich >=12.6.0",
"tld >=0.12.6",
"requests >=2.28.1",
"simplejson >=3.17.6",
"packaging >=21.3",
"appdirs >=1.4.4",
"iOSbackup >=0.9.923",
"adb-shell[usb] >=0.4.3",
"libusb1 >=3.0.0",
"cryptography >=42.0.5",
"pyyaml >=6.0",
"pyahocorasick >= 2.0.0",
"click==8.2.1",
"rich==14.0.0",
"tld==0.13.1",
"requests==2.32.4",
"simplejson==3.20.1",
"packaging==25.0",
"appdirs==1.4.4",
"iOSbackup==0.9.925",
"adb-shell[usb]==0.4.4",
"libusb1==3.3.1",
"cryptography==45.0.3",
"PyYAML>=6.0.2",
"pyahocorasick==2.1.0",
"betterproto==1.2.5",
"pydantic==2.11.5",
"pydantic-settings==2.9.1",
"NSKeyedUnArchiver==1.5.2",
"python-dateutil==2.9.0.post0",
]
requires-python = ">= 3.8"
requires-python = ">= 3.10"
[project.urls]
homepage = "https://docs.mvt.re/en/latest/"
@@ -99,4 +104,4 @@ where = ["src"]
mvt = ["ios/data/*.json"]
[tool.setuptools.dynamic]
version = {attr = "mvt.common.version.MVT_VERSION"}
version = {attr = "mvt.common.version.MVT_VERSION"}

View File

@@ -4,13 +4,14 @@
# https://license.mvt.re/1.1/
import base64
import binascii
import hashlib
from .artifact import AndroidArtifact
class DumpsysADBArtifact(AndroidArtifact):
multiline_fields = ["user_keys"]
multiline_fields = ["user_keys", "keystore"]
def indented_dump_parser(self, dump_data):
"""
@@ -67,14 +68,38 @@ class DumpsysADBArtifact(AndroidArtifact):
return res
def parse_xml(self, xml_data):
"""
Parse XML data from dumpsys ADB output
"""
import xml.etree.ElementTree as ET
keystore = []
keystore_root = ET.fromstring(xml_data)
for adb_key in keystore_root.findall("adbKey"):
key_info = self.calculate_key_info(adb_key.get("key").encode("utf-8"))
key_info["last_connected"] = adb_key.get("lastConnection")
keystore.append(key_info)
return keystore
@staticmethod
def calculate_key_info(user_key: bytes) -> str:
key_base64, user = user_key.split(b" ", 1)
key_raw = base64.b64decode(key_base64)
key_fingerprint = hashlib.md5(key_raw).hexdigest().upper()
key_fingerprint_colon = ":".join(
[key_fingerprint[i : i + 2] for i in range(0, len(key_fingerprint), 2)]
)
if b" " in user_key:
key_base64, user = user_key.split(b" ", 1)
else:
key_base64, user = user_key, b""
try:
key_raw = base64.b64decode(key_base64)
key_fingerprint = hashlib.md5(key_raw).hexdigest().upper()
key_fingerprint_colon = ":".join(
[key_fingerprint[i : i + 2] for i in range(0, len(key_fingerprint), 2)]
)
except binascii.Error:
# Impossible to parse base64
key_fingerprint_colon = ""
return {
"user": user.decode("utf-8"),
"fingerprint": key_fingerprint_colon,
@@ -115,8 +140,24 @@ class DumpsysADBArtifact(AndroidArtifact):
if parsed.get("debugging_manager") is None:
self.log.error("Unable to find expected ADB entries in dumpsys output") # noqa
return
# Keystore can be in different levels, as the basic parser
# is not always consistent due to different dumpsys formats.
if parsed.get("keystore"):
keystore_data = b"\n".join(parsed["keystore"])
elif parsed["debugging_manager"].get("keystore"):
keystore_data = b"\n".join(parsed["debugging_manager"]["keystore"])
else:
parsed = parsed["debugging_manager"]
keystore_data = None
# Keystore is in XML format on some devices and we need to parse it
if keystore_data and keystore_data.startswith(b"<?xml"):
parsed["debugging_manager"]["keystore"] = self.parse_xml(keystore_data)
else:
# Keystore is not XML format
parsed["debugging_manager"]["keystore"] = keystore_data
parsed = parsed["debugging_manager"]
# Calculate key fingerprints for better readability
key_info = []

View File

@@ -11,6 +11,10 @@ from mvt.common.utils import convert_datetime_to_iso
from .artifact import AndroidArtifact
RISKY_PERMISSIONS = ["REQUEST_INSTALL_PACKAGES"]
RISKY_PACKAGES = ["com.android.shell"]
class DumpsysAppopsArtifact(AndroidArtifact):
"""
Parser for dumpsys app ops info
@@ -45,15 +49,39 @@ class DumpsysAppopsArtifact(AndroidArtifact):
self.detected.append(result)
continue
detected_permissions = []
for perm in result["permissions"]:
if (
perm["name"] == "REQUEST_INSTALL_PACKAGES"
and perm["access"] == "allow"
perm["name"] in RISKY_PERMISSIONS
# and perm["access"] == "allow"
):
self.log.info(
"Package %s with REQUEST_INSTALL_PACKAGES permission",
result["package_name"],
)
detected_permissions.append(perm)
for entry in sorted(perm["entries"], key=lambda x: x["timestamp"]):
self.log.warning(
"Package '%s' had risky permission '%s' set to '%s' at %s",
result["package_name"],
perm["name"],
entry["access"],
entry["timestamp"],
)
elif result["package_name"] in RISKY_PACKAGES:
detected_permissions.append(perm)
for entry in sorted(perm["entries"], key=lambda x: x["timestamp"]):
self.log.warning(
"Risky package '%s' had '%s' permission set to '%s' at %s",
result["package_name"],
perm["name"],
entry["access"],
entry["timestamp"],
)
if detected_permissions:
# We clean the result to only include the risky permission, otherwise the timeline
# will be polluted with all the other irrelevant permissions
cleaned_result = result.copy()
cleaned_result["permissions"] = detected_permissions
self.detected.append(cleaned_result)
def parse(self, output: str) -> None:
self.results: List[Dict[str, Any]] = []
@@ -121,11 +149,16 @@ class DumpsysAppopsArtifact(AndroidArtifact):
if line.startswith(" "):
# Permission entry like:
# Reject: [fg-s]2021-05-19 22:02:52.054 (-314d1h25m2s33ms)
access_type = line.split(":")[0].strip()
if access_type not in ["Access", "Reject"]:
# Skipping invalid access type. Some entries are not in the format we expect
continue
if entry:
perm["entries"].append(entry)
entry = {}
entry["access"] = line.split(":")[0].strip()
entry["access"] = access_type
entry["type"] = line[line.find("[") + 1 : line.find("]")]
try:

View File

@@ -0,0 +1,43 @@
# Mobile Verification Toolkit (MVT)
# Copyright (c) 2021-2023 The MVT Authors.
# Use of this software is governed by the MVT License 1.1 that can be found at
# https://license.mvt.re/1.1/
from typing import Union
from .artifact import AndroidArtifact
class FileTimestampsArtifact(AndroidArtifact):
def serialize(self, record: dict) -> Union[dict, list]:
records = []
for ts in set(
[
record.get("access_time"),
record.get("changed_time"),
record.get("modified_time"),
]
):
if not ts:
continue
macb = ""
macb += "M" if ts == record.get("modified_time") else "-"
macb += "A" if ts == record.get("access_time") else "-"
macb += "C" if ts == record.get("changed_time") else "-"
macb += "-"
msg = record["path"]
if record.get("context"):
msg += f" ({record['context']})"
records.append(
{
"timestamp": ts,
"module": self.__class__.__name__,
"event": macb,
"data": msg,
}
)
return records

View File

@@ -42,6 +42,17 @@ class GetProp(AndroidArtifact):
entry = {"name": matches[0][0], "value": matches[0][1]}
self.results.append(entry)
def get_device_timezone(self) -> str:
"""
Get the device timezone from the getprop results
Used in other moduels to calculate the timezone offset
"""
for entry in self.results:
if entry["name"] == "persist.sys.timezone":
return entry["value"]
return None
def check_indicators(self) -> None:
for entry in self.results:
if entry["name"] in INTERESTING_PROPERTIES:

View File

@@ -16,6 +16,11 @@ ANDROID_DANGEROUS_SETTINGS = [
"key": "package_verifier_enable",
"safe_value": "1",
},
{
"description": "disabled APK package verification",
"key": "package_verifier_state",
"safe_value": "1",
},
{
"description": "disabled Google Play Protect",
"key": "package_verifier_user_consent",

File diff suppressed because it is too large Load Diff

View File

@@ -48,6 +48,37 @@ class AndroidQFModule(MVTModule):
def _get_files_by_pattern(self, pattern: str):
return fnmatch.filter(self.files, pattern)
def _get_device_timezone(self):
"""
Get the device timezone from the getprop.txt file.
This is needed to map local timestamps stored in some
Android log files to UTC/timezone-aware timestamps.
"""
get_prop_files = self._get_files_by_pattern("*/getprop.txt")
if not get_prop_files:
self.log.warning(
"Could not find getprop.txt file. "
"Some timestamps and timeline data may be incorrect."
)
return None
from mvt.android.artifacts.getprop import GetProp
properties_artifact = GetProp()
prop_data = self._get_file_content(get_prop_files[0]).decode("utf-8")
properties_artifact.parse(prop_data)
timezone = properties_artifact.get_device_timezone()
if timezone:
self.log.debug("Identified local phone timezone: %s", timezone)
return timezone
self.log.warning(
"Could not find or determine local device timezone. "
"Some timestamps and timeline data may be incorrect."
)
return None
def _get_file_content(self, file_path):
if self.archive:
handle = self.archive.open(file_path)

View File

@@ -6,6 +6,11 @@
import datetime
import json
import logging
try:
import zoneinfo
except ImportError:
from backports import zoneinfo
from typing import Optional, Union
from mvt.android.modules.androidqf.base import AndroidQFModule
@@ -106,6 +111,12 @@ class Files(AndroidQFModule):
# TODO: adds SHA1 and MD5 when available in MVT
def run(self) -> None:
if timezone := self._get_device_timezone():
device_timezone = zoneinfo.ZoneInfo(timezone)
else:
self.log.warning("Unable to determine device timezone, using UTC")
device_timezone = zoneinfo.ZoneInfo("UTC")
for file in self._get_files_by_pattern("*/files.json"):
rawdata = self._get_file_content(file).decode("utf-8", errors="ignore")
try:
@@ -120,11 +131,18 @@ class Files(AndroidQFModule):
for file_data in data:
for ts in ["access_time", "changed_time", "modified_time"]:
if ts in file_data:
file_data[ts] = convert_datetime_to_iso(
datetime.datetime.fromtimestamp(
file_data[ts], tz=datetime.timezone.utc
)
utc_timestamp = datetime.datetime.fromtimestamp(
file_data[ts], tz=datetime.timezone.utc
)
# Convert the UTC timestamp to local tiem on Android device's local timezone
local_timestamp = utc_timestamp.astimezone(device_timezone)
# HACK: We only output the UTC timestamp in convert_datetime_to_iso, we
# set the timestamp timezone to UTC, to avoid the timezone conversion again.
local_timestamp = local_timestamp.replace(
tzinfo=datetime.timezone.utc
)
file_data[ts] = convert_datetime_to_iso(local_timestamp)
self.results.append(file_data)

View File

@@ -0,0 +1,65 @@
# Mobile Verification Toolkit (MVT)
# Copyright (c) 2021-2023 The MVT Authors.
# Use of this software is governed by the MVT License 1.1 that can be found at
# https://license.mvt.re/1.1/
import os
import datetime
import logging
from typing import Optional
from mvt.common.utils import convert_datetime_to_iso
from .base import AndroidQFModule
from mvt.android.artifacts.file_timestamps import FileTimestampsArtifact
class LogsFileTimestamps(FileTimestampsArtifact, AndroidQFModule):
"""This module extracts records from battery daily updates."""
slug = "logfile_timestamps"
def __init__(
self,
file_path: Optional[str] = None,
target_path: Optional[str] = None,
results_path: Optional[str] = None,
module_options: Optional[dict] = None,
log: logging.Logger = logging.getLogger(__name__),
results: Optional[list] = None,
) -> None:
super().__init__(
file_path=file_path,
target_path=target_path,
results_path=results_path,
module_options=module_options,
log=log,
results=results,
)
def _get_file_modification_time(self, file_path: str) -> dict:
if self.archive:
file_timetuple = self.archive.getinfo(file_path).date_time
return datetime.datetime(*file_timetuple)
else:
file_stat = os.stat(os.path.join(self.parent_path, file_path))
return datetime.datetime.fromtimestamp(file_stat.st_mtime)
def run(self) -> None:
filesystem_files = self._get_files_by_pattern("*/logs/*")
self.results = []
for file in filesystem_files:
# Only the modification time is available in the zip file metadata.
# The timezone is the local timezone of the machine the phone.
modification_time = self._get_file_modification_time(file)
self.results.append(
{
"path": file,
"modified_time": convert_datetime_to_iso(modification_time),
}
)
self.log.info(
"Extracted a total of %d filesystem timestamps from AndroidQF logs directory.",
len(self.results),
)

View File

@@ -3,10 +3,11 @@
# Use of this software is governed by the MVT License 1.1 that can be found at
# https://license.mvt.re/1.1/
import os
from rich.prompt import Prompt
from mvt.common.config import settings
MVT_ANDROID_BACKUP_PASSWORD = "MVT_ANDROID_BACKUP_PASSWORD"
@@ -16,24 +17,24 @@ def cli_load_android_backup_password(log, backup_password):
Used in MVT CLI command parsers.
"""
password_from_env = os.environ.get(MVT_ANDROID_BACKUP_PASSWORD, None)
password_from_env_or_config = settings.ANDROID_BACKUP_PASSWORD
if backup_password:
log.info(
"Your password may be visible in the process table because it "
"was supplied on the command line!"
)
if password_from_env:
if password_from_env_or_config:
log.info(
"Ignoring %s environment variable, using --backup-password argument instead",
MVT_ANDROID_BACKUP_PASSWORD,
"MVT_ANDROID_BACKUP_PASSWORD",
)
return backup_password
elif password_from_env:
elif password_from_env_or_config:
log.info(
"Using backup password from %s environment variable",
MVT_ANDROID_BACKUP_PASSWORD,
"Using backup password from %s environment variable or config file",
"MVT_ANDROID_BACKUP_PASSWORD",
)
return password_from_env
return password_from_env_or_config
def prompt_or_load_android_backup_password(log, module_options):

View File

@@ -14,6 +14,8 @@ from .packages import Packages
from .platform_compat import PlatformCompat
from .receivers import Receivers
from .adb_state import DumpsysADBState
from .fs_timestamps import BugReportTimestamps
from .tombstones import Tombstones
BUGREPORT_MODULES = [
Accessibility,
@@ -27,4 +29,6 @@ BUGREPORT_MODULES = [
PlatformCompat,
Receivers,
DumpsysADBState,
BugReportTimestamps,
Tombstones,
]

View File

@@ -2,10 +2,11 @@
# Copyright (c) 2021-2023 The MVT Authors.
# See the file 'LICENSE' for usage and copying permissions, or find a copy at
# https://github.com/mvt-project/mvt/blob/main/LICENSE
import datetime
import fnmatch
import logging
import os
from typing import List, Optional
from zipfile import ZipFile
@@ -91,3 +92,11 @@ class BugReportModule(MVTModule):
return None
return self._get_file_content(dumpstate_logs[0])
def _get_file_modification_time(self, file_path: str) -> dict:
if self.zip_archive:
file_timetuple = self.zip_archive.getinfo(file_path).date_time
return datetime.datetime(*file_timetuple)
else:
file_stat = os.stat(os.path.join(self.extract_path, file_path))
return datetime.datetime.fromtimestamp(file_stat.st_mtime)

View File

@@ -0,0 +1,55 @@
# Mobile Verification Toolkit (MVT)
# Copyright (c) 2021-2023 The MVT Authors.
# Use of this software is governed by the MVT License 1.1 that can be found at
# https://license.mvt.re/1.1/
import logging
from typing import Optional
from mvt.common.utils import convert_datetime_to_iso
from .base import BugReportModule
from mvt.android.artifacts.file_timestamps import FileTimestampsArtifact
class BugReportTimestamps(FileTimestampsArtifact, BugReportModule):
"""This module extracts records from battery daily updates."""
slug = "bugreport_timestamps"
def __init__(
self,
file_path: Optional[str] = None,
target_path: Optional[str] = None,
results_path: Optional[str] = None,
module_options: Optional[dict] = None,
log: logging.Logger = logging.getLogger(__name__),
results: Optional[list] = None,
) -> None:
super().__init__(
file_path=file_path,
target_path=target_path,
results_path=results_path,
module_options=module_options,
log=log,
results=results,
)
def run(self) -> None:
filesystem_files = self._get_files_by_pattern("FS/*")
self.results = []
for file in filesystem_files:
# Only the modification time is available in the zip file metadata.
# The timezone is the local timezone of the machine the phone.
modification_time = self._get_file_modification_time(file)
self.results.append(
{
"path": file,
"modified_time": convert_datetime_to_iso(modification_time),
}
)
self.log.info(
"Extracted a total of %d filesystem timestamps from bugreport.",
len(self.results),
)

View File

@@ -0,0 +1,64 @@
# Mobile Verification Toolkit (MVT)
# Copyright (c) 2021-2023 The MVT Authors.
# Use of this software is governed by the MVT License 1.1 that can be found at
# https://license.mvt.re/1.1/
import logging
from typing import Optional
from mvt.android.artifacts.tombstone_crashes import TombstoneCrashArtifact
from .base import BugReportModule
class Tombstones(TombstoneCrashArtifact, BugReportModule):
"""This module extracts records from battery daily updates."""
slug = "tombstones"
def __init__(
self,
file_path: Optional[str] = None,
target_path: Optional[str] = None,
results_path: Optional[str] = None,
module_options: Optional[dict] = None,
log: logging.Logger = logging.getLogger(__name__),
results: Optional[list] = None,
) -> None:
super().__init__(
file_path=file_path,
target_path=target_path,
results_path=results_path,
module_options=module_options,
log=log,
results=results,
)
def run(self) -> None:
tombstone_files = self._get_files_by_pattern("*/tombstone_*")
if not tombstone_files:
self.log.error(
"Unable to find any tombstone files. "
"Did you provide a valid bugreport archive?"
)
return
for tombstone_file in sorted(tombstone_files):
tombstone_filename = tombstone_file.split("/")[-1]
modification_time = self._get_file_modification_time(tombstone_file)
tombstone_data = self._get_file_content(tombstone_file)
try:
if tombstone_file.endswith(".pb"):
self.parse_protobuf(
tombstone_filename, modification_time, tombstone_data
)
else:
self.parse(tombstone_filename, modification_time, tombstone_data)
except ValueError as e:
# Catch any exceptions raised during parsing or validation.
self.log.error(f"Error parsing tombstone file {tombstone_file}: {e}")
self.log.info(
"Extracted a total of %d tombstone files",
len(self.results),
)

View File

@@ -0,0 +1,195 @@
// tombstone.proto file from Android source
// Src: https://android.googlesource.com/platform/system/core/+/refs/heads/main/debuggerd/proto/tombstone.proto
//
// Protobuf definition for Android tombstones.
//
// An app can get hold of these for any `REASON_CRASH_NATIVE` instance of
// `android.app.ApplicationExitInfo`.
//
// https://developer.android.com/reference/android/app/ApplicationExitInfo#getTraceInputStream()
//
syntax = "proto3";
option java_package = "com.android.server.os";
option java_outer_classname = "TombstoneProtos";
// NOTE TO OEMS:
// If you add custom fields to this proto, do not use numbers in the reserved range.
message CrashDetail {
bytes name = 1;
bytes data = 2;
reserved 3 to 999;
}
message StackHistoryBufferEntry {
BacktraceFrame addr = 1;
uint64 fp = 2;
uint64 tag = 3;
reserved 4 to 999;
}
message StackHistoryBuffer {
uint64 tid = 1;
repeated StackHistoryBufferEntry entries = 2;
reserved 3 to 999;
}
message Tombstone {
Architecture arch = 1;
Architecture guest_arch = 24;
string build_fingerprint = 2;
string revision = 3;
string timestamp = 4;
uint32 pid = 5;
uint32 tid = 6;
uint32 uid = 7;
string selinux_label = 8;
repeated string command_line = 9;
// Process uptime in seconds.
uint32 process_uptime = 20;
Signal signal_info = 10;
string abort_message = 14;
repeated CrashDetail crash_details = 21;
repeated Cause causes = 15;
map<uint32, Thread> threads = 16;
map<uint32, Thread> guest_threads = 25;
repeated MemoryMapping memory_mappings = 17;
repeated LogBuffer log_buffers = 18;
repeated FD open_fds = 19;
uint32 page_size = 22;
bool has_been_16kb_mode = 23;
StackHistoryBuffer stack_history_buffer = 26;
reserved 27 to 999;
}
enum Architecture {
ARM32 = 0;
ARM64 = 1;
X86 = 2;
X86_64 = 3;
RISCV64 = 4;
NONE = 5;
reserved 6 to 999;
}
message Signal {
int32 number = 1;
string name = 2;
int32 code = 3;
string code_name = 4;
bool has_sender = 5;
int32 sender_uid = 6;
int32 sender_pid = 7;
bool has_fault_address = 8;
uint64 fault_address = 9;
// Note, may or may not contain the dump of the actual memory contents. Currently, on arm64, we
// only include metadata, and not the contents.
MemoryDump fault_adjacent_metadata = 10;
reserved 11 to 999;
}
message HeapObject {
uint64 address = 1;
uint64 size = 2;
uint64 allocation_tid = 3;
repeated BacktraceFrame allocation_backtrace = 4;
uint64 deallocation_tid = 5;
repeated BacktraceFrame deallocation_backtrace = 6;
}
message MemoryError {
enum Tool {
GWP_ASAN = 0;
SCUDO = 1;
reserved 2 to 999;
}
Tool tool = 1;
enum Type {
UNKNOWN = 0;
USE_AFTER_FREE = 1;
DOUBLE_FREE = 2;
INVALID_FREE = 3;
BUFFER_OVERFLOW = 4;
BUFFER_UNDERFLOW = 5;
reserved 6 to 999;
}
Type type = 2;
oneof location {
HeapObject heap = 3;
}
reserved 4 to 999;
}
message Cause {
string human_readable = 1;
oneof details {
MemoryError memory_error = 2;
}
reserved 3 to 999;
}
message Register {
string name = 1;
uint64 u64 = 2;
reserved 3 to 999;
}
message Thread {
int32 id = 1;
string name = 2;
repeated Register registers = 3;
repeated string backtrace_note = 7;
repeated string unreadable_elf_files = 9;
repeated BacktraceFrame current_backtrace = 4;
repeated MemoryDump memory_dump = 5;
int64 tagged_addr_ctrl = 6;
int64 pac_enabled_keys = 8;
reserved 10 to 999;
}
message BacktraceFrame {
uint64 rel_pc = 1;
uint64 pc = 2;
uint64 sp = 3;
string function_name = 4;
uint64 function_offset = 5;
string file_name = 6;
uint64 file_map_offset = 7;
string build_id = 8;
reserved 9 to 999;
}
message ArmMTEMetadata {
// One memory tag per granule (e.g. every 16 bytes) of regular memory.
bytes memory_tags = 1;
reserved 2 to 999;
}
message MemoryDump {
string register_name = 1;
string mapping_name = 2;
uint64 begin_address = 3;
bytes memory = 4;
oneof metadata {
ArmMTEMetadata arm_mte_metadata = 6;
}
reserved 5, 7 to 999;
}
message MemoryMapping {
uint64 begin_address = 1;
uint64 end_address = 2;
uint64 offset = 3;
bool read = 4;
bool write = 5;
bool execute = 6;
string mapping_name = 7;
string build_id = 8;
uint64 load_bias = 9;
reserved 10 to 999;
}
message FD {
int32 fd = 1;
string path = 2;
string owner = 3;
uint64 tag = 4;
reserved 5 to 999;
}
message LogBuffer {
string name = 1;
repeated LogMessage logs = 2;
reserved 3 to 999;
}
message LogMessage {
string timestamp = 1;
uint32 pid = 2;
uint32 tid = 3;
uint32 priority = 4;
string tag = 5;
string message = 6;
reserved 7 to 999;
}

View File

@@ -0,0 +1,208 @@
# Generated by the protocol buffer compiler. DO NOT EDIT!
# sources: tombstone.proto
# plugin: python-betterproto
from dataclasses import dataclass
from typing import Dict, List
import betterproto
class Architecture(betterproto.Enum):
ARM32 = 0
ARM64 = 1
X86 = 2
X86_64 = 3
RISCV64 = 4
NONE = 5
class MemoryErrorTool(betterproto.Enum):
GWP_ASAN = 0
SCUDO = 1
class MemoryErrorType(betterproto.Enum):
UNKNOWN = 0
USE_AFTER_FREE = 1
DOUBLE_FREE = 2
INVALID_FREE = 3
BUFFER_OVERFLOW = 4
BUFFER_UNDERFLOW = 5
@dataclass
class CrashDetail(betterproto.Message):
"""
NOTE TO OEMS: If you add custom fields to this proto, do not use numbers in
the reserved range.
"""
name: bytes = betterproto.bytes_field(1)
data: bytes = betterproto.bytes_field(2)
@dataclass
class StackHistoryBufferEntry(betterproto.Message):
addr: "BacktraceFrame" = betterproto.message_field(1)
fp: int = betterproto.uint64_field(2)
tag: int = betterproto.uint64_field(3)
@dataclass
class StackHistoryBuffer(betterproto.Message):
tid: int = betterproto.uint64_field(1)
entries: List["StackHistoryBufferEntry"] = betterproto.message_field(2)
@dataclass
class Tombstone(betterproto.Message):
arch: "Architecture" = betterproto.enum_field(1)
guest_arch: "Architecture" = betterproto.enum_field(24)
build_fingerprint: str = betterproto.string_field(2)
revision: str = betterproto.string_field(3)
timestamp: str = betterproto.string_field(4)
pid: int = betterproto.uint32_field(5)
tid: int = betterproto.uint32_field(6)
uid: int = betterproto.uint32_field(7)
selinux_label: str = betterproto.string_field(8)
command_line: List[str] = betterproto.string_field(9)
# Process uptime in seconds.
process_uptime: int = betterproto.uint32_field(20)
signal_info: "Signal" = betterproto.message_field(10)
abort_message: str = betterproto.string_field(14)
crash_details: List["CrashDetail"] = betterproto.message_field(21)
causes: List["Cause"] = betterproto.message_field(15)
threads: Dict[int, "Thread"] = betterproto.map_field(
16, betterproto.TYPE_UINT32, betterproto.TYPE_MESSAGE
)
guest_threads: Dict[int, "Thread"] = betterproto.map_field(
25, betterproto.TYPE_UINT32, betterproto.TYPE_MESSAGE
)
memory_mappings: List["MemoryMapping"] = betterproto.message_field(17)
log_buffers: List["LogBuffer"] = betterproto.message_field(18)
open_fds: List["FD"] = betterproto.message_field(19)
page_size: int = betterproto.uint32_field(22)
has_been_16kb_mode: bool = betterproto.bool_field(23)
stack_history_buffer: "StackHistoryBuffer" = betterproto.message_field(26)
@dataclass
class Signal(betterproto.Message):
number: int = betterproto.int32_field(1)
name: str = betterproto.string_field(2)
code: int = betterproto.int32_field(3)
code_name: str = betterproto.string_field(4)
has_sender: bool = betterproto.bool_field(5)
sender_uid: int = betterproto.int32_field(6)
sender_pid: int = betterproto.int32_field(7)
has_fault_address: bool = betterproto.bool_field(8)
fault_address: int = betterproto.uint64_field(9)
# Note, may or may not contain the dump of the actual memory contents.
# Currently, on arm64, we only include metadata, and not the contents.
fault_adjacent_metadata: "MemoryDump" = betterproto.message_field(10)
@dataclass
class HeapObject(betterproto.Message):
address: int = betterproto.uint64_field(1)
size: int = betterproto.uint64_field(2)
allocation_tid: int = betterproto.uint64_field(3)
allocation_backtrace: List["BacktraceFrame"] = betterproto.message_field(4)
deallocation_tid: int = betterproto.uint64_field(5)
deallocation_backtrace: List["BacktraceFrame"] = betterproto.message_field(6)
@dataclass
class MemoryError(betterproto.Message):
tool: "MemoryErrorTool" = betterproto.enum_field(1)
type: "MemoryErrorType" = betterproto.enum_field(2)
heap: "HeapObject" = betterproto.message_field(3, group="location")
@dataclass
class Cause(betterproto.Message):
human_readable: str = betterproto.string_field(1)
memory_error: "MemoryError" = betterproto.message_field(2, group="details")
@dataclass
class Register(betterproto.Message):
name: str = betterproto.string_field(1)
u64: int = betterproto.uint64_field(2)
@dataclass
class Thread(betterproto.Message):
id: int = betterproto.int32_field(1)
name: str = betterproto.string_field(2)
registers: List["Register"] = betterproto.message_field(3)
backtrace_note: List[str] = betterproto.string_field(7)
unreadable_elf_files: List[str] = betterproto.string_field(9)
current_backtrace: List["BacktraceFrame"] = betterproto.message_field(4)
memory_dump: List["MemoryDump"] = betterproto.message_field(5)
tagged_addr_ctrl: int = betterproto.int64_field(6)
pac_enabled_keys: int = betterproto.int64_field(8)
@dataclass
class BacktraceFrame(betterproto.Message):
rel_pc: int = betterproto.uint64_field(1)
pc: int = betterproto.uint64_field(2)
sp: int = betterproto.uint64_field(3)
function_name: str = betterproto.string_field(4)
function_offset: int = betterproto.uint64_field(5)
file_name: str = betterproto.string_field(6)
file_map_offset: int = betterproto.uint64_field(7)
build_id: str = betterproto.string_field(8)
@dataclass
class ArmMTEMetadata(betterproto.Message):
# One memory tag per granule (e.g. every 16 bytes) of regular memory.
memory_tags: bytes = betterproto.bytes_field(1)
@dataclass
class MemoryDump(betterproto.Message):
register_name: str = betterproto.string_field(1)
mapping_name: str = betterproto.string_field(2)
begin_address: int = betterproto.uint64_field(3)
memory: bytes = betterproto.bytes_field(4)
arm_mte_metadata: "ArmMTEMetadata" = betterproto.message_field(6, group="metadata")
@dataclass
class MemoryMapping(betterproto.Message):
begin_address: int = betterproto.uint64_field(1)
end_address: int = betterproto.uint64_field(2)
offset: int = betterproto.uint64_field(3)
read: bool = betterproto.bool_field(4)
write: bool = betterproto.bool_field(5)
execute: bool = betterproto.bool_field(6)
mapping_name: str = betterproto.string_field(7)
build_id: str = betterproto.string_field(8)
load_bias: int = betterproto.uint64_field(9)
@dataclass
class FD(betterproto.Message):
fd: int = betterproto.int32_field(1)
path: str = betterproto.string_field(2)
owner: str = betterproto.string_field(3)
tag: int = betterproto.uint64_field(4)
@dataclass
class LogBuffer(betterproto.Message):
name: str = betterproto.string_field(1)
logs: List["LogMessage"] = betterproto.message_field(2)
@dataclass
class LogMessage(betterproto.Message):
timestamp: str = betterproto.string_field(1)
pid: int = betterproto.uint32_field(2)
tid: int = betterproto.uint32_field(3)
priority: int = betterproto.uint32_field(4)
tag: str = betterproto.string_field(5)
message: str = betterproto.string_field(6)

View File

@@ -65,6 +65,10 @@ class CmdCheckIOCS(Command):
m = iocs_module.from_json(
file_path, log=logging.getLogger(iocs_module.__module__)
)
if not m:
log.warning("No result from this module, skipping it")
continue
if self.iocs.total_ioc_count > 0:
m.indicators = self.iocs
m.indicators.log = m.log

View File

@@ -17,6 +17,7 @@ from mvt.common.utils import (
generate_hashes_from_path,
get_sha256_from_file_path,
)
from mvt.common.config import settings
from mvt.common.version import MVT_VERSION
@@ -100,15 +101,25 @@ class Command:
if not self.results_path:
return
# We use local timestamps in the timeline on Android as many
# logs do not contain timezone information.
if type(self).__name__.startswith("CmdAndroid"):
is_utc = False
else:
is_utc = True
if len(self.timeline) > 0:
save_timeline(
self.timeline, os.path.join(self.results_path, "timeline.csv")
self.timeline,
os.path.join(self.results_path, "timeline.csv"),
is_utc=is_utc,
)
if len(self.timeline_detected) > 0:
save_timeline(
self.timeline_detected,
os.path.join(self.results_path, "timeline_detected.csv"),
is_utc=is_utc,
)
def _store_info(self) -> None:
@@ -132,7 +143,7 @@ class Command:
if ioc_file_path and ioc_file_path not in info["ioc_files"]:
info["ioc_files"].append(ioc_file_path)
if self.target_path and (os.environ.get("MVT_HASH_FILES") or self.hashes):
if self.target_path and (settings.HASH_FILES or self.hashes):
self.generate_hashes()
info["hashes"] = self.hash_values
@@ -141,7 +152,7 @@ class Command:
with open(info_path, "w+", encoding="utf-8") as handle:
json.dump(info, handle, indent=4)
if self.target_path and (os.environ.get("MVT_HASH_FILES") or self.hashes):
if self.target_path and (settings.HASH_FILES or self.hashes):
info_hash = get_sha256_from_file_path(info_path)
self.log.info('Reference hash of the info.json file: "%s"', info_hash)

105
src/mvt/common/config.py Normal file
View File

@@ -0,0 +1,105 @@
import os
import yaml
import json
from typing import Tuple, Type, Optional
from appdirs import user_config_dir
from pydantic import AnyHttpUrl, Field
from pydantic_settings import (
BaseSettings,
InitSettingsSource,
PydanticBaseSettingsSource,
SettingsConfigDict,
YamlConfigSettingsSource,
)
MVT_CONFIG_FOLDER = user_config_dir("mvt")
MVT_CONFIG_PATH = os.path.join(MVT_CONFIG_FOLDER, "config.yaml")
class MVTSettings(BaseSettings):
model_config = SettingsConfigDict(
env_prefix="MVT_",
env_nested_delimiter="_",
extra="ignore",
nested_model_default_partial_updates=True,
)
# Allow to decided if want to load environment variables
load_env: bool = Field(True, exclude=True)
# General settings
PYPI_UPDATE_URL: AnyHttpUrl = Field(
"https://pypi.org/pypi/mvt/json",
validate_default=False,
)
NETWORK_ACCESS_ALLOWED: bool = True
NETWORK_TIMEOUT: int = 15
# Command default settings, all can be specified by MVT_ prefixed environment variables too.
IOS_BACKUP_PASSWORD: Optional[str] = Field(
None, description="Default password to use to decrypt iOS backups"
)
ANDROID_BACKUP_PASSWORD: Optional[str] = Field(
None, description="Default password to use to decrypt Android backups"
)
STIX2: Optional[str] = Field(
None, description="List of directories where STIX2 files are stored"
)
VT_API_KEY: Optional[str] = Field(
None, description="API key to use for VirusTotal lookups"
)
PROFILE: bool = Field(False, description="Profile the execution of MVT modules")
HASH_FILES: bool = Field(False, description="Should MVT hash output files")
@classmethod
def settings_customise_sources(
cls,
settings_cls: Type[BaseSettings],
init_settings: InitSettingsSource,
env_settings: PydanticBaseSettingsSource,
dotenv_settings: PydanticBaseSettingsSource,
file_secret_settings: PydanticBaseSettingsSource,
) -> Tuple[PydanticBaseSettingsSource, ...]:
sources = (
YamlConfigSettingsSource(settings_cls, MVT_CONFIG_PATH),
init_settings,
)
# Load env variables if enabled
if init_settings.init_kwargs.get("load_env", True):
sources = (env_settings,) + sources
return sources
def save_settings(
self,
) -> None:
"""
Save the current settings to a file.
"""
if not os.path.isdir(MVT_CONFIG_FOLDER):
os.makedirs(MVT_CONFIG_FOLDER)
# Dump the settings to the YAML file
model_serializable = json.loads(self.model_dump_json(exclude_defaults=True))
with open(MVT_CONFIG_PATH, "w") as config_file:
config_file.write(yaml.dump(model_serializable, default_flow_style=False))
@classmethod
def initialise(cls) -> "MVTSettings":
"""
Initialise the settings file.
We first initialise the settings (without env variable) and then persist
them to file. This way we can update the config file with the default values.
Afterwards we load the settings again, this time including the env variables.
"""
# Set invalid env prefix to avoid loading env variables.
settings = MVTSettings(load_env=False)
settings.save_settings()
# Load the settings again with any ENV variables.
settings = MVTSettings(load_env=True)
return settings
settings = MVTSettings.initialise()

View File

@@ -14,6 +14,7 @@ import ahocorasick
from appdirs import user_data_dir
from .url import URL
from .config import settings
MVT_DATA_FOLDER = user_data_dir("mvt")
MVT_INDICATORS_FOLDER = os.path.join(MVT_DATA_FOLDER, "indicators")
@@ -41,12 +42,12 @@ class Indicators:
def _check_stix2_env_variable(self) -> None:
"""
Checks if a variable MVT_STIX2 contains path to a STIX file. Also recursively searches through dirs in MVT_STIX2
Checks if MVT_STIX2 setting or environment variable contains path to a STIX file. Also recursively searches through dirs in MVT_STIX2
"""
if "MVT_STIX2" not in os.environ:
if not settings.STIX2:
return
paths = os.environ["MVT_STIX2"].split(":")
paths = settings.STIX2.split(":")
for path in paths:
if os.path.isfile(path) and path.lower().endswith(".stix2"):
self.parse_stix2(path)

View File

@@ -29,7 +29,7 @@ def check_updates() -> None:
if latest_version:
rich_print(
f"\t\t[bold]Version {latest_version} is available! "
"Upgrade mvt with `pip3 install -U mvt`[/bold]"
"Upgrade mvt with `pip3 install -U mvt` or with `pipx upgrade mvt`[/bold]"
)
# Then we check for indicators files updates.

View File

@@ -69,10 +69,14 @@ class MVTModule:
@classmethod
def from_json(cls, json_path: str, log: logging.Logger):
with open(json_path, "r", encoding="utf-8") as handle:
results = json.load(handle)
if log:
log.info('Loaded %d results from "%s"', len(results), json_path)
return cls(results=results, log=log)
try:
results = json.load(handle)
if log:
log.info('Loaded %d results from "%s"', len(results), json_path)
return cls(results=results, log=log)
except json.decoder.JSONDecodeError as err:
log.error('Error to decode the json "%s" file: "%s"', json_path, err)
return None
@classmethod
def get_slug(cls) -> str:
@@ -227,7 +231,7 @@ def run_module(module: MVTModule) -> None:
module.save_to_json()
def save_timeline(timeline: list, timeline_path: str) -> None:
def save_timeline(timeline: list, timeline_path: str, is_utc: bool = True) -> None:
"""Save the timeline in a csv file.
:param timeline: List of records to order and store
@@ -238,7 +242,12 @@ def save_timeline(timeline: list, timeline_path: str) -> None:
csvoutput = csv.writer(
handle, delimiter=",", quotechar='"', quoting=csv.QUOTE_ALL, escapechar="\\"
)
csvoutput.writerow(["UTC Timestamp", "Plugin", "Event", "Description"])
if is_utc:
timestamp_header = "UTC Timestamp"
else:
timestamp_header = "Device Local Timestamp"
csvoutput.writerow([timestamp_header, "Plugin", "Event", "Description"])
for event in sorted(
timeline, key=lambda x: x["timestamp"] if x["timestamp"] is not None else ""

View File

@@ -14,6 +14,7 @@ from packaging import version
from .indicators import MVT_DATA_FOLDER, MVT_INDICATORS_FOLDER
from .version import MVT_VERSION
from .config import settings
log = logging.getLogger(__name__)
@@ -23,7 +24,7 @@ INDICATORS_CHECK_FREQUENCY = 12
class MVTUpdates:
def check(self) -> str:
res = requests.get("https://pypi.org/pypi/mvt/json", timeout=15)
res = requests.get(settings.PYPI_UPDATE_URL, timeout=15)
data = res.json()
latest_version = data.get("info", {}).get("version", "")

View File

@@ -13,6 +13,7 @@ import re
from typing import Any, Iterator, Union
from rich.logging import RichHandler
from mvt.common.config import settings
class CustomJSONEncoder(json.JSONEncoder):
@@ -256,7 +257,7 @@ def set_verbose_logging(verbose: bool = False):
def exec_or_profile(module, globals, locals):
"""Hook for profiling MVT modules"""
if int(os.environ.get("MVT_PROFILE", False)):
if settings.PROFILE:
cProfile.runctx(module, globals, locals)
else:
exec(module, globals, locals)

View File

@@ -3,4 +3,4 @@
# Use of this software is governed by the MVT License 1.1 that can be found at
# https://license.mvt.re/1.1/
MVT_VERSION = "2.6.0"
MVT_VERSION = "2.6.1"

Some files were not shown because too many files have changed in this diff Show More