Merge branch 'main' into fix/vt

Co-authored-by: DonnchaC <DonnchaC@users.noreply.github.com>

Makefile

@@ -23,7 +23,7 @@ install:
 	python3 -m pip install --upgrade -e .
 
 test-requirements:
-	python3 -m pip install --upgrade -r test-requirements.txt
+	python3 -m pip install --upgrade --group dev
 
 generate-proto-parsers:
 	# Generate python parsers for protobuf files

docs/requirements.txt

@@ -1,5 +1,5 @@
 mkdocs==1.6.1
-mkdocs-autorefs==1.4.2
-mkdocs-material==9.6.16
+mkdocs-autorefs==1.4.3
+mkdocs-material==9.6.20
 mkdocs-material-extensions==1.3.1
 mkdocstrings==0.30.0

pyproject.toml

@@ -1,13 +1,11 @@
 [project]
 name = "mvt"
 dynamic = ["version"]
-authors = [
-    {name = "Claudio Guarnieri", email = "nex@nex.sx"}
-]
+authors = [{ name = "Claudio Guarnieri", email = "nex@nex.sx" }]
 maintainers = [
-    {name = "Etienne Maynier", email = "tek@randhome.io"},
-    {name = "Donncha Ó Cearbhaill", email = "donncha.ocearbhaill@amnesty.org"},
-    {name = "Rory Flynn", email = "rory.flynn@amnesty.org"}
+    { name = "Etienne Maynier", email = "tek@randhome.io" },
+    { name = "Donncha Ó Cearbhaill", email = "donncha.ocearbhaill@amnesty.org" },
+    { name = "Rory Flynn", email = "rory.flynn@amnesty.org" },
 ]
 description = "Mobile Verification Toolkit"
 readme = "README.md"
@@ -16,7 +14,7 @@ classifiers = [
     "Development Status :: 5 - Production/Stable",
     "Intended Audience :: Information Technology",
     "Operating System :: OS Independent",
-    "Programming Language :: Python"
+    "Programming Language :: Python",
 ]
 dependencies = [
     "click==8.2.1",
@@ -45,20 +43,31 @@ homepage = "https://docs.mvt.re/en/latest/"
 repository = "https://github.com/mvt-project/mvt"
 
 [project.scripts]
-    mvt-ios = "mvt.ios:cli"
-    mvt-android = "mvt.android:cli"
+mvt-ios = "mvt.ios:cli"
+mvt-android = "mvt.android:cli"
+
+[dependency-groups]
+dev = [
+    "requests>=2.31.0",
+    "pytest>=7.4.3",
+    "pytest-cov>=4.1.0",
+    "pytest-github-actions-annotate-failures>=0.2.0",
+    "pytest-mock>=3.14.0",
+    "stix2>=3.0.1",
+    "ruff>=0.1.6",
+    "mypy>=1.7.1",
+    "betterproto[compiler]",
+]
 
 [build-system]
 requires = ["setuptools>=61.0"]
 build-backend = "setuptools.build_meta"
 
 [tool.coverage.run]
-omit = [
-    "tests/*",
-]
+omit = ["tests/*"]
 
 [tool.coverage.html]
-directory= "htmlcov"
+directory = "htmlcov"
 
 [tool.mypy]
 install_types = true
@@ -68,15 +77,13 @@ packages = "src"
 
 [tool.pytest.ini_options]
 addopts = "-ra -q --cov=mvt --cov-report html --junitxml=pytest.xml --cov-report=term-missing:skip-covered"
-testpaths = [
-    "tests"
-]
+testpaths = ["tests"]
 
 [tool.ruff.lint]
-select = ["C90", "E", "F", "W"]  # flake8 default set
+select = ["C90", "E", "F", "W"] # flake8 default set
 ignore = [
-    "E501",  # don't enforce line length violations
-    "C901",  # complex-structure
+    "E501", # don't enforce line length violations
+    "C901", # complex-structure
 
     # These were previously ignored but don't seem to be required:
     # "E265",  # no-space-after-block-comment
@@ -88,14 +95,14 @@ ignore = [
 ]
 
 [tool.ruff.lint.per-file-ignores]
-"__init__.py" = ["F401"]  # unused-import
+"__init__.py" = ["F401"] # unused-import
 
 [tool.ruff.lint.mccabe]
 max-complexity = 10
 
 [tool.setuptools]
 include-package-data = true
-package-dir = {"" = "src"}
+package-dir = { "" = "src" }
 
 [tool.setuptools.packages.find]
 where = ["src"]
@@ -104,4 +111,4 @@ where = ["src"]
 mvt = ["ios/data/*.json"]
 
 [tool.setuptools.dynamic]
-version = {attr = "mvt.common.version.MVT_VERSION"}
+version = { attr = "mvt.common.version.MVT_VERSION" }

src/mvt/android/modules/adb/packages.py

@@ -107,8 +107,7 @@ class Packages(AndroidExtraction):
                     result["matched_indicator"] = ioc
                     self.detected.append(result)
 
-    @staticmethod
-    def check_virustotal(packages: list) -> None:
+    def check_virustotal(self, packages: list) -> None:
         hashes = []
         for package in packages:
             for file in package.get("files", []):
@@ -143,8 +142,15 @@ class Packages(AndroidExtraction):
 
         for package in packages:
             for file in package.get("files", []):
-                row = [package["package_name"], file["path"]]
-
+                if "package_name" in package:
+                    row = [package["package_name"], file["path"]]
+                elif "name" in package:
+                    row = [package["name"], file["path"]]
+                else:
+                    self.log.error(
+                        f"Package {package} has no name or package_name. packages.json or apks.json is malformed"
+                    )
+                    continue
                 if file["sha256"] in detections:
                     detection = detections[file["sha256"]]
                     positives = detection.split("/")[0]

src/mvt/android/modules/androidqf/__init__.py

@@ -19,7 +19,6 @@ from .processes import Processes
 from .settings import Settings
 from .sms import SMS
 from .files import Files
-from .root_binaries import RootBinaries
 
 ANDROIDQF_MODULES = [
     DumpsysActivities,
@@ -38,5 +37,4 @@ ANDROIDQF_MODULES = [
     SMS,
     DumpsysPackages,
     Files,
-    RootBinaries,
 ]

src/mvt/android/modules/androidqf/root_binaries.py

@@ -1,121 +0,0 @@
-# Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021-2023 The MVT Authors.
-# Use of this software is governed by the MVT License 1.1 that can be found at
-#   https://license.mvt.re/1.1/
-
-import json
-import logging
-from typing import Optional
-
-from .base import AndroidQFModule
-
-
-class RootBinaries(AndroidQFModule):
-    """This module analyzes root_binaries.json for root binaries found by androidqf."""
-
-    def __init__(
-        self,
-        file_path: Optional[str] = None,
-        target_path: Optional[str] = None,
-        results_path: Optional[str] = None,
-        module_options: Optional[dict] = None,
-        log: logging.Logger = logging.getLogger(__name__),
-        results: Optional[list] = None,
-    ) -> None:
-        super().__init__(
-            file_path=file_path,
-            target_path=target_path,
-            results_path=results_path,
-            module_options=module_options,
-            log=log,
-            results=results,
-        )
-
-    def serialize(self, record: dict) -> dict:
-        return {
-            "timestamp": record.get("timestamp"),
-            "module": self.__class__.__name__,
-            "event": "root_binary_found",
-            "data": f"Root binary found: {record['path']} (binary: {record['binary_name']})",
-        }
-
-    def check_indicators(self) -> None:
-        """Check for indicators of device rooting."""
-        if not self.results:
-            return
-
-        # All found root binaries are considered indicators of rooting
-        for result in self.results:
-            self.log.warning(
-                'Found root binary "%s" at path "%s"',
-                result["binary_name"],
-                result["path"],
-            )
-            self.detected.append(result)
-
-        if self.detected:
-            self.log.warning(
-                "Device shows signs of rooting with %d root binaries found",
-                len(self.detected),
-            )
-
-    def run(self) -> None:
-        """Run the root binaries analysis."""
-        root_binaries_files = self._get_files_by_pattern("*/root_binaries.json")
-
-        if not root_binaries_files:
-            self.log.info("No root_binaries.json file found")
-            return
-
-        rawdata = self._get_file_content(root_binaries_files[0]).decode(
-            "utf-8", errors="ignore"
-        )
-
-        try:
-            root_binary_paths = json.loads(rawdata)
-        except json.JSONDecodeError as e:
-            self.log.error("Failed to parse root_binaries.json: %s", e)
-            return
-
-        if not isinstance(root_binary_paths, list):
-            self.log.error("Expected root_binaries.json to contain a list of paths")
-            return
-
-        # Known root binary names that might be found and their descriptions
-        # This maps the binary name to a human-readable description
-        known_root_binaries = {
-            "su": "SuperUser binary",
-            "busybox": "BusyBox utilities",
-            "supersu": "SuperSU root management",
-            "Superuser.apk": "Superuser app",
-            "KingoUser.apk": "KingRoot app",
-            "SuperSu.apk": "SuperSU app",
-            "magisk": "Magisk root framework",
-            "magiskhide": "Magisk hide utility",
-            "magiskinit": "Magisk init binary",
-            "magiskpolicy": "Magisk policy binary",
-        }
-
-        for path in root_binary_paths:
-            if not path or not isinstance(path, str):
-                continue
-
-            # Extract binary name from path
-            binary_name = path.split("/")[-1].lower()
-
-            # Check if this matches a known root binary by exact name match
-            description = "Unknown root binary"
-            for known_binary in known_root_binaries:
-                if binary_name == known_binary.lower():
-                    description = known_root_binaries[known_binary]
-                    break
-
-            result = {
-                "path": path.strip(),
-                "binary_name": binary_name,
-                "description": description,
-            }
-
-            self.results.append(result)
-
-        self.log.info("Found %d root binaries", len(self.results))

src/mvt/ios/data/ios_versions.json

@@ -895,6 +895,10 @@
         "version": "15.8.4",
         "build": "19H390"
     },
+    {
+        "version": "15.8.5",
+        "build": "19H394"
+    },
     {
         "build": "20A362",
         "version": "16.0"
@@ -1000,6 +1004,10 @@
         "version": "16.7.11",
         "build": "20H360"
     },
+    {
+        "version": "16.7.12",
+        "build": "20H364"
+    },
     {
         "version": "17.0",
         "build": "21A327"
@@ -1135,5 +1143,21 @@
     {
         "version": "18.6",
         "build": "22G86"
+    },
+    {
+        "version": "18.6.1",
+        "build": "22G90"
+    },
+    {
+        "version": "18.6.2",
+        "build": "22G100"
+    },
+    {
+        "version": "18.7",
+        "build": "22H20"
+    },
+    {
+        "version": "26",
+        "build": "23A341"
     }
 ]

test-requirements.txt

@@ -1,9 +0,0 @@
-requests>=2.31.0
-pytest>=7.4.3
-pytest-cov>=4.1.0
-pytest-github-actions-annotate-failures>=0.2.0
-pytest-mock>=3.14.0
-stix2>=3.0.1
-ruff>=0.1.6
-mypy>=1.7.1
-betterproto[compiler]

tests/android_androidqf/test_root_binaries.py

@@ -1,116 +0,0 @@
-# Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021-2023 The MVT Authors.
-# Use of this software is governed by the MVT License 1.1 that can be found at
-#   https://license.mvt.re/1.1/
-
-import logging
-from pathlib import Path
-
-import pytest
-
-from mvt.android.modules.androidqf.root_binaries import RootBinaries
-from mvt.common.module import run_module
-
-from ..utils import get_android_androidqf, list_files
-
-
-@pytest.fixture()
-def data_path():
-    return get_android_androidqf()
-
-
-@pytest.fixture()
-def parent_data_path(data_path):
-    return Path(data_path).absolute().parent.as_posix()
-
-
-@pytest.fixture()
-def file_list(data_path):
-    return list_files(data_path)
-
-
-@pytest.fixture()
-def module(parent_data_path, file_list):
-    m = RootBinaries(target_path=parent_data_path, log=logging)
-    m.from_folder(parent_data_path, file_list)
-    return m
-
-
-class TestAndroidqfRootBinaries:
-    def test_root_binaries_detection(self, module):
-        run_module(module)
-
-        # Should find 4 root binaries from the test file
-        assert len(module.results) == 4
-        assert len(module.detected) == 4
-
-        # Check that all results are detected as indicators
-        binary_paths = [result["path"] for result in module.results]
-        assert "/system/bin/su" in binary_paths
-        assert "/system/xbin/busybox" in binary_paths
-        assert "/data/local/tmp/magisk" in binary_paths
-        assert "/system/bin/magiskhide" in binary_paths
-
-    def test_root_binaries_descriptions(self, module):
-        run_module(module)
-
-        # Check that binary descriptions are correctly identified
-        su_result = next((r for r in module.results if "su" in r["binary_name"]), None)
-        assert su_result is not None
-        assert "SuperUser binary" in su_result["description"]
-
-        busybox_result = next(
-            (r for r in module.results if "busybox" in r["binary_name"]), None
-        )
-        assert busybox_result is not None
-        assert "BusyBox utilities" in busybox_result["description"]
-
-        magisk_result = next(
-            (r for r in module.results if r["binary_name"] == "magisk"), None
-        )
-        assert magisk_result is not None
-        assert "Magisk root framework" in magisk_result["description"]
-
-        magiskhide_result = next(
-            (r for r in module.results if "magiskhide" in r["binary_name"]), None
-        )
-        assert magiskhide_result is not None
-        assert "Magisk hide utility" in magiskhide_result["description"]
-
-    def test_root_binaries_warnings(self, caplog, module):
-        run_module(module)
-
-        # Check that warnings are logged for each root binary found
-        assert 'Found root binary "su" at path "/system/bin/su"' in caplog.text
-        assert (
-            'Found root binary "busybox" at path "/system/xbin/busybox"' in caplog.text
-        )
-        assert (
-            'Found root binary "magisk" at path "/data/local/tmp/magisk"' in caplog.text
-        )
-        assert (
-            'Found root binary "magiskhide" at path "/system/bin/magiskhide"'
-            in caplog.text
-        )
-        assert "Device shows signs of rooting with 4 root binaries found" in caplog.text
-
-    def test_serialize_method(self, module):
-        run_module(module)
-
-        # Test that serialize method works correctly
-        if module.results:
-            serialized = module.serialize(module.results[0])
-            assert serialized["module"] == "RootBinaries"
-            assert serialized["event"] == "root_binary_found"
-            assert "Root binary found:" in serialized["data"]
-
-    def test_no_root_binaries_file(self, parent_data_path):
-        # Test behavior when no root_binaries.json file is present
-        empty_file_list = []
-        m = RootBinaries(target_path=parent_data_path, log=logging)
-        m.from_folder(parent_data_path, empty_file_list)
-
-        run_module(m)
-
-        assert len(m.results) == 0
-        assert len(m.detected) == 0

tests/artifacts/androidqf/root_binaries.json

@@ -1,6 +0,0 @@
-[
-    "/system/bin/su",
-    "/system/xbin/busybox",
-    "/data/local/tmp/magisk",
-    "/system/bin/magiskhide"
-]

tests/common/test_utils.py

@@ -62,7 +62,7 @@ class TestHashes:
     def test_hash_from_folder(self):
         path = os.path.join(get_artifact_folder(), "androidqf")
         hashes = list(generate_hashes_from_path(path, logging))
-        assert len(hashes) == 8
+        assert len(hashes) == 7
         # Sort the files to have reliable order for tests.
         hashes = sorted(hashes, key=lambda x: x["file_path"])
         assert hashes[0]["file_path"] == os.path.join(path, "backup.ab")