Update WIP for dumpstate parser

Co-authored-by: DonnchaC <DonnchaC@users.noreply.github.com>

.safety-policy.yml

@@ -6,4 +6,6 @@
 security: # configuration for the `safety check` command
     ignore-vulnerabilities: # Here you can list multiple specific vulnerabilities you want to ignore (optionally for a time period)
         67599: # Example vulnerability ID
+            reason: disputed, inapplicable
+        70612:
             reason: disputed, inapplicable

mvt/android/artifacts/dumpstate_artifact.py

@@ -0,0 +1,165 @@
+# Mobile Verification Toolkit (MVT)
+# Copyright (c) 2021-2023 The MVT Authors.
+# Use of this software is governed by the MVT License 1.1 that can be found at
+#   https://license.mvt.re/1.1/
+import re
+
+from .artifact import AndroidArtifact
+
+
+# The AOSP dumpstate code is available at https://cs.android.com/android/platform/superproject/+/master:frameworks/native/cmds/dumpstate/
+# The dumpstate code is used to generate bugreports on Android devices. It looks like there are
+# bugs in the code that leave some sections with out ending lines. We need to handle these cases.
+#
+# The approach here is to flag probably broken section, and to search for plausible new section headers
+# to close the previous section. This is a heuristic approach, and may not work in all cases. We can't do
+# this for all sections as we will detect subsections as new sections.
+SECTION_BROKEN_TERMINATORS = [
+    b"VM TRACES AT LAST ANR",
+    b"DIGITAL_HALL",
+]
+
+
+class DumpStateArtifact(AndroidArtifact):
+    def __init__(self, *args, **kwargs):
+        self.dumpstate_sections = []
+        self.dumpstate_header = {}
+        self.unparsed_lines = []
+        super().__init__(*args, **kwargs)
+
+    def _parse_dumpstate_header(self, header_text):
+        """
+        Parse dumpstate header metadata
+        """
+        fields = {}
+        for line in header_text.splitlines():
+            if line.startswith(b"="):
+                continue
+
+            if b":" in line:
+                # Save line if it's a key-value pair.
+                key, value = line.split(b":", 1)
+                fields[key] = value[1:]
+
+            if not line and fields:
+                # Finish if we get an empty line and already parsed lines
+                break
+            else:
+                # Skip until we find lines
+                continue
+
+        self.dumpstate_header = fields
+        return fields
+
+    def _get_section_header(self, header_match):
+        """
+        Create internal dictionary to track dumpsys section.
+        """
+        section_full = header_match.group(0).strip(b"-").strip()
+        section_name = header_match.group(1).rstrip()
+
+        if header_match.group(2):
+            section_command = header_match.group(2).strip(b"()")
+        else:
+            # Some headers can missing the command
+            section_command = ""
+
+        has_broken_terminator = False
+        for broken_section in SECTION_BROKEN_TERMINATORS:
+            if broken_section in section_name:
+                has_broken_terminator = True
+                break
+
+        section = {
+            "section_name": section_name,
+            "section_command": section_command,
+            "section_full": section_full,
+            "missing_terminator": has_broken_terminator,
+            "lines": [],
+            "error": False,
+        }
+        self.dumpstate_sections.append(section)
+        return section
+
+    def parse_dumpstate(self, text: str) -> list:
+        """
+        Extract all sections from a full dumpstate file.
+
+        :param text: content of the full dumpstate file (string)
+        """
+        # Parse the header
+        self._parse_dumpstate_header(text)
+
+        header = b"------ "
+
+        # Regexes to parse headers
+        section_name_re = re.compile(rb"------ ([\w\d\s\-\/\&]+)(\(.*\))? ------")
+        end_of_section_re = re.compile(rb"------ End of .* ------")
+        missing_file_error_re = re.compile(rb"\*\*\* (.*): No such file or directory")
+        generic_error_re = re.compile(rb"\*\*\* (.*) (?<!\*\*\*)$")
+
+        section = None
+
+        # Parse each line in dumpstate and look for headers
+        for line in text.splitlines():
+            if not section:
+                # If we find an end section when not in a section, we can skip
+                # It's probably the trailing line of a section.
+                end_of_section_match = re.match(end_of_section_re, line)
+                if end_of_section_match:
+                    self.unparsed_lines.append(line)
+                    continue
+
+                possible_section_header = re.match(section_name_re, line)
+                if possible_section_header:
+                    section = self._get_section_header(possible_section_header)
+                    # print("found section", section)
+                    continue
+                else:
+                    # We continue to next line as we weren't already in a section
+                    self.unparsed_lines.append(line)
+                    continue
+
+            if line.lstrip().startswith(header):
+                # This may be an internal section, or the terminator for our current section
+                # Ending looks like: ------ 0.557s was the duration of 'DUMPSYS CRITICAL' ------
+
+                # Check that we have the end for the right command.
+                section_command_in_quotes = b"'" + section["section_name"] + b"'"
+                if (
+                    section_command_in_quotes in line
+                    or section["section_full"]
+                    in line  # Needed for 0.070s was the duration of 'KERNEL LOG (dmesg)'
+                ):
+                    # Add end line and finish up the section
+                    section["lines"].append(line)
+                    section = None
+                    continue
+
+                # If we haven't closed previous, but this matches a section header, we can try close.
+                # Probably a bug where not closed properly. We explicitly flag known broken fields.
+
+                # This fails on these blocks if we dont blacklist. Maybe we need to make a blacklist of badly closed items
+                # ------ DUMP BLOCK STAT ------
+                # ------ BLOCK STAT (/sys/block/dm-20) ------
+
+                possible_section_header = re.match(section_name_re, line)
+                if possible_section_header and section["missing_terminator"]:
+                    section = self._get_section_header(possible_section_header)
+                else:
+                    # Probably terminator for subsection, ignore and treat as a regular line.
+                    pass
+
+            # Handle lines with special meaning
+            # TODO: This is failing as sometime errors are followed by a terminator and sometimes not.
+            if re.match(missing_file_error_re, line) or re.match(
+                generic_error_re, line
+            ):
+                # The line in a failed file read which is dumped without an header end section.
+                section["failed"] = True
+                section["lines"].append(line)
+                section = None
+            else:
+                section["lines"].append(line)
+
+        return self.dumpstate_sections

mvt/android/modules/androidqf/__init__.py

@@ -12,6 +12,7 @@ from .dumpsys_dbinfo import DumpsysDBInfo
 from .dumpsys_packages import DumpsysPackages
 from .dumpsys_receivers import DumpsysReceivers
 from .getprop import Getprop
+from .packages import Packages
 from .processes import Processes
 from .settings import Settings
 from .sms import SMS
@@ -24,6 +25,7 @@ ANDROIDQF_MODULES = [
     DumpsysDBInfo,
     DumpsysBatteryDaily,
     DumpsysBatteryHistory,
+    Packages,
     Processes,
     Getprop,
     Settings,

mvt/android/modules/androidqf/packages.py

@@ -0,0 +1,97 @@
+# Mobile Verification Toolkit (MVT)
+# Copyright (c) 2021-2023 The MVT Authors.
+# Use of this software is governed by the MVT License 1.1 that can be found at
+#   https://license.mvt.re/1.1/
+
+import logging
+from typing import Optional
+import json
+
+from mvt.android.utils import (
+    ROOT_PACKAGES,
+    BROWSER_INSTALLERS,
+    PLAY_STORE_INSTALLERS,
+    THIRD_PARTY_STORE_INSTALLERS,
+)
+
+from .base import AndroidQFModule
+
+
+class Packages(AndroidQFModule):
+    """This module examines the installed packages in packages.json"""
+
+    def __init__(
+        self,
+        file_path: Optional[str] = None,
+        target_path: Optional[str] = None,
+        results_path: Optional[str] = None,
+        module_options: Optional[dict] = None,
+        log: logging.Logger = logging.getLogger(__name__),
+        results: Optional[list] = None,
+    ) -> None:
+        super().__init__(
+            file_path=file_path,
+            target_path=target_path,
+            results_path=results_path,
+            module_options=module_options,
+            log=log,
+            results=results,
+        )
+
+    def check_indicators(self) -> None:
+        for result in self.results:
+            if result["name"] in ROOT_PACKAGES:
+                self.log.warning(
+                    "Found an installed package related to "
+                    'rooting/jailbreaking: "%s"',
+                    result["name"],
+                )
+                self.detected.append(result)
+                continue
+
+            # Detections for apps installed via unusual methods
+            if result["installer"] in THIRD_PARTY_STORE_INSTALLERS:
+                self.log.warning(
+                    'Found a package installed via a third party store (installer="%s"): "%s"',
+                    result["installer"],
+                    result["name"],
+                )
+            elif result["installer"] in BROWSER_INSTALLERS:
+                self.log.warning(
+                    'Found a package installed via a browser (installer="%s"): "%s"',
+                    result["installer"],
+                    result["name"],
+                )
+            elif result["installer"] == "null" and result["system"] is False:
+                self.log.warning(
+                    'Found a non-system package installed via adb or another method: "%s"',
+                    result["name"],
+                )
+            elif result["installer"] in PLAY_STORE_INSTALLERS:
+                pass
+
+            if not self.indicators:
+                continue
+
+            ioc = self.indicators.check_app_id(result.get("name"))
+            if ioc:
+                result["matched_indicator"] = ioc
+                self.detected.append(result)
+                continue
+
+            for package_file in result.get("files", []):
+                ioc = self.indicators.check_file_hash(package_file["sha256"])
+                if ioc:
+                    result["matched_indicator"] = ioc
+                    self.detected.append(result)
+
+    def run(self) -> None:
+        packages = self._get_files_by_pattern("*/packages.json")
+        if not packages:
+            self.log.error(
+                "packages.json file not found in this androidqf bundle. Possibly malformed?"
+            )
+            return
+
+        self.results = json.loads(self._get_file_content(packages[0]))
+        self.log.info("Found %d packages in packages.json", len(self.results))

mvt/android/utils.py

@@ -91,3 +91,15 @@ SYSTEM_UPDATE_PACKAGES = [
     "com.transsion.systemupdate",
     "com.wssyncmldm",
 ]
+
+# Apps installed from the Play store have this installer
+PLAY_STORE_INSTALLERS = ["com.android.vending"]
+
+# Installer id for apps from common 3rd party stores
+THIRD_PARTY_STORE_INSTALLERS = ["com.aurora.store", "org.fdroid.fdroid"]
+
+# Packages installed via a browser have these installers
+BROWSER_INSTALLERS = [
+    "com.google.android.packageinstaller",
+    "com.android.packageinstaller",
+]

mvt/ios/data/ios_versions.json

@@ -1067,5 +1067,17 @@
     {
         "version": "17.5.1",
         "build": "21F90"
+    },
+    {
+        "version": "17.6.1",
+        "build": "21G93"
+    },
+    {
+        "version": "17.6.1",
+        "build": "21G101"
+    },
+    {
+        "version": "18",
+        "build": "22A3354"
     }
 ]

tests/android/test_artifact_dumpstate.py

@@ -0,0 +1,45 @@
+# Mobile Verification Toolkit (MVT)
+# Copyright (c) 2021-2023 The MVT Authors.
+# Use of this software is governed by the MVT License 1.1 that can be found at
+#   https://license.mvt.re/1.1/
+from mvt.android.artifacts.dumpstate_artifact import DumpStateArtifact
+
+from ..utils import get_artifact
+
+
+class TestAndroidArtifactDumpState:
+    def _parse_dump_state(self):
+        """
+        Load the test artifact
+        """
+        file = get_artifact("android_data/bugreport/dumpstate.txt")
+        with open(file, "rb") as f:
+            data = f.read()
+        dumpstate = DumpStateArtifact()
+        dumpstate.parse_dumpstate(data)
+        return dumpstate
+
+    def test_extract_dumpstate_sections(self):
+        """
+        Test parsing of dumpstate sections
+        """
+        dumpstate = self._parse_dump_state()
+        assert len(dumpstate.dumpstate_sections) == 4
+
+        assert len(dumpstate.dumpstate_header) == 4
+        assert dumpstate.dumpstate_header.get(b"Bugreport format version") == b"2.0"
+
+        for section in dumpstate.dumpstate_sections:
+            if section["section_name"] == b"SYSTEM LOG":
+                assert len(section["lines"]) == 5
+                assert section["lines"][0].startswith(b"--------- beginning of system")
+
+            elif section["section_name"] == b"MODEM CRASH HISTORY":
+                # Test parsing where section only has an error message
+                assert len(section["lines"]) == 1
+                assert (
+                    section["lines"][0]
+                    == b"*** /data/tombstones//modem/mcrash_history: No such file or directory"
+                )
+
+        assert len(dumpstate.unparsed_lines) == 11

tests/android_androidqf/test_packages.py

@@ -0,0 +1,87 @@
+# Mobile Verification Toolkit (MVT)
+# Copyright (c) 2021-2023 The MVT Authors.
+# Use of this software is governed by the MVT License 1.1 that can be found at
+#   https://license.mvt.re/1.1/
+
+import logging
+import pytest
+from pathlib import Path
+
+from mvt.android.modules.androidqf.packages import Packages
+from mvt.common.module import run_module
+
+from ..utils import get_android_androidqf, list_files
+
+
+@pytest.fixture()
+def data_path():
+    return get_android_androidqf()
+
+
+@pytest.fixture()
+def parent_data_path(data_path):
+    return Path(data_path).absolute().parent.as_posix()
+
+
+@pytest.fixture()
+def file_list(data_path):
+    return list_files(data_path)
+
+
+@pytest.fixture()
+def module(parent_data_path, file_list):
+    m = Packages(target_path=parent_data_path, log=logging)
+    m.from_folder(parent_data_path, file_list)
+    return m
+
+
+class TestAndroidqfPackages:
+    def test_packages_list(self, module):
+        run_module(module)
+
+        # There should just be 7 packages listed, no detections
+        assert len(module.results) == 7
+        assert len(module.timeline) == 0
+        assert len(module.detected) == 0
+
+    def test_non_appstore_warnings(self, caplog, module):
+        run_module(module)
+
+        # Not a super test to be searching logs for this but heuristic detections not yet formalised
+        assert (
+            'Found a non-system package installed via adb or another method: "com.whatsapp"'
+            in caplog.text
+        )
+        assert (
+            'Found a package installed via a browser (installer="com.google.android.packageinstaller"): '
+            '"app.revanced.manager.flutter"' in caplog.text
+        )
+        assert (
+            'Found a package installed via a third party store (installer="org.fdroid.fdroid"): "org.nuclearfog.apollo"'
+            in caplog.text
+        )
+
+    def test_packages_ioc_package_names(self, module, indicators_factory):
+        module.indicators = indicators_factory(app_ids=["com.malware.blah"])
+
+        run_module(module)
+
+        assert len(module.detected) == 1
+        assert module.detected[0]["name"] == "com.malware.blah"
+        assert module.detected[0]["matched_indicator"]["value"] == "com.malware.blah"
+
+    def test_packages_ioc_sha256(self, module, indicators_factory):
+        module.indicators = indicators_factory(
+            files_sha256=[
+                "31037a27af59d4914906c01ad14a318eee2f3e31d48da8954dca62a99174e3fa"
+            ]
+        )
+
+        run_module(module)
+
+        assert len(module.detected) == 1
+        assert module.detected[0]["name"] == "com.malware.muahaha"
+        assert (
+            module.detected[0]["matched_indicator"]["value"]
+            == "31037a27af59d4914906c01ad14a318eee2f3e31d48da8954dca62a99174e3fa"
+        )

tests/artifacts/android_data/bugreport/dumpstate.txt

@@ -1,3 +1,25 @@
+========================================================
+== dumpstate: 2024-04-21 10:00:11
+========================================================
+
+Build: TP1A.220624.014
+Uptime: up 0 weeks, 0 days, 0 hours, 20 minutes,  load average: 20.00, 19.92, 15.46
+Bugreport format version: 2.0
+Dumpstate info: id=1 pid=21015 dry_run=0 parallel_run=1 args=/system/bin/dumpstate -S bugreport_mode=
+
+------ DUMPSYS CRITICAL (/system/bin/dumpsys) ------
+-------------------------------------------------------------------------------
+DUMP OF SERVICE CRITICAL SurfaceFlinger:
+now = 1202781815070
+Build configuration: [sf PRESENT_TIME_OFFSET=0 FORCE_HWC_FOR_RBG_TO_YUV=1 MAX_VIRT_DISPLAY_DIM=0 RUNNING_WITHOUT_SYNC_FRAMEWORK=0 NUM_FRAMEBUFFER_SURFACE_BUFFERS=3]
+
+Display identification data:
+Display 0 (HWC display 0): no identification data
+
+Wide-Color information:
+Device has wide color built-in display: 0
+Device uses color management: 1
+
 Currently running services:
   AAS
   AODManagerService
@@ -246,6 +268,16 @@ Packages:
         com.instagram.direct.share.handler.DirectMultipleExternalMediaShareActivity
         com.instagram.share.handleractivity.ClipsShareHandlerActivity
         com.instagram.direct.share.handler.DirectMultipleExternalMediaShareActivityInterop
-
-
-
+------ 0.557s was the duration of 'DUMPSYS CRITICAL' ------
+------ 0.023s was the duration of 'DUMPSYS CRITICAL PROTO' ------
+------ SERIALIZE PERFETTO TRACE (perfetto --save-for-bugreport) ------
+------ 0.036s was the duration of 'SERIALIZE PERFETTO TRACE' ------
+------ End of SERIALIZE PERFETTO TRACE (perfetto --save-for-bugreport) ------
+------ MODEM CRASH HISTORY (/data/tombstones//modem/mcrash_history) ------
+*** /data/tombstones//modem/mcrash_history: No such file or directory
+------ SYSTEM LOG (logcat -v threadtime -v printable -v uid -d *:v) ------
+--------- beginning of system
+05-28 09:44:19.845  root   578   578 I vold    : Vold 3.0 (the awakening) firing up
+05-28 09:44:19.845  root   578   578 D vold    : Detected support for: exfat ext4 f2fs ntfs vfat
+05-28 09:44:19.849  root   578   578 W vold    : [libfs_mgr]Warning: unknown flag: resize
+------ 0.417s was the duration of 'SYSTEM LOG' ------

tests/artifacts/androidqf/packages.json

@@ -0,0 +1,233 @@
+[
+    {
+        "name": "com.whatsapp",
+        "files": [
+            {
+                "path": "/data/app/~~/com.whatsapp-~~/base.apk",
+                "local_name": "",
+                "md5": "5870bd06e642de410c54705226ecfa9a",
+                "sha1": "6cb06e9ab5619345f930c2b2096b4dd013a10ec9",
+                "sha256": "744ed47f8176ec423840344c33e88bd2c96e8988cda0797f3415bb5229efc12b",
+                "sha512": "f222f742b0bd302c82e202bc78f7ff8b2de4acfc8d606994245ffa80998b003e215cad82cae023abe4f65c0da0a56fa9890e9bb3a753af6dac848a753ac07aee",
+                "error": "",
+                "verified_certificate": true,
+                "certificate": {
+                    "Md5": "556c6019249bbc0cab70495178d3a9d1",
+                    "Sha1": "38a0f7d505fe18fec64fbf343ecaaaf310dbd799",
+                    "Sha256": "3987d043d10aefaf5a8710b3671418fe57e0e19b653c9df82558feb5ffce5d44",
+                    "ValidFrom": "2010-06-25T23:07:16Z",
+                    "ValidTo": "2044-02-15T23:07:16Z",
+                    "Issuer": "C=US, ST=California, L=Santa Clara, O=WhatsApp Inc., OU=Engineering, CN=Brian Acton",
+                    "Subject": "C=US, ST=California, L=Santa Clara, O=WhatsApp Inc., OU=Engineering, CN=Brian Acton",
+                    "SignatureAlgorithm": "DSA-SHA1",
+                    "SerialNumber": 1277507236
+                },
+                "certificate_error": "",
+                "trusted_certificate": true
+            }
+        ],
+        "installer": "null",
+        "uid": 10271,
+        "disabled": false,
+        "system": false,
+        "third_party": true
+    },
+    {
+        "name": "app.revanced.manager.flutter",
+        "files": [
+            {
+                "path": "/data/app/~~==/app.revanced.manager.flutter-==/base.apk",
+                "local_name": "",
+                "md5": "aae9b55c6f2592233518bb5a173e8505",
+                "sha1": "9185a83dae0fc8a0ba79f89f3c84fe8a038f93af",
+                "sha256": "6ddb76f6180ca8bc0a11d5b343ac9ad8f137a351f20c080e989ca4310973d319",
+                "sha512": "923a57d4cdf2e7d48539307abbd12f982d61f393a1d058ceef0f6109301d21fedf0fe73c667f8add37fb35da570ac35c6b911360d9bf0389aa0bbbd53103ff46",
+                "error": "",
+                "verified_certificate": true,
+                "certificate": {
+                    "Md5": "f822d70f449d798f0688e2c7358a429c",
+                    "Sha1": "93adc8e2bd1687644f1143e184bcbfd57912ff2c",
+                    "Sha256": "b6362c6ea7888efd15c0800f480786ad0f5b133b4f84e12d46afba5f9eac1223",
+                    "ValidFrom": "2022-09-14T11:45:44Z",
+                    "ValidTo": "2050-01-30T11:45:44Z",
+                    "Issuer": "C=Unknown, ST=Unknown, L=Unknown, O=ReVanced, OU=ReVanced, CN=ReVanced Manager",
+                    "Subject": "C=Unknown, ST=Unknown, L=Unknown, O=ReVanced, OU=ReVanced, CN=ReVanced Manager",
+                    "SignatureAlgorithm": "SHA256-RSA",
+                    "SerialNumber": 710526530
+                },
+                "certificate_error": "",
+                "trusted_certificate": false
+            }
+        ],
+        "installer": "com.google.android.packageinstaller",
+        "uid": 10266,
+        "disabled": false,
+        "system": false,
+        "third_party": true
+    },
+    {
+        "name": "com.google.android.youtube",
+        "files": [
+            {
+                "path": "/data/app/~~==/com.google.android.youtube-==/base.apk",
+                "local_name": "",
+                "md5": "3ec11b187ec6195e9ca4b5be671eba34",
+                "sha1": "33a9a89836690966498ba106283e76eff430365b",
+                "sha256": "a81b6392ab855905763272cf1a248b0d09fc675a91eabe7ef4ed589356a35241",
+                "sha512": "c736fbd07fe52539d8e96f6489a49c915c2bac472f0203f6187d167e2e3623f07db9e70b0fbc0494f6eeffb66a4cf71da56ad70503dc8138512faa3c1e847174",
+                "error": "",
+                "verified_certificate": true,
+                "certificate": {
+                    "Md5": "d046fc5d1fc3cd0e57c5444097cd5449",
+                    "Sha1": "24bb24c05e47e0aefa68a58a766179d9b613a600",
+                    "Sha256": "3d7a1223019aa39d9ea0e3436ab7c0896bfb4fb679f4de5fe7c23f326c8f994a",
+                    "ValidFrom": "2008-12-02T02:07:58Z",
+                    "ValidTo": "2036-04-19T02:07:58Z",
+                    "Issuer": "C=US, ST=CA, L=Mountain View, O=Google, Inc, OU=Google, Inc, CN=Unknown",
+                    "Subject": "C=US, ST=CA, L=Mountain View, O=Google, Inc, OU=Google, Inc, CN=Unknown",
+                    "SignatureAlgorithm": "MD5-RSA",
+                    "SerialNumber": 1228183678
+                },
+                "certificate_error": "",
+                "trusted_certificate": true
+            }
+        ],
+        "installer": "com.google.android.packageinstaller",
+        "uid": 10194,
+        "disabled": false,
+        "system": true,
+        "third_party": false
+    },
+    {
+        "name": "org.fdroid.fdroid",
+        "files": [
+            {
+                "path": "/data/app/~~-==/org.fdroid.fdroid-==/base.apk",
+                "local_name": "",
+                "md5": "1f7524d15b3d229e5e89af609551e640",
+                "sha1": "4ce271a8ac2afb9f584f1deb165f1ab4768c50b0",
+                "sha256": "dc3bb88f6419ee7dde7d1547a41569aa03282fe00e0dc43ce035efd7c9d27d75",
+                "sha512": "40e9bfaf6c2833078e370c85001adcb7493851a5146d2b4067a9909266a0d7904f80825f040c8c6e0cb59ec6e8c0825d522ff963f6db780b049a24d47f81b289",
+                "error": "",
+                "verified_certificate": true,
+                "certificate": {
+                    "Md5": "17c55c628056e193e95644e989792786",
+                    "Sha1": "05f2e65928088981b317fc9a6dbfe04b0fa13b4e",
+                    "Sha256": "43238d512c1e5eb2d6569f4a3afbf5523418b82e0a3ed1552770abb9a9c9ccab",
+                    "ValidFrom": "2010-07-23T17:10:24Z",
+                    "ValidTo": "2037-12-08T17:10:24Z",
+                    "Issuer": "C=UK, ST=Unknown, L=Wetherby, O=Unknown, OU=Unknown, CN=Ciaran Gultnieks",
+                    "Subject": "C=UK, ST=Unknown, L=Wetherby, O=Unknown, OU=Unknown, CN=Ciaran Gultnieks",
+                    "SignatureAlgorithm": "SHA1-RSA",
+                    "SerialNumber": 1279905024
+                },
+                "certificate_error": "",
+                "trusted_certificate": false
+            }
+        ],
+        "installer": "com.google.android.packageinstaller",
+        "uid": 10267,
+        "disabled": false,
+        "system": false,
+        "third_party": true
+    },
+    {
+        "name": "org.nuclearfog.apollo",
+        "files": [
+            {
+                "path": "/data/app/~~==/org.nuclearfog.apollo-==/base.apk",
+                "local_name": "",
+                "md5": "69f611758cc911f472fcabad6151684a",
+                "sha1": "1f5e450ef1901e245d4828735e0e93f0f94fb4da",
+                "sha256": "00bdfc80a397b449bef89dd2051ddd3c9d2a64e954176420b40c90a2af956799",
+                "sha512": "2af8037e0e226cba9f32227f709afc32fd8871c0077f73d00d59353d67ab843cb6641a5e0101d494699aeb91dcd136767fe9d76b30df65e1a1153f3c5b51a837",
+                "error": "",
+                "verified_certificate": true,
+                "certificate": {
+                    "Md5": "2bef3d492d62fad190a8b6b7d71d42a4",
+                    "Sha1": "cad23563b5be0c33611d827ee0da6ad5ef3be39a",
+                    "Sha256": "e1a418c51baa829917daa2e86d7509a8a10470e44280c20146b70ea550bfe1ab",
+                    "ValidFrom": "2022-01-15T20:17:10Z",
+                    "ValidTo": "2047-01-09T20:17:10Z",
+                    "Issuer": "C=DE, ST=Saarland, CN=nuclearfog",
+                    "Subject": "C=DE, ST=Saarland, CN=nuclearfog",
+                    "SignatureAlgorithm": "SHA256-RSA",
+                    "SerialNumber": 75365821
+                },
+                "certificate_error": "",
+                "trusted_certificate": false
+            }
+        ],
+        "installer": "org.fdroid.fdroid",
+        "uid": 10272,
+        "disabled": false,
+        "system": false,
+        "third_party": true
+    },
+    {
+        "name": "com.malware.blah",
+        "files": [
+            {
+                "path": "/data/app/~~-==/com.malware.blah-==/base.apk",
+                "local_name": "",
+                "md5": "349ba2de140fccaf2ed2ac20f66e711f",
+                "sha1": "2cc5b4a70ada9229fb50d30f525392f2d66f58d6",
+                "sha256": "79a3569fbb63a9167ad8a2dad963616bb01474c87d769c7640f6d6810c448eae",
+                "sha512": "df1bbbfa6e895054b36093548558ee0d9fbf61ef09e617d3b3b158ba9f9c11825dbbf7e84711331afb80fc24ea0e5aa07a9db1919932c109c34fefec3c02d184",
+                "error": "",
+                "verified_certificate": true,
+                "certificate": {
+                    "Md5": "54d5b5aca1e7e76bb1a26c61a9381b93",
+                    "Sha1": "4ba9d1f82adb7be841bcf53b03ddae857747199a",
+                    "Sha256": "c3e8cafdcd10e7cd9b2ec67f7abd4447b840431126066f6b16ed42151d2b4d64",
+                    "ValidFrom": "2021-01-15T22:03:53Z",
+                    "ValidTo": "2051-01-15T22:03:53Z",
+                    "Issuer": "C=US, ST=California, L=Mountain View, O=Google Inc., OU=Android, CN=Android",
+                    "Subject": "C=US, ST=California, L=Mountain View, O=Google Inc., OU=Android, CN=Android",
+                    "SignatureAlgorithm": "SHA256-RSA",
+                    "SerialNumber": 955466096586930338769951715633687128507538251257
+                },
+                "certificate_error": "",
+                "trusted_certificate": false
+            }
+        ],
+        "installer": "null",
+        "uid": 10058,
+        "disabled": false,
+        "system": true,
+        "third_party": false
+    },
+    {
+        "name": "com.malware.muahaha",
+        "files": [
+            {
+                "path": "/data/app/~~-==/com.malware.meh-==/base.apk",
+                "local_name": "",
+                "md5": "349ba2de140fccaf2ed2ac20f66e711f",
+                "sha1": "2cc5b4a70ada9229fb50d30f525392f2d66f58d6",
+                "sha256": "31037a27af59d4914906c01ad14a318eee2f3e31d48da8954dca62a99174e3fa",
+                "sha512": "df1bbbfa6e895054b36093548558ee0d9fbf61ef09e617d3b3b158ba9f9c11825dbbf7e84711331afb80fc24ea0e5aa07a9db1919932c109c34fefec3c02d184",
+                "error": "",
+                "verified_certificate": true,
+                "certificate": {
+                    "Md5": "54d5b5aca1e7e76bb1a26c61a9381b93",
+                    "Sha1": "4ba9d1f82adb7be841bcf53b03ddae857747199a",
+                    "Sha256": "31037a27af59d4914906c01ad14a318eee2f3e31d48da8954dca62a99174e3fa",
+                    "ValidFrom": "2021-01-15T22:03:53Z",
+                    "ValidTo": "2051-01-15T22:03:53Z",
+                    "Issuer": "C=US, ST=California, L=Mountain View, O=Google Inc., OU=Android, CN=Android",
+                    "Subject": "C=US, ST=California, L=Mountain View, O=Google Inc., OU=Android, CN=Android",
+                    "SignatureAlgorithm": "SHA256-RSA",
+                    "SerialNumber": 955466096586930338769951715633687128507538251257
+                },
+                "certificate_error": "",
+                "trusted_certificate": false
+            }
+        ],
+        "installer": "null",
+        "uid": 10058,
+        "disabled": false,
+        "system": true,
+        "third_party": false
+    }
+]

tests/common/test_utils.py

@@ -62,7 +62,7 @@ class TestHashes:
     def test_hash_from_folder(self):
         path = os.path.join(get_artifact_folder(), "androidqf")
         hashes = list(generate_hashes_from_path(path, logging))
-        assert len(hashes) == 5
+        assert len(hashes) == 6
         # Sort the files to have reliable order for tests.
         hashes = sorted(hashes, key=lambda x: x["file_path"])
         assert hashes[0]["file_path"] == os.path.join(path, "backup.ab")

tests/conftest.py

@@ -8,6 +8,8 @@ import os
 import pytest
 
 from .artifacts.generate_stix import generate_test_stix_file
+import logging
+from mvt.common.indicators import Indicators
 
 
 @pytest.fixture(scope="session", autouse=True)
@@ -24,3 +26,31 @@ def clean_test_env(request, tmp_path_factory):
         del os.environ["MVT_STIX2"]
     except KeyError:
         pass
+
+
+@pytest.fixture()
+def indicators_factory(indicator_file):
+    def f(
+        domains=[],
+        emails=[],
+        file_names=[],
+        processes=[],
+        app_ids=[],
+        android_property_names=[],
+        files_sha256=[],
+    ):
+
+        ind = Indicators(log=logging.getLogger())
+        ind.parse_stix2(indicator_file)
+
+        ind.ioc_collections[0]["domains"].extend(domains)
+        ind.ioc_collections[0]["emails"].extend(emails)
+        ind.ioc_collections[0]["file_names"].extend(file_names)
+        ind.ioc_collections[0]["processes"].extend(processes)
+        ind.ioc_collections[0]["app_ids"].extend(app_ids)
+        ind.ioc_collections[0]["android_property_names"].extend(android_property_names)
+        ind.ioc_collections[0]["files_sha256"].extend(files_sha256)
+
+        return ind
+
+    return f