Merge branch 'main' into update-check

there were characters in the revision field

docs/requirements.txt

@@ -2,4 +2,4 @@ mkdocs==1.6.1
 mkdocs-autorefs==1.4.3
 mkdocs-material==9.6.20
 mkdocs-material-extensions==1.3.1
-mkdocstrings==0.30.0
+mkdocstrings==0.30.1

pyproject.toml

@@ -35,6 +35,7 @@ dependencies = [
     "pydantic-settings==2.10.1",
     "NSKeyedUnArchiver==1.5.2",
     "python-dateutil==2.9.0.post0",
+    "tzdata==2025.2",
 ]
 requires-python = ">= 3.10"
 

src/mvt/android/artifacts/tombstone_crashes.py

@@ -53,7 +53,7 @@ class TombstoneCrashResult(pydantic.BaseModel):
     file_name: str
     file_timestamp: str  # We store the timestamp as a string to avoid timezone issues
     build_fingerprint: str
-    revision: int
+    revision: str
     arch: Optional[str] = None
     timestamp: str  # We store the timestamp as a string to avoid timezone issues
     process_uptime: Optional[int] = None
@@ -187,7 +187,7 @@ class TombstoneCrashArtifact(AndroidArtifact):
             raise ValueError(f"Expected key {key}, got {line_key}")
 
         value_clean = value.strip().strip("'")
-        if destination_key in ["uid", "revision"]:
+        if destination_key == "uid":
             tombstone[destination_key] = int(value_clean)
         elif destination_key == "process_uptime":
             # eg. "Process uptime: 40s"

src/mvt/android/cli.py

@@ -31,6 +31,8 @@ from mvt.common.help import (
     HELP_MSG_HASHES,
     HELP_MSG_CHECK_IOCS,
     HELP_MSG_STIX2,
+    HELP_MSG_DISABLE_UPDATE_CHECK,
+    HELP_MSG_DISABLE_INDICATOR_UPDATE_CHECK,
 )
 from mvt.common.logo import logo
 from mvt.common.updates import IndicatorsUpdates
@@ -53,12 +55,37 @@ log = logging.getLogger("mvt")
 CONTEXT_SETTINGS = dict(help_option_names=["-h", "--help"])
 
 
+def _get_disable_flags(ctx):
+    """Helper function to safely get disable flags from context."""
+    if ctx.obj is None:
+        return False, False
+    return (
+        ctx.obj.get("disable_version_check", False),
+        ctx.obj.get("disable_indicator_check", False),
+    )
+
+
 # ==============================================================================
 # Main
 # ==============================================================================
 @click.group(invoke_without_command=False)
-def cli():
-    logo()
+@click.option(
+    "--disable-update-check", is_flag=True, help=HELP_MSG_DISABLE_UPDATE_CHECK
+)
+@click.option(
+    "--disable-indicator-update-check",
+    is_flag=True,
+    help=HELP_MSG_DISABLE_INDICATOR_UPDATE_CHECK,
+)
+@click.pass_context
+def cli(ctx, disable_update_check, disable_indicator_update_check):
+    ctx.ensure_object(dict)
+    ctx.obj["disable_version_check"] = disable_update_check
+    ctx.obj["disable_indicator_check"] = disable_indicator_update_check
+    logo(
+        disable_version_check=disable_update_check,
+        disable_indicator_check=disable_indicator_update_check,
+    )
 
 
 # ==============================================================================
@@ -166,6 +193,8 @@ def check_adb(
         module_name=module,
         serial=serial,
         module_options=module_options,
+        disable_version_check=_get_disable_flags(ctx)[0],
+        disable_indicator_check=_get_disable_flags(ctx)[1],
     )
 
     if list_modules:
@@ -212,6 +241,8 @@ def check_bugreport(ctx, iocs, output, list_modules, module, verbose, bugreport_
         ioc_files=iocs,
         module_name=module,
         hashes=True,
+        disable_version_check=_get_disable_flags(ctx)[0],
+        disable_indicator_check=_get_disable_flags(ctx)[1],
     )
 
     if list_modules:
@@ -274,6 +305,8 @@ def check_backup(
             "interactive": not non_interactive,
             "backup_password": cli_load_android_backup_password(log, backup_password),
         },
+        disable_version_check=_get_disable_flags(ctx)[0],
+        disable_indicator_check=_get_disable_flags(ctx)[1],
     )
 
     if list_modules:
@@ -338,6 +371,8 @@ def check_androidqf(
             "interactive": not non_interactive,
             "backup_password": cli_load_android_backup_password(log, backup_password),
         },
+        disable_version_check=_get_disable_flags(ctx)[0],
+        disable_indicator_check=_get_disable_flags(ctx)[1],
     )
 
     if list_modules:
@@ -372,7 +407,13 @@ def check_androidqf(
 @click.argument("FOLDER", type=click.Path(exists=True))
 @click.pass_context
 def check_iocs(ctx, iocs, list_modules, module, folder):
-    cmd = CmdCheckIOCS(target_path=folder, ioc_files=iocs, module_name=module)
+    cmd = CmdCheckIOCS(
+        target_path=folder,
+        ioc_files=iocs,
+        module_name=module,
+        disable_version_check=_get_disable_flags(ctx)[0],
+        disable_indicator_check=_get_disable_flags(ctx)[1],
+    )
     cmd.modules = BACKUP_MODULES + ADB_MODULES + BUGREPORT_MODULES
 
     if list_modules:

src/mvt/android/cmd_check_adb.py

@@ -22,6 +22,8 @@ class CmdAndroidCheckADB(Command):
         module_name: Optional[str] = None,
         serial: Optional[str] = None,
         module_options: Optional[dict] = None,
+        disable_version_check: bool = False,
+        disable_indicator_check: bool = False,
     ) -> None:
         super().__init__(
             target_path=target_path,
@@ -31,6 +33,8 @@ class CmdAndroidCheckADB(Command):
             serial=serial,
             module_options=module_options,
             log=log,
+            disable_version_check=disable_version_check,
+            disable_indicator_check=disable_indicator_check,
         )
 
         self.name = "check-adb"

src/mvt/android/cmd_check_androidqf.py

@@ -26,6 +26,8 @@ class CmdAndroidCheckAndroidQF(Command):
         serial: Optional[str] = None,
         module_options: Optional[dict] = None,
         hashes: bool = False,
+        disable_version_check: bool = False,
+        disable_indicator_check: bool = False,
     ) -> None:
         super().__init__(
             target_path=target_path,
@@ -36,6 +38,8 @@ class CmdAndroidCheckAndroidQF(Command):
             module_options=module_options,
             hashes=hashes,
             log=log,
+            disable_version_check=disable_version_check,
+            disable_indicator_check=disable_indicator_check,
         )
 
         self.name = "check-androidqf"

src/mvt/android/cmd_check_backup.py

@@ -36,6 +36,8 @@ class CmdAndroidCheckBackup(Command):
         serial: Optional[str] = None,
         module_options: Optional[dict] = None,
         hashes: bool = False,
+        disable_version_check: bool = False,
+        disable_indicator_check: bool = False,
     ) -> None:
         super().__init__(
             target_path=target_path,
@@ -46,6 +48,8 @@ class CmdAndroidCheckBackup(Command):
             module_options=module_options,
             hashes=hashes,
             log=log,
+            disable_version_check=disable_version_check,
+            disable_indicator_check=disable_indicator_check,
         )
 
         self.name = "check-backup"

src/mvt/android/cmd_check_bugreport.py

@@ -27,6 +27,8 @@ class CmdAndroidCheckBugreport(Command):
         serial: Optional[str] = None,
         module_options: Optional[dict] = None,
         hashes: bool = False,
+        disable_version_check: bool = False,
+        disable_indicator_check: bool = False,
     ) -> None:
         super().__init__(
             target_path=target_path,
@@ -37,6 +39,8 @@ class CmdAndroidCheckBugreport(Command):
             module_options=module_options,
             hashes=hashes,
             log=log,
+            disable_version_check=disable_version_check,
+            disable_indicator_check=disable_indicator_check,
         )
 
         self.name = "check-bugreport"

src/mvt/android/modules/adb/packages.py

@@ -107,8 +107,7 @@ class Packages(AndroidExtraction):
                     result["matched_indicator"] = ioc
                     self.detected.append(result)
 
-    @staticmethod
-    def check_virustotal(packages: list) -> None:
+    def check_virustotal(self, packages: list) -> None:
         hashes = []
         for package in packages:
             for file in package.get("files", []):
@@ -143,8 +142,15 @@ class Packages(AndroidExtraction):
 
         for package in packages:
             for file in package.get("files", []):
-                row = [package["package_name"], file["path"]]
-
+                if "package_name" in package:
+                    row = [package["package_name"], file["path"]]
+                elif "name" in package:
+                    row = [package["name"], file["path"]]
+                else:
+                    self.log.error(
+                        f"Package {package} has no name or package_name. packages.json or apks.json is malformed"
+                    )
+                    continue
                 if file["sha256"] in detections:
                     detection = detections[file["sha256"]]
                     positives = detection.split("/")[0]

src/mvt/common/cmd_check_iocs.py

@@ -22,6 +22,8 @@ class CmdCheckIOCS(Command):
         module_name: Optional[str] = None,
         serial: Optional[str] = None,
         module_options: Optional[dict] = None,
+        disable_version_check: bool = False,
+        disable_indicator_check: bool = False,
     ) -> None:
         super().__init__(
             target_path=target_path,
@@ -31,6 +33,8 @@ class CmdCheckIOCS(Command):
             serial=serial,
             module_options=module_options,
             log=log,
+            disable_version_check=disable_version_check,
+            disable_indicator_check=disable_indicator_check,
         )
 
         self.name = "check-iocs"

src/mvt/common/command.py

@@ -32,6 +32,8 @@ class Command:
         module_options: Optional[dict] = None,
         hashes: bool = False,
         log: logging.Logger = logging.getLogger(__name__),
+        disable_version_check: bool = False,
+        disable_indicator_check: bool = False,
     ) -> None:
         self.name = ""
         self.modules = []
@@ -42,6 +44,8 @@ class Command:
         self.module_name = module_name
         self.serial = serial
         self.log = log
+        self.disable_version_check = disable_version_check
+        self.disable_indicator_check = disable_indicator_check
 
         # This dictionary can contain options that will be passed down from
         # the Command to all modules. This can for example be used to pass

src/mvt/common/help.py

@@ -15,6 +15,8 @@ HELP_MSG_HASHES = "Generate hashes of all the files analyzed"
 HELP_MSG_VERBOSE = "Verbose mode"
 HELP_MSG_CHECK_IOCS = "Compare stored JSON results to provided indicators"
 HELP_MSG_STIX2 = "Download public STIX2 indicators"
+HELP_MSG_DISABLE_UPDATE_CHECK = "Disable MVT version update check"
+HELP_MSG_DISABLE_INDICATOR_UPDATE_CHECK = "Disable indicators update check"
 
 # IOS Specific
 HELP_MSG_DECRYPT_BACKUP = "Decrypt an encrypted iTunes backup"

src/mvt/common/logo.py

@@ -12,74 +12,85 @@ from .updates import IndicatorsUpdates, MVTUpdates
 from .version import MVT_VERSION
 
 
-def check_updates() -> None:
+def check_updates(
+    disable_version_check: bool = False, disable_indicator_check: bool = False
+) -> None:
     log = logging.getLogger("mvt")
+
     # First we check for MVT version updates.
-    try:
-        mvt_updates = MVTUpdates()
-        latest_version = mvt_updates.check()
-    except (requests.exceptions.ConnectionError, requests.exceptions.Timeout):
-        rich_print(
-            "\t\t[bold]Note: Could not check for MVT updates.[/bold] "
-            "You may be working offline. Please update MVT regularly."
-        )
-    except Exception as e:
-        log.error("Error encountered when trying to check latest MVT version: %s", e)
-    else:
-        if latest_version:
+    if not disable_version_check:
+        try:
+            mvt_updates = MVTUpdates()
+            latest_version = mvt_updates.check()
+        except (requests.exceptions.ConnectionError, requests.exceptions.Timeout):
             rich_print(
-                f"\t\t[bold]Version {latest_version} is available! "
-                "Upgrade mvt with `pip3 install -U mvt` or with `pipx upgrade mvt`[/bold]"
+                "\t\t[bold]Note: Could not check for MVT updates.[/bold] "
+                "You may be working offline. Please update MVT regularly."
             )
-
-    # Then we check for indicators files updates.
-    ioc_updates = IndicatorsUpdates()
-
-    # Before proceeding, we check if we have downloaded an indicators index.
-    # If not, there's no point in proceeding with the updates check.
-    if ioc_updates.get_latest_update() == 0:
-        rich_print(
-            "\t\t[bold]You have not yet downloaded any indicators, check "
-            "the `download-iocs` command![/bold]"
-        )
-        return
-
-    # We only perform this check at a fixed frequency, in order to not
-    # overburden the user with too many lookups if the command is being run
-    # multiple times.
-    should_check, hours = ioc_updates.should_check()
-    if not should_check:
-        rich_print(
-            f"\t\tIndicators updates checked recently, next automatic check "
-            f"in {int(hours)} hours"
-        )
-        return
-
-    try:
-        ioc_to_update = ioc_updates.check()
-    except (requests.exceptions.ConnectionError, requests.exceptions.Timeout):
-        rich_print(
-            "\t\t[bold]Note: Could not check for indicator updates.[/bold] "
-            "You may be working offline. Please update MVT indicators regularly."
-        )
-    except Exception as e:
-        log.error("Error encountered when trying to check latest MVT indicators: %s", e)
-    else:
-        if ioc_to_update:
-            rich_print(
-                "\t\t[bold]There are updates to your indicators files! "
-                "Run the `download-iocs` command to update![/bold]"
+        except Exception as e:
+            log.error(
+                "Error encountered when trying to check latest MVT version: %s", e
             )
         else:
-            rich_print("\t\tYour indicators files seem to be up to date.")
+            if latest_version:
+                rich_print(
+                    f"\t\t[bold]Version {latest_version} is available! "
+                    "Upgrade mvt with `pip3 install -U mvt` or with `pipx upgrade mvt`[/bold]"
+                )
+
+    # Then we check for indicators files updates.
+    if not disable_indicator_check:
+        ioc_updates = IndicatorsUpdates()
+
+        # Before proceeding, we check if we have downloaded an indicators index.
+        # If not, there's no point in proceeding with the updates check.
+        if ioc_updates.get_latest_update() == 0:
+            rich_print(
+                "\t\t[bold]You have not yet downloaded any indicators, check "
+                "the `download-iocs` command![/bold]"
+            )
+            return
+
+        # We only perform this check at a fixed frequency, in order to not
+        # overburden the user with too many lookups if the command is being run
+        # multiple times.
+        should_check, hours = ioc_updates.should_check()
+        if not should_check:
+            rich_print(
+                f"\t\tIndicators updates checked recently, next automatic check "
+                f"in {int(hours)} hours"
+            )
+            return
+
+        try:
+            ioc_to_update = ioc_updates.check()
+        except (requests.exceptions.ConnectionError, requests.exceptions.Timeout):
+            rich_print(
+                "\t\t[bold]Note: Could not check for indicator updates.[/bold] "
+                "You may be working offline. Please update MVT indicators regularly."
+            )
+        except Exception as e:
+            log.error(
+                "Error encountered when trying to check latest MVT indicators: %s", e
+            )
+        else:
+            if ioc_to_update:
+                rich_print(
+                    "\t\t[bold]There are updates to your indicators files! "
+                    "Run the `download-iocs` command to update![/bold]"
+                )
+            else:
+                rich_print("\t\tYour indicators files seem to be up to date.")
 
 
-def logo() -> None:
+def logo(
+    disable_version_check: bool = False, disable_indicator_check: bool = False
+) -> None:
     rich_print("\n")
     rich_print("\t[bold]MVT[/bold] - Mobile Verification Toolkit")
     rich_print("\t\thttps://mvt.re")
     rich_print(f"\t\tVersion: {MVT_VERSION}")
 
-    check_updates()
+    check_updates(disable_version_check, disable_indicator_check)
 
     rich_print("\n")

src/mvt/common/updates.py

@@ -24,7 +24,11 @@ INDICATORS_CHECK_FREQUENCY = 12
 
 class MVTUpdates:
     def check(self) -> str:
-        res = requests.get(settings.PYPI_UPDATE_URL, timeout=15)
+        try:
+            res = requests.get(settings.PYPI_UPDATE_URL, timeout=5)
+        except requests.exceptions.RequestException as e:
+            log.error("Failed to check for updates, skipping updates: %s", e)
+            return ""
         data = res.json()
         latest_version = data.get("info", {}).get("version", "")
 
@@ -93,7 +97,12 @@ class IndicatorsUpdates:
         url = self.github_raw_url.format(
             self.index_owner, self.index_repo, self.index_branch, self.index_path
         )
-        res = requests.get(url, timeout=15)
+        try:
+            res = requests.get(url, timeout=5)
+        except requests.exceptions.RequestException as e:
+            log.error("Failed to retrieve indicators index from %s: %s", url, e)
+            return None
+
         if res.status_code != 200:
             log.error(
                 "Failed to retrieve indicators index located at %s (error %d)",
@@ -105,7 +114,12 @@ class IndicatorsUpdates:
         return yaml.safe_load(res.content)
 
     def download_remote_ioc(self, ioc_url: str) -> Optional[str]:
-        res = requests.get(ioc_url, timeout=15)
+        try:
+            res = requests.get(ioc_url, timeout=15)
+        except requests.exceptions.RequestException as e:
+            log.error("Failed to download indicators file from %s: %s", ioc_url, e)
+            return None
+
         if res.status_code != 200:
             log.error(
                 "Failed to download indicators file from %s (error %d)",
@@ -171,7 +185,12 @@ class IndicatorsUpdates:
         file_commit_url = (
             f"https://api.github.com/repos/{owner}/{repo}/commits?path={path}"
         )
-        res = requests.get(file_commit_url, timeout=15)
+        try:
+            res = requests.get(file_commit_url, timeout=5)
+        except requests.exceptions.RequestException as e:
+            log.error("Failed to get details about file %s: %s", file_commit_url, e)
+            return -1
+
         if res.status_code != 200:
             log.error(
                 "Failed to get details about file %s (error %d)",

src/mvt/ios/cli.py

@@ -37,6 +37,8 @@ from mvt.common.help import (
     HELP_MSG_CHECK_IOCS,
     HELP_MSG_STIX2,
     HELP_MSG_CHECK_IOS_BACKUP,
+    HELP_MSG_DISABLE_UPDATE_CHECK,
+    HELP_MSG_DISABLE_INDICATOR_UPDATE_CHECK,
 )
 from .cmd_check_backup import CmdIOSCheckBackup
 from .cmd_check_fs import CmdIOSCheckFS
@@ -53,12 +55,37 @@ MVT_IOS_BACKUP_PASSWORD = "MVT_IOS_BACKUP_PASSWORD"
 CONTEXT_SETTINGS = dict(help_option_names=["-h", "--help"])
 
 
+def _get_disable_flags(ctx):
+    """Helper function to safely get disable flags from context."""
+    if ctx.obj is None:
+        return False, False
+    return (
+        ctx.obj.get("disable_version_check", False),
+        ctx.obj.get("disable_indicator_check", False),
+    )
+
+
 # ==============================================================================
 # Main
 # ==============================================================================
 @click.group(invoke_without_command=False)
-def cli():
-    logo()
+@click.option(
+    "--disable-update-check", is_flag=True, help=HELP_MSG_DISABLE_UPDATE_CHECK
+)
+@click.option(
+    "--disable-indicator-update-check",
+    is_flag=True,
+    help=HELP_MSG_DISABLE_INDICATOR_UPDATE_CHECK,
+)
+@click.pass_context
+def cli(ctx, disable_update_check, disable_indicator_update_check):
+    ctx.ensure_object(dict)
+    ctx.obj["disable_version_check"] = disable_update_check
+    ctx.obj["disable_indicator_check"] = disable_indicator_update_check
+    logo(
+        disable_version_check=disable_update_check,
+        disable_indicator_check=disable_indicator_update_check,
+    )
 
 
 # ==============================================================================
@@ -219,6 +246,8 @@ def check_backup(
         module_name=module,
         module_options=module_options,
         hashes=hashes,
+        disable_version_check=_get_disable_flags(ctx)[0],
+        disable_indicator_check=_get_disable_flags(ctx)[1],
     )
 
     if list_modules:
@@ -266,6 +295,8 @@ def check_fs(ctx, iocs, output, fast, list_modules, module, hashes, verbose, dum
         module_name=module,
         module_options=module_options,
         hashes=hashes,
+        disable_version_check=_get_disable_flags(ctx)[0],
+        disable_indicator_check=_get_disable_flags(ctx)[1],
     )
 
     if list_modules:
@@ -300,7 +331,13 @@ def check_fs(ctx, iocs, output, fast, list_modules, module, hashes, verbose, dum
 @click.argument("FOLDER", type=click.Path(exists=True))
 @click.pass_context
 def check_iocs(ctx, iocs, list_modules, module, folder):
-    cmd = CmdCheckIOCS(target_path=folder, ioc_files=iocs, module_name=module)
+    cmd = CmdCheckIOCS(
+        target_path=folder,
+        ioc_files=iocs,
+        module_name=module,
+        disable_version_check=_get_disable_flags(ctx)[0],
+        disable_indicator_check=_get_disable_flags(ctx)[1],
+    )
     cmd.modules = BACKUP_MODULES + FS_MODULES + MIXED_MODULES
 
     if list_modules:

src/mvt/ios/cmd_check_backup.py

@@ -24,6 +24,8 @@ class CmdIOSCheckBackup(Command):
         serial: Optional[str] = None,
         module_options: Optional[dict] = None,
         hashes: bool = False,
+        disable_version_check: bool = False,
+        disable_indicator_check: bool = False,
     ) -> None:
         super().__init__(
             target_path=target_path,
@@ -34,6 +36,8 @@ class CmdIOSCheckBackup(Command):
             module_options=module_options,
             hashes=hashes,
             log=log,
+            disable_version_check=disable_version_check,
+            disable_indicator_check=disable_indicator_check,
         )
 
         self.name = "check-backup"

src/mvt/ios/cmd_check_fs.py

@@ -24,16 +24,19 @@ class CmdIOSCheckFS(Command):
         serial: Optional[str] = None,
         module_options: Optional[dict] = None,
         hashes: bool = False,
+        disable_version_check: bool = False,
+        disable_indicator_check: bool = False,
     ) -> None:
         super().__init__(
             target_path=target_path,
             results_path=results_path,
             ioc_files=ioc_files,
             module_name=module_name,
-            serial=serial,
             module_options=module_options,
             hashes=hashes,
             log=log,
+            disable_version_check=disable_version_check,
+            disable_indicator_check=disable_indicator_check,
         )
 
         self.name = "check-fs"

src/mvt/ios/data/ios_versions.json

@@ -895,6 +895,10 @@
         "version": "15.8.4",
         "build": "19H390"
     },
+    {
+        "version": "15.8.5",
+        "build": "19H394"
+    },
     {
         "build": "20A362",
         "version": "16.0"
@@ -1000,6 +1004,10 @@
         "version": "16.7.11",
         "build": "20H360"
     },
+    {
+        "version": "16.7.12",
+        "build": "20H364"
+    },
     {
         "version": "17.0",
         "build": "21A327"
@@ -1143,5 +1151,17 @@
     {
         "version": "18.6.2",
         "build": "22G100"
+    },
+    {
+        "version": "18.7",
+        "build": "22H20"
+    },
+    {
+        "version": "26",
+        "build": "23A341"
+    },
+    {
+        "version": "26.0.1",
+        "build": "23A355"
     }
 ]