Uniforming style in test units

This reverts commit e1efaa5467.

.github/workflows/python-package.yml

@@ -16,7 +16,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [3.7, 3.8, 3.9]
+        # python-version: [3.7, 3.8, 3.9]
+        python-version: [3.8, 3.9]
 
     steps:
     - uses: actions/checkout@v2
@@ -27,8 +28,9 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        python -m pip install flake8 pytest safety
+        python -m pip install flake8 pytest safety stix2
         if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+        python -m pip install .
     - name: Lint with flake8
       run: |
         # stop the build if there are Python syntax errors or undefined names
@@ -37,7 +39,5 @@ jobs:
         flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
     - name: Safety checks
       run: safety check
-
-    # - name: Test with pytest
-    #   run: |
-    #     pytest
+    - name: Test with pytest
+      run: pytest

docs/iocs.md

@@ -38,7 +38,9 @@ export MVT_STIX2="/home/user/IOC1.stix2:/home/user/IOC2.stix2"
 
 - The [Amnesty International investigations repository](https://github.com/AmnestyTech/investigations) contains STIX-formatted IOCs for:
     - [Pegasus](https://en.wikipedia.org/wiki/Pegasus_(spyware)) ([STIX2](https://raw.githubusercontent.com/AmnestyTech/investigations/master/2021-07-18_nso/pegasus.stix2))
-    - [Predator from Cytrox](https://citizenlab.ca/2021/12/pegasus-vs-predator-dissidents-doubly-infected-iphone-reveals-cytrox-mercenary-spyware/) ([STIX2](https://github.com/AmnestyTech/investigations/tree/master/2021-12-16_cytrox/cytrox.stix2))
-- [This repository](https://github.com/Te-k/stalkerware-indicators) contains IOCs for Android stalkerware including [a STIX MVT-compatible file](https://github.com/Te-k/stalkerware-indicators/blob/master/stalkerware.stix2).
+    - [Predator from Cytrox](https://citizenlab.ca/2021/12/pegasus-vs-predator-dissidents-doubly-infected-iphone-reveals-cytrox-mercenary-spyware/) ([STIX2](https://raw.githubusercontent.com/AmnestyTech/investigations/master/2021-12-16_cytrox/cytrox.stix2))
+- [This repository](https://github.com/Te-k/stalkerware-indicators) contains IOCs for Android stalkerware including [a STIX MVT-compatible file](https://raw.githubusercontent.com/Te-k/stalkerware-indicators/master/stalkerware.stix2).
+
+You can automaticallly download the latest public indicator files with the command `mvt-ios download-iocs` or `mvt-android download-iocs`.
 
 Please [open an issue](https://github.com/mvt-project/mvt/issues/) to suggest new sources of STIX-formatted IOCs.

mvt/android/cli.py

@@ -9,10 +9,10 @@ import os
 import click
 from rich.logging import RichHandler
 
-from mvt.common.help import HELP_MSG_MODULE, HELP_MSG_IOC
-from mvt.common.help import HELP_MSG_FAST, HELP_MSG_OUTPUT, HELP_MSG_LIST_MODULES
-from mvt.common.help import HELP_MSG_SERIAL
-from mvt.common.indicators import Indicators, IndicatorsFileBadFormat
+from mvt.common.help import (HELP_MSG_FAST, HELP_MSG_IOC,
+                             HELP_MSG_LIST_MODULES, HELP_MSG_MODULE,
+                             HELP_MSG_OUTPUT, HELP_MSG_SERIAL)
+from mvt.common.indicators import Indicators, download_indicators_files
 from mvt.common.logo import logo
 from mvt.common.module import run_module, save_timeline
 
@@ -129,13 +129,7 @@ def check_adb(ctx, iocs, output, fast, list_modules, module, serial):
             ctx.exit(1)
 
     indicators = Indicators(log=log)
-    for ioc_path in iocs:
-        try:
-            indicators.parse_stix2(ioc_path)
-        except IndicatorsFileBadFormat as e:
-            log.critical(e)
-            ctx.exit(1)
-    log.info("Loaded a total of %d indicators", indicators.ioc_count)
+    indicators.load_indicators_files(iocs)
 
     timeline = []
     timeline_detected = []
@@ -145,13 +139,12 @@ def check_adb(ctx, iocs, output, fast, list_modules, module, serial):
 
         m = adb_module(output_folder=output, fast_mode=fast,
                        log=logging.getLogger(adb_module.__module__))
+        if indicators.ioc_count:
+            m.indicators = indicators
+            m.indicators.log = m.log
         if serial:
             m.serial = serial
 
-        if iocs:
-            indicators.log = m.log
-            m.indicators = indicators
-
         run_module(m)
         timeline.extend(m.timeline)
         timeline_detected.extend(m.timeline_detected)
@@ -184,13 +177,7 @@ def check_backup(ctx, iocs, output, backup_path, serial):
             ctx.exit(1)
 
     indicators = Indicators(log=log)
-    for ioc_path in iocs:
-        try:
-            indicators.parse_stix2(ioc_path)
-        except IndicatorsFileBadFormat as e:
-            log.critical(e)
-            ctx.exit(1)
-    log.info("Loaded a total of %d indicators", indicators.ioc_count)
+    indicators.load_indicators_files(iocs)
 
     if os.path.isfile(backup_path):
         log.critical("The path you specified is a not a folder!")
@@ -203,12 +190,18 @@ def check_backup(ctx, iocs, output, backup_path, serial):
     for module in BACKUP_MODULES:
         m = module(base_folder=backup_path, output_folder=output,
                    log=logging.getLogger(module.__module__))
-
+        if indicators.ioc_count:
+            m.indicators = indicators
+            m.indicators.log = m.log
         if serial:
             m.serial = serial
 
-        if len(indicators.ioc_count) > 0:
-            indicators.log = m.log
-            m.indicators = indicators
-
         run_module(m)
+
+
+#==============================================================================
+# Command: download-iocs
+#==============================================================================
+@cli.command("download-iocs", help="Download public STIX2 indicators")
+def download_indicators():
+    download_indicators_files(log)

mvt/android/modules/adb/__init__.py

@@ -4,6 +4,7 @@
 #   https://license.mvt.re/1.1/
 
 from .chrome_history import ChromeHistory
+from .dumpsys_accessibility import DumpsysAccessibility
 from .dumpsys_batterystats import DumpsysBatterystats
 from .dumpsys_full import DumpsysFull
 from .dumpsys_packages import DumpsysPackages
@@ -18,6 +19,6 @@ from .sms import SMS
 from .whatsapp import Whatsapp
 
 ADB_MODULES = [ChromeHistory, SMS, Whatsapp, Processes,
-               DumpsysBatterystats, DumpsysProcstats,
+               DumpsysAccessibility, DumpsysBatterystats, DumpsysProcstats,
                DumpsysPackages, DumpsysReceivers, DumpsysFull,
                Packages, RootBinaries, Logcat, Files]

mvt/android/modules/adb/base.py

@@ -112,7 +112,7 @@ class AndroidExtraction(MVTModule):
         :returns: Output of command
 
         """
-        return self.device.shell(command)
+        return self.device.shell(command, read_timeout_s=200.0)
 
     def _adb_check_if_root(self):
         """Check if we have a `su` binary on the Android device.

mvt/android/modules/adb/dumpsys_accessibility.py

@@ -0,0 +1,53 @@
+# Mobile Verification Toolkit (MVT)
+# Copyright (c) 2021 The MVT Project Authors.
+# Use of this software is governed by the MVT License 1.1 that can be found at
+#   https://license.mvt.re/1.1/
+
+import io
+import logging
+import os
+
+from .base import AndroidExtraction
+
+log = logging.getLogger(__name__)
+
+
+class DumpsysAccessibility(AndroidExtraction):
+    """This module extracts stats on accessibility."""
+
+    def __init__(self, file_path=None, base_folder=None, output_folder=None,
+                 serial=None, fast_mode=False, log=None, results=[]):
+        super().__init__(file_path=file_path, base_folder=base_folder,
+                         output_folder=output_folder, fast_mode=fast_mode,
+                         log=log, results=results)
+
+    def run(self):
+        self._adb_connect()
+
+        stats = self._adb_command("dumpsys accessibility")
+
+        in_services = False
+        for line in stats.split("\n"):
+            if line.strip().startswith("installed services:"):
+                in_services = True
+                continue
+
+            if not in_services:
+                continue
+
+            if line.strip() == "}":
+                break
+
+            service = line.split(":")[1].strip()
+            log.info("Found installed accessibility service \"%s\"", service)
+
+        if self.output_folder:
+            acc_path = os.path.join(self.output_folder,
+                                      "dumpsys_accessibility.txt")
+            with io.open(acc_path, "w", encoding="utf-8") as handle:
+                handle.write(stats)
+
+            log.info("Records from dumpsys accessibility stored at %s",
+                     acc_path)
+
+        self._adb_disconnect()

mvt/android/modules/adb/files.py

@@ -3,12 +3,11 @@
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
-import logging
-import os
-import stat
 import datetime
+import logging
+import stat
 
-from mvt.common.utils import check_for_links, convert_timestamp_to_iso
+from mvt.common.utils import convert_timestamp_to_iso
 
 from .base import AndroidExtraction
 
@@ -31,8 +30,8 @@ class Files(AndroidExtraction):
         # Run find command with correct args and parse results.
 
         # Check that full file printf options are suppported on first run.
-        if self.full_find == None:
-            output = self._adb_command(f"find '/' -maxdepth 1 -printf '%T@ %m %s %u %g %p\n' 2> /dev/null")
+        if self.full_find is None:
+            output = self._adb_command("find '/' -maxdepth 1 -printf '%T@ %m %s %u %g %p\n' 2> /dev/null")
             if not (output or output.strip().splitlines()):
                 # Full  find command failed to generate output, fallback to basic file arguments
                 self.full_find = False
@@ -40,7 +39,7 @@ class Files(AndroidExtraction):
                 self.full_find = True
 
         found_files = []
-        if self.full_find == True:
+        if self.full_find is True:
             # Run full file command and collect additonal file information.
             output = self._adb_command(f"find '{file_path}' -printf '%T@ %m %s %u %g %p\n' 2> /dev/null")
             for file_line in output.splitlines():
@@ -90,7 +89,7 @@ class Files(AndroidExtraction):
             return
 
         for result in self.results:
-            if self.indicators.check_filename(result["path"]):
+            if self.indicators.check_file_name(result["path"]):
                 self.log.warning("Found a known suspicous filename at path: \"%s\"", result["path"])
                 self.detected.append(result)
 

mvt/common/indicators.py

@@ -3,24 +3,23 @@
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
+import io
 import json
 import os
 
+import requests
+from appdirs import user_data_dir
+
 from .url import URL
 
 
-class IndicatorsFileBadFormat(Exception):
-    pass
-
-
 class Indicators:
     """This class is used to parse indicators from a STIX2 file and provide
     functions to compare extracted artifacts to the indicators.
-
-
     """
 
     def __init__(self, log=None):
+        self.data_dir = user_data_dir("mvt")
         self.log = log
         self.ioc_domains = []
         self.ioc_processes = []
@@ -30,24 +29,49 @@ class Indicators:
         self.ioc_app_ids = []
         self.ios_profile_ids = []
         self.ioc_count = 0
-        self._check_env_variable()
 
     def _add_indicator(self, ioc, iocs_list):
         if ioc not in iocs_list:
             iocs_list.append(ioc)
             self.ioc_count += 1
 
-    def _check_env_variable(self):
+    def _load_downloaded_indicators(self):
+        if not os.path.isdir(self.data_dir):
+            return False
+
+        for f in os.listdir(self.data_dir):
+            if f.lower().endswith(".stix2"):
+                self.parse_stix2(os.path.join(self.data_dir, f))
+
+    def _check_stix2_env_variable(self):
         """
-        Checks if a variable MVT_STIX2 contains path to STIX Files
+        Checks if a variable MVT_STIX2 contains path to STIX Files.
         """
-        if "MVT_STIX2" in os.environ:
-            paths = os.environ["MVT_STIX2"].split(":")
-            for path in paths:
-                if os.path.isfile(path):
-                    self.parse_stix2(path)
-                else:
-                    self.log.info("Invalid STIX2 path %s in MVT_STIX2 environment variable", path)
+        if "MVT_STIX2" not in os.environ:
+            return False
+
+        paths = os.environ["MVT_STIX2"].split(":")
+        for path in paths:
+            if os.path.isfile(path):
+                self.parse_stix2(path)
+            else:
+                self.log.info("Invalid STIX2 path %s in MVT_STIX2 environment variable", path)
+
+    def load_indicators_files(self, files, load_default=True):
+        """
+        Load a list of indicators files.
+        """
+        for file_path in files:
+            if os.path.isfile(file_path):
+                self.parse_stix2(file_path)
+            else:
+                self.log.warning("This indicators file %s does not exist", file_path)
+
+        # Load downloaded indicators and any indicators from env variable
+        if load_default:
+            self._load_downloaded_indicators()
+        self._check_stix2_env_variable()
+        self.log.info("Loaded a total of %d unique indicators", self.ioc_count)
 
     def parse_stix2(self, file_path):
         """Extract indicators from a STIX2 file.
@@ -56,14 +80,13 @@ class Indicators:
         :type file_path: str
 
         """
-        self.log.info("Parsing STIX2 indicators file at path %s",
-                      file_path)
-
+        self.log.info("Parsing STIX2 indicators file at path %s", file_path)
         with open(file_path, "r") as handle:
             try:
                 data = json.load(handle)
             except json.decoder.JSONDecodeError:
-                raise IndicatorsFileBadFormat("Unable to parse STIX2 indicators file, the file seems malformed or in the wrong format")
+                self.log.critical("Unable to parse STIX2 indicator file. The file is malformed or in the wrong format.")
+                return
 
         for entry in data.get("objects", []):
             if entry.get("type", "") != "indicator":
@@ -249,7 +272,7 @@ class Indicators:
 
         return False
 
-    def check_filename(self, file_path) -> bool:
+    def check_file_name(self, file_path) -> bool:
         """Check the provided file path against the list of file indicators.
 
         :param file_path: File path or file name to check against file
@@ -268,6 +291,9 @@ class Indicators:
 
         return False
 
+    # TODO: The difference between check_file_name() and check_file_path()
+    #       needs to be more explicit and clear. Probably, the two should just
+    #       be combined into one function.
     def check_file_path(self, file_path) -> bool:
         """Check the provided file path against the list of file indicators.
 
@@ -281,10 +307,11 @@ class Indicators:
         if not file_path:
             return False
 
-        for ioc_file in  self.ioc_files:
+        for ioc_file in self.ioc_files:
             # Strip any trailing slash from indicator paths to match directories.
             if file_path.startswith(ioc_file.rstrip("/")):
                 return True
+
         return False
 
     def check_profile(self, profile_uuid) -> bool:
@@ -299,4 +326,36 @@ class Indicators:
         if profile_uuid in self.ios_profile_ids:
             return True
 
-        return False
+        return False
+
+
+def download_indicators_files(log):
+    """
+    Download indicators from repo into MVT app data directory.
+    """
+    data_dir = user_data_dir("mvt")
+    if not os.path.isdir(data_dir):
+        os.makedirs(data_dir, exist_ok=True)
+
+    # Download latest list of indicators from the MVT repo.
+    res = requests.get("https://github.com/mvt-project/mvt/raw/main/public_indicators.json")
+    if res.status_code != 200:
+        log.warning("Unable to find retrieve list of indicators from the MVT repository.")
+        return
+
+    for ioc_entry in res.json():
+        ioc_url = ioc_entry["stix2_url"]
+        log.info("Downloading indicator file '%s' from '%s'", ioc_entry["name"], ioc_url)
+
+        res = requests.get(ioc_url)
+        if res.status_code != 200:
+            log.warning("Could not find indicator file '%s'", ioc_url)
+            continue
+
+        clean_file_name = ioc_url.lstrip("https://").replace("/", "_")
+        ioc_path = os.path.join(data_dir, clean_file_name)
+
+        # Write file to disk. This will overwrite any older version of the STIX2 file.
+        with io.open(ioc_path, "w") as f:
+            f.write(res.text)
+        log.info("Saved indicator file to '%s'", os.path.basename(ioc_path))

mvt/common/module.py

@@ -30,7 +30,7 @@ class MVTModule(object):
     slug = None
 
     def __init__(self, file_path=None, base_folder=None, output_folder=None,
-                 fast_mode=False, log=None, results=[]):
+                 fast_mode=False, log=None, results=None):
         """Initialize module.
 
         :param file_path: Path to the module's database file, if there is any
@@ -51,7 +51,7 @@ class MVTModule(object):
         self.fast_mode = fast_mode
         self.log = log
         self.indicators = None
-        self.results = results
+        self.results = results if results else []
         self.detected = []
         self.timeline = []
         self.timeline_detected = []

mvt/common/version.py

@@ -6,7 +6,7 @@
 import requests
 from packaging import version
 
-MVT_VERSION = "1.4.0"
+MVT_VERSION = "1.4.4"
 
 
 def check_for_updates():

mvt/ios/cli.py

@@ -10,10 +10,10 @@ import click
 from rich.logging import RichHandler
 from rich.prompt import Prompt
 
-from mvt.common.help import HELP_MSG_MODULE, HELP_MSG_IOC
-from mvt.common.help import HELP_MSG_FAST, HELP_MSG_OUTPUT
-from mvt.common.help import HELP_MSG_LIST_MODULES
-from mvt.common.indicators import Indicators, IndicatorsFileBadFormat
+from mvt.common.help import (HELP_MSG_FAST, HELP_MSG_IOC,
+                             HELP_MSG_LIST_MODULES, HELP_MSG_MODULE,
+                             HELP_MSG_OUTPUT)
+from mvt.common.indicators import Indicators, download_indicators_files
 from mvt.common.logo import logo
 from mvt.common.module import run_module, save_timeline
 from mvt.common.options import MutuallyExclusiveOption
@@ -157,13 +157,7 @@ def check_backup(ctx, iocs, output, fast, backup_path, list_modules, module):
             ctx.exit(1)
 
     indicators = Indicators(log=log)
-    for ioc_path in iocs:
-        try:
-            indicators.parse_stix2(ioc_path)
-        except IndicatorsFileBadFormat as e:
-            log.critical(e)
-            ctx.exit(1)
-    log.info("Loaded a total of %d indicators", indicators.ioc_count)
+    indicators.load_indicators_files(iocs)
 
     timeline = []
     timeline_detected = []
@@ -174,8 +168,7 @@ def check_backup(ctx, iocs, output, fast, backup_path, list_modules, module):
         m = backup_module(base_folder=backup_path, output_folder=output, fast_mode=fast,
                           log=logging.getLogger(backup_module.__module__))
         m.is_backup = True
-
-        if indicators.ioc_count > 0:
+        if indicators.ioc_count:
             m.indicators = indicators
             m.indicators.log = m.log
 
@@ -220,13 +213,7 @@ def check_fs(ctx, iocs, output, fast, dump_path, list_modules, module):
             ctx.exit(1)
 
     indicators = Indicators(log=log)
-    for ioc_path in iocs:
-        try:
-            indicators.parse_stix2(ioc_path)
-        except IndicatorsFileBadFormat as e:
-            log.critical(e)
-            ctx.exit(1)
-    log.info("Loaded a total of %d indicators", indicators.ioc_count)
+    indicators.load_indicators_files(iocs)
 
     timeline = []
     timeline_detected = []
@@ -238,8 +225,7 @@ def check_fs(ctx, iocs, output, fast, dump_path, list_modules, module):
                       log=logging.getLogger(fs_module.__module__))
 
         m.is_fs_dump = True
-
-        if iocs:
+        if indicators.ioc_count:
             m.indicators = indicators
             m.indicators.log = m.log
 
@@ -280,13 +266,7 @@ def check_iocs(ctx, iocs, list_modules, module, folder):
     log.info("Checking stored results against provided indicators...")
 
     indicators = Indicators(log=log)
-    for ioc_path in iocs:
-        try:
-            indicators.parse_stix2(ioc_path)
-        except IndicatorsFileBadFormat as e:
-            log.critical(e)
-            ctx.exit(1)
-    log.info("Loaded a total of %d indicators", indicators.ioc_count)
+    indicators.load_indicators_files(iocs)
 
     for file_name in os.listdir(folder):
         name_only, ext = os.path.splitext(file_name)
@@ -304,11 +284,19 @@ def check_iocs(ctx, iocs, list_modules, module, folder):
 
             m = iocs_module.from_json(file_path,
                                       log=logging.getLogger(iocs_module.__module__))
-
-            m.indicators = indicators
-            m.indicators.log = m.log
+            if indicators.ioc_count:
+                m.indicators = indicators
+                m.indicators.log = m.log
 
             try:
                 m.check_indicators()
             except NotImplementedError:
                 continue
+
+
+#==============================================================================
+# Command: download-iocs
+#==============================================================================
+@cli.command("download-iocs", help="Download public STIX2 indicators")
+def download_iocs():
+    download_indicators_files(log)

mvt/ios/modules/backup/configuration_profiles.py

@@ -7,6 +7,7 @@
 import os
 import plistlib
 from base64 import b64encode
+
 from mvt.common.utils import convert_timestamp_to_iso
 
 from ..base import IOSExtraction
@@ -70,7 +71,7 @@ class ConfigurationProfiles(IOSExtraction):
             with open(conf_file_path, "rb") as handle:
                 try:
                     conf_plist = plistlib.load(handle)
-                except:
+                except Exception:
                     conf_plist = {}
 
             if "SignerCerts" in conf_plist:

mvt/ios/modules/backup/manifest.py

@@ -83,7 +83,7 @@ class Manifest(IOSExtraction):
                     self.detected.append(result)
                     continue
 
-            if self.indicators.check_filename(result["relative_path"]):
+            if self.indicators.check_file_name(result["relative_path"]):
                 self.log.warning("Found a known malicious file at path: %s", result["relative_path"])
                 self.detected.append(result)
                 continue

mvt/ios/modules/mixed/__init__.py

@@ -16,13 +16,13 @@ from .net_datausage import Datausage
 from .osanalytics_addaily import OSAnalyticsADDaily
 from .safari_browserstate import SafariBrowserState
 from .safari_history import SafariHistory
+from .shortcuts import Shortcuts
 from .sms import SMS
 from .sms_attachments import SMSAttachments
 from .tcc import TCC
 from .webkit_resource_load_statistics import WebkitResourceLoadStatistics
 from .webkit_session_resource_log import WebkitSessionResourceLog
 from .whatsapp import Whatsapp
-from .shortcuts import Shortcuts
 
 MIXED_MODULES = [Calls, ChromeFavicon, ChromeHistory, Contacts, FirefoxFavicon,
                  FirefoxHistory, IDStatusCache, InteractionC, LocationdClients,

mvt/ios/modules/mixed/shortcuts.py

@@ -3,12 +3,13 @@
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
-import sqlite3
 import io
-import plistlib
 import itertools
+import plistlib
+import sqlite3
 
-from mvt.common.utils import check_for_links, convert_mactime_to_unix, convert_timestamp_to_iso
+from mvt.common.utils import (check_for_links, convert_mactime_to_unix,
+                              convert_timestamp_to_iso)
 
 from ..base import IOSExtraction
 
@@ -33,13 +34,21 @@ class Shortcuts(IOSExtraction):
         found_urls = ""
         if record["action_urls"]:
             found_urls = "- URLs in actions: {}".format(", ".join(record["action_urls"]))
+        desc = ""
+        if record["description"]:
+            desc = record["description"].decode('utf-8', errors='ignore')
 
-        return {
+        return [{
             "timestamp": record["isodate"],
             "module": self.__class__.__name__,
-            "event": "shortcut",
-            "data": f"iOS Shortcut '{record['shortcut_name']}': {record['description']} {found_urls}"
-        }
+            "event": "shortcut_created",
+            "data": f"iOS Shortcut '{record['shortcut_name'].decode('utf-8')}': {desc} {found_urls}"
+        }, {
+            "timestamp": record["modified_date"],
+            "module": self.__class__.__name__,
+            "event": "shortcut_modified",
+            "data": f"iOS Shortcut '{record['shortcut_name'].decode('utf-8')}': {desc} {found_urls}"
+        }]
 
     def check_indicators(self):
         if not self.indicators:
@@ -57,17 +66,25 @@ class Shortcuts(IOSExtraction):
         conn = sqlite3.connect(self.file_path)
         conn.text_factory = bytes
         cur = conn.cursor()
-        cur.execute("""
-              SELECT
-                  ZSHORTCUT.Z_PK as "shortcut_id",
-                  ZSHORTCUT.ZNAME as "shortcut_name",
-                  ZSHORTCUT.ZCREATIONDATE as "created_date",
-                  ZSHORTCUT.ZMODIFICATIONDATE as "modified_date",
-                  ZSHORTCUT.ZACTIONSDESCRIPTION as "description",
-                  ZSHORTCUTACTIONS.ZDATA as "action_data"
-              FROM ZSHORTCUT
-              LEFT JOIN ZSHORTCUTACTIONS ON ZSHORTCUTACTIONS.ZSHORTCUT == ZSHORTCUT.Z_PK;
-        """)
+        try:
+            cur.execute("""
+                SELECT
+                    ZSHORTCUT.Z_PK as "shortcut_id",
+                    ZSHORTCUT.ZNAME as "shortcut_name",
+                    ZSHORTCUT.ZCREATIONDATE as "created_date",
+                    ZSHORTCUT.ZMODIFICATIONDATE as "modified_date",
+                    ZSHORTCUT.ZACTIONSDESCRIPTION as "description",
+                    ZSHORTCUTACTIONS.ZDATA as "action_data"
+                FROM ZSHORTCUT
+                LEFT JOIN ZSHORTCUTACTIONS ON ZSHORTCUTACTIONS.ZSHORTCUT == ZSHORTCUT.Z_PK;
+            """)
+        except sqlite3.OperationalError:
+            # Table ZSHORTCUT does not exist
+            self.log.info("Invalid shortcut database format, skipping...")
+            cur.close()
+            conn.close()
+            return
+
         names = [description[0] for description in cur.description]
 
         for item in cur:
@@ -83,14 +100,13 @@ class Shortcuts(IOSExtraction):
                 action["identifier"] = action_entry["WFWorkflowActionIdentifier"]
                 action["parameters"] = action_entry["WFWorkflowActionParameters"]
 
-                # URLs might be in multiple fields, do a simple regex search across the parameters
+                # URLs might be in multiple fields, do a simple regex search across the parameters.
                 extracted_urls = check_for_links(str(action["parameters"]))
 
-                # Remove quoting characters that may have been captured by the regex
+                # Remove quoting characters that may have been captured by the regex.
                 action["urls"] = [url.rstrip("',") for url in extracted_urls]
                 actions.append(action)
 
-            # pprint.pprint(actions)
             shortcut["isodate"] = convert_timestamp_to_iso(convert_mactime_to_unix(shortcut.pop("created_date")))
             shortcut["modified_date"] = convert_timestamp_to_iso(convert_mactime_to_unix(shortcut["modified_date"]))
             shortcut["parsed_actions"] = len(actions)

mvt/ios/modules/mixed/webkit_resource_load_statistics.py

@@ -77,7 +77,8 @@ class WebkitResourceLoadStatistics(IOSExtraction):
                 for backup_file in self._get_backup_files_from_manifest(relative_path=WEBKIT_RESOURCELOADSTATICS_BACKUP_RELPATH):
                     db_path = self._get_backup_file_from_id(backup_file["file_id"])
                     key = f"{backup_file['domain']}/{WEBKIT_RESOURCELOADSTATICS_BACKUP_RELPATH}"
-                    self._process_observations_db(db_path=db_path, key=key)
+                    if db_path:
+                        self._process_observations_db(db_path=db_path, key=key)
             except Exception as e:
                 self.log.info("Unable to search for WebKit observations.db: %s", e)
         elif self.is_fs_dump:

mvt/ios/versions.py

@@ -234,6 +234,8 @@ IPHONE_IOS_VERSIONS = [
     {"build": "19A404", "version": "15.0.2"},
     {"build": "19B74", "version": "15.1"},
     {"build": "19B81", "version": "15.1.1"},
+    {"build": "19C56", "version": "15.2"},
+    {"build": "19C63", "version": "15.2.1"},
 ]
 
 

public_indicators.json

@@ -0,0 +1,14 @@
+[
+    {
+        "name": "NSO Group Pegasus Indicators of Compromise",
+        "source": "Amnesty International",
+        "reference": "https://www.amnesty.org/en/latest/research/2021/07/forensic-methodology-report-how-to-catch-nso-groups-pegasus/",
+        "stix2_url": "https://raw.githubusercontent.com/AmnestyTech/investigations/master/2021-07-18_nso/pegasus.stix2"
+    },
+    {
+        "name": "Cytrox Predator Spyware Indicators of Compromise",
+        "source": "Meta, Amnesty International, Citizen Lab",
+        "reference": "https://citizenlab.ca/2021/12/pegasus-vs-predator-dissidents-doubly-infected-iphone-reveals-cytrox-mercenary-spyware/",
+        "stix2_url": "https://raw.githubusercontent.com/AmnestyTech/investigations/master/2021-12-16_cytrox/cytrox.stix2"
+    }
+]

setup.py

@@ -23,6 +23,7 @@ requires = (
     "requests>=2.26.0",
     "simplejson>=3.17.5",
     "packaging>=21.0",
+    "appdirs>=1.4.4",
     # iOS dependencies:
     "iOSbackup>=0.9.921",
     # Android dependencies:

tests/artifacts/.gitignore

@@ -0,0 +1 @@
+test.stix2

tests/artifacts/generate_stix.py

@@ -0,0 +1,50 @@
+# Mobile Verification Toolkit (MVT)
+# Copyright (c) 2021 The MVT Project Authors.
+# Use of this software is governed by the MVT License 1.1 that can be found at
+#   https://license.mvt.re/1.1/
+
+import os
+
+from stix2.v21 import Bundle, Indicator, Malware, Relationship
+
+
+def generate_test_stix_file(file_path):
+    if os.path.isfile(file_path):
+        os.remove(file_path)
+
+    domains = ["example.org"]
+    processes = ["Launch"]
+    emails = ["foobar@example.org"]
+    filenames = ["/var/foobar/txt"]
+
+    res = []
+    malware = Malware(name="TestMalware", is_family=False, description="")
+    res.append(malware)
+    for d in domains:
+        i = Indicator(indicator_types=["malicious-activity"], pattern="[domain-name:value='{}']".format(d), pattern_type="stix")
+        res.append(i)
+        res.append(Relationship(i, "indicates", malware))
+
+    for p in processes:
+        i = Indicator(indicator_types=["malicious-activity"], pattern="[process:name='{}']".format(p), pattern_type="stix")
+        res.append(i)
+        res.append(Relationship(i, "indicates", malware))
+
+    for f in filenames:
+        i = Indicator(indicator_types=["malicious-activity"], pattern="[file:name='{}']".format(f), pattern_type="stix")
+        res.append(i)
+        res.append(Relationship(i, "indicates", malware))
+
+    for e in emails:
+        i = Indicator(indicator_types=["malicious-activity"], pattern="[email-addr:value='{}']".format(e), pattern_type="stix")
+        res.append(i)
+        res.append(Relationship(i, "indicates", malware))
+
+    bundle = Bundle(objects=res)
+    with open(file_path, "w+") as f:
+        f.write(bundle.serialize(pretty=True))
+
+
+if __name__ == "__main__":
+    generate_test_stix_file("test.stix2")
+    print("test.stix2 file created")

tests/common/test_indicators.py

@@ -0,0 +1,32 @@
+# Mobile Verification Toolkit (MVT)
+# Copyright (c) 2021 The MVT Project Authors.
+# Use of this software is governed by the MVT License 1.1 that can be found at
+#   https://license.mvt.re/1.1/
+
+import logging
+import os
+
+from mvt.common.indicators import Indicators
+
+
+class TestIndicators:
+    def test_parse_stix2(self, indicator_file):
+        ind = Indicators(log=logging)
+        ind.load_indicators_files([indicator_file], load_default=False)
+        assert ind.ioc_count == 4
+        assert len(ind.ioc_domains) == 1
+        assert len(ind.ioc_emails) == 1
+        assert len(ind.ioc_files) == 1
+        assert len(ind.ioc_processes) == 1
+
+    def test_check_domain(self, indicator_file):
+        ind = Indicators(log=logging)
+        ind.load_indicators_files([indicator_file], load_default=False)
+        assert ind.check_domain("https://www.example.org/foobar")
+        assert ind.check_domain("http://example.org:8080/toto")
+
+    def test_env_stix(self, indicator_file):
+        os.environ["MVT_STIX2"] = indicator_file
+        ind = Indicators(log=logging)
+        ind.load_indicators_files([indicator_file], load_default=False)
+        assert ind.ioc_count == 4

tests/conftest.py

@@ -0,0 +1,26 @@
+# Mobile Verification Toolkit (MVT)
+# Copyright (c) 2021 The MVT Project Authors.
+# Use of this software is governed by the MVT License 1.1 that can be found at
+#   https://license.mvt.re/1.1/
+
+import os
+
+import pytest
+
+from .artifacts.generate_stix import generate_test_stix_file
+
+
+@pytest.fixture(scope="session", autouse=True)
+def indicator_file(request, tmp_path_factory):
+    indicator_dir = tmp_path_factory.mktemp("indicators")
+    stix_path = indicator_dir / "indicators.stix2"
+    generate_test_stix_file(stix_path)
+    return str(stix_path)
+
+
+@pytest.fixture(scope="session", autouse=True)
+def clean_test_env(request, tmp_path_factory):
+    try:
+        del os.environ["MVT_STIX2"]
+    except KeyError:
+        pass

tests/ios/test_backup_info.py

@@ -0,0 +1,19 @@
+# Mobile Verification Toolkit (MVT)
+# Copyright (c) 2021 The MVT Project Authors.
+# Use of this software is governed by the MVT License 1.1 that can be found at
+#   https://license.mvt.re/1.1/
+
+import logging
+
+from mvt.common.module import run_module
+from mvt.ios.modules.backup.backup_info import BackupInfo
+
+from ..utils import get_backup_folder
+
+
+class TestBackupInfoModule:
+    def test_manifest(self):
+        m = BackupInfo(base_folder=get_backup_folder(), log=logging)
+        run_module(m)
+        assert m.results["Build Version"] == "18C66"
+        assert m.results["IMEI"] == "42"

tests/ios/test_datausage.py

@@ -0,0 +1,31 @@
+# Mobile Verification Toolkit (MVT)
+# Copyright (c) 2021 The MVT Project Authors.
+# Use of this software is governed by the MVT License 1.1 that can be found at
+#   https://license.mvt.re/1.1/
+
+import logging
+
+from mvt.common.indicators import Indicators
+from mvt.common.module import run_module
+from mvt.ios.modules.mixed.net_datausage import Datausage
+
+from ..utils import get_backup_folder
+
+
+class TestDatausageModule:
+    def test_datausage(self):
+        m = Datausage(base_folder=get_backup_folder(), log=logging, results=[])
+        run_module(m)
+        assert len(m.results) == 42
+        assert len(m.timeline) == 60
+        assert len(m.detected) == 0
+
+    def test_detection(self, indicator_file):
+        m = Datausage(base_folder=get_backup_folder(), log=logging, results=[])
+        ind = Indicators(log=logging)
+        ind.parse_stix2(indicator_file)
+        # Adds a file that exists in the manifest.
+        ind.ioc_processes[0] = "CumulativeUsageTracker"
+        m.indicators = ind
+        run_module(m)
+        assert len(m.detected) == 2