Uniforming style in test units

This reverts commit e1efaa5467.

.github/workflows/python-package.yml

@@ -16,7 +16,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [3.7, 3.8, 3.9]
+        # python-version: [3.7, 3.8, 3.9]
+        python-version: [3.8, 3.9]
 
     steps:
     - uses: actions/checkout@v2
@@ -27,8 +28,9 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        python -m pip install flake8 pytest safety
+        python -m pip install flake8 pytest safety stix2
         if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+        python -m pip install .
     - name: Lint with flake8
       run: |
         # stop the build if there are Python syntax errors or undefined names
@@ -37,7 +39,5 @@ jobs:
         flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
     - name: Safety checks
       run: safety check
-
-    # - name: Test with pytest
-    #   run: |
-    #     pytest
+    - name: Test with pytest
+      run: pytest

README.md

@@ -15,15 +15,15 @@ It has been developed and released by the [Amnesty International Security Lab](h
 
 ## Installation
 
-MVT can be installed from sources or from [PyPi](https://pypi.org/project/mvt/) (you will need some dependencies, check the [documentation](https://docs.mvt.re/en/latest/install.html)):
+MVT can be installed from sources or from [PyPi](https://pypi.org/project/mvt/) (you will need some dependencies, check the [documentation](https://docs.mvt.re/en/latest/install/)):
 
 ```
 pip3 install mvt
 ```
 
-Alternatively, you can decide to run MVT and all relevant tools through a [Docker container](https://docs.mvt.re/en/latest/docker.html).
+Alternatively, you can decide to run MVT and all relevant tools through a [Docker container](https://docs.mvt.re/en/latest/docker/).
 
-**Please note:** MVT is best run on Linux or Mac systems. [It does not currently support running natively on Windows.](https://docs.mvt.re/en/latest/install.html#mvt-on-windows)
+**Please note:** MVT is best run on Linux or Mac systems. [It does not currently support running natively on Windows.](https://docs.mvt.re/en/latest/install/#mvt-on-windows)
 
 ## Usage
 
@@ -31,4 +31,4 @@ MVT provides two commands `mvt-ios` and `mvt-android`. [Check out the documentat
 
 ## License
 
-The purpose of MVT is to facilitate the ***consensual forensic analysis*** of devices of those who might be targets of sophisticated mobile spyware attacks, especially members of civil society and marginalized communities. We do not want MVT to enable privacy violations of non-consenting individuals.  In order to achieve this, MVT is released under its own license. [Read more here.](https://docs.mvt.re/en/latest/license.html)
+The purpose of MVT is to facilitate the ***consensual forensic analysis*** of devices of those who might be targets of sophisticated mobile spyware attacks, especially members of civil society and marginalized communities. We do not want MVT to enable privacy violations of non-consenting individuals.  In order to achieve this, MVT is released under its own license. [Read more here.](https://docs.mvt.re/en/latest/license/)

docs/iocs.md

@@ -28,10 +28,19 @@ The `--iocs` option can be invoked multiple times to let MVT import multiple STI
 mvt-ios check-backup --iocs ~/iocs/malware1.stix --iocs ~/iocs/malware2.stix2 /path/to/backup
 ```
 
+It is also possible to load STIX2 files automatically from the environment variable `MVT_STIX2`:
+
+```bash
+export MVT_STIX2="/home/user/IOC1.stix2:/home/user/IOC2.stix2"
+```
+
 ## Known repositories of STIX2 IOCs
 
 - The [Amnesty International investigations repository](https://github.com/AmnestyTech/investigations) contains STIX-formatted IOCs for:
     - [Pegasus](https://en.wikipedia.org/wiki/Pegasus_(spyware)) ([STIX2](https://raw.githubusercontent.com/AmnestyTech/investigations/master/2021-07-18_nso/pegasus.stix2))
-- [This repository](https://github.com/Te-k/stalkerware-indicators) contains IOCs for Android stalkerware including [a STIX MVT-compatible file](https://github.com/Te-k/stalkerware-indicators/blob/master/stalkerware.stix2).
+    - [Predator from Cytrox](https://citizenlab.ca/2021/12/pegasus-vs-predator-dissidents-doubly-infected-iphone-reveals-cytrox-mercenary-spyware/) ([STIX2](https://raw.githubusercontent.com/AmnestyTech/investigations/master/2021-12-16_cytrox/cytrox.stix2))
+- [This repository](https://github.com/Te-k/stalkerware-indicators) contains IOCs for Android stalkerware including [a STIX MVT-compatible file](https://raw.githubusercontent.com/Te-k/stalkerware-indicators/master/stalkerware.stix2).
+
+You can automaticallly download the latest public indicator files with the command `mvt-ios download-iocs` or `mvt-android download-iocs`.
 
 Please [open an issue](https://github.com/mvt-project/mvt/issues/) to suggest new sources of STIX-formatted IOCs.

docs/requirements.txt

@@ -1,4 +1,4 @@
-mkdocs==1.2.1
+mkdocs==1.2.3
 mkdocs-autorefs
 mkdocs-material
 mkdocs-material-extensions

mvt/android/cli.py

@@ -9,8 +9,10 @@ import os
 import click
 from rich.logging import RichHandler
 
-from mvt.common.help import *
-from mvt.common.indicators import Indicators, IndicatorsFileBadFormat
+from mvt.common.help import (HELP_MSG_FAST, HELP_MSG_IOC,
+                             HELP_MSG_LIST_MODULES, HELP_MSG_MODULE,
+                             HELP_MSG_OUTPUT, HELP_MSG_SERIAL)
+from mvt.common.indicators import Indicators, download_indicators_files
 from mvt.common.logo import logo
 from mvt.common.module import run_module, save_timeline
 
@@ -26,6 +28,7 @@ logging.basicConfig(level="INFO", format=LOG_FORMAT, handlers=[
     RichHandler(show_path=False, log_time_format="%X")])
 log = logging.getLogger(__name__)
 
+
 #==============================================================================
 # Main
 #==============================================================================
@@ -104,10 +107,11 @@ def download_apks(ctx, all_apks, virustotal, koodous, all_checks, output, from_f
               default=[], help=HELP_MSG_IOC)
 @click.option("--output", "-o", type=click.Path(exists=False),
               help=HELP_MSG_OUTPUT)
+@click.option("--fast", "-f", is_flag=True, help=HELP_MSG_FAST)
 @click.option("--list-modules", "-l", is_flag=True, help=HELP_MSG_LIST_MODULES)
 @click.option("--module", "-m", help=HELP_MSG_MODULE)
 @click.pass_context
-def check_adb(ctx, iocs, output, list_modules, module, serial):
+def check_adb(ctx, iocs, output, fast, list_modules, module, serial):
     if list_modules:
         log.info("Following is the list of available check-adb modules:")
         for adb_module in ADB_MODULES:
@@ -125,13 +129,7 @@ def check_adb(ctx, iocs, output, list_modules, module, serial):
             ctx.exit(1)
 
     indicators = Indicators(log=log)
-    for ioc_path in iocs:
-        try:
-            indicators.parse_stix2(ioc_path)
-        except IndicatorsFileBadFormat as e:
-            log.critical(e)
-            ctx.exit(1)
-    log.info("Loaded a total of %d indicators", indicators.ioc_count)
+    indicators.load_indicators_files(iocs)
 
     timeline = []
     timeline_detected = []
@@ -139,14 +137,14 @@ def check_adb(ctx, iocs, output, list_modules, module, serial):
         if module and adb_module.__name__ != module:
             continue
 
-        m = adb_module(output_folder=output, log=logging.getLogger(adb_module.__module__))
+        m = adb_module(output_folder=output, fast_mode=fast,
+                       log=logging.getLogger(adb_module.__module__))
+        if indicators.ioc_count:
+            m.indicators = indicators
+            m.indicators.log = m.log
         if serial:
             m.serial = serial
 
-        if iocs:
-            indicators.log = m.log
-            m.indicators = indicators
-
         run_module(m)
         timeline.extend(m.timeline)
         timeline_detected.extend(m.timeline_detected)
@@ -179,31 +177,31 @@ def check_backup(ctx, iocs, output, backup_path, serial):
             ctx.exit(1)
 
     indicators = Indicators(log=log)
-    for ioc_path in iocs:
-        try:
-            indicators.parse_stix2(ioc_path)
-        except IndicatorsFileBadFormat as e:
-            log.critical(e)
-            ctx.exit(1)
-    log.info("Loaded a total of %d indicators", indicators.ioc_count)
+    indicators.load_indicators_files(iocs)
 
     if os.path.isfile(backup_path):
         log.critical("The path you specified is a not a folder!")
 
         if os.path.basename(backup_path) == "backup.ab":
-            log.info("You can use ABE (https://github.com/nelenkov/android-backup-extractor) " \
+            log.info("You can use ABE (https://github.com/nelenkov/android-backup-extractor) "
                      "to extract 'backup.ab' files!")
         ctx.exit(1)
 
     for module in BACKUP_MODULES:
         m = module(base_folder=backup_path, output_folder=output,
                    log=logging.getLogger(module.__module__))
-
+        if indicators.ioc_count:
+            m.indicators = indicators
+            m.indicators.log = m.log
         if serial:
             m.serial = serial
 
-        if iocs:
-            indicators.log = m.log
-            m.indicators = indicators
-
         run_module(m)
+
+
+#==============================================================================
+# Command: download-iocs
+#==============================================================================
+@cli.command("download-iocs", help="Download public STIX2 indicators")
+def download_indicators():
+    download_indicators_files(log)

mvt/android/download_apks.py

@@ -7,7 +7,6 @@ import json
 import logging
 import os
 
-import pkg_resources
 from tqdm import tqdm
 
 from mvt.common.module import InsufficientPrivileges
@@ -17,6 +16,7 @@ from .modules.adb.packages import Packages
 
 log = logging.getLogger(__name__)
 
+
 # TODO: Would be better to replace tqdm with rich.progress to reduce
 #       the number of dependencies. Need to investigate whether
 #       it's possible to have a similar callback system.
@@ -138,7 +138,7 @@ class DownloadAPKs(AndroidExtraction):
                     packages_selection.append(package)
 
             log.info("Selected only %d packages which are not marked as system",
-                len(packages_selection))
+                     len(packages_selection))
 
         if len(packages_selection) == 0:
             log.info("No packages were selected for download")

mvt/android/lookups/koodous.py

@@ -13,6 +13,7 @@ from rich.text import Text
 
 log = logging.getLogger(__name__)
 
+
 def koodous_lookup(packages):
     log.info("Looking up all extracted files on Koodous (www.koodous.com)")
     log.info("This might take a while...")

mvt/android/lookups/virustotal.py

@@ -13,6 +13,7 @@ from rich.text import Text
 
 log = logging.getLogger(__name__)
 
+
 def get_virustotal_report(hashes):
     apikey = "233f22e200ca5822bd91103043ccac138b910db79f29af5616a9afe8b6f215ad"
     url = f"https://www.virustotal.com/partners/sysinternals/file-reports?apikey={apikey}"
@@ -36,6 +37,7 @@ def get_virustotal_report(hashes):
         log.error("Unexpected response from VirusTotal: %s", res.status_code)
         return None
 
+
 def virustotal_lookup(packages):
     log.info("Looking up all extracted files on VirusTotal (www.virustotal.com)")
 
@@ -48,6 +50,7 @@ def virustotal_lookup(packages):
     total_unique_hashes = len(unique_hashes)
 
     detections = {}
+
     def virustotal_query(batch):
         report = get_virustotal_report(batch)
         if not report:

mvt/android/modules/adb/__init__.py

@@ -4,6 +4,7 @@
 #   https://license.mvt.re/1.1/
 
 from .chrome_history import ChromeHistory
+from .dumpsys_accessibility import DumpsysAccessibility
 from .dumpsys_batterystats import DumpsysBatterystats
 from .dumpsys_full import DumpsysFull
 from .dumpsys_packages import DumpsysPackages
@@ -18,6 +19,6 @@ from .sms import SMS
 from .whatsapp import Whatsapp
 
 ADB_MODULES = [ChromeHistory, SMS, Whatsapp, Processes,
-               DumpsysBatterystats, DumpsysProcstats,
+               DumpsysAccessibility, DumpsysBatterystats, DumpsysProcstats,
                DumpsysPackages, DumpsysReceivers, DumpsysFull,
                Packages, RootBinaries, Logcat, Files]

mvt/android/modules/adb/base.py

@@ -25,6 +25,7 @@ log = logging.getLogger(__name__)
 ADB_KEY_PATH = os.path.expanduser("~/.android/adbkey")
 ADB_PUB_KEY_PATH = os.path.expanduser("~/.android/adbkey.pub")
 
+
 class AndroidExtraction(MVTModule):
     """This class provides a base for all Android extraction modules."""
 
@@ -89,7 +90,7 @@ class AndroidExtraction(MVTModule):
             except OSError as e:
                 if e.errno == 113 and self.serial:
                     log.critical("Unable to connect to the device %s: did you specify the correct IP addres?",
-                              self.serial)
+                                 self.serial)
                     sys.exit(-1)
             else:
                 break
@@ -111,7 +112,7 @@ class AndroidExtraction(MVTModule):
         :returns: Output of command
 
         """
-        return self.device.shell(command)
+        return self.device.shell(command, read_timeout_s=200.0)
 
     def _adb_check_if_root(self):
         """Check if we have a `su` binary on the Android device.

mvt/android/modules/adb/chrome_history.py

@@ -16,6 +16,7 @@ log = logging.getLogger(__name__)
 
 CHROME_HISTORY_PATH = "data/data/com.android.chrome/app_chrome/Default/History"
 
+
 class ChromeHistory(AndroidExtraction):
     """This module extracts records from Android's Chrome browsing history."""
 

mvt/android/modules/adb/dumpsys_accessibility.py

@@ -0,0 +1,53 @@
+# Mobile Verification Toolkit (MVT)
+# Copyright (c) 2021 The MVT Project Authors.
+# Use of this software is governed by the MVT License 1.1 that can be found at
+#   https://license.mvt.re/1.1/
+
+import io
+import logging
+import os
+
+from .base import AndroidExtraction
+
+log = logging.getLogger(__name__)
+
+
+class DumpsysAccessibility(AndroidExtraction):
+    """This module extracts stats on accessibility."""
+
+    def __init__(self, file_path=None, base_folder=None, output_folder=None,
+                 serial=None, fast_mode=False, log=None, results=[]):
+        super().__init__(file_path=file_path, base_folder=base_folder,
+                         output_folder=output_folder, fast_mode=fast_mode,
+                         log=log, results=results)
+
+    def run(self):
+        self._adb_connect()
+
+        stats = self._adb_command("dumpsys accessibility")
+
+        in_services = False
+        for line in stats.split("\n"):
+            if line.strip().startswith("installed services:"):
+                in_services = True
+                continue
+
+            if not in_services:
+                continue
+
+            if line.strip() == "}":
+                break
+
+            service = line.split(":")[1].strip()
+            log.info("Found installed accessibility service \"%s\"", service)
+
+        if self.output_folder:
+            acc_path = os.path.join(self.output_folder,
+                                      "dumpsys_accessibility.txt")
+            with io.open(acc_path, "w", encoding="utf-8") as handle:
+                handle.write(stats)
+
+            log.info("Records from dumpsys accessibility stored at %s",
+                     acc_path)
+
+        self._adb_disconnect()

mvt/android/modules/adb/dumpsys_batterystats.py

@@ -10,6 +10,7 @@ from .base import AndroidExtraction
 
 log = logging.getLogger(__name__)
 
+
 class DumpsysBatterystats(AndroidExtraction):
     """This module extracts stats on battery consumption by processes."""
 
@@ -30,7 +31,7 @@ class DumpsysBatterystats(AndroidExtraction):
                 handle.write(stats)
 
             log.info("Records from dumpsys batterystats stored at %s",
-                      stats_path)
+                     stats_path)
 
         history = self._adb_command("dumpsys batterystats --history")
         if self.output_folder:

mvt/android/modules/adb/dumpsys_full.py

@@ -10,6 +10,7 @@ from .base import AndroidExtraction
 
 log = logging.getLogger(__name__)
 
+
 class DumpsysFull(AndroidExtraction):
     """This module extracts stats on battery consumption by processes."""
 
@@ -30,6 +31,6 @@ class DumpsysFull(AndroidExtraction):
                 handle.write(stats)
 
             log.info("Full dumpsys output stored at %s",
-                      stats_path)
+                     stats_path)
 
         self._adb_disconnect()

mvt/android/modules/adb/dumpsys_procstats.py

@@ -10,6 +10,7 @@ from .base import AndroidExtraction
 
 log = logging.getLogger(__name__)
 
+
 class DumpsysProcstats(AndroidExtraction):
     """This module extracts stats on memory consumption by processes."""
 

mvt/android/modules/adb/dumpsys_receivers.py

@@ -4,7 +4,6 @@
 #   https://license.mvt.re/1.1/
 
 import logging
-import os
 
 from .base import AndroidExtraction
 
@@ -15,6 +14,7 @@ ACTION_SMS_RECEIVED = "android.provider.Telephony.SMS_RECEIVED"
 ACTION_DATA_SMS_RECEIVED = "android.intent.action.DATA_SMS_RECEIVED"
 ACTION_PHONE_STATE = "android.intent.action.PHONE_STATE"
 
+
 class DumpsysReceivers(AndroidExtraction):
     """This module extracts details on receivers for risky activities."""
 
@@ -67,16 +67,16 @@ class DumpsysReceivers(AndroidExtraction):
 
             if activity == ACTION_NEW_OUTGOING_SMS:
                 self.log.info("Found a receiver to intercept outgoing SMS messages: \"%s\"",
-                    receiver)
+                              receiver)
             elif activity == ACTION_SMS_RECEIVED:
                 self.log.info("Found a receiver to intercept incoming SMS messages: \"%s\"",
-                    receiver)
+                              receiver)
             elif activity == ACTION_DATA_SMS_RECEIVED:
                 self.log.info("Found a receiver to intercept incoming data SMS message: \"%s\"",
-                    receiver)
+                              receiver)
             elif activity == ACTION_PHONE_STATE:
                 self.log.info("Found a receiver monitoring telephony state: \"%s\"",
-                    receiver)
+                              receiver)
 
             self.results.append({
                 "activity": activity,

mvt/android/modules/adb/files.py

@@ -3,31 +3,121 @@
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
+import datetime
 import logging
-import os
+import stat
+
+from mvt.common.utils import convert_timestamp_to_iso
 
 from .base import AndroidExtraction
 
 log = logging.getLogger(__name__)
 
+
 class Files(AndroidExtraction):
-    """This module extracts the list of installed packages."""
+    """This module extracts the list of files on the device."""
 
     def __init__(self, file_path=None, base_folder=None, output_folder=None,
                  serial=None, fast_mode=False, log=None, results=[]):
         super().__init__(file_path=file_path, base_folder=base_folder,
                          output_folder=output_folder, fast_mode=fast_mode,
                          log=log, results=results)
+        self.full_find = None
+
+    def find_path(self, file_path):
+        """Checks if Android system supports full find command output"""
+        # Check find command params on first run
+        # Run find command with correct args and parse results.
+
+        # Check that full file printf options are suppported on first run.
+        if self.full_find is None:
+            output = self._adb_command("find '/' -maxdepth 1 -printf '%T@ %m %s %u %g %p\n' 2> /dev/null")
+            if not (output or output.strip().splitlines()):
+                # Full  find command failed to generate output, fallback to basic file arguments
+                self.full_find = False
+            else:
+                self.full_find = True
+
+        found_files = []
+        if self.full_find is True:
+            # Run full file command and collect additonal file information.
+            output = self._adb_command(f"find '{file_path}' -printf '%T@ %m %s %u %g %p\n' 2> /dev/null")
+            for file_line in output.splitlines():
+                [unix_timestamp, mode, size, owner, group, full_path] = file_line.rstrip().split(" ", 5)
+                mod_time = convert_timestamp_to_iso(datetime.datetime.utcfromtimestamp(int(float(unix_timestamp))))
+                found_files.append({
+                    "path": full_path,
+                    "modified_time": mod_time,
+                    "mode": mode,
+                    "is_suid": (int(mode, 8) & stat.S_ISUID) == 2048,
+                    "is_sgid": (int(mode, 8) & stat.S_ISGID) == 1024,
+                    "size": size,
+                    "owner": owner,
+                    "group": group,
+                })
+        else:
+            # Run a basic listing of file paths.
+            output = self._adb_command(f"find '{file_path}' 2> /dev/null")
+            for file_line in output.splitlines():
+                found_files.append({
+                    "path": file_line.rstrip()
+                })
+
+        return found_files
+
+    def serialize(self, record):
+        if "modified_time" in record:
+            return {
+                "timestamp": record["modified_time"],
+                "module": self.__class__.__name__,
+                "event": "file_modified",
+                "data": record["path"],
+            }
+
+    def check_suspicious(self):
+        """Check for files with suspicious permissions"""
+        for result in sorted(self.results, key=lambda item: item["path"]):
+            if result.get("is_suid"):
+                self.log.warning("Found an SUID file in a non-standard directory \"%s\".",
+                                 result["path"])
+                self.detected.append(result)
+
+    def check_indicators(self):
+        """Check file list for known suspicious files or suspicious properties"""
+        self.check_suspicious()
+        if not self.indicators:
+            return
+
+        for result in self.results:
+            if self.indicators.check_file_name(result["path"]):
+                self.log.warning("Found a known suspicous filename at path: \"%s\"", result["path"])
+                self.detected.append(result)
+
+            if self.indicators.check_file_path(result["path"]):
+                self.log.warning("Found a known suspicous file at path: \"%s\"", result["path"])
+                self.detected.append(result)
 
     def run(self):
         self._adb_connect()
+        found_file_paths = []
 
-        output = self._adb_command("find / -type f 2> /dev/null")
-        if output and self.output_folder:
-            files_txt_path = os.path.join(self.output_folder, "files.txt")
-            with open(files_txt_path, "w") as handle:
-                handle.write(output)
+        DATA_PATHS = ["/data/local/tmp/", "/sdcard/", "/tmp/"]
+        for path in DATA_PATHS:
+            file_info = self.find_path(path)
+            found_file_paths.extend(file_info)
 
-            log.info("List of visible files stored at %s", files_txt_path)
+        # Store results
+        self.results.extend(found_file_paths)
+        self.log.info("Found %s files in primary Android data directories.", len(found_file_paths))
+
+        if self.fast_mode:
+            self.log.info("Flag --fast was enabled: skipping full file listing")
+        else:
+            self.log.info("Flag --fast was not enabled: processing full file listing. "
+                          "This may take a while...")
+            output = self.find_path("/")
+            if output and self.output_folder:
+                self.results.extend(output)
+                log.info("List of visible files stored in files.json")
 
         self._adb_disconnect()

mvt/android/modules/adb/packages.py

@@ -12,6 +12,7 @@ from .base import AndroidExtraction
 
 log = logging.getLogger(__name__)
 
+
 class Packages(AndroidExtraction):
     """This module extracts the list of installed packages."""
 
@@ -49,11 +50,10 @@ class Packages(AndroidExtraction):
         root_packages = root_packages_string.decode("utf-8").split("\n")
         root_packages = [rp.strip() for rp in root_packages]
 
-
         for result in self.results:
             if result["package_name"] in root_packages:
                 self.log.warning("Found an installed package related to rooting/jailbreaking: \"%s\"",
-                                result["package_name"])
+                                 result["package_name"])
                 self.detected.append(result)
             if result["package_name"] in self.indicators.ioc_app_ids:
                 self.log.warning("Found a malicious package name: \"%s\"",

mvt/android/modules/adb/processes.py

@@ -9,6 +9,7 @@ from .base import AndroidExtraction
 
 log = logging.getLogger(__name__)
 
+
 class Processes(AndroidExtraction):
     """This module extracts details on running processes."""
 

mvt/android/modules/adb/rootbinaries.py

@@ -12,6 +12,7 @@ from .base import AndroidExtraction
 
 log = logging.getLogger(__name__)
 
+
 class RootBinaries(AndroidExtraction):
     """This module extracts the list of installed packages."""
 

mvt/android/modules/adb/sms.py

@@ -15,12 +15,12 @@ log = logging.getLogger(__name__)
 
 SMS_BUGLE_PATH = "data/data/com.google.android.apps.messaging/databases/bugle_db"
 SMS_BUGLE_QUERY = """
-SELECT 
+SELECT
     ppl.normalized_destination AS number,
     p.timestamp AS timestamp,
-CASE WHEN m.sender_id IN 
+CASE WHEN m.sender_id IN
 (SELECT _id FROM participants WHERE contact_id=-1)
-THEN 2 ELSE 1 END incoming, p.text AS text 
+THEN 2 ELSE 1 END incoming, p.text AS text
 FROM messages m, conversations c, parts p,
         participants ppl, conversation_participants cp
 WHERE (m.conversation_id = c._id)
@@ -31,14 +31,15 @@ WHERE (m.conversation_id = c._id)
 
 SMS_MMSSMS_PATH = "data/data/com.android.providers.telephony/databases/mmssms.db"
 SMS_MMSMS_QUERY = """
-SELECT 
+SELECT
     address AS number,
     date_sent AS timestamp,
     type as incoming,
-    body AS text 
+    body AS text
 FROM sms;
 """
 
+
 class SMS(AndroidExtraction):
     """This module extracts all SMS messages containing links."""
 
@@ -62,7 +63,7 @@ class SMS(AndroidExtraction):
             return
 
         for message in self.results:
-            if not "text" in message:
+            if "text" not in message:
                 continue
 
             message_links = check_for_links(message["text"])
@@ -77,7 +78,7 @@ class SMS(AndroidExtraction):
         """
         conn = sqlite3.connect(db_path)
         cur = conn.cursor()
-        
+
         if (self.SMS_DB_TYPE == 1):
             cur.execute(SMS_BUGLE_QUERY)
         elif (self.SMS_DB_TYPE == 2):

mvt/android/modules/adb/whatsapp.py

@@ -16,6 +16,7 @@ log = logging.getLogger(__name__)
 
 WHATSAPP_PATH = "data/data/com.whatsapp/databases/msgstore.db"
 
+
 class Whatsapp(AndroidExtraction):
     """This module extracts all WhatsApp messages containing links."""
 
@@ -39,7 +40,7 @@ class Whatsapp(AndroidExtraction):
             return
 
         for message in self.results:
-            if not "data" in message:
+            if "data" not in message:
                 continue
 
             message_links = check_for_links(message["data"])

mvt/android/modules/backup/__init__.py

@@ -5,4 +5,4 @@
 
 from .sms import SMS
 
-BACKUP_MODULES = [SMS,]
+BACKUP_MODULES = [SMS]

mvt/android/modules/backup/sms.py

@@ -24,7 +24,7 @@ class SMS(MVTModule):
             return
 
         for message in self.results:
-            if not "body" in message:
+            if "body" not in message:
                 continue
 
             message_links = check_for_links(message["body"])

mvt/common/indicators.py

@@ -3,23 +3,23 @@
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
+import io
 import json
 import os
 
+import requests
+from appdirs import user_data_dir
+
 from .url import URL
 
 
-class IndicatorsFileBadFormat(Exception):
-    pass
-
 class Indicators:
     """This class is used to parse indicators from a STIX2 file and provide
     functions to compare extracted artifacts to the indicators.
-
-
     """
 
     def __init__(self, log=None):
+        self.data_dir = user_data_dir("mvt")
         self.log = log
         self.ioc_domains = []
         self.ioc_processes = []
@@ -27,6 +27,7 @@ class Indicators:
         self.ioc_files = []
         self.ioc_files_sha256 = []
         self.ioc_app_ids = []
+        self.ios_profile_ids = []
         self.ioc_count = 0
 
     def _add_indicator(self, ioc, iocs_list):
@@ -34,6 +35,44 @@ class Indicators:
             iocs_list.append(ioc)
             self.ioc_count += 1
 
+    def _load_downloaded_indicators(self):
+        if not os.path.isdir(self.data_dir):
+            return False
+
+        for f in os.listdir(self.data_dir):
+            if f.lower().endswith(".stix2"):
+                self.parse_stix2(os.path.join(self.data_dir, f))
+
+    def _check_stix2_env_variable(self):
+        """
+        Checks if a variable MVT_STIX2 contains path to STIX Files.
+        """
+        if "MVT_STIX2" not in os.environ:
+            return False
+
+        paths = os.environ["MVT_STIX2"].split(":")
+        for path in paths:
+            if os.path.isfile(path):
+                self.parse_stix2(path)
+            else:
+                self.log.info("Invalid STIX2 path %s in MVT_STIX2 environment variable", path)
+
+    def load_indicators_files(self, files, load_default=True):
+        """
+        Load a list of indicators files.
+        """
+        for file_path in files:
+            if os.path.isfile(file_path):
+                self.parse_stix2(file_path)
+            else:
+                self.log.warning("This indicators file %s does not exist", file_path)
+
+        # Load downloaded indicators and any indicators from env variable
+        if load_default:
+            self._load_downloaded_indicators()
+        self._check_stix2_env_variable()
+        self.log.info("Loaded a total of %d unique indicators", self.ioc_count)
+
     def parse_stix2(self, file_path):
         """Extract indicators from a STIX2 file.
 
@@ -41,14 +80,13 @@ class Indicators:
         :type file_path: str
 
         """
-        self.log.info("Parsing STIX2 indicators file at path %s",
-                      file_path)
-
+        self.log.info("Parsing STIX2 indicators file at path %s", file_path)
         with open(file_path, "r") as handle:
             try:
                 data = json.load(handle)
             except json.decoder.JSONDecodeError:
-                raise IndicatorsFileBadFormat("Unable to parse STIX2 indicators file, the file seems malformed or in the wrong format")
+                self.log.critical("Unable to parse STIX2 indicator file. The file is malformed or in the wrong format.")
+                return
 
         for entry in data.get("objects", []):
             if entry.get("type", "") != "indicator":
@@ -74,6 +112,9 @@ class Indicators:
             elif key == "app:id":
                 self._add_indicator(ioc=value,
                                     iocs_list=self.ioc_app_ids)
+            elif key == "configuration-profile:id":
+                self._add_indicator(ioc=value,
+                                    iocs_list=self.ios_profile_ids)
             elif key == "file:hashes.sha256":
                 self._add_indicator(ioc=value,
                                     iocs_list=self.ioc_files_sha256)
@@ -115,7 +156,7 @@ class Indicators:
             else:
                 # If it's not shortened, we just use the original URL object.
                 final_url = orig_url
-        except Exception as e:
+        except Exception:
             # If URL parsing failed, we just try to do a simple substring
             # match.
             for ioc in self.ioc_domains:
@@ -231,7 +272,7 @@ class Indicators:
 
         return False
 
-    def check_file(self, file_path) -> bool:
+    def check_file_name(self, file_path) -> bool:
         """Check the provided file path against the list of file indicators.
 
         :param file_path: File path or file name to check against file
@@ -246,7 +287,75 @@ class Indicators:
 
         file_name = os.path.basename(file_path)
         if file_name in self.ioc_files:
-            self.log.warning("Found a known suspicious file: \"%s\"", file_path)
             return True
 
         return False
+
+    # TODO: The difference between check_file_name() and check_file_path()
+    #       needs to be more explicit and clear. Probably, the two should just
+    #       be combined into one function.
+    def check_file_path(self, file_path) -> bool:
+        """Check the provided file path against the list of file indicators.
+
+        :param file_path: File path or file name to check against file
+        indicators
+        :type file_path: str
+        :returns: True if the file path matched an indicator, otherwise False
+        :rtype: bool
+
+        """
+        if not file_path:
+            return False
+
+        for ioc_file in self.ioc_files:
+            # Strip any trailing slash from indicator paths to match directories.
+            if file_path.startswith(ioc_file.rstrip("/")):
+                return True
+
+        return False
+
+    def check_profile(self, profile_uuid) -> bool:
+        """Check the provided configuration profile UUID against the list of indicators.
+
+        :param profile_uuid: Profile UUID to check against configuration profile indicators
+        :type profile_uuid: str
+        :returns: True if the UUID in indicator list, otherwise False
+        :rtype: bool
+
+        """
+        if profile_uuid in self.ios_profile_ids:
+            return True
+
+        return False
+
+
+def download_indicators_files(log):
+    """
+    Download indicators from repo into MVT app data directory.
+    """
+    data_dir = user_data_dir("mvt")
+    if not os.path.isdir(data_dir):
+        os.makedirs(data_dir, exist_ok=True)
+
+    # Download latest list of indicators from the MVT repo.
+    res = requests.get("https://github.com/mvt-project/mvt/raw/main/public_indicators.json")
+    if res.status_code != 200:
+        log.warning("Unable to find retrieve list of indicators from the MVT repository.")
+        return
+
+    for ioc_entry in res.json():
+        ioc_url = ioc_entry["stix2_url"]
+        log.info("Downloading indicator file '%s' from '%s'", ioc_entry["name"], ioc_url)
+
+        res = requests.get(ioc_url)
+        if res.status_code != 200:
+            log.warning("Could not find indicator file '%s'", ioc_url)
+            continue
+
+        clean_file_name = ioc_url.lstrip("https://").replace("/", "_")
+        ioc_path = os.path.join(data_dir, clean_file_name)
+
+        # Write file to disk. This will overwrite any older version of the STIX2 file.
+        with io.open(ioc_path, "w") as f:
+            f.write(res.text)
+        log.info("Saved indicator file to '%s'", os.path.basename(ioc_path))

mvt/common/logo.py

@@ -16,7 +16,7 @@ def logo():
 
     try:
         latest_version = check_for_updates()
-    except:
+    except Exception:
         pass
     else:
         if latest_version:

mvt/common/module.py

@@ -10,18 +10,19 @@ import re
 
 import simplejson as json
 
-from .indicators import Indicators
-
 
 class DatabaseNotFoundError(Exception):
     pass
 
+
 class DatabaseCorruptedError(Exception):
     pass
 
+
 class InsufficientPrivileges(Exception):
     pass
 
+
 class MVTModule(object):
     """This class provides a base for all extraction modules."""
 
@@ -29,7 +30,7 @@ class MVTModule(object):
     slug = None
 
     def __init__(self, file_path=None, base_folder=None, output_folder=None,
-                 fast_mode=False, log=None, results=[]):
+                 fast_mode=False, log=None, results=None):
         """Initialize module.
 
         :param file_path: Path to the module's database file, if there is any
@@ -50,7 +51,7 @@ class MVTModule(object):
         self.fast_mode = fast_mode
         self.log = log
         self.indicators = None
-        self.results = results
+        self.results = results if results else []
         self.detected = []
         self.timeline = []
         self.timeline_detected = []

mvt/common/url.py

@@ -250,6 +250,7 @@ SHORTENER_DOMAINS = [
     "zz.gd",
 ]
 
+
 class URL:
 
     def __init__(self, url):
@@ -273,7 +274,7 @@ class URL:
         # TODO: Properly handle exception.
         try:
             return get_tld(self.url, as_object=True, fix_protocol=True).parsed_url.netloc.lower().lstrip("www.")
-        except:
+        except Exception:
             return None
 
     def get_top_level(self):
@@ -288,7 +289,7 @@ class URL:
         # TODO: Properly handle exception.
         try:
             return get_tld(self.url, as_object=True, fix_protocol=True).fld.lower()
-        except:
+        except Exception:
             return None
 
     def check_if_shortened(self) -> bool:

mvt/common/utils.py

@@ -45,7 +45,7 @@ def convert_chrometime_to_unix(timestamp):
     :returns: Unix epoch timestamp.
 
     """
-    epoch_start = datetime.datetime(1601, 1 , 1)
+    epoch_start = datetime.datetime(1601, 1, 1)
     delta = datetime.timedelta(microseconds=timestamp)
     return epoch_start + delta
 
@@ -64,6 +64,7 @@ def convert_timestamp_to_iso(timestamp):
     except Exception:
         return None
 
+
 def check_for_links(text):
     """Checks if a given text contains HTTP links.
 
@@ -74,6 +75,7 @@ def check_for_links(text):
     """
     return re.findall("(?P<url>https?://[^\s]+)", text, re.IGNORECASE)
 
+
 def get_sha256_from_file_path(file_path):
     """Calculate the SHA256 hash of a file from a file path.
 
@@ -88,6 +90,7 @@ def get_sha256_from_file_path(file_path):
 
     return sha256_hash.hexdigest()
 
+
 # Note: taken from here:
 # https://stackoverflow.com/questions/57014259/json-dumps-on-dictionary-with-bytes-for-keys
 def keys_bytes_to_string(obj):

mvt/common/version.py

@@ -6,7 +6,8 @@
 import requests
 from packaging import version
 
-MVT_VERSION = "1.2.11"
+MVT_VERSION = "1.4.4"
+
 
 def check_for_updates():
     res = requests.get("https://pypi.org/pypi/mvt/json")

mvt/ios/cli.py

@@ -10,8 +10,10 @@ import click
 from rich.logging import RichHandler
 from rich.prompt import Prompt
 
-from mvt.common.help import *
-from mvt.common.indicators import Indicators, IndicatorsFileBadFormat
+from mvt.common.help import (HELP_MSG_FAST, HELP_MSG_IOC,
+                             HELP_MSG_LIST_MODULES, HELP_MSG_MODULE,
+                             HELP_MSG_OUTPUT)
+from mvt.common.indicators import Indicators, download_indicators_files
 from mvt.common.logo import logo
 from mvt.common.module import run_module, save_timeline
 from mvt.common.options import MutuallyExclusiveOption
@@ -30,6 +32,7 @@ log = logging.getLogger(__name__)
 # Set this environment variable to a password if needed.
 PASSWD_ENV = "MVT_IOS_BACKUP_PASSWORD"
 
+
 #==============================================================================
 # Main
 #==============================================================================
@@ -154,13 +157,7 @@ def check_backup(ctx, iocs, output, fast, backup_path, list_modules, module):
             ctx.exit(1)
 
     indicators = Indicators(log=log)
-    for ioc_path in iocs:
-        try:
-            indicators.parse_stix2(ioc_path)
-        except IndicatorsFileBadFormat as e:
-            log.critical(e)
-            ctx.exit(1)
-    log.info("Loaded a total of %d indicators", indicators.ioc_count)
+    indicators.load_indicators_files(iocs)
 
     timeline = []
     timeline_detected = []
@@ -171,8 +168,7 @@ def check_backup(ctx, iocs, output, fast, backup_path, list_modules, module):
         m = backup_module(base_folder=backup_path, output_folder=output, fast_mode=fast,
                           log=logging.getLogger(backup_module.__module__))
         m.is_backup = True
-
-        if iocs:
+        if indicators.ioc_count:
             m.indicators = indicators
             m.indicators.log = m.log
 
@@ -217,13 +213,7 @@ def check_fs(ctx, iocs, output, fast, dump_path, list_modules, module):
             ctx.exit(1)
 
     indicators = Indicators(log=log)
-    for ioc_path in iocs:
-        try:
-            indicators.parse_stix2(ioc_path)
-        except IndicatorsFileBadFormat as e:
-            log.critical(e)
-            ctx.exit(1)
-    log.info("Loaded a total of %d indicators", indicators.ioc_count)
+    indicators.load_indicators_files(iocs)
 
     timeline = []
     timeline_detected = []
@@ -235,8 +225,7 @@ def check_fs(ctx, iocs, output, fast, dump_path, list_modules, module):
                       log=logging.getLogger(fs_module.__module__))
 
         m.is_fs_dump = True
-
-        if iocs:
+        if indicators.ioc_count:
             m.indicators = indicators
             m.indicators.log = m.log
 
@@ -277,13 +266,7 @@ def check_iocs(ctx, iocs, list_modules, module, folder):
     log.info("Checking stored results against provided indicators...")
 
     indicators = Indicators(log=log)
-    for ioc_path in iocs:
-        try:
-            indicators.parse_stix2(ioc_path)
-        except IndicatorsFileBadFormat as e:
-            log.critical(e)
-            ctx.exit(1)
-    log.info("Loaded a total of %d indicators", indicators.ioc_count)
+    indicators.load_indicators_files(iocs)
 
     for file_name in os.listdir(folder):
         name_only, ext = os.path.splitext(file_name)
@@ -301,11 +284,19 @@ def check_iocs(ctx, iocs, list_modules, module, folder):
 
             m = iocs_module.from_json(file_path,
                                       log=logging.getLogger(iocs_module.__module__))
-
-            m.indicators = indicators
-            m.indicators.log = m.log
+            if indicators.ioc_count:
+                m.indicators = indicators
+                m.indicators.log = m.log
 
             try:
                 m.check_indicators()
             except NotImplementedError:
                 continue
+
+
+#==============================================================================
+# Command: download-iocs
+#==============================================================================
+@cli.command("download-iocs", help="Download public STIX2 indicators")
+def download_iocs():
+    download_indicators_files(log)