Uniforming style in test units

This reverts commit e1efaa5467.

.github/workflows/python-package.yml

@@ -16,7 +16,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [3.7, 3.8, 3.9]
+        # python-version: [3.7, 3.8, 3.9]
+        python-version: [3.8, 3.9]
 
     steps:
     - uses: actions/checkout@v2
@@ -27,8 +28,9 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        python -m pip install flake8 pytest safety
+        python -m pip install flake8 pytest safety stix2
         if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+        python -m pip install .
     - name: Lint with flake8
       run: |
         # stop the build if there are Python syntax errors or undefined names
@@ -37,7 +39,5 @@ jobs:
         flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
     - name: Safety checks
       run: safety check
-
-    # - name: Test with pytest
-    #   run: |
-    #     pytest
+    - name: Test with pytest
+      run: pytest

mvt/common/indicators.py

@@ -45,7 +45,7 @@ class Indicators:
 
     def _check_stix2_env_variable(self):
         """
-        Checks if a variable MVT_STIX2 contains path to STIX Files
+        Checks if a variable MVT_STIX2 contains path to STIX Files.
         """
         if "MVT_STIX2" not in os.environ:
             return False
@@ -57,9 +57,9 @@ class Indicators:
             else:
                 self.log.info("Invalid STIX2 path %s in MVT_STIX2 environment variable", path)
 
-    def load_indicators_files(self, files):
+    def load_indicators_files(self, files, load_default=True):
         """
-        Load a list of indicators files
+        Load a list of indicators files.
         """
         for file_path in files:
             if os.path.isfile(file_path):
@@ -68,7 +68,8 @@ class Indicators:
                 self.log.warning("This indicators file %s does not exist", file_path)
 
         # Load downloaded indicators and any indicators from env variable
-        self._load_downloaded_indicators()
+        if load_default:
+            self._load_downloaded_indicators()
         self._check_stix2_env_variable()
         self.log.info("Loaded a total of %d unique indicators", self.ioc_count)
 
@@ -330,7 +331,7 @@ class Indicators:
 
 def download_indicators_files(log):
     """
-    Download indicators from repo into MVT app data directory
+    Download indicators from repo into MVT app data directory.
     """
     data_dir = user_data_dir("mvt")
     if not os.path.isdir(data_dir):

mvt/common/module.py

@@ -30,7 +30,7 @@ class MVTModule(object):
     slug = None
 
     def __init__(self, file_path=None, base_folder=None, output_folder=None,
-                 fast_mode=False, log=None, results=[]):
+                 fast_mode=False, log=None, results=None):
         """Initialize module.
 
         :param file_path: Path to the module's database file, if there is any
@@ -51,7 +51,7 @@ class MVTModule(object):
         self.fast_mode = fast_mode
         self.log = log
         self.indicators = None
-        self.results = results
+        self.results = results if results else []
         self.detected = []
         self.timeline = []
         self.timeline_detected = []

mvt/common/version.py

@@ -6,7 +6,7 @@
 import requests
 from packaging import version
 
-MVT_VERSION = "1.4.3"
+MVT_VERSION = "1.4.4"
 
 
 def check_for_updates():

mvt/ios/modules/mixed/shortcuts.py

@@ -34,13 +34,21 @@ class Shortcuts(IOSExtraction):
         found_urls = ""
         if record["action_urls"]:
             found_urls = "- URLs in actions: {}".format(", ".join(record["action_urls"]))
+        desc = ""
+        if record["description"]:
+            desc = record["description"].decode('utf-8', errors='ignore')
 
-        return {
+        return [{
             "timestamp": record["isodate"],
             "module": self.__class__.__name__,
-            "event": "shortcut",
-            "data": f"iOS Shortcut '{record['shortcut_name']}': {record['description']} {found_urls}"
-        }
+            "event": "shortcut_created",
+            "data": f"iOS Shortcut '{record['shortcut_name'].decode('utf-8')}': {desc} {found_urls}"
+        }, {
+            "timestamp": record["modified_date"],
+            "module": self.__class__.__name__,
+            "event": "shortcut_modified",
+            "data": f"iOS Shortcut '{record['shortcut_name'].decode('utf-8')}': {desc} {found_urls}"
+        }]
 
     def check_indicators(self):
         if not self.indicators:
@@ -92,14 +100,13 @@ class Shortcuts(IOSExtraction):
                 action["identifier"] = action_entry["WFWorkflowActionIdentifier"]
                 action["parameters"] = action_entry["WFWorkflowActionParameters"]
 
-                # URLs might be in multiple fields, do a simple regex search across the parameters
+                # URLs might be in multiple fields, do a simple regex search across the parameters.
                 extracted_urls = check_for_links(str(action["parameters"]))
 
-                # Remove quoting characters that may have been captured by the regex
+                # Remove quoting characters that may have been captured by the regex.
                 action["urls"] = [url.rstrip("',") for url in extracted_urls]
                 actions.append(action)
 
-            # pprint.pprint(actions)
             shortcut["isodate"] = convert_timestamp_to_iso(convert_mactime_to_unix(shortcut.pop("created_date")))
             shortcut["modified_date"] = convert_timestamp_to_iso(convert_mactime_to_unix(shortcut["modified_date"]))
             shortcut["parsed_actions"] = len(actions)

tests/artifacts/.gitignore

@@ -0,0 +1 @@
+test.stix2

tests/artifacts/generate_stix.py

@@ -0,0 +1,50 @@
+# Mobile Verification Toolkit (MVT)
+# Copyright (c) 2021 The MVT Project Authors.
+# Use of this software is governed by the MVT License 1.1 that can be found at
+#   https://license.mvt.re/1.1/
+
+import os
+
+from stix2.v21 import Bundle, Indicator, Malware, Relationship
+
+
+def generate_test_stix_file(file_path):
+    if os.path.isfile(file_path):
+        os.remove(file_path)
+
+    domains = ["example.org"]
+    processes = ["Launch"]
+    emails = ["foobar@example.org"]
+    filenames = ["/var/foobar/txt"]
+
+    res = []
+    malware = Malware(name="TestMalware", is_family=False, description="")
+    res.append(malware)
+    for d in domains:
+        i = Indicator(indicator_types=["malicious-activity"], pattern="[domain-name:value='{}']".format(d), pattern_type="stix")
+        res.append(i)
+        res.append(Relationship(i, "indicates", malware))
+
+    for p in processes:
+        i = Indicator(indicator_types=["malicious-activity"], pattern="[process:name='{}']".format(p), pattern_type="stix")
+        res.append(i)
+        res.append(Relationship(i, "indicates", malware))
+
+    for f in filenames:
+        i = Indicator(indicator_types=["malicious-activity"], pattern="[file:name='{}']".format(f), pattern_type="stix")
+        res.append(i)
+        res.append(Relationship(i, "indicates", malware))
+
+    for e in emails:
+        i = Indicator(indicator_types=["malicious-activity"], pattern="[email-addr:value='{}']".format(e), pattern_type="stix")
+        res.append(i)
+        res.append(Relationship(i, "indicates", malware))
+
+    bundle = Bundle(objects=res)
+    with open(file_path, "w+") as f:
+        f.write(bundle.serialize(pretty=True))
+
+
+if __name__ == "__main__":
+    generate_test_stix_file("test.stix2")
+    print("test.stix2 file created")

tests/common/test_indicators.py

@@ -0,0 +1,32 @@
+# Mobile Verification Toolkit (MVT)
+# Copyright (c) 2021 The MVT Project Authors.
+# Use of this software is governed by the MVT License 1.1 that can be found at
+#   https://license.mvt.re/1.1/
+
+import logging
+import os
+
+from mvt.common.indicators import Indicators
+
+
+class TestIndicators:
+    def test_parse_stix2(self, indicator_file):
+        ind = Indicators(log=logging)
+        ind.load_indicators_files([indicator_file], load_default=False)
+        assert ind.ioc_count == 4
+        assert len(ind.ioc_domains) == 1
+        assert len(ind.ioc_emails) == 1
+        assert len(ind.ioc_files) == 1
+        assert len(ind.ioc_processes) == 1
+
+    def test_check_domain(self, indicator_file):
+        ind = Indicators(log=logging)
+        ind.load_indicators_files([indicator_file], load_default=False)
+        assert ind.check_domain("https://www.example.org/foobar")
+        assert ind.check_domain("http://example.org:8080/toto")
+
+    def test_env_stix(self, indicator_file):
+        os.environ["MVT_STIX2"] = indicator_file
+        ind = Indicators(log=logging)
+        ind.load_indicators_files([indicator_file], load_default=False)
+        assert ind.ioc_count == 4

tests/conftest.py

@@ -0,0 +1,26 @@
+# Mobile Verification Toolkit (MVT)
+# Copyright (c) 2021 The MVT Project Authors.
+# Use of this software is governed by the MVT License 1.1 that can be found at
+#   https://license.mvt.re/1.1/
+
+import os
+
+import pytest
+
+from .artifacts.generate_stix import generate_test_stix_file
+
+
+@pytest.fixture(scope="session", autouse=True)
+def indicator_file(request, tmp_path_factory):
+    indicator_dir = tmp_path_factory.mktemp("indicators")
+    stix_path = indicator_dir / "indicators.stix2"
+    generate_test_stix_file(stix_path)
+    return str(stix_path)
+
+
+@pytest.fixture(scope="session", autouse=True)
+def clean_test_env(request, tmp_path_factory):
+    try:
+        del os.environ["MVT_STIX2"]
+    except KeyError:
+        pass

tests/ios/test_backup_info.py

@@ -0,0 +1,19 @@
+# Mobile Verification Toolkit (MVT)
+# Copyright (c) 2021 The MVT Project Authors.
+# Use of this software is governed by the MVT License 1.1 that can be found at
+#   https://license.mvt.re/1.1/
+
+import logging
+
+from mvt.common.module import run_module
+from mvt.ios.modules.backup.backup_info import BackupInfo
+
+from ..utils import get_backup_folder
+
+
+class TestBackupInfoModule:
+    def test_manifest(self):
+        m = BackupInfo(base_folder=get_backup_folder(), log=logging)
+        run_module(m)
+        assert m.results["Build Version"] == "18C66"
+        assert m.results["IMEI"] == "42"

tests/ios/test_datausage.py

@@ -0,0 +1,31 @@
+# Mobile Verification Toolkit (MVT)
+# Copyright (c) 2021 The MVT Project Authors.
+# Use of this software is governed by the MVT License 1.1 that can be found at
+#   https://license.mvt.re/1.1/
+
+import logging
+
+from mvt.common.indicators import Indicators
+from mvt.common.module import run_module
+from mvt.ios.modules.mixed.net_datausage import Datausage
+
+from ..utils import get_backup_folder
+
+
+class TestDatausageModule:
+    def test_datausage(self):
+        m = Datausage(base_folder=get_backup_folder(), log=logging, results=[])
+        run_module(m)
+        assert len(m.results) == 42
+        assert len(m.timeline) == 60
+        assert len(m.detected) == 0
+
+    def test_detection(self, indicator_file):
+        m = Datausage(base_folder=get_backup_folder(), log=logging, results=[])
+        ind = Indicators(log=logging)
+        ind.parse_stix2(indicator_file)
+        # Adds a file that exists in the manifest.
+        ind.ioc_processes[0] = "CumulativeUsageTracker"
+        m.indicators = ind
+        run_module(m)
+        assert len(m.detected) == 2

tests/ios/test_manifest.py

@@ -0,0 +1,31 @@
+# Mobile Verification Toolkit (MVT)
+# Copyright (c) 2021 The MVT Project Authors.
+# Use of this software is governed by the MVT License 1.1 that can be found at
+#   https://license.mvt.re/1.1/
+
+import logging
+
+from mvt.common.indicators import Indicators
+from mvt.common.module import run_module
+from mvt.ios.modules.backup.manifest import Manifest
+
+from ..utils import get_backup_folder
+
+
+class TestManifestModule:
+    def test_manifest(self):
+        m = Manifest(base_folder=get_backup_folder(), log=logging, results=[])
+        run_module(m)
+        assert len(m.results) == 3721
+        assert len(m.timeline) == 5881
+        assert len(m.detected) == 0
+
+    def test_detection(self, indicator_file):
+        m = Manifest(base_folder=get_backup_folder(), log=logging, results=[])
+        ind = Indicators(log=logging)
+        ind.parse_stix2(indicator_file)
+        # Adds a file that exists in the manifest
+        ind.ioc_files[0] = "com.apple.CoreBrightness.plist"
+        m.indicators = ind
+        run_module(m)
+        assert len(m.detected) == 1

tests/ios/test_tcc.py

@@ -0,0 +1,31 @@
+# Mobile Verification Toolkit (MVT)
+# Copyright (c) 2021 The MVT Project Authors.
+# Use of this software is governed by the MVT License 1.1 that can be found at
+#   https://license.mvt.re/1.1/
+
+import logging
+
+from mvt.common.module import run_module
+from mvt.ios.modules.mixed.tcc import TCC
+
+from ..utils import get_backup_folder
+
+
+class TestManifestModule:
+    def test_manifest(self):
+        m = TCC(base_folder=get_backup_folder(), log=logging, results=[])
+        run_module(m)
+        assert len(m.results) == 11
+        assert len(m.timeline) == 11
+        assert len(m.detected) == 0
+        assert m.results[0]["service"] == "kTCCServiceUbiquity"
+        assert m.results[0]["auth_value"] == "allowed"
+
+    def test_manifest_2(self):
+        m = TCC(base_folder=get_backup_folder(), log=logging, results=[])
+        run_module(m)
+        assert len(m.results) == 11
+        assert len(m.timeline) == 11
+        assert len(m.detected) == 0
+        assert m.results[0]["service"] == "kTCCServiceUbiquity"
+        assert m.results[0]["auth_value"] == "allowed"

tests/utils.py

@@ -0,0 +1,28 @@
+# Mobile Verification Toolkit (MVT)
+# Copyright (c) 2021 The MVT Project Authors.
+# Use of this software is governed by the MVT License 1.1 that can be found at
+#   https://license.mvt.re/1.1/
+
+import os
+
+
+def get_artifact(fname):
+    """
+    Return the artifact path in the artifact folder
+    """
+    fpath = os.path.join(get_artifact_folder(), fname)
+    if os.path.isfile(fpath):
+        return fpath
+    return
+
+
+def get_artifact_folder():
+    return os.path.join(os.path.dirname(__file__), "artifacts")
+
+
+def get_backup_folder():
+    return os.path.join(os.path.dirname(__file__), "artifacts", "ios_backup")
+
+
+def get_indicator_file():
+    print("PYTEST env", os.getenv("PYTEST_CURRENT_TEST"))