Reverted version number to minor

Checking idstatuscache.plist in a dump for iOS>14.7

mvt/android/cli.py

@@ -11,6 +11,7 @@ from rich.logging import RichHandler
 
 from mvt.common.help import *
 from mvt.common.indicators import Indicators, IndicatorsFileBadFormat
+from mvt.common.logo import logo
 from mvt.common.module import run_module, save_timeline
 
 from .download_apks import DownloadAPKs
@@ -30,7 +31,7 @@ log = logging.getLogger(__name__)
 #==============================================================================
 @click.group(invoke_without_command=False)
 def cli():
-    return
+    logo()
 
 
 #==============================================================================

mvt/android/lookups/koodous.py

@@ -27,12 +27,12 @@ def koodous_lookup(packages):
     total_packages = len(packages)
     for i in track(range(total_packages), description=f"Looking up {total_packages} packages..."):
         package = packages[i]
-        for file in package.files:
+        for file in package.get("files", []):
             url = f"https://api.koodous.com/apks/{file['sha256']}"
             res = requests.get(url)
             report = res.json()
 
-            row = [package.name, file["local_name"]]
+            row = [package["package_name"], file["local_name"]]
 
             if "package_name" in report:
                 trusted = "no"

mvt/android/lookups/virustotal.py

@@ -41,7 +41,7 @@ def virustotal_lookup(packages):
 
     unique_hashes = []
     for package in packages:
-        for file in package.files:
+        for file in package.get("files", []):
             if file["sha256"] not in unique_hashes:
                 unique_hashes.append(file["sha256"])
 
@@ -74,8 +74,8 @@ def virustotal_lookup(packages):
     table.add_column("Detections")
 
     for package in packages:
-        for file in package.files:
-            row = [package.name, file["local_name"]]
+        for file in package.get("files", []):
+            row = [package["package_name"], file["local_name"]]
 
             if file["sha256"] in detections:
                 detection = detections[file["sha256"]]

mvt/android/modules/adb/__init__.py

@@ -5,8 +5,12 @@
 
 from .chrome_history import ChromeHistory
 from .dumpsys_batterystats import DumpsysBatterystats
+from .dumpsys_full import DumpsysFull
 from .dumpsys_packages import DumpsysPackages
 from .dumpsys_procstats import DumpsysProcstats
+from .dumpsys_receivers import DumpsysReceivers
+from .files import Files
+from .logcat import Logcat
 from .packages import Packages
 from .processes import Processes
 from .rootbinaries import RootBinaries
@@ -15,4 +19,5 @@ from .whatsapp import Whatsapp
 
 ADB_MODULES = [ChromeHistory, SMS, Whatsapp, Processes,
                DumpsysBatterystats, DumpsysProcstats,
-               DumpsysPackages, Packages, RootBinaries]
+               DumpsysPackages, DumpsysReceivers, DumpsysFull,
+               Packages, RootBinaries, Logcat, Files]

mvt/android/modules/adb/base.py

@@ -37,9 +37,13 @@ class AndroidExtraction(MVTModule):
         self.device = None
         self.serial = None
 
-    def _adb_check_keys(self):
+    @staticmethod
+    def _adb_check_keys():
         """Make sure Android adb keys exist.
         """
+        if not os.path.isdir(os.path.dirname(ADB_KEY_PATH)):
+            os.path.makedirs(os.path.dirname(ADB_KEY_PATH))
+
         if not os.path.exists(ADB_KEY_PATH):
             keygen(ADB_KEY_PATH)
 

mvt/android/modules/adb/dumpsys_full.py

@@ -0,0 +1,35 @@
+# Mobile Verification Toolkit (MVT)
+# Copyright (c) 2021 The MVT Project Authors.
+# Use of this software is governed by the MVT License 1.1 that can be found at
+#   https://license.mvt.re/1.1/
+
+import logging
+import os
+
+from .base import AndroidExtraction
+
+log = logging.getLogger(__name__)
+
+class DumpsysFull(AndroidExtraction):
+    """This module extracts stats on battery consumption by processes."""
+
+    def __init__(self, file_path=None, base_folder=None, output_folder=None,
+                 serial=None, fast_mode=False, log=None, results=[]):
+        super().__init__(file_path=file_path, base_folder=base_folder,
+                         output_folder=output_folder, fast_mode=fast_mode,
+                         log=log, results=results)
+
+    def run(self):
+        self._adb_connect()
+
+        stats = self._adb_command("dumpsys")
+        if self.output_folder:
+            stats_path = os.path.join(self.output_folder,
+                                      "dumpsys.txt")
+            with open(stats_path, "w") as handle:
+                handle.write(stats)
+
+            log.info("Full dumpsys output stored at %s",
+                      stats_path)
+
+        self._adb_disconnect()

mvt/android/modules/adb/dumpsys_packages.py

@@ -10,8 +10,9 @@ from .base import AndroidExtraction
 
 log = logging.getLogger(__name__)
 
+
 class DumpsysPackages(AndroidExtraction):
-    """This module extracts stats on installed packages."""
+    """This module extracts details on installed packages."""
 
     def __init__(self, file_path=None, base_folder=None, output_folder=None,
                  serial=None, fast_mode=False, log=None, results=[]):
@@ -23,6 +24,7 @@ class DumpsysPackages(AndroidExtraction):
         self._adb_connect()
 
         output = self._adb_command("dumpsys package")
+
         if self.output_folder:
             packages_path = os.path.join(self.output_folder,
                                          "dumpsys_packages.txt")

mvt/android/modules/adb/dumpsys_receivers.py

@@ -0,0 +1,87 @@
+# Mobile Verification Toolkit (MVT)
+# Copyright (c) 2021 The MVT Project Authors.
+# Use of this software is governed by the MVT License 1.1 that can be found at
+#   https://license.mvt.re/1.1/
+
+import logging
+import os
+
+from .base import AndroidExtraction
+
+log = logging.getLogger(__name__)
+
+ACTION_NEW_OUTGOING_SMS = "android.provider.Telephony.NEW_OUTGOING_SMS"
+ACTION_SMS_RECEIVED = "android.provider.Telephony.SMS_RECEIVED"
+ACTION_DATA_SMS_RECEIVED = "android.intent.action.DATA_SMS_RECEIVED"
+ACTION_PHONE_STATE = "android.intent.action.PHONE_STATE"
+
+class DumpsysReceivers(AndroidExtraction):
+    """This module extracts details on receivers for risky activities."""
+
+    def __init__(self, file_path=None, base_folder=None, output_folder=None,
+                 serial=None, fast_mode=False, log=None, results=[]):
+        super().__init__(file_path=file_path, base_folder=base_folder,
+                         output_folder=output_folder, fast_mode=fast_mode,
+                         log=log, results=results)
+
+    def run(self):
+        self._adb_connect()
+
+        output = self._adb_command("dumpsys package")
+        if not output:
+            return
+
+        activity = None
+        for line in output.split("\n"):
+            # Find activity block markers.
+            if line.strip().startswith(ACTION_NEW_OUTGOING_SMS):
+                activity = ACTION_NEW_OUTGOING_SMS
+                continue
+            elif line.strip().startswith(ACTION_SMS_RECEIVED):
+                activity = ACTION_SMS_RECEIVED
+                continue
+            elif line.strip().startswith(ACTION_PHONE_STATE):
+                activity = ACTION_PHONE_STATE
+                continue
+            elif line.strip().startswith(ACTION_DATA_SMS_RECEIVED):
+                activity = ACTION_DATA_SMS_RECEIVED
+                continue
+
+            # If we are not in an activity block yet, skip.
+            if not activity:
+                continue
+
+            # If we are in a block but the line does not start with 8 spaces
+            # it means the block ended a new one started, so we reset and
+            # continue.
+            if not line.startswith(" " * 8):
+                activity = None
+                continue
+
+            # If we got this far, we are processing receivers for the
+            # activities we are interested in.
+            receiver = line.strip().split(" ")[1]
+            package_name = receiver.split("/")[0]
+            if package_name == "com.google.android.gms":
+                continue
+
+            if activity == ACTION_NEW_OUTGOING_SMS:
+                self.log.info("Found a receiver to intercept outgoing SMS messages: \"%s\"",
+                    receiver)
+            elif activity == ACTION_SMS_RECEIVED:
+                self.log.info("Found a receiver to intercept incoming SMS messages: \"%s\"",
+                    receiver)
+            elif activity == ACTION_DATA_SMS_RECEIVED:
+                self.log.info("Found a receiver to intercept incoming data SMS message: \"%s\"",
+                    receiver)
+            elif activity == ACTION_PHONE_STATE:
+                self.log.info("Found a receiver monitoring telephony state: \"%s\"",
+                    receiver)
+
+            self.results.append({
+                "activity": activity,
+                "package_name": package_name,
+                "receiver": receiver,
+            })
+
+        self._adb_disconnect()

mvt/android/modules/adb/files.py

@@ -0,0 +1,33 @@
+# Mobile Verification Toolkit (MVT)
+# Copyright (c) 2021 The MVT Project Authors.
+# Use of this software is governed by the MVT License 1.1 that can be found at
+#   https://license.mvt.re/1.1/
+
+import logging
+import os
+
+from .base import AndroidExtraction
+
+log = logging.getLogger(__name__)
+
+class Files(AndroidExtraction):
+    """This module extracts the list of installed packages."""
+
+    def __init__(self, file_path=None, base_folder=None, output_folder=None,
+                 serial=None, fast_mode=False, log=None, results=[]):
+        super().__init__(file_path=file_path, base_folder=base_folder,
+                         output_folder=output_folder, fast_mode=fast_mode,
+                         log=log, results=results)
+
+    def run(self):
+        self._adb_connect()
+
+        output = self._adb_command("find / -type f 2> /dev/null")
+        if output and self.output_folder:
+            files_txt_path = os.path.join(self.output_folder, "files.txt")
+            with open(files_txt_path, "w") as handle:
+                handle.write(output)
+
+            log.info("List of visible files stored at %s", files_txt_path)
+
+        self._adb_disconnect()

mvt/android/modules/adb/logcat.py

@@ -0,0 +1,48 @@
+# Mobile Verification Toolkit (MVT)
+# Copyright (c) 2021 The MVT Project Authors.
+# Use of this software is governed by the MVT License 1.1 that can be found at
+#   https://license.mvt.re/1.1/
+
+import logging
+import os
+
+from .base import AndroidExtraction
+
+log = logging.getLogger(__name__)
+
+
+class Logcat(AndroidExtraction):
+    """This module extracts details on installed packages."""
+
+    def __init__(self, file_path=None, base_folder=None, output_folder=None,
+                 serial=None, fast_mode=False, log=None, results=[]):
+        super().__init__(file_path=file_path, base_folder=base_folder,
+                         output_folder=output_folder, fast_mode=fast_mode,
+                         log=log, results=results)
+
+    def run(self):
+        self._adb_connect()
+
+        # Get the current logcat.
+        output = self._adb_command("logcat -d")
+        # Get the locat prior to last reboot.
+        last_output = self._adb_command("logcat -L")
+
+        if self.output_folder:
+            logcat_path = os.path.join(self.output_folder,
+                                       "logcat.txt")
+            with open(logcat_path, "w") as handle:
+                handle.write(output)
+
+            log.info("Current logcat logs stored at %s",
+                     logcat_path)
+
+            logcat_last_path = os.path.join(self.output_folder,
+                                            "logcat_last.txt")
+            with open(logcat_last_path, "w") as handle:
+                handle.write(last_output)
+
+            log.info("Logcat logs prior to last reboot stored at %s",
+                     logcat_last_path)
+
+        self._adb_disconnect()

mvt/common/logo.py

@@ -0,0 +1,25 @@
+# Mobile Verification Toolkit (MVT)
+# Copyright (c) 2021 The MVT Project Authors.
+# Use of this software is governed by the MVT License 1.1 that can be found at
+#   https://license.mvt.re/1.1/
+
+from rich import print
+
+from .version import MVT_VERSION, check_for_updates
+
+
+def logo():
+    print("\n")
+    print("\t[bold]MVT[/bold] - Mobile Verification Toolkit")
+    print("\t\thttps://mvt.re")
+    print(f"\t\tVersion: {MVT_VERSION}")
+
+    try:
+        latest_version = check_for_updates()
+    except:
+        pass
+    else:
+        if latest_version:
+            print(f"\t\t[bold]Version {latest_version} is available! Upgrade mvt![/bold]")
+
+    print("\n")

mvt/common/module.py

@@ -100,6 +100,14 @@ class MVTModule(object):
     def serialize(self, record):
         raise NotImplementedError
 
+    @staticmethod
+    def _deduplicate_timeline(timeline):
+        """Serialize entry as JSON to deduplicate repeated entries"""
+        timeline_set = set()
+        for record in timeline:
+            timeline_set.add(json.dumps(record, sort_keys=True))
+        return [json.loads(record) for record in timeline_set]
+
     def to_timeline(self):
         """Convert results into a timeline.
         """
@@ -120,15 +128,8 @@ class MVTModule(object):
                     self.timeline_detected.append(record)
 
         # De-duplicate timeline entries.
-        self.timeline = self.timeline_deduplicate(self.timeline)
-        self.timeline_detected = self.timeline_deduplicate(self.timeline_detected)
-
-    def timeline_deduplicate(self, timeline):
-        """Serialize entry as JSON to deduplicate repeated entries"""
-        timeline_set = set()
-        for record in timeline:
-            timeline_set.add(json.dumps(record, sort_keys=True))
-        return [json.loads(record) for record in timeline_set]
+        self.timeline = self._deduplicate_timeline(self.timeline)
+        self.timeline_detected = self._deduplicate_timeline(self.timeline_detected)
 
     def run(self):
         """Run the main module procedure.
@@ -150,7 +151,7 @@ def run_module(module):
         module.log.info("There might be no data to extract by module %s: %s",
                         module.__class__.__name__, e)
     except DatabaseCorruptedError as e:
-        module.log.error("The %s module database seems to be corrupted and recovery failed: %s",
+        module.log.error("The %s module database seems to be corrupted: %s",
                          module.__class__.__name__, e)
     except Exception as e:
         module.log.exception("Error in running extraction from module %s: %s",

mvt/common/version.py

@@ -0,0 +1,19 @@
+# Mobile Verification Toolkit (MVT)
+# Copyright (c) 2021 The MVT Project Authors.
+# Use of this software is governed by the MVT License 1.1 that can be found at
+#   https://license.mvt.re/1.1/
+
+import requests
+from packaging import version
+
+MVT_VERSION = "1.2.6"
+
+def check_for_updates():
+    res = requests.get("https://pypi.org/pypi/mvt/json")
+    data = res.json()
+    latest_version = data.get("info", {}).get("version", "")
+
+    if version.parse(latest_version) > version.parse(MVT_VERSION):
+        return latest_version
+
+    return None

mvt/ios/cli.py

@@ -12,6 +12,7 @@ from rich.prompt import Prompt
 
 from mvt.common.help import *
 from mvt.common.indicators import Indicators, IndicatorsFileBadFormat
+from mvt.common.logo import logo
 from mvt.common.module import run_module, save_timeline
 from mvt.common.options import MutuallyExclusiveOption
 
@@ -34,7 +35,7 @@ PASSWD_ENV = "MVT_IOS_BACKUP_PASSWORD"
 #==============================================================================
 @click.group(invoke_without_command=False)
 def cli():
-    return
+    logo()
 
 
 #==============================================================================

mvt/ios/decrypt.py

@@ -32,7 +32,8 @@ class DecryptBackup:
     def can_process(self) -> bool:
         return self._backup is not None
 
-    def is_encrypted(self, backup_path) -> bool:
+    @staticmethod
+    def is_encrypted(backup_path) -> bool:
         """Query Manifest.db file to see if it's encrypted or not.
         :param backup_path: Path to the backup to decrypt
         """

mvt/ios/modules/backup/backup_info.py

@@ -30,9 +30,9 @@ class BackupInfo(IOSExtraction):
         with open(info_path, "rb") as handle:
             info = plistlib.load(handle)
 
-        fields = ["Build Version", "Device Name", "Display Name", "GUID",
+        fields = ["Build Version", "Device Name", "Display Name",
                   "GUID", "ICCID", "IMEI", "MEID", "Installed Applications",
-                  "Last Backup Data", "Phone Number", "Product Name",
+                  "Last Backup Date", "Phone Number", "Product Name",
                   "Product Type", "Product Version", "Serial Number",
                   "Target Identifier", "Target Type", "Unique Identifier",
                   "iTunes Version"]

mvt/ios/modules/backup/manifest.py

@@ -30,7 +30,8 @@ class Manifest(IOSExtraction):
         """
         return dictionary.get(key.encode("utf-8"), None) or dictionary.get(key, None)
 
-    def _convert_timestamp(self, timestamp_or_unix_time_int):
+    @staticmethod
+    def _convert_timestamp(timestamp_or_unix_time_int):
         """Older iOS versions stored the manifest times as unix timestamps.
         """
         if isinstance(timestamp_or_unix_time_int, datetime.datetime):
@@ -90,7 +91,7 @@ class Manifest(IOSExtraction):
     def run(self):
         manifest_db_path = os.path.join(self.base_folder, "Manifest.db")
         if not os.path.isfile(manifest_db_path):
-            raise DatabaseNotFoundError("Impossible to find the module's database file")
+            raise DatabaseNotFoundError("unable to find backup's Manifest.db")
 
         self.log.info("Found Manifest.db database at path: %s", manifest_db_path)
 

mvt/ios/modules/base.py

@@ -49,9 +49,9 @@ class IOSExtraction(MVTModule):
         self.log.info("Database at path %s is malformed. Trying to recover...", file_path)
 
         if not shutil.which("sqlite3"):
-            raise DatabaseCorruptedError("Unable to recover without sqlite3 binary. Please install sqlite3!")
+            raise DatabaseCorruptedError("failed to recover without sqlite3 binary: please install sqlite3!")
         if '"' in file_path:
-            raise DatabaseCorruptedError(f"Database at path '{file_path}' is corrupted. unable to recover because it has a quotation mark (\") in its name.")
+            raise DatabaseCorruptedError(f"database at path '{file_path}' is corrupted. unable to recover because it has a quotation mark (\") in its name")
 
         bak_path = f"{file_path}.bak"
         shutil.move(file_path, bak_path)
@@ -59,7 +59,7 @@ class IOSExtraction(MVTModule):
         ret = subprocess.call(["sqlite3", bak_path, f".clone \"{file_path}\""],
                               stdout=subprocess.PIPE, stderr=subprocess.PIPE)
         if ret != 0:
-            raise DatabaseCorruptedError("Recovery of database failed")
+            raise DatabaseCorruptedError("failed to recover database")
 
         self.log.info("Database at path %s recovered successfully!", file_path)
 
@@ -70,7 +70,7 @@ class IOSExtraction(MVTModule):
         """
         manifest_db_path = os.path.join(self.base_folder, "Manifest.db")
         if not os.path.exists(manifest_db_path):
-            raise Exception("Unable to find backup's Manifest.db")
+            raise DatabaseNotFoundError("unable to find backup's Manifest.db")
 
         base_sql = "SELECT fileID, domain, relativePath FROM Files WHERE "
 
@@ -86,7 +86,7 @@ class IOSExtraction(MVTModule):
                 elif domain:
                     cur.execute(f"{base_sql} domain = ?;", (domain,))
         except Exception as e:
-            raise Exception("Query to Manifest.db failed: %s", e)
+            raise DatabaseCorruptedError("failed to query Manifest.db: %s", e)
 
         for row in cur:
             yield {
@@ -144,6 +144,6 @@ class IOSExtraction(MVTModule):
         if file_path:
             self.file_path = file_path
         else:
-            raise DatabaseNotFoundError("Unable to find the module's database file")
+            raise DatabaseNotFoundError("unable to find the module's database file")
 
         self._recover_sqlite_db_if_needed(self.file_path)

mvt/ios/modules/mixed/idstatuscache.py

@@ -15,6 +15,7 @@ IDSTATUSCACHE_BACKUP_IDS = [
 ]
 IDSTATUSCACHE_ROOT_PATHS = [
     "private/var/mobile/Library/Preferences/com.apple.identityservices.idstatuscache.plist",
+    "private/var/mobile/Library/IdentityServices/idstatuscache.plist",
 ]
 
 class IDStatusCache(IOSExtraction):

mvt/ios/modules/mixed/safari_history.py

@@ -80,6 +80,7 @@ class SafariHistory(IOSExtraction):
                 self.detected.append(result)
 
     def _process_history_db(self, history_path):
+        self._recover_sqlite_db_if_needed(history_path)
         conn = sqlite3.connect(history_path)
         cur = conn.cursor()
         cur.execute("""

mvt/ios/modules/mixed/webkit_resource_load_statistics.py

@@ -75,7 +75,7 @@ class WebkitResourceLoadStatistics(IOSExtraction):
         if self.is_backup:
             try:
                 for backup_file in self._get_backup_files_from_manifest(relative_path=WEBKIT_RESOURCELOADSTATICS_BACKUP_RELPATH):
-                    db_path = os.path.join(self.base_folder, backup_file["file_id"][0:2], backup_file["file_id"])
+                    db_path = self._get_backup_file_from_id(backup_file["file_id"])
                     key = f"{backup_file['domain']}/{WEBKIT_RESOURCELOADSTATICS_BACKUP_RELPATH}"
                     self._process_observations_db(db_path=db_path, key=key)
             except Exception as e:

mvt/ios/modules/mixed/webkit_session_resource_log.py

@@ -113,7 +113,10 @@ class WebkitSessionResourceLog(IOSExtraction):
 
     def run(self):
         if self.is_backup:
-            for log_path in self._get_backup_files_from_manifest(relative_path=WEBKIT_SESSION_RESOURCE_LOG_BACKUP_RELPATH):
+            for log_file in self._get_backup_files_from_manifest(relative_path=WEBKIT_SESSION_RESOURCE_LOG_BACKUP_RELPATH):
+                log_path = self._get_backup_file_from_id(log_file["file_id"])
+                if not log_path:
+                    continue
                 self.log.info("Found Safari browsing session resource log at path: %s", log_path)
                 self.results[log_path] = self._extract_browsing_stats(log_path)
         elif self.is_fs_dump:

setup.py

@@ -7,9 +7,7 @@ import os
 
 from setuptools import find_packages, setup
 
-__package_name__ = "mvt"
-__version__ = "1.2.2"
-__description__ = "Mobile Verification Toolkit"
+from mvt.common.version import MVT_VERSION
 
 this_directory = os.path.abspath(os.path.dirname(__file__))
 readme_path = os.path.join(this_directory, "README.md")
@@ -24,6 +22,7 @@ requires = (
     "tqdm>=4.61.2",
     "requests>=2.26.0",
     "simplejson>=3.17.3",
+    "packaging>=21.0",
     # iOS dependencies:
     "iOSbackup>=0.9.912",
     # Android dependencies:
@@ -43,9 +42,9 @@ def get_package_data(package):
     return {package: filepaths}
 
 setup(
-    name=__package_name__,
-    version=__version__,
-    description=__description__,
+    name="mvt",
+    version=MVT_VERSION,
+    description="Mobile Verification Toolkit",
     long_description=long_description,
     long_description_content_type="text/markdown",
     url="https://github.com/mvt-project/mvt",