Mark release 2.6.0 (#601)

* Adds link in install instructions to the command completion docs added in #597
* Small visual tweaks

docs/command_completion.md

@@ -0,0 +1,43 @@
+# Command Completion 
+
+MVT utilizes the [Click](https://click.palletsprojects.com/en/stable/) library for creating its command line interface. 
+
+Click provides tab completion support for Bash (version 4.4 and up), Zsh, and Fish.
+
+To enable it, you need to manually register a special function with your shell, which varies depending on the shell you are using.
+
+The following describes how to generate the command completion scripts and add them to your shell configuration. 
+
+> **Note: You will need to start a new shell for the changes to take effect.**
+
+### For Bash
+
+```bash
+# Generates bash completion scripts
+echo "$(_MVT_IOS_COMPLETE=bash_source mvt-ios)" > ~/.mvt-ios-complete.bash &&
+echo "$(_MVT_ANDROID_COMPLETE=bash_source mvt-android)" > ~/.mvt-android-complete.bash
+```
+
+Add the following to `~/.bashrc`:
+```bash
+# source mvt completion scripts
+. ~/.mvt-ios-complete.bash && .  ~/.mvt-android-complete.bash
+```
+
+### For Zsh
+
+```bash
+# Generates zsh completion scripts
+echo "$(_MVT_IOS_COMPLETE=zsh_source mvt-ios)" >  ~/.mvt-ios-complete.zsh &&
+echo "$(_MVT_ANDROID_COMPLETE=zsh_source mvt-android)" > ~/.mvt-android-complete.zsh
+```
+
+Add the following to `~/.zshrc`:
+```bash
+# source mvt completion scripts
+.  ~/.mvt-ios-complete.zsh  && .  ~/.mvt-android-complete.zsh
+```
+
+For more information, visit the official [Click Docs](https://click.palletsprojects.com/en/stable/shell-completion/#enabling-completion).
+
+

docs/install.md

@@ -98,3 +98,7 @@ You now should have the `mvt-ios` and `mvt-android` utilities installed.
 **Notes:**
 1. The `--force` flag is necessary to force the reinstallation of the package.
 2. To revert to using a PyPI version, it will be necessary to `pipx uninstall mvt` first.
+
+## Setting up command completions
+
+See ["Command completions"](command_completion.md)

src/mvt/android/artifacts/dumpsys_appops.py

@@ -51,7 +51,7 @@ class DumpsysAppopsArtifact(AndroidArtifact):
                     and perm["access"] == "allow"
                 ):
                     self.log.info(
-                        "Package %s with REQUEST_INSTALL_PACKAGES " "permission",
+                        "Package %s with REQUEST_INSTALL_PACKAGES permission",
                         result["package_name"],
                     )
 

src/mvt/android/artifacts/dumpsys_packages.py

@@ -16,8 +16,7 @@ class DumpsysPackagesArtifact(AndroidArtifact):
         for result in self.results:
             if result["package_name"] in ROOT_PACKAGES:
                 self.log.warning(
-                    "Found an installed package related to "
-                    'rooting/jailbreaking: "%s"',
+                    'Found an installed package related to rooting/jailbreaking: "%s"',
                     result["package_name"],
                 )
                 self.detected.append(result)

src/mvt/android/modules/adb/base.py

@@ -326,8 +326,7 @@ class AndroidExtraction(MVTModule):
 
         if not header["backup"]:
             self.log.error(
-                "Extracting SMS via Android backup failed. "
-                "No valid backup data found."
+                "Extracting SMS via Android backup failed. No valid backup data found."
             )
             return None
 

src/mvt/android/modules/adb/packages.py

@@ -75,8 +75,7 @@ class Packages(AndroidExtraction):
         for result in self.results:
             if result["package_name"] in ROOT_PACKAGES:
                 self.log.warning(
-                    "Found an installed package related to "
-                    'rooting/jailbreaking: "%s"',
+                    'Found an installed package related to rooting/jailbreaking: "%s"',
                     result["package_name"],
                 )
                 self.detected.append(result)

src/mvt/android/modules/adb/sms.py

@@ -70,7 +70,7 @@ class SMS(AndroidExtraction):
             "timestamp": record["isodate"],
             "module": self.__class__.__name__,
             "event": f"sms_{record['direction']}",
-            "data": f"{record.get('address', 'unknown source')}: \"{body}\"",
+            "data": f'{record.get("address", "unknown source")}: "{body}"',
         }
 
     def check_indicators(self) -> None:

src/mvt/android/modules/androidqf/packages.py

@@ -44,8 +44,7 @@ class Packages(AndroidQFModule):
         for result in self.results:
             if result["name"] in ROOT_PACKAGES:
                 self.log.warning(
-                    "Found an installed package related to "
-                    'rooting/jailbreaking: "%s"',
+                    'Found an installed package related to rooting/jailbreaking: "%s"',
                     result["name"],
                 )
                 self.detected.append(result)

src/mvt/common/command.py

@@ -81,7 +81,7 @@ class Command:
             os.path.join(self.results_path, "command.log")
         )
         formatter = logging.Formatter(
-            "%(asctime)s - %(name)s - " "%(levelname)s - %(message)s"
+            "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
         )
         file_handler.setLevel(logging.DEBUG)
         file_handler.setFormatter(formatter)

src/mvt/common/indicators.py

@@ -383,8 +383,7 @@ class Indicators:
         for ioc in self.get_iocs("urls"):
             if ioc["value"] == url:
                 self.log.warning(
-                    "Found a known suspicious URL %s "
-                    'matching indicator "%s" from "%s"',
+                    'Found a known suspicious URL %s matching indicator "%s" from "%s"',
                     url,
                     ioc["value"],
                     ioc["name"],

src/mvt/common/version.py

@@ -3,4 +3,4 @@
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
-MVT_VERSION = "2.5.4"
+MVT_VERSION = "2.6.0"

src/mvt/ios/cli.py

@@ -100,7 +100,7 @@ def decrypt_backup(ctx, destination, password, key_file, hashes, backup_path):
     if key_file:
         if MVT_IOS_BACKUP_PASSWORD in os.environ:
             log.info(
-                "Ignoring %s environment variable, using --key-file" "'%s' instead",
+                "Ignoring %s environment variable, using --key-file'%s' instead",
                 MVT_IOS_BACKUP_PASSWORD,
                 key_file,
             )
@@ -114,7 +114,7 @@ def decrypt_backup(ctx, destination, password, key_file, hashes, backup_path):
 
         if MVT_IOS_BACKUP_PASSWORD in os.environ:
             log.info(
-                "Ignoring %s environment variable, using --password" "argument instead",
+                "Ignoring %s environment variable, using --passwordargument instead",
                 MVT_IOS_BACKUP_PASSWORD,
             )
 
@@ -168,8 +168,7 @@ def extract_key(password, key_file, backup_path):
 
         if MVT_IOS_BACKUP_PASSWORD in os.environ:
             log.info(
-                "Ignoring %s environment variable, using --password "
-                "argument instead",
+                "Ignoring %s environment variable, using --password argument instead",
                 MVT_IOS_BACKUP_PASSWORD,
             )
     elif MVT_IOS_BACKUP_PASSWORD in os.environ:

src/mvt/ios/data/ios_versions.json

@@ -1095,5 +1095,9 @@
     {
         "version": "18.2",
         "build": "22C152"
+    },
+    {
+        "version": "18.2.1",
+        "build": "22C161"
     }
 ]

src/mvt/ios/modules/backup/backup_info.py

@@ -41,7 +41,7 @@ class BackupInfo(IOSExtraction):
         info_path = os.path.join(self.target_path, "Info.plist")
         if not os.path.exists(info_path):
             raise DatabaseNotFoundError(
-                "No Info.plist at backup path, unable to extract device " "information"
+                "No Info.plist at backup path, unable to extract device information"
             )
 
         with open(info_path, "rb") as handle:

src/mvt/ios/modules/backup/manifest.py

@@ -110,8 +110,7 @@ class Manifest(IOSExtraction):
                 ioc = self.indicators.check_url(part)
                 if ioc:
                     self.log.warning(
-                        'Found mention of domain "%s" in a backup file with '
-                        "path: %s",
+                        'Found mention of domain "%s" in a backup file with path: %s',
                         ioc["value"],
                         rel_path,
                     )

src/mvt/ios/modules/base.py

@@ -74,7 +74,7 @@ class IOSExtraction(MVTModule):
 
         if not shutil.which("sqlite3"):
             raise DatabaseCorruptedError(
-                "failed to recover without sqlite3 binary: please install " "sqlite3!"
+                "failed to recover without sqlite3 binary: please install sqlite3!"
             )
         if '"' in file_path:
             raise DatabaseCorruptedError(

src/mvt/ios/modules/mixed/sms.py

@@ -43,7 +43,7 @@ class SMS(IOSExtraction):
 
     def serialize(self, record: dict) -> Union[dict, list]:
         text = record["text"].replace("\n", "\\n")
-        sms_data = f"{record['service']}: {record['guid']} \"{text}\" from {record['phone_number']} ({record['account']})"
+        sms_data = f'{record["service"]}: {record["guid"]} "{text}" from {record["phone_number"]} ({record["account"]})'
         records = [
             {
                 "timestamp": record["isodate"],

src/mvt/ios/modules/mixed/webkit_session_resource_log.py

@@ -100,7 +100,7 @@ class WebkitSessionResourceLog(IOSExtraction):
                         redirect_path += ", ".join(source_domains)
                         redirect_path += " -> "
 
-                    redirect_path += f"ORIGIN: \"{entry['origin']}\""
+                    redirect_path += f'ORIGIN: "{entry["origin"]}"'
 
                     if len(destination_domains) > 0:
                         redirect_path += " -> "

src/mvt/ios/modules/net_base.py

@@ -38,44 +38,70 @@ class NetBase(IOSExtraction):
 
     def _extract_net_data(self):
         conn = sqlite3.connect(self.file_path)
+        conn.row_factory = sqlite3.Row
         cur = conn.cursor()
-        cur.execute(
+        try:
+            cur.execute(
+                """
+                SELECT
+                    ZPROCESS.ZFIRSTTIMESTAMP,
+                    ZPROCESS.ZTIMESTAMP,
+                    ZPROCESS.ZPROCNAME,
+                    ZPROCESS.ZBUNDLENAME,
+                    ZPROCESS.Z_PK AS ZPROCESS_PK,
+                    ZLIVEUSAGE.ZWIFIIN,
+                    ZLIVEUSAGE.ZWIFIOUT,
+                    ZLIVEUSAGE.ZWWANIN,
+                    ZLIVEUSAGE.ZWWANOUT,
+                    ZLIVEUSAGE.Z_PK AS ZLIVEUSAGE_PK,
+                    ZLIVEUSAGE.ZHASPROCESS,
+                    ZLIVEUSAGE.ZTIMESTAMP AS ZL_TIMESTAMP
+                FROM ZLIVEUSAGE
+                LEFT JOIN ZPROCESS ON ZLIVEUSAGE.ZHASPROCESS = ZPROCESS.Z_PK
+                UNION
+                SELECT ZFIRSTTIMESTAMP, ZTIMESTAMP, ZPROCNAME, ZBUNDLENAME, Z_PK,
+                    NULL, NULL, NULL, NULL, NULL, NULL, NULL
+                FROM ZPROCESS WHERE Z_PK NOT IN
+                    (SELECT ZHASPROCESS FROM ZLIVEUSAGE);
             """
-            SELECT
-                ZPROCESS.ZFIRSTTIMESTAMP,
-                ZPROCESS.ZTIMESTAMP,
-                ZPROCESS.ZPROCNAME,
-                ZPROCESS.ZBUNDLENAME,
-                ZPROCESS.Z_PK,
-                ZLIVEUSAGE.ZWIFIIN,
-                ZLIVEUSAGE.ZWIFIOUT,
-                ZLIVEUSAGE.ZWWANIN,
-                ZLIVEUSAGE.ZWWANOUT,
-                ZLIVEUSAGE.Z_PK,
-                ZLIVEUSAGE.ZHASPROCESS,
-                ZLIVEUSAGE.ZTIMESTAMP
-            FROM ZLIVEUSAGE
-            LEFT JOIN ZPROCESS ON ZLIVEUSAGE.ZHASPROCESS = ZPROCESS.Z_PK
-            UNION
-            SELECT ZFIRSTTIMESTAMP, ZTIMESTAMP, ZPROCNAME, ZBUNDLENAME, Z_PK,
-                   NULL, NULL, NULL, NULL, NULL, NULL, NULL
-            FROM ZPROCESS WHERE Z_PK NOT IN
-                (SELECT ZHASPROCESS FROM ZLIVEUSAGE);
-        """
-        )
+            )
+        except sqlite3.OperationalError:
+            # Recent phones don't have ZWIFIIN and ZWIFIOUT columns
+            cur.execute(
+                """
+                SELECT
+                    ZPROCESS.ZFIRSTTIMESTAMP,
+                    ZPROCESS.ZTIMESTAMP,
+                    ZPROCESS.ZPROCNAME,
+                    ZPROCESS.ZBUNDLENAME,
+                    ZPROCESS.Z_PK AS ZPROCESS_PK,
+                    ZLIVEUSAGE.ZWWANIN,
+                    ZLIVEUSAGE.ZWWANOUT,
+                    ZLIVEUSAGE.Z_PK AS ZLIVEUSAGE_PK,
+                    ZLIVEUSAGE.ZHASPROCESS,
+                    ZLIVEUSAGE.ZTIMESTAMP AS ZL_TIMESTAMP
+                FROM ZLIVEUSAGE
+                LEFT JOIN ZPROCESS ON ZLIVEUSAGE.ZHASPROCESS = ZPROCESS.Z_PK
+                UNION
+                SELECT ZFIRSTTIMESTAMP, ZTIMESTAMP, ZPROCNAME, ZBUNDLENAME, Z_PK,
+                    NULL, NULL, NULL, NULL, NULL
+                FROM ZPROCESS WHERE Z_PK NOT IN
+                    (SELECT ZHASPROCESS FROM ZLIVEUSAGE);
+            """
+            )
 
         for row in cur:
             # ZPROCESS records can be missing after the JOIN.
             # Handle NULL timestamps.
-            if row[0] and row[1]:
-                first_isodate = convert_mactime_to_iso(row[0])
-                isodate = convert_mactime_to_iso(row[1])
+            if row["ZFIRSTTIMESTAMP"] and row["ZTIMESTAMP"]:
+                first_isodate = convert_mactime_to_iso(row["ZFIRSTTIMESTAMP"])
+                isodate = convert_mactime_to_iso(row["ZTIMESTAMP"])
             else:
-                first_isodate = row[0]
-                isodate = row[1]
+                first_isodate = row["ZFIRSTTIMESTAMP"]
+                isodate = row["ZTIMESTAMP"]
 
-            if row[11]:
-                live_timestamp = convert_mactime_to_iso(row[11])
+            if row["ZL_TIMESTAMP"]:
+                live_timestamp = convert_mactime_to_iso(row["ZL_TIMESTAMP"])
             else:
                 live_timestamp = ""
 
@@ -83,16 +109,18 @@ class NetBase(IOSExtraction):
                 {
                     "first_isodate": first_isodate,
                     "isodate": isodate,
-                    "proc_name": row[2],
-                    "bundle_id": row[3],
-                    "proc_id": row[4],
-                    "wifi_in": row[5],
-                    "wifi_out": row[6],
-                    "wwan_in": row[7],
-                    "wwan_out": row[8],
-                    "live_id": row[9],
-                    "live_proc_id": row[10],
-                    "live_isodate": live_timestamp if row[11] else first_isodate,
+                    "proc_name": row["ZPROCNAME"],
+                    "bundle_id": row["ZBUNDLENAME"],
+                    "proc_id": row["ZPROCESS_PK"],
+                    "wifi_in": row["ZWIFIIN"] if "ZWIFIIN" in row.keys() else None,
+                    "wifi_out": row["ZWIFIOUT"] if "ZWIFIOUT" in row.keys() else None,
+                    "wwan_in": row["ZWWANIN"],
+                    "wwan_out": row["ZWWANOUT"],
+                    "live_id": row["ZLIVEUSAGE_PK"],
+                    "live_proc_id": row["ZHASPROCESS"],
+                    "live_isodate": live_timestamp
+                    if row["ZL_TIMESTAMP"]
+                    else first_isodate,
                 }
             )
 
@@ -108,8 +136,6 @@ class NetBase(IOSExtraction):
         )
         record_data_usage = (
             record_data + " "
-            f"WIFI IN: {record['wifi_in']}, "
-            f"WIFI OUT: {record['wifi_out']} - "
             f"WWAN IN: {record['wwan_in']}, "
             f"WWAN OUT: {record['wwan_out']}"
         )