Bumped version

GitHub Action to lint Python code

.github/workflows/lint-python.yml

@@ -0,0 +1,21 @@
+name: lint_python
+on: [pull_request, push]
+jobs:
+  lint_python:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+      - run: pip install bandit black codespell flake8 isort mypy pytest pyupgrade safety
+      - run: bandit --recursive --skip B108,B112,B404,B602 .
+      - run: black --check .  || true
+      - run: codespell
+      - run: flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+      - run: flake8 . --count --exit-zero --max-complexity=10 --max-line-length=88 --show-source --statistics
+      - run: isort --check-only --profile black . || true
+      - run: pip install -r requirements.txt || true
+      - run: mypy --install-types --non-interactive . || true
+      - run: pytest . || true
+      - run: pytest --doctest-modules . || true
+      - run: shopt -s globstar && pyupgrade --py36-plus **/*.py || true
+      - run: safety check

.github/workflows/python-publish.yml

@@ -0,0 +1,36 @@
+# This workflow will upload a Python Package using Twine when a release is created
+# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries
+
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+
+name: Upload Python Package
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  deploy:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.x'
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install build
+    - name: Build package
+      run: python -m build
+    - name: Publish package
+      uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29
+      with:
+        user: __token__
+        password: ${{ secrets.PYPI_API_TOKEN }}

Dockerfile

@@ -0,0 +1,65 @@
+FROM ubuntu:20.04
+
+# Ref. https://github.com/mvt-project/mvt
+
+# Fixing major OS dependencies
+# ----------------------------
+RUN apt update \
+  && apt install -y python3 python3-pip libusb-1.0-0-dev \
+  && apt install -y wget \ 
+  && apt install -y adb \ 
+  && DEBIAN_FRONTEND=noninteractive apt-get -y install default-jre-headless
+
+# Install build tools for libimobiledevice
+# ----------------------------------------
+RUN apt install -y build-essential \
+	checkinstall \
+	git \
+	autoconf \
+	automake \
+	libtool-bin \
+	libplist-dev \
+	libusbmuxd-dev \
+	libssl-dev \
+    pkg-config
+
+# Clean up
+# --------
+RUN apt-get clean \
+  && rm -rf /var/lib/apt/lists/*
+
+
+# Build libimobiledevice
+# ----------------------
+RUN git clone https://github.com/libimobiledevice/libplist
+RUN git clone https://github.com/libimobiledevice/libusbmuxd
+RUN git clone https://github.com/libimobiledevice/libimobiledevice
+RUN git clone https://github.com/libimobiledevice/usbmuxd
+
+RUN cd libplist && ./autogen.sh && make && make install && ldconfig
+
+RUN cd libusbmuxd && PKG_CONFIG_PATH=/usr/local/lib/pkgconfig ./autogen.sh && make && make install && ldconfig
+
+RUN cd libimobiledevice && PKG_CONFIG_PATH=/usr/local/lib/pkgconfig ./autogen.sh --enable-debug && make && make install && ldconfig
+
+RUN cd usbmuxd && PKG_CONFIG_PATH=/usr/local/lib/pkgconfig ./autogen.sh --prefix=/usr --sysconfdir=/etc --localstatedir=/var --runstatedir=/run && make && make install
+
+# Installing MVT
+# --------------
+RUN pip3 install mvt
+
+# Installing ABE
+# --------------
+RUN mkdir /opt/abe
+RUN wget https://github.com/nelenkov/android-backup-extractor/releases/download/20210709062403-4c55371/abe.jar -O /opt/abe/abe.jar
+# Create alias for abe
+RUN echo 'alias abe="java -jar /opt/abe/abe.jar"' >> ~/.bashrc
+
+# Setup investigations environment
+# --------------------------------
+RUN mkdir /home/cases
+WORKDIR /home/cases 
+RUN echo 'echo "Mobile Verification Toolkit @ Docker\n------------------------------------\n\nYou can find information about how to use this image for Android (https://github.com/mvt-project/mvt/tree/master/docs/android) and iOS (https://github.com/mvt-project/mvt/tree/master/docs/ios) in the official docs of the project.\n"' >> ~/.bashrc
+RUN echo 'echo "Note that to perform the debug via USB you might need to give the Docker image access to the USB using \"docker run -it --privileged -v /dev/bus/usb:/dev/bus/usb mvt\" or, preferably, the \"--device=\" parameter.\n"' >> ~/.bashrc
+
+CMD /bin/bash

LICENSE

@@ -1,4 +1,4 @@
-MVT License 1.0
+MVT License 1.1
 ===============
 
 1. Definitions
@@ -35,7 +35,7 @@ MVT License 1.0
     means any form of the work other than Source Code Form.
 
 1.7. "Larger Work"
-    means a work that combines Covered Software with other material, in 
+    means a work that combines Covered Software with other material, in
     a separate file or files, that is not Covered Software.
 
 1.8. "License"
@@ -89,16 +89,16 @@ MVT License 1.0
     a Larger Work.
 
 1.16. "Device Owner" (or "Device Owners")
-    means an individal or a legal entity with legal ownership of an
+    means an individual or a legal entity with legal ownership of an
     electronic device which is being analysed through the use of
     Covered Software or a Larger Work, or from which Data was extracted
     for subsequent analysis.
 
 1.17. "Data Owner" (or "Data Owners")
-    means an individial or group of individuals who made use of the
-    electronic device from which Data that is extracted and/or analyzed
-    originated. "Data Owner" might or might not differ from "Device
-    Owner".
+    means an individual or group of individuals who made legitimate use
+    of the electronic device from which Data that is extracted and/or
+    analyzed originated. "Data Owner" might or might not differ from
+    "Device Owner".
 
 2. License Grants and Conditions
 --------------------------------
@@ -381,8 +381,8 @@ Exhibit A - Source Code Form License Notice
 -------------------------------------------
 
   This Source Code Form is subject to the terms of the MVT License,
-  v. 1.0. If a copy of the MVT License was not distributed with this
-  file, You can obtain one at TODO.
+  v. 1.1. If a copy of the MVT License was not distributed with this
+  file, You can obtain one at https://license.mvt.re/1.1/.
 
 If it is not possible or desirable to put the notice in a particular
 file, then You may include the notice in a location (such as a LICENSE
@@ -395,7 +395,7 @@ Exhibit B - "Incompatible With Secondary Licenses" Notice
 ---------------------------------------------------------
 
   This Source Code Form is "Incompatible With Secondary Licenses", as
-  defined by the MVT License, v. 1.0.
+  defined by the MVT License, v. 1.1.
 
 
 This license is an adaption of Mozilla Public License, v. 2.0.

README.md

@@ -5,21 +5,26 @@
 # Mobile Verification Toolkit
 
 [![](https://img.shields.io/pypi/v/mvt)](https://pypi.org/project/mvt/)
+[![](https://img.shields.io/badge/docs-blue.svg)](https://mvt.readthedocs.io)
 
 Mobile Verification Toolkit (MVT) is a collection of utilities to simplify and automate the process of gathering forensic traces helpful to identify a potential compromise of Android and iOS devices.
 
-[Please check out the documentation.](https://mvt.readthedocs.io/en/latest/)
+It has been developed and released by the [Amnesty International Security Lab](https://www.amnesty.org/en/tech/) in July 2021 in the context of the [Pegasus project](https://forbiddenstories.org/about-the-pegasus-project/) along with [a technical forensic methodology and forensic evidence](https://www.amnesty.org/en/latest/research/2021/07/forensic-methodology-report-how-to-catch-nso-groups-pegasus/).
+
+*Warning*: MVT is a forensic research tool intended for technologists and investigators. Using it requires understanding the basics of forensic analysis and using command-line tools. This is not intended for end-user self-assessment. If you are concerned with the security of your device please seek expert assistance.
 
 ## Installation
 
-First you need to install dependencies, on Linux `sudo apt install python3 python3-pip libusb-1.0-0` or on MacOS `brew install python3 libusb`.
+MVT can be installed from sources or conveniently using:
 
-Then you can install mvt from pypi with `pip install mvt`, or directly form sources:
-```bash
-git clone https://github.com/mvt-project/mvt.git
-cd mvt
-pip3 install .
 ```
+pip3 install mvt
+```
+
+You will need some dependencies, so please check the [documentation](https://mvt.readthedocs.io/en/latest/install.html).
+
+Alternatively, you can decide to run MVT and all relevant tools through a [Docker container](https://mvt.readthedocs.io/en/latest/docker.html).
+
 
 ## Usage
 
@@ -36,6 +41,7 @@ MVT provides two commands `mvt-ios` and `mvt-android` with the following subcomm
 
 Check out [the documentation to see how to use them](https://mvt.readthedocs.io/en/latest/).
 
+
 ## License
 
 The purpose of MVT is to facilitate the ***consensual forensic analysis*** of devices of those who might be targets of sophisticated mobile spyware attacks, especially members of civil society and marginalized communities. We do not want MVT to enable privacy violations of non-consenting individuals. Therefore, the goal of this license is to prohibit the use of MVT (and any other software licensed the same) for the purpose of *adversarial forensics*.

docs/android/backup.md

@@ -25,7 +25,7 @@ tar xvf backup.tar
 You can then extract SMSs containing links with MVT:
 
 ```bash
-$ mvt-android check-backup --output sms .
+$ mvt-android check-backup --output . .
 16:18:38 INFO     [mvt.android.cli] Checking ADB backup located at: .
          INFO     [mvt.android.modules.backup.sms] Running module SMS...
          INFO     [mvt.android.modules.backup.sms] Processing SMS backup

docs/android/download_apks.md

@@ -10,7 +10,7 @@ Now you can launch `mvt-android` and specify the `download-apks` command and the
 mvt-android download-apks --output /path/to/folder
 ```
 
-Optionally, you can decide to enable lookups of the SHA256 hash of all the extracted APKs on [VirusTotal](https://www.virustotal.com) and/or [Koodous](https://www.koodous.com). While these lookups do not provide any conclusive assessment on all of the extracted APKs, they might highlight any known malicious ones:
+Optionally, you can decide to enable lookups of the SHA256 hash of all the extracted APKs on [VirusTotal](https://www.virustotal.com) and/or [Koodous](https://koodous.com). While these lookups do not provide any conclusive assessment on all of the extracted APKs, they might highlight any known malicious ones:
 
 ```bash
 mvt-android download-apks --output /path/to/folder --virustotal

docs/docker.md

@@ -0,0 +1,35 @@
+## Using Docker
+
+Using Docker simplifies having all the required dependencies and tools (including most recent versions of [libimobiledevice](https://libimobiledevice.org)) readily installed.
+
+Install Docker following the [official documentation](https://docs.docker.com/get-docker/).
+
+Once installed, you can clone MVT's repository and build its Docker image:
+
+```bash
+git clone https://github.com/mvt-project/mvt.git
+cd mvt
+docker build -t mvt .
+```
+
+Test if the image was created successfully:
+
+```bash
+docker run -it mvt
+```
+
+If a prompt is spawned successfully, you can close it with `exit`.
+
+If you wish to use MVT to test an Android device you will need to enable the container's access to the host's USB devices. You can do so by enabling the `--privileged` flag and mounting the USB bus device as a volume:
+
+```bash
+docker run -it --privileged -v /dev/bus/usb:/dev/bus/usb mvt
+```
+
+**Please note:** the `--privileged` parameter is generally regarded as a security risk. If you want to learn more about this check out [this explainer on container escapes](https://blog.trailofbits.com/2019/07/19/understanding-docker-container-escapes/) as it gives access to the whole system.
+
+Recent versions of Docker provide a `--device` parameter allowing to specify a precise USB device without enabling `--privileged`:
+
+```bash
+docker run -it --device=/dev/<your_usb_port> mvt
+```

docs/install.md

@@ -7,7 +7,7 @@ Before proceeding, please note that mvt requires Python 3.6+ to run. While it sh
 First install some basic dependencies that will be necessary to build all required tools:
 
 ```bash
-sudo apt install python3 python3-pip libusb-1.0-0
+sudo apt install python3 python3-pip libusb-1.0-0 sqlite3
 ```
 
 *libusb-1.0-0* is not required if you intend to only use `mvt-ios` and not `mvt-android`.
@@ -19,7 +19,7 @@ Running MVT on Mac requires Xcode and [homebrew](https://brew.sh) to be installe
 In order to install dependencies use:
 
 ```bash
-brew install python3 libusb
+brew install python3 libusb sqlite3
 ```
 
 *libusb* is not required if you intend to only use `mvt-ios` and not `mvt-android`.

docs/introduction.md

@@ -14,4 +14,4 @@ Mobile Verification Toolkit (MVT) is a collection of utilities designed to facil
 
 While MVT is capable of extracting and processing various types of very personal records typically found on a mobile phone (such as calls history, SMS and WhatsApp messages, etc.), this is intended to help identify potential attack vectors such as malicious SMS messages leading to exploitation.
 
-MVT's purpose is not to facilitate adversial forensics of non-consenting individuals' devices. The use of MVT and derivative products to extract and/or analyse data originating from devices used by individuals not consenting to the procedure is explicitly prohibited in the [license](license.md).
+MVT's purpose is not to facilitate adversarial forensics of non-consenting individuals' devices. The use of MVT and derivative products to extract and/or analyse data originating from devices used by individuals not consenting to the procedure is explicitly prohibited in the [license](license.md).

docs/ios/filesystem/dump.md

@@ -1,6 +1,6 @@
 # Dumping the filesystem
 
-While iTunes backup provide a lot of very useful databases and diagnistic data, in some cases you might want to jailbreak the device and perform a full filesystem dump. In that case, you should take a look at [checkra1n](https://checkra.in/), which provides an easy way to obtain root on most recent iPhone models.
+While iTunes backup provide a lot of very useful databases and diagnostic data, in some cases you might want to jailbreak the device and perform a full filesystem dump. In that case, you should take a look at [checkra1n](https://checkra.in/), which provides an easy way to obtain root on most recent iPhone models.
 
 !!! warning
     Before you checkra1n any device, make sure you take a full backup, and that you are prepared to do a full factory reset before restoring it. Even after using checkra1n's "Restore System", some traces of the jailbreak are still left on the device and [apps with anti-jailbreaks will be able to detect them](https://github.com/checkra1n/BugTracker/issues/279) and stop functioning.

docs/ios/methodology.md

@@ -6,7 +6,7 @@ Before jumping into acquiring and analyzing data from an iOS device, you should
 
 You will need to decide whether to attempt to jailbreak the device and obtain a full filesystem dump, or not.
 
-While access the full filesystem allows to extact data that would otherwise be unavailable, it might not always be possible to jailbreak a certain iPhone model or version of iOS. In addition, depending on the type of jailbreak available, doing so might compromise some important records, pollute others, or potentially cause unintended malfunctioning of the device later in case it is used again.
+While access the full filesystem allows to extract data that would otherwise be unavailable, it might not always be possible to jailbreak a certain iPhone model or version of iOS. In addition, depending on the type of jailbreak available, doing so might compromise some important records, pollute others, or potentially cause unintended malfunctioning of the device later in case it is used again.
 
 If you are not expected to return the phone, you might want to consider to attempt a jailbreak after having exhausted all other options, including a backup.
 

docs/ios/records.md

@@ -236,7 +236,7 @@ If indicators are provided through the command-line, they are checked against th
     Backup: :material-close:  
     Full filesystem dump: :material-check:
 
-This JSON file is created by mvt-ios' `WebkitLocalStorage` module. The module extracts a lsit of file and folder names located at the following path */private/var/mobile/Containers/Data/Application/\*/Library/WebKit/WebsiteData/LocalStorage/*, which contains local storage files created by any app installed on the device.
+This JSON file is created by mvt-ios' `WebkitLocalStorage` module. The module extracts a list of file and folder names located at the following path */private/var/mobile/Containers/Data/Application/\*/Library/WebKit/WebsiteData/LocalStorage/*, which contains local storage files created by any app installed on the device.
 
 If indicators are provided through the command-line, they are checked against the extracted names. Any matches are stored in *webkit_local_storage_detected.json*.
 

mkdocs.yml

@@ -28,6 +28,7 @@ nav:
     - Welcome: "index.md"
     - Introduction: "introduction.md"
     - Installation: "install.md"
+    - Using Docker: "docker.md"
     - MVT for iOS:
         - iOS Forensic Methodology: "ios/methodology.md"
         - Install libimobiledevice: "ios/install.md"

mvt/android/download_apks.py

@@ -16,7 +16,7 @@ log = logging.getLogger(__name__)
 
 # TODO: Would be better to replace tqdm with rich.progress to reduce
 #       the number of dependencies. Need to investigate whether
-#       it's possible to have a simialr callback system.
+#       it's possible to have a similar callback system.
 class PullProgress(tqdm):
     """PullProgress is a tqdm update system for APK downloads."""
 
@@ -42,7 +42,7 @@ class DownloadAPKs(AndroidExtraction):
         """Initialize module.
         :param output_folder: Path to the folder where data should be stored
         :param all_apks: Boolean indicating whether to download all packages
-                         or filter known-goods 
+                         or filter known-goods
         :param packages: Provided list of packages, typically for JSON checks
         """
         super().__init__(file_path=None, base_folder=None,

mvt/android/modules/adb/base.py

@@ -4,6 +4,8 @@
 #   https://github.com/mvt-project/mvt/blob/main/LICENSE
 
 import os
+import random
+import string
 import sys
 import time
 import logging
@@ -96,7 +98,7 @@ class AndroidExtraction(MVTModule):
         """Check if we have a `su` binary on the Android device.
         :returns: Boolean indicating whether a `su` binary is present or not
         """
-        return bool(self._adb_command("[ ! -f /sbin/su ] || echo 1"))
+        return bool(self._adb_command("command -v su"))
 
     def _adb_root_or_die(self):
         """Check if we have a `su` binary, otherwise raise an Exception.
@@ -111,7 +113,7 @@ class AndroidExtraction(MVTModule):
         """
         return self._adb_command(f"su -c {command}")
 
-    def _adb_download(self, remote_path, local_path, progress_callback=None):
+    def _adb_download(self, remote_path, local_path, progress_callback=None, retry_root=True):
         """Download a file form the device.
         :param remote_path: Path to download from the device
         :param local_path: Path to where to locally store the copy of the file
@@ -119,6 +121,37 @@ class AndroidExtraction(MVTModule):
         """
         try:
             self.device.pull(remote_path, local_path, progress_callback)
+        except AdbCommandFailureException as e:
+            if retry_root:
+                self._adb_download_root(remote_path, local_path, progress_callback)
+            else:
+                raise Exception(f"Unable to download file {remote_path}: {e}")
+    
+    def _adb_download_root(self, remote_path, local_path, progress_callback=None):
+        try:
+            # Check if we have root, if not raise an Exception.
+            self._adb_root_or_die()
+
+            # We generate a random temporary filename.
+            tmp_filename = "tmp_" + ''.join(random.choices(string.ascii_uppercase + string.ascii_lowercase + string.digits, k=10))
+
+            # We create a temporary local file.
+            new_remote_path = f"/sdcard/{tmp_filename}"
+
+            # We copy the file from the data folder to /sdcard/.
+            cp = self._adb_command_as_root(f"cp {remote_path} {new_remote_path}")
+            if cp.startswith("cp: ") and "No such file or directory" in cp:
+                raise Exception(f"Unable to process file {remote_path}: File not found")
+            elif cp.startswith("cp: ") and "Permission denied" in cp:
+                raise Exception(f"Unable to process file {remote_path}: Permission denied")
+
+            # We download from /sdcard/ to the local temporary file.
+            # If it doesn't work now, don't try again (retry_root=False)
+            self._adb_download(new_remote_path, local_path, retry_root=False)
+
+            # Delete the copy on /sdcard/.
+            self._adb_command(f"rm -rf {new_remote_path}")
+            
         except AdbCommandFailureException as e:
             raise Exception(f"Unable to download file {remote_path}: {e}")
 

mvt/android/modules/adb/whatsapp.py

@@ -6,6 +6,7 @@
 import os
 import sqlite3
 import logging
+import base64
 
 from .base import AndroidExtraction
 from mvt.common.utils import convert_timestamp_to_iso, check_for_links
@@ -69,6 +70,8 @@ class Whatsapp(AndroidExtraction):
 
             # If we find links in the messages or if they are empty we add them to the list.
             if check_for_links(message["data"]) or message["data"].strip() == "":
+                if (message.get('thumb_image') is not None):
+                    message['thumb_image'] = base64.b64encode(message['thumb_image'])
                 messages.append(message)
 
         cur.close()

mvt/common/module.py

@@ -3,6 +3,7 @@
 # See the file 'LICENSE' for usage and copying permissions, or find a copy at
 #   https://github.com/mvt-project/mvt/blob/main/LICENSE
 
+import io
 import os
 import re
 import csv
@@ -12,6 +13,12 @@ import simplejson as json
 
 from .indicators import Indicators
 
+class DatabaseNotFoundError(Exception):
+    pass
+
+class DatabaseCorruptedError(Exception):
+    pass
+
 class MVTModule(object):
     """This class provides a base for all extraction modules."""
 
@@ -66,7 +73,7 @@ class MVTModule(object):
         self.indicators = Indicators(file_path, self.log)
 
     def check_indicators(self):
-        """Check the results of this module againt a provided list of
+        """Check the results of this module against a provided list of
         indicators."""
         raise NotImplementedError
 
@@ -82,7 +89,11 @@ class MVTModule(object):
             results_file_name = f"{name}.json"
             results_json_path = os.path.join(self.output_folder, results_file_name)
             with open(results_json_path, "w") as handle:
-                json.dump(self.results, handle, indent=4)
+                try:
+                    json.dump(self.results, handle, indent=4)
+                except Exception as e:
+                    self.log.error("Unable to store results of module %s to file %s: %s",
+                                   self.__class__.__name__, results_file_name, e)
 
         if self.detected:
             detected_file_name = f"{name}_detected.json"
@@ -98,17 +109,19 @@ class MVTModule(object):
         """
         for result in self.results:
             record = self.serialize(result)
-            if type(record) == list:
-                self.timeline.extend(record)
-            else:
-                self.timeline.append(record)
+            if record:
+                if type(record) == list:
+                    self.timeline.extend(record)
+                else:
+                    self.timeline.append(record)
 
         for detected in self.detected:
             record = self.serialize(detected)
-            if type(record) == list:
-                self.timeline_detected.extend(record)
-            else:
-                self.timeline_detected.append(record)
+            if record:
+                if type(record) == list:
+                    self.timeline_detected.extend(record)
+                else:
+                    self.timeline_detected.append(record)
 
         # De-duplicate timeline entries
         self.timeline = self.timeline_deduplicate(self.timeline)
@@ -135,8 +148,11 @@ def run_module(module):
     except NotImplementedError:
         module.log.exception("The run() procedure of module %s was not implemented yet!",
                              module.__class__.__name__)
-    except FileNotFoundError as e:
-        module.log.error("There might be no data to extract by module %s: %s",
+    except DatabaseNotFoundError as e:
+        module.log.info("There might be no data to extract by module %s: %s",
+                        module.__class__.__name__, e)
+    except DatabaseCorruptedError as e:
+        module.log.error("The %s module database seems to be corrupted and recovery failed: %s",
                          module.__class__.__name__, e)
     except Exception as e:
         module.log.exception("Error in running extraction from module %s: %s",
@@ -160,7 +176,7 @@ def save_timeline(timeline, timeline_path):
     :param timeline: List of records to order and store.
     :param timeline_path: Path to the csv file to store the timeline to.
     """
-    with open(timeline_path, "a+") as handle:
+    with io.open(timeline_path, "a+", encoding="utf-8") as handle:
         csvoutput = csv.writer(handle, delimiter=",", quotechar="\"")
         csvoutput.writerow(["UTC Timestamp", "Plugin", "Event", "Description"])
         for event in sorted(timeline, key=lambda x: x["timestamp"] if x["timestamp"] is not None else ""):

mvt/ios/cli.py

@@ -43,6 +43,7 @@ def cli():
               help="Path to the folder where to store the decrypted backup")
 @click.option("--password", "-p", cls=MutuallyExclusiveOption,
               help="Password to use to decrypt the backup",
+              prompt="Enter backup password", hide_input=True, prompt_required=False,
               mutually_exclusive=["key_file"])
 @click.option("--key-file", "-k", cls=MutuallyExclusiveOption,
               type=click.Path(exists=True),
@@ -172,7 +173,7 @@ def check_fs(iocs, output, fast, dump_path, list_modules, module):
 @click.argument("FOLDER", type=click.Path(exists=True))
 def check_iocs(iocs, list_modules, module, folder):
     all_modules = []
-    for entry in BACKUP_MODULES + FS_MODULES + SYSDIAGNOSE_MODULES:
+    for entry in BACKUP_MODULES + FS_MODULES:
         if entry not in all_modules:
             all_modules.append(entry)
 

mvt/ios/modules/fs/base.py

@@ -3,10 +3,15 @@
 # See the file 'LICENSE' for usage and copying permissions, or find a copy at
 #   https://github.com/mvt-project/mvt/blob/main/LICENSE
 
+import io
 import os
 import glob
+import shutil
+import sqlite3
+import subprocess
 
 from mvt.common.module import MVTModule
+from mvt.common.module import DatabaseNotFoundError, DatabaseCorruptedError
 
 class IOSExtraction(MVTModule):
     """This class provides a base for all iOS filesystem/backup extraction modules."""
@@ -15,6 +20,31 @@ class IOSExtraction(MVTModule):
     is_fs_dump = False
     is_sysdiagnose = False
 
+    def _recover_database(self, file_path):
+        """Tries to recover a malformed database by running a .clone command.
+        :param file_path: Path to the malformed database file.
+        """
+        # TODO: Find a better solution.
+
+        self.log.info("Database at path %s is malformed. Trying to recover...", file_path)
+
+        if not os.path.exists(file_path):
+            return
+
+        if not shutil.which("sqlite3"):
+            raise DatabaseCorruptedError("Unable to recover without sqlite3 binary. Please install sqlite3!")
+
+        bak_path = f"{file_path}.bak"
+        shutil.move(file_path, bak_path)
+
+        cmd = f"sqlite3 {bak_path} '.clone {file_path}'"
+        ret = subprocess.call(cmd, shell=True, stdout=subprocess.PIPE,
+                              stderr=subprocess.PIPE)
+        if ret != 0:
+            raise DatabaseCorruptedError("Recovery of database failed")
+
+        self.log.info("Database at path %s recovered successfully!", file_path)
+
     def _find_ios_database(self, backup_ids=None, root_paths=[]):
         """Try to locate the module's database file from either an iTunes
         backup or a full filesystem dump.
@@ -52,4 +82,20 @@ class IOSExtraction(MVTModule):
         if file_path:
             self.file_path = file_path
         else:
-            raise FileNotFoundError("Unable to find the module's database file")
+            raise DatabaseNotFoundError("Unable to find the module's database file")
+
+        # Check if the database is corrupted.
+        conn = sqlite3.connect(self.file_path)
+        cur = conn.cursor()
+
+        try:
+            recover = False
+            cur.execute("SELECT name FROM sqlite_master WHERE type='table';")
+        except sqlite3.DatabaseError as e:
+            if "database disk image is malformed" in str(e):
+                recover = True
+        finally:
+            conn.close()
+
+        if recover:
+            self._recover_database(self.file_path)

mvt/ios/modules/fs/locationd.py

@@ -43,10 +43,10 @@ class LocationdClients(IOSExtraction):
         for ts in self.timestamps:
             if ts in record.keys():
                 records.append({
-                    "timestamp": entry[ts],
+                    "timestamp": record[ts],
                     "module": self.__class__.__name__,
                     "event": ts,
-                    "data": f"{ts} from {entry['package']}"
+                    "data": f"{ts} from {record['package']}"
                 })
 
         return records
@@ -62,7 +62,7 @@ class LocationdClients(IOSExtraction):
                 result["package"] = app
                 for ts in self.timestamps:
                     if ts in result.keys():
-                        result[ts] = convert_timestamp_to_iso(convert_mactime_to_unix(result[date]))
+                        result[ts] = convert_timestamp_to_iso(convert_mactime_to_unix(result[ts]))
 
                 self.results.append(result)
 

mvt/ios/modules/fs/manifest.py

@@ -40,6 +40,8 @@ class Manifest(IOSExtraction):
 
     def serialize(self, record):
         records = []
+        if "modified" not in record or "statusChanged" not in record:
+            return
         for ts in set([record["created"], record["modified"], record["statusChanged"]]):
             macb = ""
             macb += "M" if ts == record["modified"] else "-"
@@ -63,12 +65,15 @@ class Manifest(IOSExtraction):
         for result in self.results:
             if not "relativePath" in result:
                 continue
-
-            if os.path.basename(result["relativePath"]) == "com.apple.CrashReporter.plist" and result["domain"] == "RootDomain":
-                self.log.warning("Found a potentially suspicious \"com.apple.CrashReporter.plist\" file created in RootDomain")
-                self.detected.append(result)
+            if not result["relativePath"]:
                 continue
 
+            if result["domain"]:
+                if os.path.basename(result["relativePath"]) == "com.apple.CrashReporter.plist" and result["domain"] == "RootDomain":
+                    self.log.warning("Found a potentially suspicious \"com.apple.CrashReporter.plist\" file created in RootDomain")
+                    self.detected.append(result)
+                    continue
+
             if self.indicators.check_file(result["relativePath"]):
                 self.log.warning("Found a known malicious file at path: %s", result["relativePath"])
                 self.detected.append(result)

mvt/ios/modules/fs/net_base.py

@@ -167,6 +167,10 @@ class NetBase(IOSExtraction):
         results_by_proc = {proc["proc_id"]: proc for proc in self.results if proc["proc_id"]}
         all_proc_id = sorted(results_by_proc.keys())
 
+        # Fix issue #108
+        if not all_proc_id:
+            return
+
         missing_procs, last_proc_id = {}, None
         for proc_id in range(min(all_proc_id), max(all_proc_id)):
             if proc_id not in all_proc_id:

mvt/ios/modules/fs/webkit_session_resource_log.py

@@ -70,6 +70,9 @@ class WebkitSessionResourceLog(IOSExtraction):
         return domains
 
     def check_indicators(self):
+        if not self.indicators:
+            return
+
         for key, entries in self.results.items():
             for entry in entries:
                 source_domains = self._extract_domains(entry["redirect_source"])

mvt/ios/modules/fs/whatsapp.py

@@ -73,7 +73,7 @@ class Whatsapp(IOSExtraction):
             # Extract links from the WhatsApp message.
             message_links = check_for_links(new_message["ZTEXT"])
 
-            # If we find mesages, or if there's an empty message we add it to the list.
+            # If we find messages, or if there's an empty message we add it to the list.
             if new_message["ZTEXT"] and (message_links or new_message["ZTEXT"].strip() == ""):
                 self.results.append(new_message)
 

mvt/ios/versions.py

@@ -102,6 +102,7 @@ IPHONE_IOS_VERSIONS = [
     {"build": "10A405", "version": "6.0"},
     {"build": "11B601", "version": "7.0.5"},
     {"build": "18F72", "version": "14.6"},
+    {"build": "18G69", "version": "14.7"},
     {"build": "18E199", "version": "14.5"},
     {"build": "18E212", "version": "14.5.1"},
     {"build": "18D52", "version": "14.4"},

setup.py

@@ -7,7 +7,7 @@ import os
 from setuptools import setup, find_packages
 
 __package_name__ = "mvt"
-__version__ = "1.0.11"
+__version__ = "1.0.13"
 __description__ = "Mobile Verification Toolkit"
 
 this_directory = os.path.abspath(os.path.dirname(__file__))
@@ -17,18 +17,18 @@ with open(readme_path, encoding="utf-8") as handle:
 
 requires = (
     # Base dependencies:
-    "click",
-    "rich",
-    "tld",
-    "tqdm",
-    "requests",
-    "simplejson",
+    "click>=8.0.1",
+    "rich>=10.6.0",
+    "tld>=0.12.6",
+    "tqdm>=4.61.2",
+    "requests>=2.26.0",
+    "simplejson>=3.17.3",
     # iOS dependencies:
-    "biplist",
-    "iOSbackup",
+    "biplist>=1.0.3",
+    "iOSbackup>=0.9.912",
     # Android dependencies:
-    "adb-shell",
-    "libusb1",
+    "adb-shell>=0.4.0",
+    "libusb1>=1.9.3",
 )
 
 def get_package_data(package):