Bumped version

This offers generic documentation, to show how MVT can be used with
arbitrary STIX-formatted IOCs, while still pointing users at some
known-to-be-useful sample files.

.github/workflows/python-package.yml

@@ -0,0 +1,43 @@
+# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
+
+name: Python package
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: [3.7, 3.8, 3.9]
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install flake8 pytest safety
+        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+    - name: Lint with flake8
+      run: |
+        # stop the build if there are Python syntax errors or undefined names
+        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+    - name: Safety checks
+      run: safety check
+
+    # - name: Test with pytest
+    #   run: |
+    #     pytest

.gitignore

@@ -129,3 +129,5 @@ dmypy.json
 .pyre/
 *.pyc
 
+# Temporal files
+*~

Dockerfile

@@ -0,0 +1,66 @@
+FROM ubuntu:20.04
+
+# Ref. https://github.com/mvt-project/mvt
+
+# Fixing major OS dependencies
+# ----------------------------
+RUN apt update \
+  && apt install -y python3 python3-pip libusb-1.0-0-dev \
+  && apt install -y wget \ 
+  && apt install -y adb \ 
+  && DEBIAN_FRONTEND=noninteractive apt-get -y install default-jre-headless
+
+# Install build tools for libimobiledevice
+# ----------------------------------------
+RUN apt install -y build-essential \
+	checkinstall \
+	git \
+	autoconf \
+	automake \
+	libtool-bin \
+	libplist-dev \
+	libusbmuxd-dev \
+	libssl-dev \
+	sqlite3 \
+    pkg-config
+
+# Clean up
+# --------
+RUN apt-get clean \
+  && rm -rf /var/lib/apt/lists/*
+
+
+# Build libimobiledevice
+# ----------------------
+RUN git clone https://github.com/libimobiledevice/libplist
+RUN git clone https://github.com/libimobiledevice/libusbmuxd
+RUN git clone https://github.com/libimobiledevice/libimobiledevice
+RUN git clone https://github.com/libimobiledevice/usbmuxd
+
+RUN cd libplist && ./autogen.sh && make && make install && ldconfig
+
+RUN cd libusbmuxd && PKG_CONFIG_PATH=/usr/local/lib/pkgconfig ./autogen.sh && make && make install && ldconfig
+
+RUN cd libimobiledevice && PKG_CONFIG_PATH=/usr/local/lib/pkgconfig ./autogen.sh --enable-debug && make && make install && ldconfig
+
+RUN cd usbmuxd && PKG_CONFIG_PATH=/usr/local/lib/pkgconfig ./autogen.sh --prefix=/usr --sysconfdir=/etc --localstatedir=/var --runstatedir=/run && make && make install
+
+# Installing MVT
+# --------------
+RUN pip3 install mvt
+
+# Installing ABE
+# --------------
+RUN mkdir /opt/abe
+RUN wget https://github.com/nelenkov/android-backup-extractor/releases/download/20210709062403-4c55371/abe.jar -O /opt/abe/abe.jar
+# Create alias for abe
+RUN echo 'alias abe="java -jar /opt/abe/abe.jar"' >> ~/.bashrc
+
+# Setup investigations environment
+# --------------------------------
+RUN mkdir /home/cases
+WORKDIR /home/cases 
+RUN echo 'echo "Mobile Verification Toolkit @ Docker\n------------------------------------\n\nYou can find information about how to use this image for Android (https://github.com/mvt-project/mvt/tree/master/docs/android) and iOS (https://github.com/mvt-project/mvt/tree/master/docs/ios) in the official docs of the project.\n"' >> ~/.bashrc
+RUN echo 'echo "Note that to perform the debug via USB you might need to give the Docker image access to the USB using \"docker run -it --privileged -v /dev/bus/usb:/dev/bus/usb mvt\" or, preferably, the \"--device=\" parameter.\n"' >> ~/.bashrc
+
+CMD /bin/bash

LICENSE

@@ -1,4 +1,4 @@
-MVT License 1.0
+MVT License 1.1
 ===============
 
 1. Definitions
@@ -35,7 +35,7 @@ MVT License 1.0
     means any form of the work other than Source Code Form.
 
 1.7. "Larger Work"
-    means a work that combines Covered Software with other material, in 
+    means a work that combines Covered Software with other material, in
     a separate file or files, that is not Covered Software.
 
 1.8. "License"
@@ -89,16 +89,16 @@ MVT License 1.0
     a Larger Work.
 
 1.16. "Device Owner" (or "Device Owners")
-    means an individal or a legal entity with legal ownership of an
+    means an individual or a legal entity with legal ownership of an
     electronic device which is being analysed through the use of
     Covered Software or a Larger Work, or from which Data was extracted
     for subsequent analysis.
 
 1.17. "Data Owner" (or "Data Owners")
-    means an individial or group of individuals who made use of the
-    electronic device from which Data that is extracted and/or analyzed
-    originated. "Data Owner" might or might not differ from "Device
-    Owner".
+    means an individual or group of individuals who made legitimate use
+    of the electronic device from which Data that is extracted and/or
+    analyzed originated. "Data Owner" might or might not differ from
+    "Device Owner".
 
 2. License Grants and Conditions
 --------------------------------
@@ -381,8 +381,8 @@ Exhibit A - Source Code Form License Notice
 -------------------------------------------
 
   This Source Code Form is subject to the terms of the MVT License,
-  v. 1.0. If a copy of the MVT License was not distributed with this
-  file, You can obtain one at TODO.
+  v. 1.1. If a copy of the MVT License was not distributed with this
+  file, You can obtain one at https://license.mvt.re/1.1/.
 
 If it is not possible or desirable to put the notice in a particular
 file, then You may include the notice in a location (such as a LICENSE
@@ -395,7 +395,7 @@ Exhibit B - "Incompatible With Secondary Licenses" Notice
 ---------------------------------------------------------
 
   This Source Code Form is "Incompatible With Secondary Licenses", as
-  defined by the MVT License, v. 1.0.
+  defined by the MVT License, v. 1.1.
 
 
 This license is an adaption of Mozilla Public License, v. 2.0.

README.md

@@ -5,21 +5,26 @@
 # Mobile Verification Toolkit
 
 [![](https://img.shields.io/pypi/v/mvt)](https://pypi.org/project/mvt/)
+[![Documentation Status](https://readthedocs.org/projects/mvt/badge/?version=latest)](https://docs.mvt.re/en/latest/?badge=latest)
 
 Mobile Verification Toolkit (MVT) is a collection of utilities to simplify and automate the process of gathering forensic traces helpful to identify a potential compromise of Android and iOS devices.
 
-[Please check out the documentation.](https://mvt.readthedocs.io/en/latest/)
+It has been developed and released by the [Amnesty International Security Lab](https://www.amnesty.org/en/tech/) in July 2021 in the context of the [Pegasus project](https://forbiddenstories.org/about-the-pegasus-project/) along with [a technical forensic methodology and forensic evidence](https://www.amnesty.org/en/latest/research/2021/07/forensic-methodology-report-how-to-catch-nso-groups-pegasus/).
+
+*Warning*: MVT is a forensic research tool intended for technologists and investigators. Using it requires understanding the basics of forensic analysis and using command-line tools. This is not intended for end-user self-assessment. If you are concerned with the security of your device please seek expert assistance.
 
 ## Installation
 
-First you need to install dependencies, on Linux `sudo apt install python3 python3-pip libusb-1.0-0` or on MacOS `brew install python3 libusb`.
+MVT can be installed from sources or conveniently using:
 
-Then you can install mvt from pypi with `pip install mvt`, or directly form sources:
-```bash
-git clone https://github.com/mvt-project/mvt.git
-cd mvt
-pip3 install .
 ```
+pip3 install mvt
+```
+
+You will need some dependencies, so please check the [documentation](https://docs.mvt.re/en/latest/install.html).
+
+Alternatively, you can decide to run MVT and all relevant tools through a [Docker container](https://docs.mvt.re/en/latest/docker.html).
+
 
 ## Usage
 
@@ -30,11 +35,13 @@ MVT provides two commands `mvt-ios` and `mvt-android` with the following subcomm
     * `check-fs`: Extract artifacts from a full filesystem dump
     * `check-iocs`: Compare stored JSON results to provided indicators
     * `decrypt-backup`:  Decrypt an encrypted iTunes backup
+    * `extract-key`: Extract decryption key from an iTunes backup
 * `mvt-android`:
     * `check-backup`: Check an Android Backup
     * `download-apks`: Download all or non-safelisted installed APKs
 
-Check out [the documentation to see how to use them](https://mvt.readthedocs.io/en/latest/).
+Check out [the documentation to see how to use them](https://docs.mvt.re/).
+
 
 ## License
 

dev/mvt-android

@@ -10,4 +10,5 @@ import sys
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 from mvt import android
+
 android.cli()

dev/mvt-ios

@@ -10,4 +10,5 @@ import sys
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 from mvt import ios
+
 ios.cli()

docs/android/backup.md

@@ -25,7 +25,7 @@ tar xvf backup.tar
 You can then extract SMSs containing links with MVT:
 
 ```bash
-$ mvt-android check-backup --output sms .
+$ mvt-android check-backup --output . .
 16:18:38 INFO     [mvt.android.cli] Checking ADB backup located at: .
          INFO     [mvt.android.modules.backup.sms] Running module SMS...
          INFO     [mvt.android.modules.backup.sms] Processing SMS backup

docs/android/download_apks.md

@@ -10,7 +10,7 @@ Now you can launch `mvt-android` and specify the `download-apks` command and the
 mvt-android download-apks --output /path/to/folder
 ```
 
-Optionally, you can decide to enable lookups of the SHA256 hash of all the extracted APKs on [VirusTotal](https://www.virustotal.com) and/or [Koodous](https://www.koodous.com). While these lookups do not provide any conclusive assessment on all of the extracted APKs, they might highlight any known malicious ones:
+Optionally, you can decide to enable lookups of the SHA256 hash of all the extracted APKs on [VirusTotal](https://www.virustotal.com) and/or [Koodous](https://koodous.com). While these lookups do not provide any conclusive assessment on all of the extracted APKs, they might highlight any known malicious ones:
 
 ```bash
 mvt-android download-apks --output /path/to/folder --virustotal

docs/docker.md

@@ -0,0 +1,33 @@
+Using Docker simplifies having all the required dependencies and tools (including most recent versions of [libimobiledevice](https://libimobiledevice.org)) readily installed.
+
+Install Docker following the [official documentation](https://docs.docker.com/get-docker/).
+
+Once installed, you can clone MVT's repository and build its Docker image:
+
+```bash
+git clone https://github.com/mvt-project/mvt.git
+cd mvt
+docker build -t mvt .
+```
+
+Test if the image was created successfully:
+
+```bash
+docker run -it mvt
+```
+
+If a prompt is spawned successfully, you can close it with `exit`.
+
+If you wish to use MVT to test an Android device you will need to enable the container's access to the host's USB devices. You can do so by enabling the `--privileged` flag and mounting the USB bus device as a volume:
+
+```bash
+docker run -it --privileged -v /dev/bus/usb:/dev/bus/usb mvt
+```
+
+**Please note:** the `--privileged` parameter is generally regarded as a security risk. If you want to learn more about this check out [this explainer on container escapes](https://blog.trailofbits.com/2019/07/19/understanding-docker-container-escapes/) as it gives access to the whole system.
+
+Recent versions of Docker provide a `--device` parameter allowing to specify a precise USB device without enabling `--privileged`:
+
+```bash
+docker run -it --device=/dev/<your_usb_port> mvt
+```

docs/install.md

@@ -7,7 +7,7 @@ Before proceeding, please note that mvt requires Python 3.6+ to run. While it sh
 First install some basic dependencies that will be necessary to build all required tools:
 
 ```bash
-sudo apt install python3 python3-pip libusb-1.0-0
+sudo apt install python3 python3-pip libusb-1.0-0 sqlite3
 ```
 
 *libusb-1.0-0* is not required if you intend to only use `mvt-ios` and not `mvt-android`.
@@ -19,7 +19,7 @@ Running MVT on Mac requires Xcode and [homebrew](https://brew.sh) to be installe
 In order to install dependencies use:
 
 ```bash
-brew install python3 libusb
+brew install python3 libusb sqlite3
 ```
 
 *libusb* is not required if you intend to only use `mvt-ios` and not `mvt-android`.
@@ -35,7 +35,7 @@ export PATH=$PATH:~/.local/bin
 Then you can install MVT directly from [pypi](https://pypi.org/project/mvt/)
 
 ```bash
-pip install mvt
+pip3 install mvt
 ```
 
 Or from the source code:

docs/introduction.md

@@ -14,4 +14,4 @@ Mobile Verification Toolkit (MVT) is a collection of utilities designed to facil
 
 While MVT is capable of extracting and processing various types of very personal records typically found on a mobile phone (such as calls history, SMS and WhatsApp messages, etc.), this is intended to help identify potential attack vectors such as malicious SMS messages leading to exploitation.
 
-MVT's purpose is not to facilitate adversial forensics of non-consenting individuals' devices. The use of MVT and derivative products to extract and/or analyse data originating from devices used by individuals not consenting to the procedure is explicitly prohibited in the [license](license.md).
+MVT's purpose is not to facilitate adversarial forensics of non-consenting individuals' devices. The use of MVT and derivative products to extract and/or analyse data originating from devices used by individuals not consenting to the procedure is explicitly prohibited in the [license](license.md).

docs/iocs.md

@@ -0,0 +1,32 @@
+# Indicators of Compromise (IOCs)
+
+MVT uses [Structured Threat Information Expression (STIX)](https://oasis-open.github.io/cti-documentation/stix/intro.html) files to identify potential traces of compromise.
+
+These indicators of compromise are contained in a file with a particular structure of [JSON](https://en.wikipedia.org/wiki/JSON) with the `.stix2` or `.json` extensions.
+
+You can indicate a path to a STIX2 indicators file when checking iPhone backups or filesystem dumps. For example:
+
+```bash
+mvt-ios check-backup --iocs ~/ios/malware.stix2 --output /path/to/iphone/output /path/to/backup
+```
+
+Or, with data from an Android backup:
+
+```bash
+mvt-android check-backup --iocs ~/iocs/malware.stix2 /path/to/android/backup/
+```
+
+After extracting forensics data from a device, you are also able to compare it with any STIX2 file you indicate:
+
+```bash
+mvt-ios check-iocs --iocs ~/iocs/malware.stix2 /path/to/iphone/output/
+```
+
+If you're looking for indicators of compromise for a specific piece of malware or adversary, please ask investigators or anti-malware researchers who have the relevant expertise for a STIX file.
+
+## Known repositories of STIX2 IOCs
+
+- The [Amnesty International investigations repository](https://github.com/AmnestyTech/investigations) contains STIX-formatted IOCs for:
+    - [Pegasus](https://en.wikipedia.org/wiki/Pegasus_(spyware)) ([STIX2](https://raw.githubusercontent.com/AmnestyTech/investigations/master/2021-07-18_nso/pegasus.stix2))
+
+Please [open an issue](https://github.com/mvt-project/mvt/issues/) to suggest new sources of STIX-formatted IOCs.

docs/ios/backup/check.md

@@ -2,6 +2,32 @@
 
 The backup might take some time. It is best to make sure the phone remains unlocked during the backup process. Afterwards, a new folder will be created under the path you specified using the UDID of the iPhone you backed up.
 
+## Extracting and saving the decryption key (optional)
+
+If you do not wish to enter a password every time when decrypting a backup, MVT can accept a key file instead. This key can be used with the `decrypt-backup` command.
+
+To generate a key file, you will need your device backup and the backup password:
+
+    $ mvt-ios extract-key --help
+    Usage: mvt-ios extract-key [OPTIONS] BACKUP_PATH
+
+      Extract decryption key from an iTunes backup
+
+    Options:
+      -p, --password TEXT  Password to use to decrypt the backup  [required]
+      -k, --key-file FILE  Key file to be written (if unset, will print to STDOUT)
+      --help               Show this message and exit.
+
+You can specify the password on the command line, or omit the `-p` option to have MVT prompt for a password. The `-k` option specifies where to save the file containing the decryption key. If `-k` is omitted, MVT will display the decryption key without saving.
+
+_Note_: This decryption key is sensitive data! Keep the file safe.
+
+To extract the key and have MVT prompt for a password:
+
+```bash
+mvt-ios extract-key -k /path/to/save/key /path/to/backup
+```
+
 ## Decrypting a backup
 
 In case you have an encrypted backup, you will need to decrypt it first. This can be done with `mvt-ios` as well:
@@ -15,9 +41,10 @@ In case you have an encrypted backup, you will need to decrypt it first. This ca
       -d, --destination TEXT  Path to the folder where to store the decrypted
                               backup  [required]
 
-      -p, --password TEXT     Password to use to decrypt the backup NOTE: This
-                              argument is mutually exclusive with arguments:
-                              [key_file].
+      -p, --password TEXT     Password to use to decrypt the backup (or, set
+                              MVT_IOS_BACKUP_PASSWORD environment variable)
+                              NOTE: This argument is mutually exclusive with
+                              arguments: [key_file].
 
       -k, --key-file PATH     File containing raw encryption key to use to decrypt
                               the backup NOTE: This argument is mutually exclusive
@@ -25,10 +52,10 @@ In case you have an encrypted backup, you will need to decrypt it first. This ca
 
       --help                  Show this message and exit.
 
-You can specify either a password via command-line or pass a key file, and you need to specify a destination path where the decrypted backup will be stored. Following is an example usage of `decrypt-backup`:
+You can specify the password in the environment variable `MVT_IOS_BACKUP_PASSWORD`, or via command-line argument, or you can pass a key file.  You need to specify a destination path where the decrypted backup will be stored. If a password cannot be found and no key file is specified, MVT will ask for a password. Following is an example usage of `decrypt-backup` sending the password via an environment variable:
 
 ```bash
-mvt-ios decrypt-backup -p password -d /path/to/decrypted /path/to/backup
+MVT_IOS_BACKUP_PASSWORD="mypassword" mvt-ios decrypt-backup -d /path/to/decrypted /path/to/backup
 ```
 
 ## Run `mvt-ios` on a Backup

docs/ios/backup/libimobiledevice.md

@@ -3,10 +3,14 @@
 If you have correctly [installed libimobiledevice](../install.md) you can easily generate an iTunes backup using the `idevicebackup2` tool included in the suite. First, you might want to ensure that backup encryption is enabled (**note: encrypted backup contain more data than unencrypted backups**):
 
 ```bash
-idevicebackup2 backup encryption on
+idevicebackup2 -i backup encryption on
 ```
 
-Note that if a backup password was previously set on this device, you might need to use the same or change it. You can try changing password using `idevicebackup2 backup changepw` or resetting the password by resetting only the settings through the iPhone's Settings app.
+Note that if a backup password was previously set on this device, you might need to use the same or change it. You can try changing password using `idevicebackup2 -i backup changepw`, or by turning off encryption (`idevicebackup2 -i backup encryption off`) and turning it back on again.
+
+If you are not able to recover or change the password, you should try to disable encryption and obtain an unencrypted backup.
+
+If all else fails, as a *last resort* you can try resetting the password by [resetting all the settings through the iPhone's Settings app](https://support.apple.com/en-us/HT205220), via `Settings » General » Reset » Reset All Settings`.  Note that resetting the settings through the iPhone's Settings app will wipe some of the files that contain useful forensic traces, so try the options explained above first.
 
 Once ready, you can proceed performing the backup:
 

docs/ios/filesystem/dump.md

@@ -1,6 +1,6 @@
 # Dumping the filesystem
 
-While iTunes backup provide a lot of very useful databases and diagnistic data, in some cases you might want to jailbreak the device and perform a full filesystem dump. In that case, you should take a look at [checkra1n](https://checkra.in/), which provides an easy way to obtain root on most recent iPhone models.
+While iTunes backup provide a lot of very useful databases and diagnostic data, in some cases you might want to jailbreak the device and perform a full filesystem dump. In that case, you should take a look at [checkra1n](https://checkra.in/), which provides an easy way to obtain root on most recent iPhone models.
 
 !!! warning
     Before you checkra1n any device, make sure you take a full backup, and that you are prepared to do a full factory reset before restoring it. Even after using checkra1n's "Restore System", some traces of the jailbreak are still left on the device and [apps with anti-jailbreaks will be able to detect them](https://github.com/checkra1n/BugTracker/issues/279) and stop functioning.

docs/ios/methodology.md

@@ -6,7 +6,7 @@ Before jumping into acquiring and analyzing data from an iOS device, you should
 
 You will need to decide whether to attempt to jailbreak the device and obtain a full filesystem dump, or not.
 
-While access the full filesystem allows to extact data that would otherwise be unavailable, it might not always be possible to jailbreak a certain iPhone model or version of iOS. In addition, depending on the type of jailbreak available, doing so might compromise some important records, pollute others, or potentially cause unintended malfunctioning of the device later in case it is used again.
+While access the full filesystem allows to extract data that would otherwise be unavailable, it might not always be possible to jailbreak a certain iPhone model or version of iOS. In addition, depending on the type of jailbreak available, doing so might compromise some important records, pollute others, or potentially cause unintended malfunctioning of the device later in case it is used again.
 
 If you are not expected to return the phone, you might want to consider to attempt a jailbreak after having exhausted all other options, including a backup.
 

docs/ios/records.md

@@ -236,7 +236,7 @@ If indicators are provided through the command-line, they are checked against th
     Backup: :material-close:  
     Full filesystem dump: :material-check:
 
-This JSON file is created by mvt-ios' `WebkitLocalStorage` module. The module extracts a lsit of file and folder names located at the following path */private/var/mobile/Containers/Data/Application/\*/Library/WebKit/WebsiteData/LocalStorage/*, which contains local storage files created by any app installed on the device.
+This JSON file is created by mvt-ios' `WebkitLocalStorage` module. The module extracts a list of file and folder names located at the following path */private/var/mobile/Containers/Data/Application/\*/Library/WebKit/WebsiteData/LocalStorage/*, which contains local storage files created by any app installed on the device.
 
 If indicators are provided through the command-line, they are checked against the extracted names. Any matches are stored in *webkit_local_storage_detected.json*.
 

mkdocs.yml

@@ -28,6 +28,7 @@ nav:
     - Welcome: "index.md"
     - Introduction: "introduction.md"
     - Installation: "install.md"
+    - Using Docker: "docker.md"
     - MVT for iOS:
         - iOS Forensic Methodology: "ios/methodology.md"
         - Install libimobiledevice: "ios/install.md"
@@ -43,4 +44,5 @@ nav:
         - Android Forensic Methodology: "android/methodology.md"
         - Check APKs: "android/download_apks.md"
         - Check an Android Backup: "android/backup.md"
+    - Indicators of Compromise: "iocs.md"
     - License: "license.md"

mvt/android/cli.py

@@ -3,15 +3,17 @@
 # See the file 'LICENSE' for usage and copying permissions, or find a copy at
 #   https://github.com/mvt-project/mvt/blob/main/LICENSE
 
-import os
-import sys
-import click
 import argparse
 import logging
+import os
+import sys
+
+import click
 from rich.logging import RichHandler
 
-from mvt.common.module import run_module, save_timeline
 from mvt.common.indicators import Indicators
+from mvt.common.module import run_module, save_timeline
+
 from .download_apks import DownloadAPKs
 from .lookups.koodous import koodous_lookup
 from .lookups.virustotal import virustotal_lookup
@@ -45,8 +47,8 @@ def cli():
 @click.option("--virustotal", "-v", is_flag=True, help="Check packages on VirusTotal")
 @click.option("--koodous", "-k", is_flag=True, help="Check packages on Koodous")
 @click.option("--all-checks", "-A", is_flag=True, help="Run all available checks")
-@click.option("--output", "-o", type=click.Path(exists=True),
-              help="Specify a path to a folder where you want to store JSON results")
+@click.option("--output", "-o", type=click.Path(exists=False),
+              help="Specify a path to a folder where you want to store the APKs")
 @click.option("--from-file", "-f", type=click.Path(exists=True),
               help="Instead of acquiring from phone, load an existing packages.json file for lookups (mainly for debug purposes)")
 def download_apks(all_apks, virustotal, koodous, all_checks, output, from_file):
@@ -54,9 +56,12 @@ def download_apks(all_apks, virustotal, koodous, all_checks, output, from_file):
         if from_file:
             download = DownloadAPKs.from_json(from_file)
         else:
-            if not output:
-                log.critical("You need to specify an output folder (with --output, -o) when extracting APKs from a device")
-                sys.exit(-1)
+            if output and not os.path.exists(output):
+                try:
+                    os.makedirs(output)
+                except Exception as e:
+                    log.critical("Unable to create output folder %s: %s", output, e)
+                    sys.exit(-1)
 
             download = DownloadAPKs(output_folder=output, all_apks=all_apks)
             download.run()
@@ -81,7 +86,7 @@ def download_apks(all_apks, virustotal, koodous, all_checks, output, from_file):
 #==============================================================================
 @cli.command("check-adb", help="Check an Android device over adb")
 @click.option("--iocs", "-i", type=click.Path(exists=True), help="Path to indicators file")
-@click.option("--output", "-o", type=click.Path(exists=True),
+@click.option("--output", "-o", type=click.Path(exists=False),
               help="Specify a path to a folder where you want to store JSON results")
 @click.option("--list-modules", "-l", is_flag=True, help="Print list of available modules and exit")
 @click.option("--module", "-m", help="Name of a single module you would like to run instead of all")
@@ -95,6 +100,13 @@ def check_adb(iocs, output, list_modules, module):
 
     log.info("Checking Android through adb bridge")
 
+    if output and not os.path.exists(output):
+        try:
+            os.makedirs(output)
+        except Exception as e:
+            log.critical("Unable to create output folder %s: %s", output, e)
+            sys.exit(-1)
+
     if iocs:
         # Pre-load indicators for performance reasons.
         log.info("Loading indicators from provided file at %s", iocs)
@@ -127,11 +139,18 @@ def check_adb(iocs, output, list_modules, module):
 #==============================================================================
 @cli.command("check-backup", help="Check an Android Backup")
 @click.option("--iocs", "-i", type=click.Path(exists=True), help="Path to indicators file")
-@click.option("--output", "-o", type=click.Path(exists=True), help=OUTPUT_HELP_MESSAGE)
+@click.option("--output", "-o", type=click.Path(exists=False), help=OUTPUT_HELP_MESSAGE)
 @click.argument("BACKUP_PATH", type=click.Path(exists=True))
 def check_backup(iocs, output, backup_path):
     log.info("Checking ADB backup located at: %s", backup_path)
 
+    if output and not os.path.exists(output):
+        try:
+            os.makedirs(output)
+        except Exception as e:
+            log.critical("Unable to create output folder %s: %s", output, e)
+            sys.exit(-1)
+
     if iocs:
         # Pre-load indicators for performance reasons.
         log.info("Loading indicators from provided file at %s", iocs)

mvt/android/download_apks.py

@@ -3,20 +3,22 @@
 # See the file 'LICENSE' for usage and copying permissions, or find a copy at
 #   https://github.com/mvt-project/mvt/blob/main/LICENSE
 
-import os
 import json
 import logging
+import os
+
 import pkg_resources
 from tqdm import tqdm
 
 from mvt.common.utils import get_sha256_from_file_path
+
 from .modules.adb.base import AndroidExtraction
 
 log = logging.getLogger(__name__)
 
 # TODO: Would be better to replace tqdm with rich.progress to reduce
 #       the number of dependencies. Need to investigate whether
-#       it's possible to have a simialr callback system.
+#       it's possible to have a similar callback system.
 class PullProgress(tqdm):
     """PullProgress is a tqdm update system for APK downloads."""
 
@@ -42,7 +44,7 @@ class DownloadAPKs(AndroidExtraction):
         """Initialize module.
         :param output_folder: Path to the folder where data should be stored
         :param all_apks: Boolean indicating whether to download all packages
-                         or filter known-goods 
+                         or filter known-goods
         :param packages: Provided list of packages, typically for JSON checks
         """
         super().__init__(file_path=None, base_folder=None,

mvt/android/lookups/koodous.py

@@ -3,13 +3,13 @@
 # See the file 'LICENSE' for usage and copying permissions, or find a copy at
 #   https://github.com/mvt-project/mvt/blob/main/LICENSE
 
-import requests
 import logging
 
-from rich.text import Text
-from rich.table import Table
-from rich.progress import track
+import requests
 from rich.console import Console
+from rich.progress import track
+from rich.table import Table
+from rich.text import Text
 
 log = logging.getLogger(__name__)
 

mvt/android/lookups/virustotal.py

@@ -3,12 +3,13 @@
 # See the file 'LICENSE' for usage and copying permissions, or find a copy at
 #   https://github.com/mvt-project/mvt/blob/main/LICENSE
 
-import requests
 import logging
-from rich.text import Text
-from rich.table import Table
-from rich.progress import track
+
+import requests
 from rich.console import Console
+from rich.progress import track
+from rich.table import Table
+from rich.text import Text
 
 log = logging.getLogger(__name__)
 

mvt/android/modules/adb/__init__.py

@@ -7,11 +7,11 @@ from .chrome_history import ChromeHistory
 from .dumpsys_batterystats import DumpsysBatterystats
 from .dumpsys_packages import DumpsysPackages
 from .dumpsys_procstats import DumpsysProcstats
+from .packages import Packages
 from .processes import Processes
+from .rootbinaries import RootBinaries
 from .sms import SMS
 from .whatsapp import Whatsapp
-from .packages import Packages
-from .rootbinaries import RootBinaries
 
 ADB_MODULES = [ChromeHistory, SMS, Whatsapp, Processes,
                DumpsysBatterystats, DumpsysProcstats,

mvt/android/modules/adb/base.py

@@ -3,16 +3,19 @@
 # See the file 'LICENSE' for usage and copying permissions, or find a copy at
 #   https://github.com/mvt-project/mvt/blob/main/LICENSE
 
-import os
-import sys
-import time
 import logging
+import os
+import random
+import string
+import sys
 import tempfile
+import time
+
 from adb_shell.adb_device import AdbDeviceUsb
 from adb_shell.auth.keygen import keygen, write_public_keyfile
 from adb_shell.auth.sign_pythonrsa import PythonRSASigner
-from adb_shell.exceptions import DeviceAuthError, AdbCommandFailureException
-from usb1 import USBErrorBusy, USBErrorAccess
+from adb_shell.exceptions import AdbCommandFailureException, DeviceAuthError
+from usb1 import USBErrorAccess, USBErrorBusy
 
 from mvt.common.module import MVTModule
 
@@ -96,7 +99,7 @@ class AndroidExtraction(MVTModule):
         """Check if we have a `su` binary on the Android device.
         :returns: Boolean indicating whether a `su` binary is present or not
         """
-        return bool(self._adb_command("[ ! -f /sbin/su ] || echo 1"))
+        return bool(self._adb_command("command -v su"))
 
     def _adb_root_or_die(self):
         """Check if we have a `su` binary, otherwise raise an Exception.
@@ -110,8 +113,21 @@ class AndroidExtraction(MVTModule):
         :returns: Output of command
         """
         return self._adb_command(f"su -c {command}")
+    
+    def _adb_check_file_exists(self, file):
+        """Verify that a file exists.
+        :param file: Path of the file
+        :returns: Boolean indicating whether the file exists or not
+        """
 
-    def _adb_download(self, remote_path, local_path, progress_callback=None):
+        # Connect to the device over adb.
+        self._adb_connect()
+        # Check if we have root, if not raise an Exception.
+        self._adb_root_or_die()
+
+        return bool(self._adb_command_as_root(f"[ ! -f {file} ] || echo 1"))
+
+    def _adb_download(self, remote_path, local_path, progress_callback=None, retry_root=True):
         """Download a file form the device.
         :param remote_path: Path to download from the device
         :param local_path: Path to where to locally store the copy of the file
@@ -119,6 +135,37 @@ class AndroidExtraction(MVTModule):
         """
         try:
             self.device.pull(remote_path, local_path, progress_callback)
+        except AdbCommandFailureException as e:
+            if retry_root:
+                self._adb_download_root(remote_path, local_path, progress_callback)
+            else:
+                raise Exception(f"Unable to download file {remote_path}: {e}")
+    
+    def _adb_download_root(self, remote_path, local_path, progress_callback=None):
+        try:
+            # Check if we have root, if not raise an Exception.
+            self._adb_root_or_die()
+
+            # We generate a random temporary filename.
+            tmp_filename = "tmp_" + ''.join(random.choices(string.ascii_uppercase + string.ascii_lowercase + string.digits, k=10))
+
+            # We create a temporary local file.
+            new_remote_path = f"/sdcard/{tmp_filename}"
+
+            # We copy the file from the data folder to /sdcard/.
+            cp = self._adb_command_as_root(f"cp {remote_path} {new_remote_path}")
+            if cp.startswith("cp: ") and "No such file or directory" in cp:
+                raise Exception(f"Unable to process file {remote_path}: File not found")
+            elif cp.startswith("cp: ") and "Permission denied" in cp:
+                raise Exception(f"Unable to process file {remote_path}: Permission denied")
+
+            # We download from /sdcard/ to the local temporary file.
+            # If it doesn't work now, don't try again (retry_root=False)
+            self._adb_download(new_remote_path, local_path, retry_root=False)
+
+            # Delete the copy on /sdcard/.
+            self._adb_command(f"rm -rf {new_remote_path}")
+            
         except AdbCommandFailureException as e:
             raise Exception(f"Unable to download file {remote_path}: {e}")
 

mvt/android/modules/adb/chrome_history.py

@@ -3,11 +3,12 @@
 # See the file 'LICENSE' for usage and copying permissions, or find a copy at
 #   https://github.com/mvt-project/mvt/blob/main/LICENSE
 
+import logging
 import os
 import sqlite3
-import logging
 
-from mvt.common.utils import convert_chrometime_to_unix, convert_timestamp_to_iso
+from mvt.common.utils import (convert_chrometime_to_unix,
+                              convert_timestamp_to_iso)
 
 from .base import AndroidExtraction
 

mvt/android/modules/adb/dumpsys_batterystats.py

@@ -3,8 +3,8 @@
 # See the file 'LICENSE' for usage and copying permissions, or find a copy at
 #   https://github.com/mvt-project/mvt/blob/main/LICENSE
 
-import os
 import logging
+import os
 
 from .base import AndroidExtraction
 

mvt/android/modules/adb/dumpsys_packages.py

@@ -3,8 +3,8 @@
 # See the file 'LICENSE' for usage and copying permissions, or find a copy at
 #   https://github.com/mvt-project/mvt/blob/main/LICENSE
 
-import os
 import logging
+import os
 
 from .base import AndroidExtraction
 

mvt/android/modules/adb/dumpsys_procstats.py

@@ -3,8 +3,8 @@
 # See the file 'LICENSE' for usage and copying permissions, or find a copy at
 #   https://github.com/mvt-project/mvt/blob/main/LICENSE
 
-import os
 import logging
+import os
 
 from .base import AndroidExtraction
 

mvt/android/modules/adb/packages.py

@@ -3,8 +3,9 @@
 # See the file 'LICENSE' for usage and copying permissions, or find a copy at
 #   https://github.com/mvt-project/mvt/blob/main/LICENSE
 
-import os
 import logging
+import os
+
 import pkg_resources
 
 from .base import AndroidExtraction

mvt/android/modules/adb/rootbinaries.py

@@ -3,8 +3,9 @@
 # See the file 'LICENSE' for usage and copying permissions, or find a copy at
 #   https://github.com/mvt-project/mvt/blob/main/LICENSE
 
-import os
 import logging
+import os
+
 import pkg_resources
 
 from .base import AndroidExtraction

mvt/android/modules/adb/sms.py

@@ -3,16 +3,41 @@
 # See the file 'LICENSE' for usage and copying permissions, or find a copy at
 #   https://github.com/mvt-project/mvt/blob/main/LICENSE
 
+import logging
 import os
 import sqlite3
-import logging
+
+from mvt.common.utils import check_for_links, convert_timestamp_to_iso
 
 from .base import AndroidExtraction
-from mvt.common.utils import convert_timestamp_to_iso, check_for_links
 
 log = logging.getLogger(__name__)
 
-SMS_PATH = "data/data/com.google.android.apps.messaging/databases/bugle_db"
+SMS_BUGLE_PATH = "data/data/com.google.android.apps.messaging/databases/bugle_db"
+SMS_BUGLE_QUERY = """
+SELECT 
+    ppl.normalized_destination AS number,
+    p.timestamp AS timestamp,
+CASE WHEN m.sender_id IN 
+(SELECT _id FROM participants WHERE contact_id=-1)
+THEN 2 ELSE 1 END incoming, p.text AS text 
+FROM messages m, conversations c, parts p,
+        participants ppl, conversation_participants cp
+WHERE (m.conversation_id = c._id)
+    AND (m._id = p.message_id)
+    AND (cp.conversation_id = c._id)
+    AND (cp.participant_id = ppl._id);
+"""
+
+SMS_MMSSMS_PATH = "data/data/com.android.providers.telephony/databases/mmssms.db"
+SMS_MMSMS_QUERY = """
+SELECT 
+    address AS number,
+    date_sent AS timestamp,
+    type as incoming,
+    body AS text 
+FROM sms;
+"""
 
 class SMS(AndroidExtraction):
     """This module extracts all SMS messages containing links."""
@@ -50,20 +75,12 @@ class SMS(AndroidExtraction):
         """
         conn = sqlite3.connect(db_path)
         cur = conn.cursor()
-        cur.execute("""
-            SELECT 
-                ppl.normalized_destination AS number,
-                p.timestamp AS timestamp,
-            CASE WHEN m.sender_id IN 
-            (SELECT _id FROM participants WHERE contact_id=-1)
-            THEN 2 ELSE 1 END incoming, p.text AS text 
-            FROM messages m, conversations c, parts p,
-                 participants ppl, conversation_participants cp
-            WHERE (m.conversation_id = c._id)
-                AND (m._id = p.message_id)
-                AND (cp.conversation_id = c._id)
-                AND (cp.participant_id = ppl._id);
-        """)
+        
+        if (self.SMS_DB_TYPE == 1):
+            cur.execute(SMS_BUGLE_QUERY)
+        elif (self.SMS_DB_TYPE == 2):
+            cur.execute(SMS_MMSMS_QUERY)
+
         names = [description[0] for description in cur.description]
 
         for item in cur:
@@ -85,7 +102,15 @@ class SMS(AndroidExtraction):
         log.info("Extracted a total of %d SMS messages containing links", len(self.results))
 
     def run(self):
+        # Checking the SMS database path
         try:
-            self._adb_process_file(os.path.join("/", SMS_PATH), self._parse_db)
+            if (self._adb_check_file_exists(os.path.join("/", SMS_BUGLE_PATH))):
+                self.SMS_DB_TYPE = 1
+                self._adb_process_file(os.path.join("/", SMS_BUGLE_PATH), self._parse_db)
+            elif (self._adb_check_file_exists(os.path.join("/", SMS_MMSSMS_PATH))):
+                self.SMS_DB_TYPE = 2
+                self._adb_process_file(os.path.join("/", SMS_MMSSMS_PATH), self._parse_db)
+            else:
+                self.log.error("No SMS database found")
         except Exception as e:
             self.log.error(e)