Bumped version

Create adb keys (Fixes #165)

.github/workflows/lint-python.yml

@@ -1,21 +0,0 @@
-name: lint_python
-on: [pull_request, push]
-jobs:
-  lint_python:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
-      - run: pip install bandit black codespell flake8 isort mypy pytest pyupgrade safety
-      - run: bandit --recursive --skip B108,B112,B404,B602 .
-      - run: black --check .  || true
-      - run: codespell
-      - run: flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
-      - run: flake8 . --count --exit-zero --max-complexity=10 --max-line-length=88 --show-source --statistics
-      - run: isort --check-only --profile black . || true
-      - run: pip install -r requirements.txt || true
-      - run: mypy --install-types --non-interactive . || true
-      - run: pytest . || true
-      - run: pytest --doctest-modules . || true
-      - run: shopt -s globstar && pyupgrade --py36-plus **/*.py || true
-      - run: safety check

.github/workflows/python-package.yml

@@ -0,0 +1,43 @@
+# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
+
+name: Python package
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: [3.7, 3.8, 3.9]
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install flake8 pytest safety
+        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+    - name: Lint with flake8
+      run: |
+        # stop the build if there are Python syntax errors or undefined names
+        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+    - name: Safety checks
+      run: safety check
+
+    # - name: Test with pytest
+    #   run: |
+    #     pytest

.github/workflows/python-publish.yml

@@ -1,36 +0,0 @@
-# This workflow will upload a Python Package using Twine when a release is created
-# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries
-
-# This workflow uses actions that are not certified by GitHub.
-# They are provided by a third-party and are governed by
-# separate terms of service, privacy policy, and support
-# documentation.
-
-name: Upload Python Package
-
-on:
-  release:
-    types: [published]
-
-jobs:
-  deploy:
-
-    runs-on: ubuntu-latest
-
-    steps:
-    - uses: actions/checkout@v2
-    - name: Set up Python
-      uses: actions/setup-python@v2
-      with:
-        python-version: '3.x'
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install build
-    - name: Build package
-      run: python -m build
-    - name: Publish package
-      uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29
-      with:
-        user: __token__
-        password: ${{ secrets.PYPI_API_TOKEN }}

.gitignore

@@ -129,3 +129,5 @@ dmypy.json
 .pyre/
 *.pyc
 
+# Temporal files
+*~

AUTHORS

@@ -0,0 +1,7 @@
+MVT was originally authored by Claudio Guarnieri <nex@nex.sx>.
+
+For an up-to-date list of all contributors visit:
+    https://github.com/mvt-project/mvt/graphs/contributors
+
+Or run:
+    git shortlog -s -n

Dockerfile

@@ -2,47 +2,56 @@ FROM ubuntu:20.04
 
 # Ref. https://github.com/mvt-project/mvt
 
+LABEL url="https://mvt.re"
+LABEL vcs-url="https://github.com/mvt-project/mvt"
+LABEL description="MVT is a forensic tool to look for signs of infection in smartphone devices."
+
+ENV PIP_NO_CACHE_DIR=1
+
 # Fixing major OS dependencies
 # ----------------------------
 RUN apt update \
   && apt install -y python3 python3-pip libusb-1.0-0-dev \
-  && apt install -y wget \ 
-  && apt install -y adb \ 
-  && DEBIAN_FRONTEND=noninteractive apt-get -y install default-jre-headless
+  && apt install -y wget unzip\ 
+  && DEBIAN_FRONTEND=noninteractive apt-get -y install default-jre-headless \
 
 # Install build tools for libimobiledevice
 # ----------------------------------------
-RUN apt install -y build-essential \
-	checkinstall \
-	git \
-	autoconf \
-	automake \
-	libtool-bin \
-	libplist-dev \
-	libusbmuxd-dev \
-	libssl-dev \
-    pkg-config
+  build-essential \
+  checkinstall \
+  git \
+  autoconf \
+  automake \
+  libtool-bin \
+  libplist-dev \
+  libusbmuxd-dev \
+  libssl-dev \
+  sqlite3 \
+  pkg-config \
 
 # Clean up
 # --------
-RUN apt-get clean \
-  && rm -rf /var/lib/apt/lists/*
+  && apt-get clean \
+  && rm -rf /var/lib/apt/lists/* /var/cache/apt
 
 
 # Build libimobiledevice
 # ----------------------
-RUN git clone https://github.com/libimobiledevice/libplist
-RUN git clone https://github.com/libimobiledevice/libusbmuxd
-RUN git clone https://github.com/libimobiledevice/libimobiledevice
-RUN git clone https://github.com/libimobiledevice/usbmuxd
+RUN git clone https://github.com/libimobiledevice/libplist \
+  && git clone https://github.com/libimobiledevice/libusbmuxd \
+  && git clone https://github.com/libimobiledevice/libimobiledevice \
+  && git clone https://github.com/libimobiledevice/usbmuxd \
 
-RUN cd libplist && ./autogen.sh && make && make install && ldconfig
+  && cd libplist && ./autogen.sh && make && make install && ldconfig \
 
-RUN cd libusbmuxd && PKG_CONFIG_PATH=/usr/local/lib/pkgconfig ./autogen.sh && make && make install && ldconfig
+  && cd ../libusbmuxd && PKG_CONFIG_PATH=/usr/local/lib/pkgconfig ./autogen.sh && make && make install && ldconfig \
 
-RUN cd libimobiledevice && PKG_CONFIG_PATH=/usr/local/lib/pkgconfig ./autogen.sh --enable-debug && make && make install && ldconfig
+  && cd ../libimobiledevice && PKG_CONFIG_PATH=/usr/local/lib/pkgconfig ./autogen.sh --enable-debug && make && make install && ldconfig \
 
-RUN cd usbmuxd && PKG_CONFIG_PATH=/usr/local/lib/pkgconfig ./autogen.sh --prefix=/usr --sysconfdir=/etc --localstatedir=/var --runstatedir=/run && make && make install
+  && cd ../usbmuxd && PKG_CONFIG_PATH=/usr/local/lib/pkgconfig ./autogen.sh --prefix=/usr --sysconfdir=/etc --localstatedir=/var --runstatedir=/run && make && make install \
+
+  # Clean up.
+  && cd .. && rm -rf libplist libusbmuxd libimobiledevice usbmuxd
 
 # Installing MVT
 # --------------
@@ -50,16 +59,29 @@ RUN pip3 install mvt
 
 # Installing ABE
 # --------------
-RUN mkdir /opt/abe
-RUN wget https://github.com/nelenkov/android-backup-extractor/releases/download/20210709062403-4c55371/abe.jar -O /opt/abe/abe.jar
+RUN mkdir /opt/abe \
+  && wget https://github.com/nelenkov/android-backup-extractor/releases/download/20210709062403-4c55371/abe.jar -O /opt/abe/abe.jar \
 # Create alias for abe
-RUN echo 'alias abe="java -jar /opt/abe/abe.jar"' >> ~/.bashrc
+  && echo 'alias abe="java -jar /opt/abe/abe.jar"' >> ~/.bashrc
+
+# Install Android Platform Tools 
+# ------------------------------ 
+
+RUN mkdir /opt/android \
+  && wget -q https://dl.google.com/android/repository/platform-tools-latest-linux.zip \
+  && unzip platform-tools-latest-linux.zip -d /opt/android \
+# Create alias for adb 
+  && echo 'alias adb="/opt/android/platform-tools/adb"' >> ~/.bashrc 
+
+# Generate adb key folder 
+# ------------------------------ 
+RUN mkdir /root/.android && /opt/android/platform-tools/adb keygen /root/.android/adbkey
 
 # Setup investigations environment
 # --------------------------------
 RUN mkdir /home/cases
 WORKDIR /home/cases 
-RUN echo 'echo "Mobile Verification Toolkit @ Docker\n------------------------------------\n\nYou can find information about how to use this image for Android (https://github.com/mvt-project/mvt/tree/master/docs/android) and iOS (https://github.com/mvt-project/mvt/tree/master/docs/ios) in the official docs of the project.\n"' >> ~/.bashrc
-RUN echo 'echo "Note that to perform the debug via USB you might need to give the Docker image access to the USB using \"docker run -it --privileged -v /dev/bus/usb:/dev/bus/usb mvt\" or, preferably, the \"--device=\" parameter.\n"' >> ~/.bashrc
+RUN echo 'echo "Mobile Verification Toolkit @ Docker\n------------------------------------\n\nYou can find information about how to use this image for Android (https://github.com/mvt-project/mvt/tree/master/docs/android) and iOS (https://github.com/mvt-project/mvt/tree/master/docs/ios) in the official docs of the project.\n"' >> ~/.bashrc \
+  && echo 'echo "Note that to perform the debug via USB you might need to give the Docker image access to the USB using \"docker run -it --privileged -v /dev/bus/usb:/dev/bus/usb mvt\" or, preferably, the \"--device=\" parameter.\n"' >> ~/.bashrc
 
 CMD /bin/bash

README.md

@@ -1,11 +1,11 @@
 <p align="center">
-     <img src="./docs/mvt.png" width="300" />
+     <img src="./docs/mvt.png" width="200" />
 </p>
 
 # Mobile Verification Toolkit
 
 [![](https://img.shields.io/pypi/v/mvt)](https://pypi.org/project/mvt/)
-[![](https://img.shields.io/badge/docs-blue.svg)](https://mvt.readthedocs.io)
+[![Documentation Status](https://readthedocs.org/projects/mvt/badge/?version=latest)](https://docs.mvt.re/en/latest/?badge=latest)
 
 Mobile Verification Toolkit (MVT) is a collection of utilities to simplify and automate the process of gathering forensic traces helpful to identify a potential compromise of Android and iOS devices.
 
@@ -15,37 +15,20 @@ It has been developed and released by the [Amnesty International Security Lab](h
 
 ## Installation
 
-MVT can be installed from sources or conveniently using:
+MVT can be installed from sources or from [PyPi](https://pypi.org/project/mvt/) (you will need some dependencies, check the [documentation](https://docs.mvt.re/en/latest/install.html)):
 
 ```
 pip3 install mvt
 ```
 
-You will need some dependencies, so please check the [documentation](https://mvt.readthedocs.io/en/latest/install.html).
-
-Alternatively, you can decide to run MVT and all relevant tools through a [Docker container](https://mvt.readthedocs.io/en/latest/docker.html).
+Alternatively, you can decide to run MVT and all relevant tools through a [Docker container](https://docs.mvt.re/en/latest/docker.html).
 
+**Please note:** MVT is best run on Linux or Mac systems. [It does not currently support running natively on Windows.](https://docs.mvt.re/en/latest/install.html#mvt-on-windows)
 
 ## Usage
 
-MVT provides two commands `mvt-ios` and `mvt-android` with the following subcommands available:
-
-* `mvt-ios`:
-    * `check-backup`: Extract artifacts from an iTunes backup
-    * `check-fs`: Extract artifacts from a full filesystem dump
-    * `check-iocs`: Compare stored JSON results to provided indicators
-    * `decrypt-backup`:  Decrypt an encrypted iTunes backup
-* `mvt-android`:
-    * `check-backup`: Check an Android Backup
-    * `download-apks`: Download all or non-safelisted installed APKs
-
-Check out [the documentation to see how to use them](https://mvt.readthedocs.io/en/latest/).
-
+MVT provides two commands `mvt-ios` and `mvt-android`. [Check out the documentation to learn how to use them!](https://docs.mvt.re/)
 
 ## License
 
-The purpose of MVT is to facilitate the ***consensual forensic analysis*** of devices of those who might be targets of sophisticated mobile spyware attacks, especially members of civil society and marginalized communities. We do not want MVT to enable privacy violations of non-consenting individuals. Therefore, the goal of this license is to prohibit the use of MVT (and any other software licensed the same) for the purpose of *adversarial forensics*.
-
-In order to achieve this, MVT is released under an adaptation of [Mozilla Public License v2.0](https://www.mozilla.org/MPL). This modified license includes a new clause 3.0, "Consensual Use Restriction" which permits the use of the licensed software (and any *"Larger Work"* derived from it) exclusively with the explicit consent of the person/s whose data is being extracted and/or analysed (*"Data Owner"*).
-
-[Read the LICENSE](https://github.com/mvt-project/mvt/blob/main/LICENSE)
+The purpose of MVT is to facilitate the ***consensual forensic analysis*** of devices of those who might be targets of sophisticated mobile spyware attacks, especially members of civil society and marginalized communities. We do not want MVT to enable privacy violations of non-consenting individuals.  In order to achieve this, MVT is released under its own license. [Read more here.](https://docs.mvt.re/en/latest/license.html)

dev/mvt-android

@@ -1,8 +1,8 @@
 #!/usr/bin/env python3
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021 MVT Project Developers.
-# See the file 'LICENSE' for usage and copying permissions, or find a copy at
-#   https://github.com/mvt-project/mvt/blob/main/LICENSE
+# Copyright (c) 2021 The MVT Project Authors.
+# Use of this software is governed by the MVT License 1.1 that can be found at
+#   https://license.mvt.re/1.1/
 
 import os
 import sys
@@ -10,4 +10,5 @@ import sys
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 from mvt import android
+
 android.cli()

dev/mvt-ios

@@ -1,8 +1,8 @@
 #!/usr/bin/env python3
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021 MVT Project Developers.
-# See the file 'LICENSE' for usage and copying permissions, or find a copy at
-#   https://github.com/mvt-project/mvt/blob/main/LICENSE
+# Copyright (c) 2021 The MVT Project Authors.
+# Use of this software is governed by the MVT License 1.1 that can be found at
+#   https://license.mvt.re/1.1/
 
 import os
 import sys
@@ -10,4 +10,5 @@ import sys
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 from mvt import ios
+
 ios.cli()

docs/android/adb.md

@@ -0,0 +1,42 @@
+# Check over ADB
+
+In order to check an Android device over the [Android Debug Bridge (adb)](https://developer.android.com/studio/command-line/adb) you will first need to install [Android SDK Platform Tools](https://developer.android.com/studio/releases/platform-tools). If you have installed [Android Studio](https://developer.android.com/studio/) you should already have access to `adb` and other utilities.
+
+While many Linux distributions already package Android Platform Tools (for example `android-platform-tools-base` on Debian), it is preferable to install the most recent version from the official website. Packaged versions might be outdated and incompatible with most recent Android handsets.
+
+Next you will need to enable debugging on the Android device you are testing. [Please follow the official instructions on how to do so.](https://developer.android.com/studio/command-line/adb)
+
+## Connecting over USB
+
+The easiest way to check the device is over a USB transport. You will need to have USB debugging enabled and the device plugged into your computer. If everything is configured appropriately you should see your device when launching the command `adb devices`.
+
+Now you can try launching MVT with:
+
+```bash
+mvt-android check-adb --output /path/to/results
+```
+
+If you have previously started an adb daemon MVT will alert you and require you to kill it with `adb kill-server` and relaunch the command.
+
+!!! warning
+    MVT relies on the Python library [adb-shell](https://pypi.org/project/adb-shell/) to connect to an Android device, which relies on libusb for the USB transport. Because of known driver issues, Windows users [are recommended](https://github.com/JeffLIrion/adb_shell/issues/118) to install appropriate drivers using [Zadig](https://zadig.akeo.ie/). Alternatively, an easier option might be to use the TCP transport and connect over Wi-Fi as describe next.
+
+## Connecting over Wi-FI
+
+When connecting to the device over USB is not possible or not working properly, an alternative option is to connect over the network. In order to do so, first launch an adb daemon at a fixed port number:
+
+```bash
+adb tcpip 5555
+```
+
+Then you can specify the IP address of the phone with the adb port number to MVT like so:
+
+```bash
+mvt-android check-adb --serial 192.168.1.20:5555 --output /path/to/results
+```
+
+Where `192.168.1.20` is the correct IP address of your device.
+
+## MVT modules requiring root privileges
+
+Of the currently available `mvt-android check-adb` modules a handful require root privileges to function correctly. This is because certain files, such as browser history and SMS messages databases are not accessible with user privileges through adb. These modules are to be considered OPTIONALLY available in case the device was already jailbroken. **Do NOT jailbreak your own device unless you are sure of what you are doing!** Jailbreaking your phone exposes it to considerable security risks!

docs/android/backup.md

@@ -1,38 +1,47 @@
-# Checking SMSs from Android backup
+# Check an Android Backup (SMS messages)
 
-Some attacks against Android phones are done by sending malicious links by SMS. The Android backup feature does not allow to gather much information that can be interesting for a forensic analysis, but it can be used to extract SMSs and check them with MVT.
+Android supports generating a backup archive of all the installed applications which supports it. However, over the years this functionality has been increasingly abandoned in favor of enabling users to remotely backup their personal data over the cloud. App developers can therefore decide to opt out from allowing the apps' data from being exported locally.
 
-To do so, you need to connect your Android device to your computer. You will then need to [enable USB debugging](https://developer.android.com/studio/debug/dev-options#enable>) on the Android device.
+At the time of writing, the Android Debug Bridge (adb) command to generate backups is still available but marked as deprecated.
 
-If this is the first time you connect to this device, you will need to approve the authentication keys through a prompt that will appear on your Android device.
+That said, most versions of Android should still allow to locally backup SMS messages, and since messages are still a prime vehicle for phishing and malware attacks, you might still want to take advantage of this functionality while it is supported.
 
-Then you can use adb to extract the backup for SMS only with the following command:
+## Generate a backup
+
+Because `mvt-android check-backup` currently only supports checking SMS messages, you can indicate to backup only those:
 
 ```bash
 adb backup com.android.providers.telephony
 ```
 
-You will need to approve the backup on the phone and potentially enter a password to encrypt the backup. The backup will then be stored in a file named `backup.ab`.
+In case you nonetheless wish to take a full backup, you can do so with
 
-You will need to use [Android Backup Extractor](https://github.com/nelenkov/android-backup-extractor) to convert it to a readable file format. Make sure that java is installed on your system and use the following command:
 ```bash
-java -jar ~/Download/abe.jar unpack backup.ab backup.tar
+adb backup -all
+```
+
+## Unpack the backup
+
+In order to reliable unpack th [Android Backup Extractor (ABE)](https://github.com/nelenkov/android-backup-extractor) to convert it to a readable file format. Make sure that java is installed on your system and use the following command:
+
+```bash
+java -jar ~/path/to/abe.jar unpack backup.ab backup.tar
 tar xvf backup.tar
 ```
 
-(If the backup is encrypted, the password will be asked by Android Backup Extractor).
+If the backup is encrypted, ABE will prompt you to enter the password.
+
+## Check the backup
 
 You can then extract SMSs containing links with MVT:
 
 ```bash
-$ mvt-android check-backup --output . .
+$ mvt-android check-backup --output /path/to/results/ /path/to/backup/
 16:18:38 INFO     [mvt.android.cli] Checking ADB backup located at: .
          INFO     [mvt.android.modules.backup.sms] Running module SMS...
-         INFO     [mvt.android.modules.backup.sms] Processing SMS backup
-                  file at ./apps/com.android.providers.telephony/d_f/000
-                  000_sms_backup
+         INFO     [mvt.android.modules.backup.sms] Processing SMS backup file at /path/to/backup/apps/com.android.providers.telephony/d_f/000000_sms_backup
 16:18:39 INFO     [mvt.android.modules.backup.sms] Extracted a total of
                   64 SMS messages containing links
 ```
 
-Through the `--iocs` argument you can specify a [STIX2](https://oasis-open.github.io/cti-documentation/stix/intro) file defining a list of malicious indicators to check against the records extracted from the backup by mvt. Any matches will be highlighted in the terminal output.
+Through the `--iocs` argument you can specify a [STIX2](https://oasis-open.github.io/cti-documentation/stix/intro) file defining a list of malicious indicators to check against the records extracted from the backup by MVT. Any matches will be highlighted in the terminal output.

docs/android/download_apks.md

@@ -1,15 +1,18 @@
 # Downloading APKs from an Android phone
 
-In order to use `mvt-android` you need to connect your Android device to your computer. You will then need to [enable USB debugging](https://developer.android.com/studio/debug/dev-options#enable>) on the Android device.
+MVT allows to attempt to download all available installed packages (APKs) in order to further inspect them and potentially identify any which might be malicious in nature.
 
-If this is the first time you connect to this device, you will need to approve the authentication keys through a prompt that will appear on your Android device.
-
-Now you can launch `mvt-android` and specify the `download-apks` command and the path to the folder where you want to store the extracted data:
+You can do so by launching the following command:
 
 ```bash
 mvt-android download-apks --output /path/to/folder
 ```
 
+It might take several minutes to complete.
+
+!!! info
+    MVT will likely warn you it was unable to download certain installed packages. There is no reason to be alarmed: this is typically expected behavior when MVT attempts to download a system package it has no privileges to access.
+
 Optionally, you can decide to enable lookups of the SHA256 hash of all the extracted APKs on [VirusTotal](https://www.virustotal.com) and/or [Koodous](https://koodous.com). While these lookups do not provide any conclusive assessment on all of the extracted APKs, they might highlight any known malicious ones:
 
 ```bash
@@ -17,8 +20,15 @@ mvt-android download-apks --output /path/to/folder --virustotal
 mvt-android download-apks --output /path/to/folder --koodous
 ```
 
-Or, to launch all available lookups::
+Or, to launch all available lookups:
 
 ```bash
 mvt-android download-apks --output /path/to/folder --all-checks
 ```
+
+In case you have a previous extraction of APKs you want to later check against VirusTotal and Koodous, you can do so with the following arguments:
+
+```bash
+mvt-android download-apks --from-file /path/to/folder/apks.json --all-checks
+```
+

docs/android/methodology.md

@@ -1,8 +1,21 @@
 # Methodology for Android forensic
 
-For different technical reasons, it is more complex to do a forensic analysis of an Android phone.
+Unfortunately Android devices provide much less observability than their iOS cousins. Android stores very little diagnostic information useful to triage potential compromises, and because of this `mvt-android` capabilities are limited as well.
 
-Currently MVT allows to perform two different checks on an Android phone:
+However, not all is lost.
 
-* Download APKs installed in order to analyze them
-* Extract Android backup in order to look for suspicious SMS
+## Check installed Apps
+
+Because malware attacks over Android typically take the form of malicious or backdoored apps, the very first thing you might want to do is to extract and verify all installed Android packages and triage quickly if there are any which stand out as malicious or which might be atypical.
+
+While it is out of the scope of this documentation to dwell into details on how to analyze Android apps, MVT does allow to easily and automatically extract information about installed apps, download copies of them, and quickly lookup services such as [VirusTotal](https://www.virustotal.com) or [Koodous](https://www.koodous.com) which might quickly indicate known bad apps.
+
+
+## Check the device over Android Debug Bridge
+
+Some additional diagnostic information can be extracted from the phone using the [Android Debug Bridge (adb)](https://developer.android.com/studio/command-line/adb). `mvt-android` allows to automatically extract information including [dumpsys](https://developer.android.com/studio/command-line/dumpsys) results, details on installed packages (without download), running processes, presence of root binaries and packages, and more.
+
+
+## Check an Android Backup (SMS messages)
+
+Although Android backups are becoming deprecated, it is still possible to generate one. Unfortunately, because apps these days typically favor backup over the cloud, the amount of data available is limited. Currently, `mvt-android check-backup` only supports checking SMS messages containing links.

docs/docker.md

@@ -1,5 +1,3 @@
-## Using Docker
-
 Using Docker simplifies having all the required dependencies and tools (including most recent versions of [libimobiledevice](https://libimobiledevice.org)) readily installed.
 
 Install Docker following the [official documentation](https://docs.docker.com/get-docker/).

docs/index.md

@@ -10,4 +10,4 @@ In this documentation you will find instructions on how to install and run the `
 
 ## Resources
 
-[:fontawesome-brands-python: Python Package](https://pypi.org/project/mvt){: .md-button .md-button--primary } [:fontawesome-brands-github: GitHub](https://github.com/mvt-project/mvt){: .md-button }
+[:fontawesome-brands-github: GitHub](https://github.com/mvt-project/mvt){: .md-button .md-button--primary } [:fontawesome-brands-python: Python Package](https://pypi.org/project/mvt){: .md-button }

docs/install.md

@@ -1,6 +1,6 @@
 # Installation
 
-Before proceeding, please note that mvt requires Python 3.6+ to run. While it should be available on most operating systems, please make sure of that before proceeding.
+Before proceeding, please note that MVT requires Python 3.6+ to run. While it should be available on most operating systems, please make sure of that before proceeding.
 
 ## Dependencies on Linux
 
@@ -12,9 +12,11 @@ sudo apt install python3 python3-pip libusb-1.0-0 sqlite3
 
 *libusb-1.0-0* is not required if you intend to only use `mvt-ios` and not `mvt-android`.
 
-## Dependencies on Mac
+When working with Android devices you should additionally install [Android SDK Platform Tools](https://developer.android.com/studio/releases/platform-tools). If you prefer to install a package made available by your distribution of choice, please make sure the version is recent to ensure compatibility with modern Android devices.
 
-Running MVT on Mac requires Xcode and [homebrew](https://brew.sh) to be installed.
+## Dependencies on macOS
+
+Running MVT on macOS requires Xcode and [homebrew](https://brew.sh) to be installed.
 
 In order to install dependencies use:
 
@@ -24,6 +26,20 @@ brew install python3 libusb sqlite3
 
 *libusb* is not required if you intend to only use `mvt-ios` and not `mvt-android`.
 
+When working with Android devices you should additionally install [Android SDK Platform Tools](https://developer.android.com/studio/releases/platform-tools):
+
+```bash
+brew install --cask android-platform-tools
+```
+
+Or by downloading the [official binary releases](https://developer.android.com/studio/releases/platform-tools).
+
+## MVT on Windows
+
+MVT does not currently officially support running natively on Windows. While most functionality should work out of the box, there are known issues especially with `mvt-android`.
+
+It is recommended to try installing and running MVT from [Windows Subsystem Linux (WSL)](https://docs.microsoft.com/en-us/windows/wsl/about) and follow Linux installation instructions for your distribution of choice.
+
 ## Installing MVT
 
 If you haven't done so, you can add this to your `.bashrc` or `.zshrc` file in order to add locally installed Pypi binaries to your `$PATH`:
@@ -35,7 +51,7 @@ export PATH=$PATH:~/.local/bin
 Then you can install MVT directly from [pypi](https://pypi.org/project/mvt/)
 
 ```bash
-pip install mvt
+pip3 install mvt
 ```
 
 Or from the source code:

docs/introduction.md

@@ -10,6 +10,8 @@ Mobile Verification Toolkit (MVT) is a collection of utilities designed to facil
 - Generate JSON logs of extracted records, and separate JSON logs of all detected malicious traces.
 - Generate a unified chronological timeline of extracted records, along with a timeline all detected malicious traces.
 
+MVT is a forensic research tool intended for technologists and investigators. Using it requires understanding the basics of forensic analysis and using command-line tools. MVT is not intended for end-user self-assessment. If you are concerned with the security of your device please seek expert assistance.
+
 ## Consensual Forensics
 
 While MVT is capable of extracting and processing various types of very personal records typically found on a mobile phone (such as calls history, SMS and WhatsApp messages, etc.), this is intended to help identify potential attack vectors such as malicious SMS messages leading to exploitation.

docs/iocs.md

@@ -0,0 +1,36 @@
+# Indicators of Compromise (IOCs)
+
+MVT uses [Structured Threat Information Expression (STIX)](https://oasis-open.github.io/cti-documentation/stix/intro.html) files to identify potential traces of compromise.
+
+These indicators of compromise are contained in a file with a particular structure of [JSON](https://en.wikipedia.org/wiki/JSON) with the `.stix2` or `.json` extensions.
+
+You can indicate a path to a STIX2 indicators file when checking iPhone backups or filesystem dumps. For example:
+
+```bash
+mvt-ios check-backup --iocs ~/ios/malware.stix2 --output /path/to/iphone/output /path/to/backup
+```
+
+Or, with data from an Android backup:
+
+```bash
+mvt-android check-backup --iocs ~/iocs/malware.stix2 /path/to/android/backup/
+```
+
+After extracting forensics data from a device, you are also able to compare it with any STIX2 file you indicate:
+
+```bash
+mvt-ios check-iocs --iocs ~/iocs/malware.stix2 /path/to/iphone/output/
+```
+
+The `--iocs` option can be invoked multiple times to let MVT import multiple STIX2 files at once. For example:
+
+```bash
+mvt-ios check-backup --iocs ~/iocs/malware1.stix --iocs ~/iocs/malware2.stix2 /path/to/backup
+```
+
+## Known repositories of STIX2 IOCs
+
+- The [Amnesty International investigations repository](https://github.com/AmnestyTech/investigations) contains STIX-formatted IOCs for:
+    - [Pegasus](https://en.wikipedia.org/wiki/Pegasus_(spyware)) ([STIX2](https://raw.githubusercontent.com/AmnestyTech/investigations/master/2021-07-18_nso/pegasus.stix2))
+
+Please [open an issue](https://github.com/mvt-project/mvt/issues/) to suggest new sources of STIX-formatted IOCs.

docs/ios/backup/check.md

@@ -2,6 +2,32 @@
 
 The backup might take some time. It is best to make sure the phone remains unlocked during the backup process. Afterwards, a new folder will be created under the path you specified using the UDID of the iPhone you backed up.
 
+## Extracting and saving the decryption key (optional)
+
+If you do not wish to enter a password every time when decrypting a backup, MVT can accept a key file instead. This key can be used with the `decrypt-backup` command.
+
+To generate a key file, you will need your device backup and the backup password:
+
+    $ mvt-ios extract-key --help
+    Usage: mvt-ios extract-key [OPTIONS] BACKUP_PATH
+
+      Extract decryption key from an iTunes backup
+
+    Options:
+      -p, --password TEXT  Password to use to decrypt the backup  [required]
+      -k, --key-file FILE  Key file to be written (if unset, will print to STDOUT)
+      --help               Show this message and exit.
+
+You can specify the password on the command line, or omit the `-p` option to have MVT prompt for a password. The `-k` option specifies where to save the file containing the decryption key. If `-k` is omitted, MVT will display the decryption key without saving.
+
+_Note_: This decryption key is sensitive data! Keep the file safe.
+
+To extract the key and have MVT prompt for a password:
+
+```bash
+mvt-ios extract-key -k /path/to/save/key /path/to/backup
+```
+
 ## Decrypting a backup
 
 In case you have an encrypted backup, you will need to decrypt it first. This can be done with `mvt-ios` as well:
@@ -15,9 +41,10 @@ In case you have an encrypted backup, you will need to decrypt it first. This ca
       -d, --destination TEXT  Path to the folder where to store the decrypted
                               backup  [required]
 
-      -p, --password TEXT     Password to use to decrypt the backup NOTE: This
-                              argument is mutually exclusive with arguments:
-                              [key_file].
+      -p, --password TEXT     Password to use to decrypt the backup (or, set
+                              MVT_IOS_BACKUP_PASSWORD environment variable)
+                              NOTE: This argument is mutually exclusive with
+                              arguments: [key_file].
 
       -k, --key-file PATH     File containing raw encryption key to use to decrypt
                               the backup NOTE: This argument is mutually exclusive
@@ -25,10 +52,10 @@ In case you have an encrypted backup, you will need to decrypt it first. This ca
 
       --help                  Show this message and exit.
 
-You can specify either a password via command-line or pass a key file, and you need to specify a destination path where the decrypted backup will be stored. Following is an example usage of `decrypt-backup`:
+You can specify the password in the environment variable `MVT_IOS_BACKUP_PASSWORD`, or via command-line argument, or you can pass a key file.  You need to specify a destination path where the decrypted backup will be stored. If a password cannot be found and no key file is specified, MVT will ask for a password. Following is an example usage of `decrypt-backup` sending the password via an environment variable:
 
 ```bash
-mvt-ios decrypt-backup -p password -d /path/to/decrypted /path/to/backup
+MVT_IOS_BACKUP_PASSWORD="mypassword" mvt-ios decrypt-backup -d /path/to/decrypted /path/to/backup
 ```
 
 ## Run `mvt-ios` on a Backup

docs/ios/backup/itunes.md

@@ -1,16 +1,16 @@
 # Backup with iTunes app
 
-It is possible to do an iPhone backup by using iTunes on Windows or Mac computers (in most recent versions of Mac OS, this feature is included in Finder).
+It is possible to do an iPhone backup by using iTunes on Windows or macOS computers (in most recent versions of macOS, this feature is included in Finder).
 
 To do that:
 
 * Make sure iTunes is installed.
 * Connect your iPhone to your computer using a Lightning/USB cable.
-* Open the device in iTunes (or Finder on Mac OS).
+* Open the device in iTunes (or Finder on macOS).
 * If you want to have a more accurate detection, ensure that the encrypted backup option is activated and choose a secure password for the backup.
 * Start the backup and wait for it to finish (this may take up to 30 minutes).
 
 ![](../../../img/macos-backup.jpg)
 _Source: [Apple Support](https://support.apple.com/en-us/HT211229)_
 
-* Once the backup is done, find its location and copy it to a place where it can be analyzed by `mvt`. On Windows, the backup can be stored either in `%USERPROFILE%\Apple\MobileSync\` or `%USERPROFILE%\AppData\Roaming\Apple Computer\MobileSync\`. On Mac OS, the backup is stored in `~/Library/Application Support/MobileSync/`.
+* Once the backup is done, find its location and copy it to a place where it can be analyzed by MVT. On Windows, the backup can be stored either in `%USERPROFILE%\Apple\MobileSync\` or `%USERPROFILE%\AppData\Roaming\Apple Computer\MobileSync\`. On macOS, the backup is stored in `~/Library/Application Support/MobileSync/`.

docs/ios/backup/libimobiledevice.md

@@ -3,10 +3,14 @@
 If you have correctly [installed libimobiledevice](../install.md) you can easily generate an iTunes backup using the `idevicebackup2` tool included in the suite. First, you might want to ensure that backup encryption is enabled (**note: encrypted backup contain more data than unencrypted backups**):
 
 ```bash
-idevicebackup2 backup encryption on
+idevicebackup2 -i backup encryption on
 ```
 
-Note that if a backup password was previously set on this device, you might need to use the same or change it. You can try changing password using `idevicebackup2 backup changepw` or resetting the password by resetting only the settings through the iPhone's Settings app.
+Note that if a backup password was previously set on this device, you might need to use the same or change it. You can try changing password using `idevicebackup2 -i backup changepw`, or by turning off encryption (`idevicebackup2 -i backup encryption off`) and turning it back on again.
+
+If you are not able to recover or change the password, you should try to disable encryption and obtain an unencrypted backup.
+
+If all else fails, as a *last resort* you can try resetting the password by [resetting all the settings through the iPhone's Settings app](https://support.apple.com/en-us/HT205220), via `Settings » General » Reset » Reset All Settings`.  Note that resetting the settings through the iPhone's Settings app will wipe some of the files that contain useful forensic traces, so try the options explained above first.
 
 Once ready, you can proceed performing the backup:
 

docs/ios/methodology.md

@@ -12,4 +12,4 @@ If you are not expected to return the phone, you might want to consider to attem
 
 #### iTunes Backup
 
-An alternative option is to generate an iTunes backup (in most recent version of mac OS, they are no longer launched from iTunes, but directly from Finder). While backups only provide a subset of the files stored on the device, in many cases it might be sufficient to at least detect some suspicious artifacts. Backups encrypted with a password will have some additional interesting records not available in unencrypted ones, such as Safari history, Safari state, etc.
+An alternative option is to generate an iTunes backup (in most recent version of macOS, they are no longer launched from iTunes, but directly from Finder). While backups only provide a subset of the files stored on the device, in many cases it might be sufficient to at least detect some suspicious artifacts. Backups encrypted with a password will have some additional interesting records not available in unencrypted ones, such as Safari history, Safari state, etc.

docs/ios/records.md

@@ -4,6 +4,16 @@ In this page you can find a (reasonably) up-to-date breakdown of the files creat
 
 ## Records extracted by `check-fs` or `check-backup`
 
+### `backup_info.json`
+
+!!! info "Availabiliy"
+    Backup: :material-check:  
+    Full filesystem dump: :material-close:
+
+This JSON file is created by mvt-ios' `BackupInfo` module. The module extracts some details about the backup and the device, such as name, phone number, IMEI, product type and version.
+
+---
+
 ### `cache_files.json`
 
 !!! info "Availability"
@@ -19,7 +29,7 @@ If indicators are provided through the command-line, they are checked against th
 ### `calls.json`
 
 !!! info "Availability"
-    Backup: :material-check:  
+    Backup (if encrypted): :material-check:  
     Full filesystem dump: :material-check:
 
 This JSON file is created by mvt-ios' `Calls` module. The module extracts records from a SQLite database located at */private/var/mobile/Library/CallHistoryDB/CallHistory.storedata*, which contains records of incoming and outgoing calls, including from messaging apps such as WhatsApp or Skype.
@@ -50,6 +60,16 @@ If indicators a provided through the command-line, they are checked against the
 
 ---
 
+### `configuration_profiles.json`
+
+!!! info "Availability"
+    Backup: :material-check:  
+    Full filesystem dump: :material-close:
+
+This JSON file is created by mvt-ios' `ConfigurationProfiles` module. The module extracts details about iOS configuration profiles that have been installed on the device. These should include both default iOS as well as third-party profiles.
+
+---
+
 ### `contacts.json`
 
 !!! info "Availability"
@@ -87,17 +107,19 @@ If indicators a provided through the command-line, they are checked against the
 ### `id_status_cache.json`
 
 !!! info "Availability"
-    Backup: :material-check:  
+    Backup (before iOS 14.7): :material-check:  
     Full filesystem dump: :material-check:
 
 This JSON file is created by mvt-ios' `IDStatusCache` module. The module extracts records from a plist file located at */private/var/mobile/Library/Preferences/com.apple.identityservices.idstatuscache.plist*, which contains a cache of Apple user ID authentication. This chance will indicate when apps like Facetime and iMessage first established contacts with other registered Apple IDs. This is significant because it might contain traces of malicious accounts involved in exploitation of those apps.
 
+Starting from iOS 14.7.0, this file is empty or absent.
+
 ---
 
 ### `interaction_c.json`
 
 !!! info "Availability"
-    Backup: :material-check:  
+    Backup (if encrypted): :material-check:  
     Full filesystem dump: :material-check:
 
 This JSON file is created by mvt-ios' `InteractionC` module. The module extracts records from a SQLite database located at */private/var/mobile/Library/CoreDuet/People/interactionC.db*, which contains details about user interactions with installed apps.
@@ -150,10 +172,20 @@ If indicators are provided through the command-line, they are checked against th
 
 ---
 
-### `safari_browser_state.json`
+### `profile_events.json`
 
 !!! info "Availability"
     Backup: :material-check:  
+    Full filesystem dump: :material-close:
+
+This JSON file is created by mvt-ios' `ProfileEvents` module. The module extracts a timeline of configuration profile operations. For example, it should indicate when a new profile was installed from the Settings app, or when one was removed.
+
+---
+
+### `safari_browser_state.json`
+
+!!! info "Availability"
+    Backup (if encrypted): :material-check:  
     Full filesystem dump: :material-check:
 
 This JSON file is created by mvt-ios' `SafariBrowserState` module. The module extracts records from the SQLite databases located at */private/var/mobile/Library/Safari/BrowserState.db* or */private/var/mobile/Containers/Data/Application/\*/Library/Safari/BrowserState.db*, which contain records of opened tabs.
@@ -177,7 +209,7 @@ If indicators are provided through the command-line, they are checked against bo
 ### `safari_history.json`
 
 !!! info "Availability"
-    Backup: :material-check:  
+    Backup (if encrypted): :material-check:  
     Full filesystem dump: :material-check:
 
 This JSON file is created by mvt-ios' `SafariHistory` module. The module extracts records from the SQLite databases located at */private/var/mobile/Library/Safari/History.db* or */private/var/mobile/Containers/Data/Application/\*/Library/Safari/History.db*, which contain a history of URL visits.
@@ -242,6 +274,18 @@ If indicators are provided through the command-line, they are checked against th
 
 ---
 
+### `webkit_resource_load_statistics.json`
+
+!!! info "Availability"
+    Backup: :material-check:  
+    Full filesystem dump: :material-check:
+
+This JSON file is created by mvt-ios `WebkitResourceLoadStatistics` module. The module extracts records from available WebKit ResourceLoadStatistics *observations.db* SQLite3 databases. These records should indicate domain names contacted by apps, including a timestamp.
+
+If indicators are provided through the command-line, they are checked against the extracted domain names. Any matches are stored in *webkit_resource_load_statistics_detected.json*.
+
+---
+
 ### `webkit_safari_view_service.json`
 
 !!! info "Availability"

mkdocs.yml

@@ -42,6 +42,8 @@ nav:
         - Records extracted by mvt-ios: "ios/records.md"
     - MVT for Android:
         - Android Forensic Methodology: "android/methodology.md"
-        - Check APKs: "android/download_apks.md"
-        - Check an Android Backup: "android/backup.md"
+        - Check over ADB: "android/adb.md"
+        - Check an Android Backup (SMS messages): "android/backup.md"
+        - Download APKs: "android/download_apks.md"
+    - Indicators of Compromise: "iocs.md"
     - License: "license.md"

mvt/__init__.py

@@ -1,4 +1,4 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021 MVT Project Developers.
-# See the file 'LICENSE' for usage and copying permissions, or find a copy at
-#   https://github.com/mvt-project/mvt/blob/main/LICENSE
+# Copyright (c) 2021 The MVT Project Authors.
+# Use of this software is governed by the MVT License 1.1 that can be found at
+#   https://license.mvt.re/1.1/

mvt/android/__init__.py

@@ -1,6 +1,6 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021 MVT Project Developers.
-# See the file 'LICENSE' for usage and copying permissions, or find a copy at
-#   https://github.com/mvt-project/mvt/blob/main/LICENSE
+# Copyright (c) 2021 The MVT Project Authors.
+# Use of this software is governed by the MVT License 1.1 that can be found at
+#   https://license.mvt.re/1.1/
 
 from .cli import cli

mvt/android/cli.py

@@ -1,17 +1,18 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021 MVT Project Developers.
-# See the file 'LICENSE' for usage and copying permissions, or find a copy at
-#   https://github.com/mvt-project/mvt/blob/main/LICENSE
+# Copyright (c) 2021 The MVT Project Authors.
+# Use of this software is governed by the MVT License 1.1 that can be found at
+#   https://license.mvt.re/1.1/
 
-import os
-import sys
-import click
-import argparse
 import logging
+import os
+
+import click
 from rich.logging import RichHandler
 
+from mvt.common.help import *
+from mvt.common.indicators import Indicators, IndicatorsFileBadFormat
 from mvt.common.module import run_module, save_timeline
-from mvt.common.indicators import Indicators
+
 from .download_apks import DownloadAPKs
 from .lookups.koodous import koodous_lookup
 from .lookups.virustotal import virustotal_lookup
@@ -24,10 +25,6 @@ logging.basicConfig(level="INFO", format=LOG_FORMAT, handlers=[
     RichHandler(show_path=False, log_time_format="%X")])
 log = logging.getLogger(__name__)
 
-# Help messages of repeating options.
-OUTPUT_HELP_MESSAGE = "Specify a path to a folder where you want to store JSON results"
-
-
 #==============================================================================
 # Main
 #==============================================================================
@@ -40,25 +37,38 @@ def cli():
 # Download APKs
 #==============================================================================
 @cli.command("download-apks", help="Download all or non-safelisted installed APKs installed on the device")
+@click.option("--serial", "-s", type=str, help=HELP_MSG_SERIAL)
 @click.option("--all-apks", "-a", is_flag=True,
-              help="Extract all packages installed on the phone, even those marked as safe")
+              help="Extract all packages installed on the phone, including system packages")
 @click.option("--virustotal", "-v", is_flag=True, help="Check packages on VirusTotal")
 @click.option("--koodous", "-k", is_flag=True, help="Check packages on Koodous")
 @click.option("--all-checks", "-A", is_flag=True, help="Run all available checks")
-@click.option("--output", "-o", type=click.Path(exists=True),
-              help="Specify a path to a folder where you want to store JSON results")
+@click.option("--output", "-o", type=click.Path(exists=False),
+              help="Specify a path to a folder where you want to store the APKs")
 @click.option("--from-file", "-f", type=click.Path(exists=True),
               help="Instead of acquiring from phone, load an existing packages.json file for lookups (mainly for debug purposes)")
-def download_apks(all_apks, virustotal, koodous, all_checks, output, from_file):
+@click.pass_context
+def download_apks(ctx, all_apks, virustotal, koodous, all_checks, output, from_file, serial):
     try:
         if from_file:
             download = DownloadAPKs.from_json(from_file)
         else:
+            # TODO: Do we actually want to be able to run without storing any file?
             if not output:
-                log.critical("You need to specify an output folder (with --output, -o) when extracting APKs from a device")
-                sys.exit(-1)
+                log.critical("You need to specify an output folder with --output!")
+                ctx.exit(1)
 
-            download = DownloadAPKs(output_folder=output, all_apks=all_apks)
+            if not os.path.exists(output):
+                try:
+                    os.makedirs(output)
+                except Exception as e:
+                    log.critical("Unable to create output folder %s: %s", output, e)
+                    ctx.exit(1)
+
+            download = DownloadAPKs(output_folder=output, all_apks=all_apks,
+                                    log=logging.getLogger(DownloadAPKs.__module__))
+            if serial:
+                download.serial = serial
             download.run()
 
         packages = download.packages
@@ -73,19 +83,22 @@ def download_apks(all_apks, virustotal, koodous, all_checks, output, from_file):
             koodous_lookup(packages)
     except KeyboardInterrupt:
         print("")
-        sys.exit(-1)
+        ctx.exit(1)
 
 
 #==============================================================================
 # Checks through ADB
 #==============================================================================
 @cli.command("check-adb", help="Check an Android device over adb")
-@click.option("--iocs", "-i", type=click.Path(exists=True), help="Path to indicators file")
-@click.option("--output", "-o", type=click.Path(exists=True),
-              help="Specify a path to a folder where you want to store JSON results")
-@click.option("--list-modules", "-l", is_flag=True, help="Print list of available modules and exit")
-@click.option("--module", "-m", help="Name of a single module you would like to run instead of all")
-def check_adb(iocs, output, list_modules, module):
+@click.option("--serial", "-s", type=str, help=HELP_MSG_SERIAL)
+@click.option("--iocs", "-i", type=click.Path(exists=True), multiple=True,
+              default=[], help=HELP_MSG_IOC)
+@click.option("--output", "-o", type=click.Path(exists=False),
+              help=HELP_MSG_OUTPUT)
+@click.option("--list-modules", "-l", is_flag=True, help=HELP_MSG_LIST_MODULES)
+@click.option("--module", "-m", help=HELP_MSG_MODULE)
+@click.pass_context
+def check_adb(ctx, iocs, output, list_modules, module, serial):
     if list_modules:
         log.info("Following is the list of available check-adb modules:")
         for adb_module in ADB_MODULES:
@@ -95,10 +108,21 @@ def check_adb(iocs, output, list_modules, module):
 
     log.info("Checking Android through adb bridge")
 
-    if iocs:
-        # Pre-load indicators for performance reasons.
-        log.info("Loading indicators from provided file at %s", iocs)
-        indicators = Indicators(iocs)
+    if output and not os.path.exists(output):
+        try:
+            os.makedirs(output)
+        except Exception as e:
+            log.critical("Unable to create output folder %s: %s", output, e)
+            ctx.exit(1)
+
+    indicators = Indicators(log=log)
+    for ioc_path in iocs:
+        try:
+            indicators.parse_stix2(ioc_path)
+        except IndicatorsFileBadFormat as e:
+            log.critical(e)
+            ctx.exit(1)
+    log.info("Loaded a total of %d indicators", indicators.ioc_count)
 
     timeline = []
     timeline_detected = []
@@ -107,6 +131,8 @@ def check_adb(iocs, output, list_modules, module):
             continue
 
         m = adb_module(output_folder=output, log=logging.getLogger(adb_module.__module__))
+        if serial:
+            m.serial = serial
 
         if iocs:
             indicators.log = m.log
@@ -122,20 +148,35 @@ def check_adb(iocs, output, list_modules, module):
         if len(timeline_detected) > 0:
             save_timeline(timeline_detected, os.path.join(output, "timeline_detected.csv"))
 
+
 #==============================================================================
 # Check ADB backup
 #==============================================================================
 @cli.command("check-backup", help="Check an Android Backup")
-@click.option("--iocs", "-i", type=click.Path(exists=True), help="Path to indicators file")
-@click.option("--output", "-o", type=click.Path(exists=True), help=OUTPUT_HELP_MESSAGE)
+@click.option("--serial", "-s", type=str, help=HELP_MSG_SERIAL)
+@click.option("--iocs", "-i", type=click.Path(exists=True), multiple=True,
+              default=[], help=HELP_MSG_IOC)
+@click.option("--output", "-o", type=click.Path(exists=False), help=HELP_MSG_OUTPUT)
 @click.argument("BACKUP_PATH", type=click.Path(exists=True))
-def check_backup(iocs, output, backup_path):
+@click.pass_context
+def check_backup(ctx, iocs, output, backup_path, serial):
     log.info("Checking ADB backup located at: %s", backup_path)
 
-    if iocs:
-        # Pre-load indicators for performance reasons.
-        log.info("Loading indicators from provided file at %s", iocs)
-        indicators = Indicators(iocs)
+    if output and not os.path.exists(output):
+        try:
+            os.makedirs(output)
+        except Exception as e:
+            log.critical("Unable to create output folder %s: %s", output, e)
+            ctx.exit(1)
+
+    indicators = Indicators(log=log)
+    for ioc_path in iocs:
+        try:
+            indicators.parse_stix2(ioc_path)
+        except IndicatorsFileBadFormat as e:
+            log.critical(e)
+            ctx.exit(1)
+    log.info("Loaded a total of %d indicators", indicators.ioc_count)
 
     if os.path.isfile(backup_path):
         log.critical("The path you specified is a not a folder!")
@@ -143,12 +184,15 @@ def check_backup(iocs, output, backup_path):
         if os.path.basename(backup_path) == "backup.ab":
             log.info("You can use ABE (https://github.com/nelenkov/android-backup-extractor) " \
                      "to extract 'backup.ab' files!")
-        sys.exit(-1)
+        ctx.exit(1)
 
     for module in BACKUP_MODULES:
         m = module(base_folder=backup_path, output_folder=output,
                    log=logging.getLogger(module.__module__))
 
+        if serial:
+            m.serial = serial
+
         if iocs:
             indicators.log = m.log
             m.indicators = indicators

mvt/android/data/safe_packages.txt

@@ -1,182 +0,0 @@
-android
-android.auto_generated_rro__
-android.autoinstalls.config.google.nexus
-com.android.backupconfirm
-com.android.bips
-com.android.bluetooth
-com.android.bluetoothmidiservice
-com.android.bookmarkprovider
-com.android.calllogbackup
-com.android.captiveportallogin
-com.android.carrierconfig
-com.android.carrierdefaultapp
-com.android.cellbroadcastreceiver
-com.android.certinstaller
-com.android.chrome
-com.android.companiondevicemanager
-com.android.connectivity.metrics
-com.android.cts.ctsshim
-com.android.cts.priv.ctsshim
-com.android.defcontainer
-com.android.documentsui
-com.android.dreams.basic
-com.android.egg
-com.android.emergency
-com.android.externalstorage
-com.android.facelock
-com.android.hotwordenrollment
-com.android.hotwordenrollment.okgoogle
-com.android.hotwordenrollment.tgoogle
-com.android.hotwordenrollment.xgoogle
-com.android.htmlviewer
-com.android.inputdevices
-com.android.keychain
-com.android.location.fused
-com.android.managedprovisioning
-com.android.mms.service
-com.android.mtp
-com.android.musicfx
-com.android.nfc
-com.android.omadm.service
-com.android.pacprocessor
-com.android.phone
-com.android.printspooler
-com.android.providers.blockednumber
-com.android.providers.calendar
-com.android.providers.contacts
-com.android.providers.downloads
-com.android.providers.downloads.ui
-com.android.providers.media
-com.android.providers.partnerbookmarks
-com.android.providers.settings
-com.android.providers.telephony
-com.android.providers.userdictionary
-com.android.proxyhandler
-com.android.retaildemo
-com.android.safetyregulatoryinfo
-com.android.sdm.plugins.connmo
-com.android.sdm.plugins.dcmo
-com.android.sdm.plugins.diagmon
-com.android.sdm.plugins.sprintdm
-com.android.server.telecom
-com.android.service.ims
-com.android.service.ims.presence
-com.android.settings
-com.android.sharedstoragebackup
-com.android.shell
-com.android.statementservice
-com.android.stk
-com.android.systemui
-com.android.systemui.theme.dark
-com.android.vending
-com.android.vpndialogs
-com.android.vzwomatrigger
-com.android.wallpaperbackup
-com.android.wallpaper.livepicker
-com.breel.wallpapers
-com.customermobile.preload.vzw
-com.google.android.apps.cloudprint
-com.google.android.apps.docs
-com.google.android.apps.docs.editors.docs
-com.google.android.apps.enterprise.dmagent
-com.google.android.apps.gcs
-com.google.android.apps.helprtc
-com.google.android.apps.inputmethod.hindi
-com.google.android.apps.maps
-com.google.android.apps.messaging
-com.google.android.apps.nexuslauncher
-com.google.android.apps.photos
-com.google.android.apps.pixelmigrate
-com.google.android.apps.tachyon
-com.google.android.apps.turbo
-com.google.android.apps.tycho
-com.google.android.apps.wallpaper
-com.google.android.apps.wallpaper.nexus
-com.google.android.apps.work.oobconfig
-com.google.android.apps.youtube.vr
-com.google.android.asdiv
-com.google.android.backuptransport
-com.google.android.calculator
-com.google.android.calendar
-com.google.android.carrier
-com.google.android.carrier.authdialog
-com.google.android.carrierentitlement
-com.google.android.carriersetup
-com.google.android.configupdater
-com.google.android.contacts
-com.google.android.deskclock
-com.google.android.dialer
-com.google.android.euicc
-com.google.android.ext.services
-com.google.android.ext.shared
-com.google.android.feedback
-com.google.android.gm
-com.google.android.gms
-com.google.android.gms.policy_auth
-com.google.android.gms.policy_sidecar_o
-com.google.android.gms.setup
-com.google.android.GoogleCamera
-com.google.android.googlequicksearchbox
-com.google.android.gsf
-com.google.android.gsf.login
-com.google.android.hardwareinfo
-com.google.android.hiddenmenu
-com.google.android.ims
-com.google.android.inputmethod.japanese
-com.google.android.inputmethod.korean
-com.google.android.inputmethod.latin
-com.google.android.inputmethod.pinyin
-com.google.android.instantapps.supervisor
-com.google.android.keep
-com.google.android.marvin.talkback
-com.google.android.music
-com.google.android.nexusicons
-com.google.android.onetimeinitializer
-com.google.android.packageinstaller
-com.google.android.partnersetup
-com.google.android.printservice.recommendation
-com.google.android.setupwizard
-com.google.android.soundpicker
-com.google.android.storagemanager
-com.google.android.syncadapters.contacts
-com.google.android.tag
-com.google.android.talk
-com.google.android.tetheringentitlement
-com.google.android.theme.pixel
-com.google.android.tts
-com.google.android.videos
-com.google.android.vr.home
-com.google.android.vr.inputmethod
-com.google.android.webview
-com.google.android.wfcactivation
-com.google.android.youtube
-com.google.ar.core
-com.google.intelligence.sense
-com.google.modemservice
-com.google.pixel.wahoo.gfxdrv
-com.google.SSRestartDetector
-com.google.tango
-com.google.vr.apps.ornament
-com.google.vr.vrcore
-com.htc.omadm.trigger
-com.qti.qualcomm.datastatusnotification
-com.qualcomm.atfwd
-com.qualcomm.embms
-com.qualcomm.fastdormancy
-com.qualcomm.ltebc_vzw
-com.qualcomm.qcrilmsgtunnel
-com.qualcomm.qti.ims
-com.qualcomm.qti.networksetting
-com.qualcomm.qti.telephonyservice
-com.qualcomm.qti.uceShimService
-com.qualcomm.shutdownlistner
-com.qualcomm.timeservice
-com.qualcomm.vzw_api
-com.quicinc.cne.CNEService
-com.verizon.llkagent
-com.verizon.mips.services
-com.verizon.obdm
-com.verizon.obdm_permissions
-com.verizon.services
-com.vzw.apnlib
-qualcomm.com.vzw_msdc_api

mvt/android/download_apks.py

@@ -1,16 +1,20 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021 MVT Project Developers.
-# See the file 'LICENSE' for usage and copying permissions, or find a copy at
-#   https://github.com/mvt-project/mvt/blob/main/LICENSE
+# Copyright (c) 2021 The MVT Project Authors.
+# Use of this software is governed by the MVT License 1.1 that can be found at
+#   https://license.mvt.re/1.1/
 
-import os
 import json
 import logging
+import os
+
 import pkg_resources
 from tqdm import tqdm
 
+from mvt.common.module import InsufficientPrivileges
 from mvt.common.utils import get_sha256_from_file_path
+
 from .modules.adb.base import AndroidExtraction
+from .modules.adb.packages import Packages
 
 log = logging.getLogger(__name__)
 
@@ -26,89 +30,33 @@ class PullProgress(tqdm):
         self.update(current - self.n)
 
 
-class Package:
-    """Package indicates a package name and all the files associated with it."""
-
-    def __init__(self, name, files=None):
-        self.name = name
-        self.files = files or []
-
-
 class DownloadAPKs(AndroidExtraction):
     """DownloadAPKs is the main class operating the download of APKs
     from the device."""
 
-    def __init__(self, output_folder=None, all_apks=False, packages=None):
+    def __init__(self, output_folder=None, all_apks=False, log=None,
+                 packages=None):
         """Initialize module.
         :param output_folder: Path to the folder where data should be stored
         :param all_apks: Boolean indicating whether to download all packages
                          or filter known-goods
         :param packages: Provided list of packages, typically for JSON checks
         """
-        super().__init__(file_path=None, base_folder=None,
-                         output_folder=output_folder)
+        super().__init__(output_folder=output_folder, log=log)
 
-        self.output_folder_apk = None
-        self.packages = packages or []
+        self.packages = packages
         self.all_apks = all_apks
-
-        self._safe_packages = []
+        self.output_folder_apk = None
 
     @classmethod
     def from_json(cls, json_path):
-        """Initialize this class from an existing packages.json file.
-        :param json_path: Path to the packages.json file to parse.
+        """Initialize this class from an existing apks.json file.
+        :param json_path: Path to the apks.json file to parse.
         """
         with open(json_path, "r") as handle:
-            data = json.load(handle)
-
-            packages = []
-            for entry in data:
-                package = Package(entry["name"], entry["files"])
-                packages.append(package)
-
+            packages = json.load(handle)
             return cls(packages=packages)
 
-    def _load_safe_packages(self):
-        """Load known-good package names.
-        """
-        safe_packages_path = os.path.join("data", "safe_packages.txt")
-        safe_packages_string = pkg_resources.resource_string(__name__, safe_packages_path)
-        safe_packages_list = safe_packages_string.decode("utf-8").split("\n")
-        self._safe_packages.extend(safe_packages_list)
-
-    def _clean_output(self, output):
-        """Clean adb shell command output.
-        :param output: Command output to clean.
-        """
-        return output.strip().replace("package:", "")
-
-    def get_packages(self):
-        """Retrieve package names from the device using adb.
-        """
-        log.info("Retrieving package names ...")
-
-        if not self.all_apks:
-            self._load_safe_packages()
-
-        output = self._adb_command("pm list packages")
-        total = 0
-        for line in output.split("\n"):
-            package_name = self._clean_output(line)
-            if package_name == "":
-                continue
-
-            total += 1
-
-            if not self.all_apks and package_name in self._safe_packages:
-                continue
-
-            if package_name not in self.packages:
-                self.packages.append(Package(package_name))
-
-        log.info("There are %d packages installed on the device. I selected %d for inspection.",
-                 total, len(self.packages))
-
     def pull_package_file(self, package_name, remote_path):
         """Pull files related to specific package from the device.
         :param package_name: Name of the package to download
@@ -137,6 +85,11 @@ class DownloadAPKs(AndroidExtraction):
                               miniters=1) as pp:
                 self._adb_download(remote_path, local_path,
                                    progress_callback=pp.update_to)
+        except InsufficientPrivileges:
+            log.warn("Unable to pull package file from %s: insufficient privileges, it might be a system app",
+                     remote_path)
+            self._adb_reconnect()
+            return None
         except Exception as e:
             log.exception("Failed to pull package file from %s: %s",
                           remote_path, e)
@@ -145,6 +98,18 @@ class DownloadAPKs(AndroidExtraction):
 
         return local_path
 
+    def get_packages(self):
+        """Use the Packages adb module to retrieve the list of packages.
+        We reuse the same extraction logic to then download the APKs.
+        """
+        self.log.info("Retrieving list of installed packages...")
+
+        m = Packages()
+        m.log = self.log
+        m.run()
+
+        self.packages = m.results
+
     def pull_packages(self):
         """Download all files of all selected packages from the device.
         """
@@ -153,26 +118,47 @@ class DownloadAPKs(AndroidExtraction):
         if not os.path.exists(self.output_folder):
             os.mkdir(self.output_folder)
 
+        # If the user provided the flag --all-apks we select all packages.
+        packages_selection = []
+        if self.all_apks:
+            log.info("Selected all %d available packages", len(self.packages))
+            packages_selection = self.packages
+        else:
+            # Otherwise we loop through the packages and get only those that
+            # are not marked as system.
+            for package in self.packages:
+                if not package.get("system", False):
+                    packages_selection.append(package)
+
+            log.info("Selected only %d packages which are not marked as system",
+                len(packages_selection))
+
+        if len(packages_selection) == 0:
+            log.info("No packages were selected for download")
+            return
+
         log.info("Downloading packages from device. This might take some time ...")
 
         self.output_folder_apk = os.path.join(self.output_folder, "apks")
         if not os.path.exists(self.output_folder_apk):
             os.mkdir(self.output_folder_apk)
 
-        total_packages = len(self.packages)
         counter = 0
-        for package in self.packages:
+        for package in packages_selection:
             counter += 1
 
-            log.info("[%d/%d] Package: %s", counter, total_packages, package.name)
+            log.info("[%d/%d] Package: %s", counter, len(packages_selection),
+                     package["package_name"])
 
+            # Get the file path for the specific package.
             try:
-                output = self._adb_command(f"pm path {package.name}")
-                output = self._clean_output(output)
+                output = self._adb_command(f"pm path {package['package_name']}")
+                output = output.strip().replace("package:", "")
                 if not output:
                     continue
             except Exception as e:
-                log.exception("Failed to get path of package %s: %s", package.name, e)
+                log.exception("Failed to get path of package %s: %s",
+                              package["package_name"], e)
                 self._adb_reconnect()
                 continue
 
@@ -180,33 +166,36 @@ class DownloadAPKs(AndroidExtraction):
             # We loop through each line and download each file.
             for path in output.split("\n"):
                 device_path = path.strip()
-                file_path = self.pull_package_file(package.name, device_path)
+                file_path = self.pull_package_file(package["package_name"],
+                                                   device_path)
                 if not file_path:
                     continue
 
-                # We add the apk metadata to the package object.
-                package.files.append({
+                file_info = {
                     "path": device_path,
                     "local_name": file_path,
                     "sha256": get_sha256_from_file_path(file_path),
-                })
+                }
+
+                if "files" not in package:
+                    package["files"] = [file_info,]
+                else:
+                    package["files"].append(file_info)
+
+        log.info("Download of selected packages completed")
 
     def save_json(self):
         """Save the results to the package.json file.
         """
-        json_path = os.path.join(self.output_folder, "packages.json")
-        packages = []
-        for package in self.packages:
-            packages.append(package.__dict__)
-
+        json_path = os.path.join(self.output_folder, "apks.json")
         with open(json_path, "w") as handle:
-            json.dump(packages, handle, indent=4)
+            json.dump(self.packages, handle, indent=4)
 
     def run(self):
         """Run all steps of fetch-apk.
         """
-        self._adb_connect()
         self.get_packages()
+        self._adb_connect()
         self.pull_packages()
         self.save_json()
         self._adb_disconnect()

mvt/android/lookups/__init__.py

@@ -1,4 +1,4 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021 MVT Project Developers.
-# See the file 'LICENSE' for usage and copying permissions, or find a copy at
-#   https://github.com/mvt-project/mvt/blob/main/LICENSE
+# Copyright (c) 2021 The MVT Project Authors.
+# Use of this software is governed by the MVT License 1.1 that can be found at
+#   https://license.mvt.re/1.1/

mvt/android/lookups/koodous.py

@@ -1,15 +1,15 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021 MVT Project Developers.
-# See the file 'LICENSE' for usage and copying permissions, or find a copy at
-#   https://github.com/mvt-project/mvt/blob/main/LICENSE
+# Copyright (c) 2021 The MVT Project Authors.
+# Use of this software is governed by the MVT License 1.1 that can be found at
+#   https://license.mvt.re/1.1/
 
-import requests
 import logging
 
-from rich.text import Text
-from rich.table import Table
-from rich.progress import track
+import requests
 from rich.console import Console
+from rich.progress import track
+from rich.table import Table
+from rich.text import Text
 
 log = logging.getLogger(__name__)
 
@@ -27,12 +27,12 @@ def koodous_lookup(packages):
     total_packages = len(packages)
     for i in track(range(total_packages), description=f"Looking up {total_packages} packages..."):
         package = packages[i]
-        for file in package.files:
+        for file in package.get("files", []):
             url = f"https://api.koodous.com/apks/{file['sha256']}"
             res = requests.get(url)
             report = res.json()
 
-            row = [package.name, file["local_name"]]
+            row = [package["package_name"], file["local_name"]]
 
             if "package_name" in report:
                 trusted = "no"

mvt/android/lookups/virustotal.py

@@ -1,14 +1,15 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021 MVT Project Developers.
-# See the file 'LICENSE' for usage and copying permissions, or find a copy at
-#   https://github.com/mvt-project/mvt/blob/main/LICENSE
+# Copyright (c) 2021 The MVT Project Authors.
+# Use of this software is governed by the MVT License 1.1 that can be found at
+#   https://license.mvt.re/1.1/
+
+import logging
 
 import requests
-import logging
-from rich.text import Text
-from rich.table import Table
-from rich.progress import track
 from rich.console import Console
+from rich.progress import track
+from rich.table import Table
+from rich.text import Text
 
 log = logging.getLogger(__name__)
 
@@ -40,7 +41,7 @@ def virustotal_lookup(packages):
 
     unique_hashes = []
     for package in packages:
-        for file in package.files:
+        for file in package.get("files", []):
             if file["sha256"] not in unique_hashes:
                 unique_hashes.append(file["sha256"])
 
@@ -73,8 +74,8 @@ def virustotal_lookup(packages):
     table.add_column("Detections")
 
     for package in packages:
-        for file in package.files:
-            row = [package.name, file["local_name"]]
+        for file in package.get("files", []):
+            row = [package["package_name"], file["local_name"]]
 
             if file["sha256"] in detections:
                 detection = detections[file["sha256"]]