1
mirror of https://github.com/mvt-project/mvt synced 2025-10-21 22:42:15 +02:00

Compare commits

..

11 Commits

Author SHA1 Message Date
Nex
cce9159eda Adding indicator to matched results 2022-01-23 15:01:49 +01:00
Nex
e1211991aa Bumped version 2022-01-23 14:17:43 +01:00
Nex
8ae9ca328c Added log line at the end to highlight number of detections 2022-01-21 16:50:32 +01:00
Nex
0e2eb51732 Fixed checking of indicators in filesystem module 2022-01-21 16:30:34 +01:00
Nex
b35cd4bc73 Added support for context-aware indicators.
This way when a detection is logged, the user can know which STIX2
file was matched by the module
2022-01-21 16:26:58 +01:00
Nex
1b4f99a31d Trying to catch missing argument error (ref: #211) 2022-01-21 12:20:22 +01:00
tek
e4e1716729 Bumped version 2022-01-20 15:28:42 +01:00
tek
083bc12351 Merge branch 'feature/check-file-path' 2022-01-20 15:19:37 +01:00
tek
cf6d392460 Adds more details on the download-iocs command 2022-01-20 13:29:50 +01:00
tek
95205d8e17 Adds indicators check to iOS TCC module 2022-01-18 17:12:20 +01:00
tek
38bb583a9e Improves management of file path indicators 2022-01-18 15:50:31 +01:00
36 changed files with 429 additions and 257 deletions

View File

@@ -41,6 +41,6 @@ export MVT_STIX2="/home/user/IOC1.stix2:/home/user/IOC2.stix2"
- [Predator from Cytrox](https://citizenlab.ca/2021/12/pegasus-vs-predator-dissidents-doubly-infected-iphone-reveals-cytrox-mercenary-spyware/) ([STIX2](https://raw.githubusercontent.com/AmnestyTech/investigations/master/2021-12-16_cytrox/cytrox.stix2))
- [This repository](https://github.com/Te-k/stalkerware-indicators) contains IOCs for Android stalkerware including [a STIX MVT-compatible file](https://raw.githubusercontent.com/Te-k/stalkerware-indicators/master/stalkerware.stix2).
You can automaticallly download the latest public indicator files with the command `mvt-ios download-iocs` or `mvt-android download-iocs`.
You can automaticallly download the latest public indicator files with the command `mvt-ios download-iocs` or `mvt-android download-iocs`. These commands download the list of indicators listed [here](https://github.com/mvt-project/mvt/blob/main/public_indicators.json) and store them in the [appdir](https://pypi.org/project/appdirs/) folder. They are then loaded automatically by mvt.
Please [open an issue](https://github.com/mvt-project/mvt/issues/) to suggest new sources of STIX-formatted IOCs.

View File

@@ -139,7 +139,7 @@ def check_adb(ctx, iocs, output, fast, list_modules, module, serial):
m = adb_module(output_folder=output, fast_mode=fast,
log=logging.getLogger(adb_module.__module__))
if indicators.ioc_count:
if indicators.total_ioc_count:
m.indicators = indicators
m.indicators.log = m.log
if serial:
@@ -190,7 +190,7 @@ def check_backup(ctx, iocs, output, backup_path, serial):
for module in BACKUP_MODULES:
m = module(base_folder=backup_path, output_folder=output,
log=logging.getLogger(module.__module__))
if indicators.ioc_count:
if indicators.total_ioc_count:
m.indicators = indicators
m.indicators.log = m.log
if serial:

View File

@@ -89,10 +89,6 @@ class Files(AndroidExtraction):
return
for result in self.results:
if self.indicators.check_file_name(result["path"]):
self.log.warning("Found a known suspicous filename at path: \"%s\"", result["path"])
self.detected.append(result)
if self.indicators.check_file_path(result["path"]):
self.log.warning("Found a known suspicous file at path: \"%s\"", result["path"])
self.detected.append(result)

View File

@@ -95,6 +95,9 @@ class Packages(AndroidExtraction):
self._adb_connect()
packages = self._adb_command("pm list packages -U -u -i -f")
if packages.strip() == "Error: Unknown option: -U":
packages = self._adb_command("pm list packages -u -i -f")
for line in packages.split("\n"):
line = line.strip()
if not line.startswith("package:"):

File diff suppressed because it is too large Load Diff

View File

@@ -6,7 +6,7 @@
import requests
from packaging import version
MVT_VERSION = "1.4.4"
MVT_VERSION = "1.4.7"
def check_for_updates():

View File

@@ -168,7 +168,7 @@ def check_backup(ctx, iocs, output, fast, backup_path, list_modules, module):
m = backup_module(base_folder=backup_path, output_folder=output, fast_mode=fast,
log=logging.getLogger(backup_module.__module__))
m.is_backup = True
if indicators.ioc_count:
if indicators.total_ioc_count > 0:
m.indicators = indicators
m.indicators.log = m.log
@@ -182,6 +182,10 @@ def check_backup(ctx, iocs, output, fast, backup_path, list_modules, module):
if len(timeline_detected) > 0:
save_timeline(timeline_detected, os.path.join(output, "timeline_detected.csv"))
if len(timeline_detected) > 0:
log.warning("The analysis of the backup produced %d detections!",
len(timeline_detected))
#==============================================================================
# Command: check-fs
@@ -225,7 +229,7 @@ def check_fs(ctx, iocs, output, fast, dump_path, list_modules, module):
log=logging.getLogger(fs_module.__module__))
m.is_fs_dump = True
if indicators.ioc_count:
if indicators.total_ioc_count > 0:
m.indicators = indicators
m.indicators.log = m.log
@@ -239,20 +243,23 @@ def check_fs(ctx, iocs, output, fast, dump_path, list_modules, module):
if len(timeline_detected) > 0:
save_timeline(timeline_detected, os.path.join(output, "timeline_detected.csv"))
if len(timeline_detected) > 0:
log.warning("The analysis of the filesystem produced %d detections!",
len(timeline_detected))
#==============================================================================
# Command: check-iocs
#==============================================================================
@cli.command("check-iocs", help="Compare stored JSON results to provided indicators")
@click.option("--iocs", "-i", type=click.Path(exists=True), multiple=True,
default=[], required=True, help=HELP_MSG_IOC)
default=[], help=HELP_MSG_IOC)
@click.option("--list-modules", "-l", is_flag=True, help=HELP_MSG_LIST_MODULES)
@click.option("--module", "-m", help=HELP_MSG_MODULE)
@click.argument("FOLDER", type=click.Path(exists=True))
@click.pass_context
def check_iocs(ctx, iocs, list_modules, module, folder):
all_modules = []
for entry in BACKUP_MODULES + FS_MODULES:
for entry in BACKUP_MODULES + FS_MODULES + MIXED_MODULES:
if entry not in all_modules:
all_modules.append(entry)
@@ -268,6 +275,7 @@ def check_iocs(ctx, iocs, list_modules, module, folder):
indicators = Indicators(log=log)
indicators.load_indicators_files(iocs)
total_detections = 0
for file_name in os.listdir(folder):
name_only, ext = os.path.splitext(file_name)
file_path = os.path.join(folder, file_name)
@@ -284,7 +292,7 @@ def check_iocs(ctx, iocs, list_modules, module, folder):
m = iocs_module.from_json(file_path,
log=logging.getLogger(iocs_module.__module__))
if indicators.ioc_count:
if indicators.total_ioc_count > 0:
m.indicators = indicators
m.indicators.log = m.log
@@ -292,6 +300,12 @@ def check_iocs(ctx, iocs, list_modules, module, folder):
m.check_indicators()
except NotImplementedError:
continue
else:
total_detections += len(m.detected)
if total_detections > 0:
log.warning("The check of the results produced %d detections!",
total_detections)
#==============================================================================

View File

@@ -46,8 +46,10 @@ class ConfigurationProfiles(IOSExtraction):
payload_content = result["plist"]["PayloadContent"][0]
# Alert on any known malicious configuration profiles in the indicator list.
if self.indicators.check_profile(result["plist"]["PayloadUUID"]):
ioc = self.indicators.check_profile(result["plist"]["PayloadUUID"])
if ioc:
self.log.warning(f"Found a known malicious configuration profile \"{result['plist']['PayloadDisplayName']}\" with UUID '{result['plist']['PayloadUUID']}'.")
result["matched_indicator"] = ioc
self.detected.append(result)
continue

View File

@@ -72,9 +72,7 @@ class Manifest(IOSExtraction):
return
for result in self.results:
if "relative_path" not in result:
continue
if not result["relative_path"]:
if not result.get("relative_path"):
continue
if result["domain"]:
@@ -83,16 +81,15 @@ class Manifest(IOSExtraction):
self.detected.append(result)
continue
if self.indicators.check_file_name(result["relative_path"]):
self.log.warning("Found a known malicious file at path: %s", result["relative_path"])
if self.indicators.check_file_path("/" + result["relative_path"]):
self.detected.append(result)
continue
relPath = result["relative_path"].lower()
for ioc in self.indicators.ioc_domains:
if ioc.lower() in relPath:
rel_path = result["relative_path"].lower()
for ioc in self.indicators.get_iocs("domains"):
if ioc["value"].lower() in rel_path:
self.log.warning("Found mention of domain \"%s\" in a backup file with path: %s",
ioc, relPath)
ioc["value"], rel_path)
self.detected.append(result)
def run(self):

View File

@@ -37,20 +37,24 @@ class Analytics(IOSExtraction):
return
for result in self.results:
for ioc in self.indicators.ioc_processes:
for key in result.keys():
if ioc == result[key]:
self.log.warning("Found mention of a malicious process \"%s\" in %s file at %s",
ioc, result["artifact"], result["timestamp"])
self.detected.append(result)
break
for ioc in self.indicators.ioc_domains:
for key in result.keys():
if ioc in str(result[key]):
self.log.warning("Found mention of a malicious domain \"%s\" in %s file at %s",
ioc, result["artifact"], result["timestamp"])
self.detected.append(result)
break
for value in result.values():
if not isinstance(value, str):
continue
ioc = self.indicators.check_process(value)
if ioc:
self.log.warning("Found mention of a malicious process \"%s\" in %s file at %s",
value, result["artifact"], result["timestamp"])
result["matched_indicator"] = ioc
self.detected.append(result)
continue
ioc = self.indicators.check_domain(value)
if ioc:
self.log.warning("Found mention of a malicious domain \"%s\" in %s file at %s",
value, result["artifact"], result["timestamp"])
result["matched_indicator"] = ioc
self.detected.append(result)
def _extract_analytics_data(self):
artifact = self.file_path.split("/")[-1]
@@ -101,6 +105,7 @@ class Analytics(IOSExtraction):
timestamp = ""
data = plistlib.loads(row[1])
data["timestamp"] = timestamp
data["artifact"] = artifact
self.results.append(data)

View File

@@ -34,13 +34,15 @@ class CacheFiles(IOSExtraction):
return
self.detected = {}
for key, items in self.results.items():
for item in items:
if self.indicators.check_domain(item["url"]):
for key, values in self.results.items():
for value in values:
ioc = self.indicators.check_domain(value["url"])
if ioc:
value["matched_indicator"] = ioc
if key not in self.detected:
self.detected[key] = [item, ]
self.detected[key] = [value, ]
else:
self.detected[key].append(item)
self.detected[key].append(value)
def _process_cache_file(self, file_path):
self.log.info("Processing cache file at path: %s", file_path)

View File

@@ -37,23 +37,25 @@ class Filesystem(IOSExtraction):
return
for result in self.results:
if self.indicators.check_file(result["path"]):
self.log.warning("Found a known malicious file name at path: %s", result["path"])
if "path" not in result:
continue
ioc = self.indicators.check_file_path(result["path"])
if ioc:
result["matched_indicator"] = ioc
self.detected.append(result)
if self.indicators.check_file_path(result["path"]):
self.log.warning("Found a known malicious file path at path: %s", result["path"])
self.detected.append(result)
# If we are instructed to run fast, we skip this.
# If we are instructed to run fast, we skip the rest.
if self.fast_mode:
self.log.info("Flag --fast was enabled: skipping extended search for suspicious files/processes")
else:
for ioc in self.indicators.ioc_processes:
parts = result["path"].split("/")
if ioc in parts:
self.log.warning("Found a known malicious file/process at path: %s", result["path"])
self.detected.append(result)
continue
for ioc in self.indicators.get_iocs("processes"):
parts = result["path"].split("/")
if ioc["value"] in parts:
self.log.warning("Found known suspicious process name mentioned in file at path \"%s\" matching indicators from \"%s\"",
result["path"], ioc["name"])
result["matched_indicator"] = ioc
self.detected.append(result)
def run(self):
for root, dirs, files in os.walk(self.base_folder):

View File

@@ -37,7 +37,12 @@ class SafariFavicon(IOSExtraction):
return
for result in self.results:
if self.indicators.check_domain(result["url"]) or self.indicators.check_domain(result["icon_url"]):
ioc = self.indicators.check_domain(result["url"])
if not ioc:
ioc = self.indicators.check_domain(result["icon_url"])
if ioc:
result["matched_indicator"] = ioc
self.detected.append(result)
def _process_favicon_db(self, file_path):

View File

@@ -34,12 +34,20 @@ class ShutdownLog(IOSExtraction):
return
for result in self.results:
for ioc in self.indicators.ioc_processes:
ioc = self.indicators.check_file_path(result["client"])
if ioc:
result["matched_indicator"] = ioc
self.detected.append(result)
continue
for ioc in self.indicators.get_iocs("processes"):
parts = result["client"].split("/")
if ioc in parts:
self.log.warning("Found mention of a known malicious process \"%s\" in shutdown.log",
ioc)
result["matched_indicator"] = ioc
self.detected.append(result)
continue
def process_shutdownlog(self, content):
current_processes = []

View File

@@ -18,9 +18,11 @@ class WebkitBase(IOSExtraction):
if not self.indicators:
return
for item in self.results:
if self.indicators.check_domain(item["url"]):
self.detected.append(item)
for result in self.results:
ioc = self.indicators.check_domain(result["url"])
if ioc:
result["matched_indicator"] = ioc
self.detected.append(result)
def _process_webkit_folder(self, root_paths):
for found_path in self._get_fs_files_from_patterns(root_paths):

View File

@@ -42,7 +42,12 @@ class ChromeFavicon(IOSExtraction):
return
for result in self.results:
if self.indicators.check_domain(result["url"]) or self.indicators.check_domain(result["icon_url"]):
ioc = self.indicators.check_domain(result["url"])
if not ioc:
ioc = self.indicators.check_domain(result["icon_url"])
if ioc:
result["matched_indicator"] = ioc
self.detected.append(result)
def run(self):

View File

@@ -41,7 +41,9 @@ class ChromeHistory(IOSExtraction):
return
for result in self.results:
if self.indicators.check_domain(result["url"]):
ioc = self.indicators.check_domain(result["url"])
if ioc:
result["matched_indicator"] = ioc
self.detected.append(result)
def run(self):

View File

@@ -40,8 +40,12 @@ class FirefoxFavicon(IOSExtraction):
return
for result in self.results:
if (self.indicators.check_domain(result.get("url", "")) or
self.indicators.check_domain(result.get("history_url", ""))):
ioc = self.indicators.check_domain(result.get("url", ""))
if not ioc:
ioc = self.indicators.check_domain(result.get("history_url", ""))
if ioc:
result["matched_indicator"] = ioc
self.detected.append(result)
def run(self):

View File

@@ -44,7 +44,9 @@ class FirefoxHistory(IOSExtraction):
return
for result in self.results:
if self.indicators.check_domain(result["url"]):
ioc = self.indicators.check_domain(result["url"])
if ioc:
result["matched_indicator"] = ioc
self.detected.append(result)
def run(self):

View File

@@ -43,7 +43,9 @@ class IDStatusCache(IOSExtraction):
for result in self.results:
if result.get("user", "").startswith("mailto:"):
email = result["user"][7:].strip("'")
if self.indicators.check_email(email):
ioc = self.indicators.check_email(email)
if ioc:
result["matched_indicator"] = ioc
self.detected.append(result)
continue

View File

@@ -41,13 +41,13 @@ class LocationdClients(IOSExtraction):
def serialize(self, record):
records = []
for ts in self.timestamps:
if ts in record.keys():
for timestamp in self.timestamps:
if timestamp in record.keys():
records.append({
"timestamp": record[ts],
"timestamp": record[timestamp],
"module": self.__class__.__name__,
"event": ts,
"data": f"{ts} from {record['package']}"
"event": timestamp,
"data": f"{timestamp} from {record['package']}"
})
return records
@@ -60,8 +60,40 @@ class LocationdClients(IOSExtraction):
parts = result["package"].split("/")
proc_name = parts[len(parts)-1]
if self.indicators.check_process(proc_name):
ioc = self.indicators.check_process(proc_name)
if ioc:
self.log.warning("Found a suspicious process name in LocationD entry %s",
result["package"])
result["matched_indicator"] = ioc
self.detected.append(result)
continue
if "BundlePath" in result:
ioc = self.indicators.check_file_path(result["BundlePath"])
if ioc:
self.log.warning("Found a suspicious file path in Location D: %s",
result["BundlePath"])
result["matched_indicator"] = ioc
self.detected.append(result)
continue
if "Executable" in result:
ioc = self.indicators.check_file_path(result["Executable"])
if ioc:
self.log.warning("Found a suspicious file path in Location D: %s",
result["Executable"])
result["matched_indicator"] = ioc
self.detected.append(result)
continue
if "Registered" in result:
ioc = self.indicators.check_file_path(result["Registered"])
if ioc:
self.log.warning("Found a suspicious file path in Location D: %s",
result["Registered"])
result["matched_indicator"] = ioc
self.detected.append(result)
continue
def _extract_locationd_entries(self, file_path):
with open(file_path, "rb") as handle:

View File

@@ -41,7 +41,9 @@ class OSAnalyticsADDaily(IOSExtraction):
return
for result in self.results:
if self.indicators.check_process(result["package"]):
ioc = self.indicators.check_process(result["package"])
if ioc:
result["matched_indicator"] = ioc
self.detected.append(result)
def run(self):

View File

@@ -44,16 +44,22 @@ class SafariBrowserState(IOSExtraction):
return
for result in self.results:
if "tab_url" in result and self.indicators.check_domain(result["tab_url"]):
self.detected.append(result)
continue
if "tab_url" in result:
ioc = self.indicators.check_domain(result["tab_url"])
if ioc:
result["matched_indicator"] = ioc
self.detected.append(result)
continue
if "session_data" not in result:
continue
for session_entry in result["session_data"]:
if "entry_url" in session_entry and self.indicators.check_domain(session_entry["entry_url"]):
self.detected.append(result)
if "entry_url" in session_entry:
ioc = self.indicators.check_domain(session_entry["entry_url"])
if ioc:
result["matched_indicator"] = ioc
self.detected.append(result)
def _process_browser_state_db(self, db_path):
conn = sqlite3.connect(db_path)

View File

@@ -80,7 +80,9 @@ class SafariHistory(IOSExtraction):
return
for result in self.results:
if self.indicators.check_domain(result["url"]):
ioc = self.indicators.check_domain(result["url"])
if ioc:
result["matched_indicator"] = ioc
self.detected.append(result)
def _process_history_db(self, history_path):

View File

@@ -54,9 +54,11 @@ class Shortcuts(IOSExtraction):
if not self.indicators:
return
for action in self.results:
if self.indicators.check_domains(action["action_urls"]):
self.detected.append(action)
for result in self.results:
ioc = self.indicators.check_domains(result["action_urls"])
if ioc:
result["matched_indicator"] = ioc
self.detected.append(result)
def run(self):
self._find_ios_database(backup_ids=SHORTCUT_BACKUP_IDS,

View File

@@ -41,10 +41,12 @@ class SMS(IOSExtraction):
if not self.indicators:
return
for message in self.results:
message_links = check_for_links(message.get("text", ""))
if self.indicators.check_domains(message_links):
self.detected.append(message)
for result in self.results:
message_links = check_for_links(result.get("text", ""))
ioc = self.indicators.check_domains(message_links)
if ioc:
result["matched_indicator"] = ioc
self.detected.append(result)
def run(self):
self._find_ios_database(backup_ids=SMS_BACKUP_IDS,

View File

@@ -66,6 +66,16 @@ class TCC(IOSExtraction):
"data": msg
}
def check_indicators(self):
if not self.indicators:
return
for result in self.results:
ioc = self.indicators.check_process(result["client"])
if ioc:
result["matched_indicator"] = ioc
self.detected.append(result)
def process_db(self, file_path):
conn = sqlite3.connect(file_path)
cur = conn.cursor()

View File

@@ -37,7 +37,9 @@ class WebkitResourceLoadStatistics(IOSExtraction):
self.detected = {}
for key, items in self.results.items():
for item in items:
if self.indicators.check_domain(item["registrable_domain"]):
ioc = self.indicators.check_domain(item["registrable_domain"])
if ioc:
item["matched_indicator"] = ioc
if key not in self.detected:
self.detected[key] = [item, ]
else:

View File

@@ -66,7 +66,9 @@ class WebkitSessionResourceLog(IOSExtraction):
all_origins = set([entry["origin"]] + source_domains + destination_domains)
if self.indicators.check_domains(all_origins):
ioc = self.indicators.check_domains(all_origins)
if ioc:
entry["matched_indicator"] = ioc
self.detected.append(entry)
redirect_path = ""

View File

@@ -35,6 +35,7 @@ class Whatsapp(IOSExtraction):
links_text = ""
if record["links"]:
links_text = " - Embedded links: " + ", ".join(record["links"])
return {
"timestamp": record.get("isodate"),
"module": self.__class__.__name__,
@@ -46,9 +47,11 @@ class Whatsapp(IOSExtraction):
if not self.indicators:
return
for message in self.results:
if self.indicators.check_domains(message["links"]):
self.detected.append(message)
for result in self.results:
ioc = self.indicators.check_domains(result.get("links", []))
if ioc:
result["matched_indicator"] = ioc
self.detected.append(result)
def run(self):
self._find_ios_database(backup_ids=WHATSAPP_BACKUP_IDS,
@@ -83,14 +86,15 @@ class Whatsapp(IOSExtraction):
message["isodate"] = convert_timestamp_to_iso(convert_mactime_to_unix(message.get("ZMESSAGEDATE")))
message["ZTEXT"] = message["ZTEXT"] if message["ZTEXT"] else ""
# Extract links from the WhatsApp message. URLs can be stored in multiple fields/columns. Check each of them!
# Extract links from the WhatsApp message. URLs can be stored in multiple fields/columns.
# Check each of them!
message_links = []
fields_with_links = ["ZTEXT", "ZMATCHEDTEXT", "ZMEDIAURL", "ZCONTENT1", "ZCONTENT2"]
for field in fields_with_links:
if message.get(field):
message_links.extend(check_for_links(message.get(field, "")))
# Remove WhatsApp internal media URLs
# Remove WhatsApp internal media URLs.
filtered_links = []
for link in message_links:
if not (link.startswith("https://mmg-fna.whatsapp.net/") or link.startswith("https://mmg.whatsapp.net/")):

View File

@@ -237,5 +237,7 @@ class NetBase(IOSExtraction):
if not result["proc_id"]:
continue
if self.indicators.check_process(proc_name):
ioc = self.indicators.check_process(proc_name)
if ioc:
result["matched_indicator"] = ioc
self.detected.append(result)

Some files were not shown because too many files have changed in this diff Show More