Load Android device timezone info and add additional file modificatio…

…n logs (#567)

* Use local timestamp for Files module timeline.

Most other Android timestamps appear to be local time. The
results timeline is more useful if all the timestamps
are consistent. I would prefer to use UTC, but that would
mean converting all the other timestamps to UTC as well. We probably
do not have sufficient information to do that accurately,
especially if the device is moving between timezones..

* Add file timestamp modules to add logs into timeline

* Handle case were we cannot load device timezone

* Fix crash if prop file does not exist

* Move _get_file_modification_time to BugReportModule

* Add backport for timezone and fix Tombstone module to use local time.

* Fix import for backported Zoneinfo

* Fix ruff error

pyproject.toml

@@ -35,6 +35,7 @@ dependencies = [
     "betterproto >=1.2.0",
     "pydantic >= 2.10.0",
     "pydantic-settings >= 2.7.0",
+    'backports.zoneinfo; python_version < "3.9"',
 ]
 requires-python = ">= 3.8"
 

src/mvt/android/artifacts/file_timestamps.py

@@ -0,0 +1,43 @@
+# Mobile Verification Toolkit (MVT)
+# Copyright (c) 2021-2023 The MVT Authors.
+# Use of this software is governed by the MVT License 1.1 that can be found at
+#   https://license.mvt.re/1.1/
+from typing import Union
+
+from .artifact import AndroidArtifact
+
+
+class FileTimestampsArtifact(AndroidArtifact):
+    def serialize(self, record: dict) -> Union[dict, list]:
+        records = []
+
+        for ts in set(
+            [
+                record.get("access_time"),
+                record.get("changed_time"),
+                record.get("modified_time"),
+            ]
+        ):
+            if not ts:
+                continue
+
+            macb = ""
+            macb += "M" if ts == record.get("modified_time") else "-"
+            macb += "A" if ts == record.get("access_time") else "-"
+            macb += "C" if ts == record.get("changed_time") else "-"
+            macb += "-"
+
+            msg = record["path"]
+            if record.get("context"):
+                msg += f" ({record['context']})"
+
+            records.append(
+                {
+                    "timestamp": ts,
+                    "module": self.__class__.__name__,
+                    "event": macb,
+                    "data": msg,
+                }
+            )
+
+        return records

src/mvt/android/artifacts/getprop.py

@@ -42,6 +42,17 @@ def parse(self, entry: str) -> None:
             entry = {"name": matches[0][0], "value": matches[0][1]}
             self.results.append(entry)
 
+    def get_device_timezone(self) -> str:
+        """
+        Get the device timezone from the getprop results
+
+        Used in other moduels to calculate the timezone offset
+        """
+        for entry in self.results:
+            if entry["name"] == "persist.sys.timezone":
+                return entry["value"]
+        return None
+
     def check_indicators(self) -> None:
         for entry in self.results:
             if entry["name"] in INTERESTING_PROPERTIES:

src/mvt/android/artifacts/tombstone_crashes.py

@@ -81,8 +81,8 @@ def serialize(self, record: dict) -> Union[dict, list]:
             "module": self.__class__.__name__,
             "event": "Tombstone",
             "data": (
-                f"Crash in '{record['process_name']}' process running as UID '{record['uid']}' at "
-                f"{record['timestamp']}. Crash type '{record['signal_info']['name']}' with code '{record['signal_info']['code_name']}'"
+                f"Crash in '{record['process_name']}' process running as UID '{record['uid']}' in file '{record['file_name']}' "
+                f"Crash type '{record['signal_info']['name']}' with code '{record['signal_info']['code_name']}'"
             ),
         }
 
@@ -258,7 +258,10 @@ def _parse_timestamp_string(timestamp: str) -> str:
         timestamp_parsed = datetime.datetime.strptime(
             timestamp_without_micro, "%Y-%m-%d %H:%M:%S%z"
         )
-        return convert_datetime_to_iso(timestamp_parsed)
+
+        # HACK: Swap the local timestamp to UTC, so keep the original time and avoid timezone conversion.
+        local_timestamp = timestamp_parsed.replace(tzinfo=datetime.timezone.utc)
+        return convert_datetime_to_iso(local_timestamp)
 
     @staticmethod
     def _proccess_name_from_thread(tombstone_dict: dict) -> str:

src/mvt/android/modules/androidqf/base.py

@@ -48,6 +48,37 @@ def from_zip_file(self, archive: zipfile.ZipFile, files: List[str]):
     def _get_files_by_pattern(self, pattern: str):
         return fnmatch.filter(self.files, pattern)
 
+    def _get_device_timezone(self):
+        """
+        Get the device timezone from the getprop.txt file.
+
+        This is needed to map local timestamps stored in some
+        Android log files to UTC/timezone-aware timestamps.
+        """
+        get_prop_files = self._get_files_by_pattern("*/getprop.txt")
+        if not get_prop_files:
+            self.log.warning(
+                "Could not find getprop.txt file. "
+                "Some timestamps and timeline data may be incorrect."
+            )
+            return None
+
+        from mvt.android.artifacts.getprop import GetProp
+
+        properties_artifact = GetProp()
+        prop_data = self._get_file_content(get_prop_files[0]).decode("utf-8")
+        properties_artifact.parse(prop_data)
+        timezone = properties_artifact.get_device_timezone()
+        if timezone:
+            self.log.debug("Identified local phone timezone: %s", timezone)
+            return timezone
+
+        self.log.warning(
+            "Could not find or determine local device timezone. "
+            "Some timestamps and timeline data may be incorrect."
+        )
+        return None
+
     def _get_file_content(self, file_path):
         if self.archive:
             handle = self.archive.open(file_path)

src/mvt/android/modules/androidqf/files.py

@@ -6,6 +6,11 @@
 import datetime
 import json
 import logging
+
+try:
+    import zoneinfo
+except ImportError:
+    from backports import zoneinfo
 from typing import Optional, Union
 
 from mvt.android.modules.androidqf.base import AndroidQFModule
@@ -106,6 +111,12 @@ def check_indicators(self) -> None:
             # TODO: adds SHA1 and MD5 when available in MVT
 
     def run(self) -> None:
+        if timezone := self._get_device_timezone():
+            device_timezone = zoneinfo.ZoneInfo(timezone)
+        else:
+            self.log.warning("Unable to determine device timezone, using UTC")
+            device_timezone = zoneinfo.ZoneInfo("UTC")
+
         for file in self._get_files_by_pattern("*/files.json"):
             rawdata = self._get_file_content(file).decode("utf-8", errors="ignore")
             try:
@@ -120,11 +131,18 @@ def run(self) -> None:
             for file_data in data:
                 for ts in ["access_time", "changed_time", "modified_time"]:
                     if ts in file_data:
-                        file_data[ts] = convert_datetime_to_iso(
-                            datetime.datetime.fromtimestamp(
-                                file_data[ts], tz=datetime.timezone.utc
-                            )
+                        utc_timestamp = datetime.datetime.fromtimestamp(
+                            file_data[ts], tz=datetime.timezone.utc
+                        )
+                        # Convert the UTC timestamp to local tiem on Android device's local timezone
+                        local_timestamp = utc_timestamp.astimezone(device_timezone)
+
+                        # HACK: We only output the UTC timestamp in convert_datetime_to_iso, we
+                        # set the timestamp timezone to UTC, to avoid the timezone conversion again.
+                        local_timestamp = local_timestamp.replace(
+                            tzinfo=datetime.timezone.utc
                         )
+                        file_data[ts] = convert_datetime_to_iso(local_timestamp)
 
                 self.results.append(file_data)
 

src/mvt/android/modules/androidqf/logfile_timestamps.py

@@ -0,0 +1,65 @@
+# Mobile Verification Toolkit (MVT)
+# Copyright (c) 2021-2023 The MVT Authors.
+# Use of this software is governed by the MVT License 1.1 that can be found at
+#   https://license.mvt.re/1.1/
+
+import os
+import datetime
+import logging
+from typing import Optional
+
+from mvt.common.utils import convert_datetime_to_iso
+from .base import AndroidQFModule
+from mvt.android.artifacts.file_timestamps import FileTimestampsArtifact
+
+
+class LogsFileTimestamps(FileTimestampsArtifact, AndroidQFModule):
+    """This module extracts records from battery daily updates."""
+
+    slug = "logfile_timestamps"
+
+    def __init__(
+        self,
+        file_path: Optional[str] = None,
+        target_path: Optional[str] = None,
+        results_path: Optional[str] = None,
+        module_options: Optional[dict] = None,
+        log: logging.Logger = logging.getLogger(__name__),
+        results: Optional[list] = None,
+    ) -> None:
+        super().__init__(
+            file_path=file_path,
+            target_path=target_path,
+            results_path=results_path,
+            module_options=module_options,
+            log=log,
+            results=results,
+        )
+
+    def _get_file_modification_time(self, file_path: str) -> dict:
+        if self.archive:
+            file_timetuple = self.archive.getinfo(file_path).date_time
+            return datetime.datetime(*file_timetuple)
+        else:
+            file_stat = os.stat(os.path.join(self.parent_path, file_path))
+            return datetime.datetime.fromtimestamp(file_stat.st_mtime)
+
+    def run(self) -> None:
+        filesystem_files = self._get_files_by_pattern("*/logs/*")
+
+        self.results = []
+        for file in filesystem_files:
+            # Only the modification time is available in the zip file metadata.
+            # The timezone is the local timezone of the machine the phone.
+            modification_time = self._get_file_modification_time(file)
+            self.results.append(
+                {
+                    "path": file,
+                    "modified_time": convert_datetime_to_iso(modification_time),
+                }
+            )
+
+        self.log.info(
+            "Extracted a total of %d filesystem timestamps from AndroidQF logs directory.",
+            len(self.results),
+        )

src/mvt/android/modules/bugreport/__init__.py

@@ -14,6 +14,7 @@
 from .platform_compat import PlatformCompat
 from .receivers import Receivers
 from .adb_state import DumpsysADBState
+from .fs_timestamps import BugReportTimestamps
 from .tombstones import Tombstones
 
 BUGREPORT_MODULES = [
@@ -28,5 +29,6 @@
     PlatformCompat,
     Receivers,
     DumpsysADBState,
+    BugReportTimestamps,
     Tombstones,
 ]

src/mvt/android/modules/bugreport/base.py

@@ -6,6 +6,7 @@
 import fnmatch
 import logging
 import os
+
 from typing import List, Optional
 from zipfile import ZipFile
 

src/mvt/android/modules/bugreport/fs_timestamps.py

@@ -0,0 +1,55 @@
+# Mobile Verification Toolkit (MVT)
+# Copyright (c) 2021-2023 The MVT Authors.
+# Use of this software is governed by the MVT License 1.1 that can be found at
+#   https://license.mvt.re/1.1/
+
+import logging
+from typing import Optional
+
+from mvt.common.utils import convert_datetime_to_iso
+from .base import BugReportModule
+from mvt.android.artifacts.file_timestamps import FileTimestampsArtifact
+
+
+class BugReportTimestamps(FileTimestampsArtifact, BugReportModule):
+    """This module extracts records from battery daily updates."""
+
+    slug = "bugreport_timestamps"
+
+    def __init__(
+        self,
+        file_path: Optional[str] = None,
+        target_path: Optional[str] = None,
+        results_path: Optional[str] = None,
+        module_options: Optional[dict] = None,
+        log: logging.Logger = logging.getLogger(__name__),
+        results: Optional[list] = None,
+    ) -> None:
+        super().__init__(
+            file_path=file_path,
+            target_path=target_path,
+            results_path=results_path,
+            module_options=module_options,
+            log=log,
+            results=results,
+        )
+
+    def run(self) -> None:
+        filesystem_files = self._get_files_by_pattern("FS/*")
+
+        self.results = []
+        for file in filesystem_files:
+            # Only the modification time is available in the zip file metadata.
+            # The timezone is the local timezone of the machine the phone.
+            modification_time = self._get_file_modification_time(file)
+            self.results.append(
+                {
+                    "path": file,
+                    "modified_time": convert_datetime_to_iso(modification_time),
+                }
+            )
+
+        self.log.info(
+            "Extracted a total of %d filesystem timestamps from bugreport.",
+            len(self.results),
+        )

src/mvt/common/command.py

@@ -101,15 +101,25 @@ def _store_timeline(self) -> None:
         if not self.results_path:
             return
 
+        # We use local timestamps in the timeline on Android as many
+        # logs do not contain timezone information.
+        if type(self).__name__.startswith("CmdAndroid"):
+            is_utc = False
+        else:
+            is_utc = True
+
         if len(self.timeline) > 0:
             save_timeline(
-                self.timeline, os.path.join(self.results_path, "timeline.csv")
+                self.timeline,
+                os.path.join(self.results_path, "timeline.csv"),
+                is_utc=is_utc,
             )
 
         if len(self.timeline_detected) > 0:
             save_timeline(
                 self.timeline_detected,
                 os.path.join(self.results_path, "timeline_detected.csv"),
+                is_utc=is_utc,
             )
 
     def _store_info(self) -> None:

src/mvt/common/module.py

@@ -227,7 +227,7 @@ def run_module(module: MVTModule) -> None:
         module.save_to_json()
 
 
-def save_timeline(timeline: list, timeline_path: str) -> None:
+def save_timeline(timeline: list, timeline_path: str, is_utc: bool = True) -> None:
     """Save the timeline in a csv file.
 
     :param timeline: List of records to order and store
@@ -238,7 +238,12 @@ def save_timeline(timeline: list, timeline_path: str) -> None:
         csvoutput = csv.writer(
             handle, delimiter=",", quotechar='"', quoting=csv.QUOTE_ALL, escapechar="\\"
         )
-        csvoutput.writerow(["UTC Timestamp", "Plugin", "Event", "Description"])
+
+        if is_utc:
+            timestamp_header = "UTC Timestamp"
+        else:
+            timestamp_header = "Device Local Timestamp"
+        csvoutput.writerow([timestamp_header, "Plugin", "Event", "Description"])
 
         for event in sorted(
             timeline, key=lambda x: x["timestamp"] if x["timestamp"] is not None else ""

tests/android/test_artifact_tombstones.py

@@ -60,6 +60,8 @@ def validate_tombstone_result(self, tombstone_result: dict):
         assert tombstone_result.get("pid") == 25541
         assert tombstone_result.get("process_name") == "mtk.ape.decoder"
 
-        # Check if the timestamp is correctly parsed, and converted to UTC
-        # Original is in +0200: 2023-04-12 12:32:40.518290770+0200, result should be 2023-04-12 10:32:40.000000+0000
-        assert tombstone_result.get("timestamp") == "2023-04-12 10:32:40.000000"
+        # With Android logs we want to keep timestamps as device local time for consistency.
+        # We often don't know the time offset for a log entry and so can't convert everything to UTC.
+        # MVT should output the local time only:
+        # So original 2023-04-12 12:32:40.518290770+0200 -> 2023-04-12 12:32:40.000000
+        assert tombstone_result.get("timestamp") == "2023-04-12 12:32:40.000000"