fix: real-time axis, non-UTF-8 file support, and OOM on large logs

dosview/__init__.py

@@ -61,31 +61,70 @@ def run(self):
 
 
 
+class SmartDateAxisItem(pg.DateAxisItem):
+    """DateAxisItem that prefixes the date whenever the calendar day changes."""
+
+    def tickStrings(self, values, scale, spacing):
+        strings = super().tickStrings(values, scale, spacing)
+        if not values:
+            return strings
+        result = []
+        prev_date = None
+        for v, s in zip(values, strings):
+            dt = datetime.datetime.fromtimestamp(v, tz=datetime.timezone.utc)
+            current_date = dt.date()
+            if current_date != prev_date:
+                result.append(f"{current_date.strftime('%Y-%m-%d')}\n{s}")
+                prev_date = current_date
+            else:
+                result.append(s)
+        return result
+
+
 class PlotCanvas(pg.GraphicsLayoutWidget):
     def __init__(self, parent=None, file_path=None):
         super().__init__(parent)
         self.data = []
         self.file_path = file_path
-        self.telemetry_lines = {'temperature_0': None, 'humidity_0': None, 'temperature_1': None, 'humidity_1': None, 'temperature_2': None, 'pressure_3': None, 
+        self._real_time = False
+        self.telemetry_lines = {'temperature_0': None, 'humidity_0': None, 'temperature_1': None, 'humidity_1': None, 'temperature_2': None, 'pressure_3': None,
                                 'voltage': None, 'current': None, 'capacity_remaining': None, 'capacity_full': None, 'temperature': None}
 
+    @staticmethod
+    def _has_real_time(data) -> bool:
+        try:
+            return data[3].get("log_info", {}).get("has_real_time", False)
+        except (IndexError, AttributeError):
+            return False
+
+    def _x(self, t_seconds):
+        """Convert RTC/Unix seconds to plot x-axis units."""
+        return t_seconds if self._real_time else t_seconds / 60
+
     def plot(self, data):
         start_time = time.time()
 
         self.data = data
+        self._real_time = self._has_real_time(data)
         window_size = 20
 
         self.clear()
 
-        plot_evolution = self.addPlot(row=0, col=0)
+        if self._real_time:
+            bottom_axis = SmartDateAxisItem(orientation='bottom')
+            plot_evolution = self.addPlot(row=0, col=0, axisItems={'bottom': bottom_axis})
+        else:
+            plot_evolution = self.addPlot(row=0, col=0)
         plot_spectrum = self.addPlot(row=1, col=0)
 
-
         plot_evolution.showGrid(x=True, y=True)
         plot_evolution.setLabel("left",  "Total count per exposition", units="#")
-        plot_evolution.setLabel("bottom","Time", units="min")
+        if self._real_time:
+            plot_evolution.setLabel("bottom", "Time (UTC)")
+        else:
+            plot_evolution.setLabel("bottom", "Time", units="min")
 
-        time_axis = self.data[0]/60
+        time_axis = self._x(self.data[0])
         self._curve_evolution = plot_evolution.plot(time_axis, self.data[1],
                         symbol ='o', symbolPen ='pink', name ='Channel', pen=None)
 
@@ -124,13 +163,20 @@ def plot(self, data):
     }
 
     def _add_telemetry_plot(self, telemetry):
-        plot_telemetry = self.addPlot(row=2, col=0)
+        if self._real_time:
+            bottom_axis = SmartDateAxisItem(orientation='bottom')
+            plot_telemetry = self.addPlot(row=2, col=0, axisItems={'bottom': bottom_axis})
+        else:
+            plot_telemetry = self.addPlot(row=2, col=0)
         plot_telemetry.showGrid(x=True, y=True)
-        plot_telemetry.setLabel("bottom", "Time", units="min")
+        if self._real_time:
+            plot_telemetry.setLabel("bottom", "Time (UTC)")
+        else:
+            plot_telemetry.setLabel("bottom", "Time", units="min")
         plot_telemetry.addLegend()
         for key, (t, vals) in telemetry.items():
             pen = pg.mkPen(color=self._telemetry_colors.get(key, (200, 200, 200)), width=2)
-            line = plot_telemetry.plot(t / 60, vals, pen=pen, name=key)
+            line = plot_telemetry.plot(self._x(t), vals, pen=pen, name=key)
             self.telemetry_lines[key] = line
         self._has_telemetry_plot = True
 
@@ -152,7 +198,7 @@ def update_data(self, data):
 
         self.data = data
         window_size = 20
-        time_axis = data[0] / 60
+        time_axis = self._x(data[0])
         sums = data[1]
 
         self._curve_evolution.setData(time_axis, sums)
@@ -166,7 +212,7 @@ def update_data(self, data):
         if has_telemetry:
             for key, (t, vals) in data[4].items():
                 if self.telemetry_lines.get(key) is not None:
-                    self.telemetry_lines[key].setData(t / 60, vals)
+                    self.telemetry_lines[key].setData(self._x(t), vals)
 
     def telemetry_toggle(self, key, value):
         if self.telemetry_lines[key] is not None:

dosview/parsers.py

@@ -35,7 +35,7 @@ class AirdosV2LogParser(BaseLogParser):
     @staticmethod
     def detect(file_path: str | Path) -> bool:
         has_dos = False
-        with open(file_path, "r") as f:
+        with open(file_path, "r", encoding="ascii", errors="replace") as f:
             for line in f:
                 if line.startswith("$DOS"):
                     has_dos = True
@@ -48,6 +48,27 @@ def detect(file_path: str | Path) -> bool:
                     return True
         return False
 
+    @staticmethod
+    def _find_max_channel(file_path: str) -> int:
+        """Quick pre-scan to find the highest channel index used in $E and $STOP records."""
+        max_ch = 0
+        with open(file_path, "r", encoding="ascii", errors="replace") as f:
+            for line in f:
+                if line.startswith("$E,"):
+                    comma2 = line.index(",", 3) if "," in line[3:] else -1
+                    if comma2 > 0:
+                        try:
+                            ch = int(line[comma2 + 1:].split(",")[0])
+                            if ch > max_ch:
+                                max_ch = ch
+                        except ValueError:
+                            pass
+                elif line.startswith("$STOP,"):
+                    n_bins = line.count(",") - 4  # parts[5:] count
+                    if n_bins - 1 > max_ch:
+                        max_ch = n_bins - 1
+        return max_ch
+
     def parse(self):
         start_time = time.time()
         print("AIRDOS v2 parser start")
@@ -56,7 +77,8 @@ def parse(self):
             "log_device_info": {},
             "log_info": {},
         }
-        hist = np.zeros(65536, dtype=int)
+        n_channels = max(self._find_max_channel(self.file_path) + 1, 256)
+        hist = np.zeros(n_channels, dtype=np.int32)
         total_counts = 0
         sums: List[int] = []
         time_axis: List[float] = []
@@ -67,8 +89,9 @@ def parse(self):
         device_type = "unknown"
         env_records: List[Tuple[float, ...]] = []
         batt_records: List[Tuple[float, ...]] = []
+        rtc_unix_offset: float | None = None
 
-        with open(self.file_path, "r") as file:
+        with open(self.file_path, "r", encoding="ascii", errors="replace") as file:
             for line in file:
                 parts = line.strip().split(",")
                 match parts[0]:
@@ -138,6 +161,14 @@ def parse(self):
                                 ))
                             except ValueError:
                                 pass
+                    case "$TIME":
+                        # $TIME,rtc_counter,rtc_unix_offset,unix_timestamp,...
+                        # rtc_unix_offset = unix_timestamp - rtc_counter
+                        if len(parts) >= 3 and rtc_unix_offset is None:
+                            try:
+                                rtc_unix_offset = float(parts[2])
+                            except ValueError:
+                                pass
                     case "$BATT":
                         # $BATT,count,tm.tm_s100,voltage_mV,current_mA,remaining_mAh,full_mAh,temp_C
                         if len(parts) >= 8:
@@ -155,11 +186,19 @@ def parse(self):
                     case _:
                         continue
 
+        if rtc_unix_offset is not None:
+            time_axis = [t + rtc_unix_offset for t in time_axis]
+            env_records = [(r[0] + rtc_unix_offset,) + r[1:] for r in env_records]
+            batt_records = [(r[0] + rtc_unix_offset,) + r[1:] for r in batt_records]
+
         metadata["log_info"]["histogram_channels"] = hist.shape[0]
         metadata["log_info"]["events_total"] = int(total_counts)
         metadata["log_info"]["log_type_version"] = "2.0"
         metadata["log_info"]["log_type"] = "xDOS_SPECTRAL"
         metadata["log_info"]["detector_type"] = device_type
+        metadata["log_info"]["has_real_time"] = rtc_unix_offset is not None
+        if rtc_unix_offset is not None:
+            metadata["log_info"]["rtc_unix_offset"] = rtc_unix_offset
 
         telemetry: dict = {}
         if env_records:
@@ -192,7 +231,7 @@ class OldLogParser(BaseLogParser):
 
     @staticmethod
     def detect(file_path: str | Path) -> bool:
-        with open(file_path, "r") as f:
+        with open(file_path, "r", encoding="ascii", errors="replace") as f:
             for line in f:
                 if line.startswith("$DOS") and "AIRDOS04C" not in line:
                     return True
@@ -213,7 +252,7 @@ def parse(self):
         df_lines: List[Sequence[str]] = []
         df_metadata: List[Sequence[str]] = []
         unique_events: List[Tuple[float, int]] = []
-        with open(self.file_path, "r") as file:
+        with open(self.file_path, "r", encoding="ascii", errors="replace") as file:
             for line in file:
                 parts = line.strip().split(",")
                 match parts[0]: