flyitalyadsb: --api-update-every, a faster API without writing the json files more often

.github/workflows/flyitalyadsb-test.yaml

@@ -0,0 +1,26 @@
+name: flyitalyadsb tests
+
+# flyitalyadsb: behaviour this fork adds on top of wiedehopf/readsb, tested on
+# a real binary (build.yaml is upstream's and only runs `make test`).
+on:
+  push:
+    branches: [dev]
+  pull_request:
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  test:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y --no-install-recommends pkg-config libncurses-dev zlib1g-dev libzstd-dev
+      - name: Build
+        run: make -j"$(nproc)"
+      - name: --api-update-every
+        run: flyitalyadsb/test-api-update-every.sh ./readsb

api.c

@@ -845,7 +845,7 @@ static inline int apiAdd(struct apiBuffer *buffer, struct aircraft *a, int64_t n
 
 #if defined(WITH_UUIDS)
     // only include receivers if they have received the plane within the 2x the update interval
-    int64_t printNewer = now - 2 * Modes.json_interval;
+    int64_t printNewer = now - 2 * Modes.api_interval;
     for (int i = 0; i < RECENT_RECEIVER_IDS; i++) {
         entry->recentReceiverIds[i] = (a->recentReceiverIds[i].time > printNewer) ? a->recentReceiverIds[i].id : 0;
     }
@@ -1035,8 +1035,17 @@ static int apiUpdate() {
         atomic_store(&Modes.apiFlip[i], flip);
     }
 
-    pthread_cond_signal(&Threads.json.cond);
-    pthread_cond_signal(&Threads.globeJson.cond);
+    // flyitalyadsb: with --api-update-every below --write-json-every, wake the json
+    // writers once per json interval only, so the files keep their rate while the
+    // API (and the map through --tar1090-use-api) gets the faster one. Equal
+    // intervals: every update, as upstream. Only this thread (and the initial
+    // apiUpdate before it starts) gets here.
+    static int64_t lastJsonWake;
+    if (now - lastJsonWake >= Modes.json_interval - Modes.api_interval / 2) {
+        lastJsonWake = now;
+        pthread_cond_signal(&Threads.json.cond);
+        pthread_cond_signal(&Threads.globeJson.cond);
+    }
 
     return buffer->len;
 }
@@ -2065,7 +2074,7 @@ static void *apiUpdateEntryPoint(void *arg) {
 
         int64_t now = mstime();
 
-        int64_t ival = Modes.json_interval;
+        int64_t ival = Modes.api_interval;
         int64_t remaining = ival - (now % ival);
 
         if (remaining < 5) {

flyitalyadsb/test-api-update-every.sh

@@ -0,0 +1,80 @@
+#!/bin/bash
+# flyitalyadsb: --api-update-every keeps the API (and the map, through
+# --tar1090-use-api) fresher than the json files, which keep --write-json-every.
+#
+#   flyitalyadsb/test-api-update-every.sh [path/to/readsb]   (default ./readsb)
+#
+# Starts readsb with no input (net-only), samples the API's "now" and
+# aircraft.json's "now" ~every 0.1 s for 8 s, and takes the median gap between
+# distinct values: readsb's own timestamps of each update, so the sampling's
+# speed doesn't matter. Also checks that without the option nothing changes.
+set -euo pipefail
+BIN=$(realpath "${1:-./readsb}")
+WORK=$(mktemp -d)
+PID=
+trap '[ -n "$PID" ] && kill "$PID" 2>/dev/null; wait 2>/dev/null; rm -rf "$WORK"' EXIT
+fail() { echo "FAIL: $*"; exit 1; }
+
+# run NAME ARGS...: start readsb in $WORK/NAME, wait for receiver.json and the API socket
+run() {
+    local d="$WORK/$1"; shift
+    mkdir -p "$d"
+    "$BIN" --net-only --quiet --net-ri-port=0 --net-ro-port=0 --net-sbs-port=0 --net-bi-port=0 --net-bo-port=0 \
+        --write-json="$d" --net-api-port="unix:$d/api.sock" --tar1090-use-api "$@" > "$d/log" 2>&1 &
+    PID=$!
+    for _ in $(seq 100); do
+        [ -S "$d/api.sock" ] && [ -s "$d/receiver.json" ] && [ -s "$d/aircraft.json" ] && return 0
+        sleep 0.1
+    done
+    cat "$d/log"; fail "$1: readsb didn't start"
+}
+stop() { kill "$PID"; wait "$PID" 2>/dev/null || true; PID=; }
+now_of() { python3 -c 'import json,sys; print(json.load(sys.stdin)["now"])'; }
+refresh() { python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["refresh"])' "$WORK/$1/receiver.json"; }
+
+# sample NAME: prints "<API gap> <aircraft.json gap>", the median seconds between
+# distinct "now" values seen over 8 s
+sample() {
+    local d="$WORK/$1" end=$((SECONDS + 8))
+    : > "$d/api.now"; : > "$d/file.now"
+    while [ "$SECONDS" -lt "$end" ]; do
+        curl -s --unix-socket "$d/api.sock" 'http://localhost/?all' | now_of >> "$d/api.now"
+        now_of < "$d/aircraft.json" >> "$d/file.now" 2>/dev/null || true
+        sleep 0.1
+    done
+    python3 - "$d/api.now" "$d/file.now" <<'PY'
+import statistics, sys
+def gap(path):
+    v = sorted({float(x) for x in open(path).read().split()})
+    return statistics.median(b - a for a, b in zip(v, v[1:])) if len(v) > 2 else 0
+print("%.2f %.2f" % (gap(sys.argv[1]), gap(sys.argv[2])))
+PY
+}
+# between LOW HIGH VALUE
+between() { python3 -c 'import sys; lo, hi, v = map(float, sys.argv[1:]); sys.exit(not lo <= v <= hi)' "$@"; }
+
+# 1. API every 0.5 s, files every 2 s
+run fast --write-json-every=2 --api-update-every=0.5
+[ "$(refresh fast)" = 500 ] || fail "receiver.json refresh is $(refresh fast), expected 500"
+read -r api file < <(sample fast)
+echo "api-update-every 0.5, write-json-every 2: API every $api s, aircraft.json every $file s"
+between 0.4 0.6 "$api" || fail "API updated every $api s, expected 0.5"
+between 1.8 2.2 "$file" || fail "aircraft.json written every $file s, expected 2"
+stop
+
+# 2. Without the option: as upstream, both every 2 s
+run same --write-json-every=2
+[ "$(refresh same)" = 2000 ] || fail "receiver.json refresh is $(refresh same), expected 2000"
+read -r api file < <(sample same)
+echo "write-json-every 2 only: API every $api s, aircraft.json every $file s"
+between 1.8 2.2 "$api" || fail "API updated every $api s, expected 2"
+between 1.8 2.2 "$file" || fail "aircraft.json written every $file s, expected 2"
+stop
+
+# 3. Longer than write-json-every: clamped, with a warning
+run slow --write-json-every=2 --api-update-every=5
+[ "$(refresh slow)" = 2000 ] || fail "receiver.json refresh is $(refresh slow), expected 2000 (clamped)"
+grep -q "api-update-every can't be longer" "$WORK/slow/log" || fail "no warning for --api-update-every 5 > 2"
+stop
+
+echo "ok"

help.h

@@ -157,6 +157,7 @@ static struct argp_option optionsReadsb[] = {
     {"net-json-port-wind-triggered", OptNetJsonPortWindTriggered, 0, 0, "TCP json position output: instead of emitting when a new position is calculated, only emit when the wind was calculated now based on data from the last 2.5 seconds and the position is no older than 2.5 seconds", 2},
     {"net-api-port", OptNetApiPorts, "<port>", 0, "http API listen port (in contrast to other listeners, only a single port is allowed) (update frequency controlled by write-json-every parameter) (default: 0)", 2},
     {"api-shutdown-delay", OptApiShutdownDelay, "<seconds>", 0, "Shutdown delay to server remaining API queries, new queries get a 503 response (default: 0)", 2},
+    {"api-update-every", OptApiUpdateTime, "<sec>", 0, "Update the API data every sec seconds, without writing the json files more often (default: the write-json-every value, never slower). With --tar1090-use-api, receiver.json advertises this as the map refresh", 2},
     {"tar1090-use-api", OptTar1090UseApi, 0, 0, "when running with globe-index, signal tar1090 use the readsb API to get data, requires webserver mapping of /tar1090/re-api to proxy_pass the requests to the --net-api-port, see nginx-readsb-api.conf in the tar1090 repository for details", 2},
     {"net-beast-reduce-out-port", OptNetBeastReducePorts, "<ports>", 0, "TCP BeastReduce output listen port / TCP server (default: 0)", 2},
     {"net-beast-reduce-interval", OptNetBeastReduceInterval, "<seconds>", 0, "BeastReduce data update interval, longer means less data (default: 0.250, valid range: 0.000 - 14.999)", 2},

json_out.c

@@ -1924,7 +1924,8 @@ struct char_buffer generateReceiverJson() {
     p = safe_snprintf(p, end, "{ "
             "\"refresh\": %.0f, "
             "\"history\": %d",
-            1.0 * Modes.json_interval, Modes.json_aircraft_history_next + 1);
+            // flyitalyadsb: with --tar1090-use-api the map polls the API, so advertise its rate
+            1.0 * (Modes.tar1090_use_api ? Modes.api_interval : Modes.json_interval), Modes.json_aircraft_history_next + 1);
 
 
     if (Modes.json_location_accuracy != 0 && Modes.userLocationValid) {

readsb.c

@@ -2076,6 +2076,11 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) {
         case OptTar1090UseApi:
             Modes.tar1090_use_api = 1;
             break;
+        case OptApiUpdateTime:
+            Modes.api_interval = (int64_t) (1000.0 * atof(arg));
+            if (Modes.api_interval < 100) // 0.1s, as --write-json-every
+                Modes.api_interval = 100;
+            break;
         case OptNetApiPorts:
             sfree(Modes.net_output_api_ports);
             Modes.net_output_api_ports = strdup(arg);
@@ -2576,6 +2581,15 @@ int parseCommandLine(int argc, char **argv) {
 }
 
 static void configAfterParse() {
+    // flyitalyadsb: --api-update-every defaults to --write-json-every and is never slower:
+    // the API update is what wakes the json writers (apiUpdate in api.c)
+    if (Modes.api_interval > Modes.json_interval) {
+        fprintf(stderr, "api-update-every can't be longer than write-json-every, using %.1f s\n", Modes.json_interval / 1000.0);
+    }
+    if (Modes.api_interval <= 0 || Modes.api_interval > Modes.json_interval) {
+        Modes.api_interval = Modes.json_interval;
+    }
+
     Modes.sdr_buf_samples = Modes.sdr_buf_size / 2;
     Modes.trackExpireMax = Modes.trackExpireJaero + TRACK_EXPIRE_LONG + 1 * MINUTES;
 

readsb.h

@@ -861,6 +861,7 @@ struct _Modes
     double sample_rate; // actual sample rate in use (in hz)
     int64_t interactive_display_ttl; // Interactive mode: TTL display
     int64_t json_interval; // Interval between rewriting the json aircraft file, in milliseconds; also the advertised map refresh interval
+    int64_t api_interval; // flyitalyadsb: interval between API buffer updates (--api-update-every), ms; 0 = json_interval
     int64_t stats_display_interval; // Interval (millis) between stats dumps,
     int64_t range_outline_duration;
     int64_t writeTracesActualDuration; // how long the trace writing cycle took
@@ -1361,6 +1362,7 @@ enum {
     OptNetApiPorts,
     OptApiShutdownDelay,
     OptTar1090UseApi,
+    OptApiUpdateTime,
     OptNetRoSize,
     OptNetRoInterval,
     OptNetRoIntervalBeastReduce,