Development

.gitignore

@@ -167,3 +167,7 @@ torn-war-checker.html
 
 # Data Exports
 data-exports/
+
+# GitNexus updates
+AGENTS.md
+CLAUDE.md

api/TornTools.Api/Weav3rPython/bazaar_server.py

@@ -3,14 +3,17 @@
 
 Protocol (line-delimited JSON over stdin/stdout):
   Request  → {"url": "...", "headers": {...}}
-  Response ← {"ok": true,  "body": "..."}
-           ← {"ok": false, "error": "..."}
+  Response ← {"ok": true,  "status": 200, "body": "..."}
+           ← {"ok": false, "status": 429, "error": "...", "retry_after_seconds": 12.0}
+           ← {"ok": false, "error": "..."}                       (transport failure, no status)
 
 One request at a time; the C# side serialises concurrent calls with a SemaphoreSlim.
 """
+import datetime
 import json
 import os
 import sys
+from email.utils import parsedate_to_datetime
 
 sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "libs"))
 
@@ -20,6 +23,26 @@
 # PYTHONUNBUFFERED=1 is set by the host, but be explicit about line-buffering.
 sys.stdout.reconfigure(line_buffering=True)
 
+
+def parse_retry_after(value):
+    """RFC 7231: Retry-After is either delta-seconds or an HTTP-date."""
+    if not value:
+        return None
+    value = value.strip()
+    try:
+        return max(0.0, float(value))
+    except ValueError:
+        pass
+    try:
+        when = parsedate_to_datetime(value)
+        if when.tzinfo is None:
+            when = when.replace(tzinfo=datetime.timezone.utc)
+        delta = (when - datetime.datetime.now(datetime.timezone.utc)).total_seconds()
+        return max(0.0, delta)
+    except (TypeError, ValueError):
+        return None
+
+
 while True:
     line = sys.stdin.readline()
     if not line:  # EOF - C# closed stdin, exit cleanly
@@ -34,11 +57,22 @@
         url = req["url"]
         headers = req.get("headers", {})
 
-        response = cffi_requests.get(url, headers=headers, impersonate="chrome124")
-        response.raise_for_status()
+        try:
+            response = cffi_requests.get(url, headers=headers, impersonate="chrome124")
+        except RequestException as e:
+            print(json.dumps({"ok": False, "error": f"curl_cffi error: {e}"}), flush=True)
+            continue
 
-        print(json.dumps({"ok": True, "body": response.text}), flush=True)
-    except RequestException as e:
-        print(json.dumps({"ok": False, "error": f"curl_cffi error: {e}"}), flush=True)
+        status = response.status_code
+        if response.ok:
+            print(json.dumps({"ok": True, "status": status, "body": response.text}), flush=True)
+        else:
+            retry_after = parse_retry_after(response.headers.get("Retry-After"))
+            print(json.dumps({
+                "ok": False,
+                "status": status,
+                "error": f"HTTP {status}",
+                "retry_after_seconds": retry_after,
+            }), flush=True)
     except Exception as e:
         print(json.dumps({"ok": False, "error": f"Unexpected error: {e}"}), flush=True)

api/TornTools.Application/Services/Weav3rPythonServer.cs

@@ -14,12 +14,18 @@ namespace TornTools.Application.Services;
 /// </summary>
 public sealed class Weav3rPythonServer : IDisposable
 {
+  // Default pause when upstream returns 429 with no Retry-After header.
+  private static readonly TimeSpan DefaultCooldown = TimeSpan.FromSeconds(30);
+  // Sanity cap so a malformed or hostile Retry-After can't park us indefinitely.
+  private static readonly TimeSpan MaxCooldown = TimeSpan.FromMinutes(5);
+
   private readonly ILogger<Weav3rPythonServer> _logger;
   private readonly string _compiledPath;
   private readonly string _scriptPath;
   private readonly string _pythonExe;
   private Process? _process;
   private readonly SemaphoreSlim _requestLock = new(1, 1);
+  private DateTime _cooldownUntilUtc = DateTime.MinValue;
 
   public Weav3rPythonServer(ILogger<Weav3rPythonServer> logger)
   {
@@ -35,6 +41,15 @@ public Weav3rPythonServer(ILogger<Weav3rPythonServer> logger)
     await _requestLock.WaitAsync(ct);
     try
     {
+      var remaining = _cooldownUntilUtc - DateTime.UtcNow;
+      if (remaining > TimeSpan.Zero)
+      {
+        _logger.LogInformation(
+            "bazaar_server in cooldown; waiting {Seconds:F1}s before next request.",
+            remaining.TotalSeconds);
+        await Task.Delay(remaining, ct);
+      }
+
       EnsureProcessRunning();
 
       var requestLine = JsonSerializer.Serialize(new RequestPayload(url, headers ?? []));
@@ -56,7 +71,25 @@ public Weav3rPythonServer(ILogger<Weav3rPythonServer> logger)
       var response = JsonSerializer.Deserialize<ResponsePayload>(responseLine);
       if (response?.Ok == true) return response.Body;
 
-      _logger.LogError("bazaar_server returned error: {Error}", response?.Error);
+      if (response?.Status == 429)
+      {
+        var cooldown = DefaultCooldown;
+        if (response.RetryAfterSeconds is double seconds && seconds > 0)
+        {
+          cooldown = TimeSpan.FromSeconds(seconds);
+        }
+        if (cooldown > MaxCooldown) cooldown = MaxCooldown;
+
+        _cooldownUntilUtc = DateTime.UtcNow + cooldown;
+        _logger.LogWarning(
+            "bazaar_server got HTTP 429; pausing all Weav3r calls for {Seconds:F1}s (Retry-After: {RetryAfter}).",
+            cooldown.TotalSeconds,
+            response.RetryAfterSeconds?.ToString("F1") ?? "absent");
+      }
+      else
+      {
+        _logger.LogError("bazaar_server returned error: {Error}", response?.Error);
+      }
       return null;
     }
     catch (OperationCanceledException) when (!ct.IsCancellationRequested)
@@ -132,6 +165,8 @@ private record RequestPayload(
   private record ResponsePayload(
       [property: JsonPropertyName("ok")] bool Ok,
       [property: JsonPropertyName("body")] string? Body,
-      [property: JsonPropertyName("error")] string? Error
+      [property: JsonPropertyName("error")] string? Error,
+      [property: JsonPropertyName("status")] int? Status,
+      [property: JsonPropertyName("retry_after_seconds")] double? RetryAfterSeconds
   );
 }