From a33d15aa31ca9b41d9ef4be8909c5fedbfd7ec72 Mon Sep 17 00:00:00 2001 From: Drew Morgan Date: Sun, 26 Apr 2026 21:57:45 +0100 Subject: [PATCH] feat: update bazaar server response handling and add cooldown logic for HTTP 429 status --- .gitignore | 4 + AGENTS.md | 101 -------------- CLAUDE.md | 131 ------------------ .../Weav3rPython/bazaar_server.py | 48 ++++++- .../Services/Weav3rPythonServer.cs | 39 +++++- 5 files changed, 82 insertions(+), 241 deletions(-) delete mode 100644 AGENTS.md delete mode 100644 CLAUDE.md diff --git a/.gitignore b/.gitignore index 6f3e8ed..62af7ca 100644 --- a/.gitignore +++ b/.gitignore @@ -167,3 +167,7 @@ torn-war-checker.html # Data Exports data-exports/ + +# GitNexus updates +AGENTS.md +CLAUDE.md diff --git a/AGENTS.md b/AGENTS.md deleted file mode 100644 index 7fe0eca..0000000 --- a/AGENTS.md +++ /dev/null @@ -1,101 +0,0 @@ - -# GitNexus — Code Intelligence - -This project is indexed by GitNexus as **TornTools** (2096 symbols, 6153 relationships, 147 execution flows). Use the GitNexus MCP tools to understand code, assess impact, and navigate safely. - -> If any GitNexus tool warns the index is stale, run `npx gitnexus analyze` in terminal first. - -## Always Do - -- **MUST run impact analysis before editing any symbol.** Before modifying a function, class, or method, run `gitnexus_impact({target: "symbolName", direction: "upstream"})` and report the blast radius (direct callers, affected processes, risk level) to the user. -- **MUST run `gitnexus_detect_changes()` before committing** to verify your changes only affect expected symbols and execution flows. -- **MUST warn the user** if impact analysis returns HIGH or CRITICAL risk before proceeding with edits. -- When exploring unfamiliar code, use `gitnexus_query({query: "concept"})` to find execution flows instead of grepping. It returns process-grouped results ranked by relevance. -- When you need full context on a specific symbol — callers, callees, which execution flows it participates in — use `gitnexus_context({name: "symbolName"})`. - -## When Debugging - -1. `gitnexus_query({query: ""})` — find execution flows related to the issue -2. `gitnexus_context({name: ""})` — see all callers, callees, and process participation -3. `READ gitnexus://repo/TornTools/process/{processName}` — trace the full execution flow step by step -4. For regressions: `gitnexus_detect_changes({scope: "compare", base_ref: "main"})` — see what your branch changed - -## When Refactoring - -- **Renaming**: MUST use `gitnexus_rename({symbol_name: "old", new_name: "new", dry_run: true})` first. Review the preview — graph edits are safe, text_search edits need manual review. Then run with `dry_run: false`. -- **Extracting/Splitting**: MUST run `gitnexus_context({name: "target"})` to see all incoming/outgoing refs, then `gitnexus_impact({target: "target", direction: "upstream"})` to find all external callers before moving code. -- After any refactor: run `gitnexus_detect_changes({scope: "all"})` to verify only expected files changed. - -## Never Do - -- NEVER edit a function, class, or method without first running `gitnexus_impact` on it. -- NEVER ignore HIGH or CRITICAL risk warnings from impact analysis. -- NEVER rename symbols with find-and-replace — use `gitnexus_rename` which understands the call graph. -- NEVER commit changes without running `gitnexus_detect_changes()` to check affected scope. - -## Tools Quick Reference - -| Tool | When to use | Command | -|------|-------------|---------| -| `query` | Find code by concept | `gitnexus_query({query: "auth validation"})` | -| `context` | 360-degree view of one symbol | `gitnexus_context({name: "validateUser"})` | -| `impact` | Blast radius before editing | `gitnexus_impact({target: "X", direction: "upstream"})` | -| `detect_changes` | Pre-commit scope check | `gitnexus_detect_changes({scope: "staged"})` | -| `rename` | Safe multi-file rename | `gitnexus_rename({symbol_name: "old", new_name: "new", dry_run: true})` | -| `cypher` | Custom graph queries | `gitnexus_cypher({query: "MATCH ..."})` | - -## Impact Risk Levels - -| Depth | Meaning | Action | -|-------|---------|--------| -| d=1 | WILL BREAK — direct callers/importers | MUST update these | -| d=2 | LIKELY AFFECTED — indirect deps | Should test | -| d=3 | MAY NEED TESTING — transitive | Test if critical path | - -## Resources - -| Resource | Use for | -|----------|---------| -| `gitnexus://repo/TornTools/context` | Codebase overview, check index freshness | -| `gitnexus://repo/TornTools/clusters` | All functional areas | -| `gitnexus://repo/TornTools/processes` | All execution flows | -| `gitnexus://repo/TornTools/process/{name}` | Step-by-step execution trace | - -## Self-Check Before Finishing - -Before completing any code modification task, verify: -1. `gitnexus_impact` was run for all modified symbols -2. No HIGH/CRITICAL risk warnings were ignored -3. `gitnexus_detect_changes()` confirms changes match expected scope -4. All d=1 (WILL BREAK) dependents were updated - -## Keeping the Index Fresh - -After committing code changes, the GitNexus index becomes stale. Re-run analyze to update it: - -```bash -npx gitnexus analyze -``` - -If the index previously included embeddings, preserve them by adding `--embeddings`: - -```bash -npx gitnexus analyze --embeddings -``` - -To check whether embeddings exist, inspect `.gitnexus/meta.json` — the `stats.embeddings` field shows the count (0 means no embeddings). **Running analyze without `--embeddings` will delete any previously generated embeddings.** - -> Claude Code users: A PostToolUse hook handles this automatically after `git commit` and `git merge`. - -## CLI - -| Task | Read this skill file | -|------|---------------------| -| Understand architecture / "How does X work?" | `.claude/skills/gitnexus/gitnexus-exploring/SKILL.md` | -| Blast radius / "What breaks if I change X?" | `.claude/skills/gitnexus/gitnexus-impact-analysis/SKILL.md` | -| Trace bugs / "Why is X failing?" | `.claude/skills/gitnexus/gitnexus-debugging/SKILL.md` | -| Rename / extract / split / refactor | `.claude/skills/gitnexus/gitnexus-refactoring/SKILL.md` | -| Tools, resources, schema reference | `.claude/skills/gitnexus/gitnexus-guide/SKILL.md` | -| Index, status, clean, wiki CLI commands | `.claude/skills/gitnexus/gitnexus-cli/SKILL.md` | - - diff --git a/CLAUDE.md b/CLAUDE.md deleted file mode 100644 index 5e7e09e..0000000 --- a/CLAUDE.md +++ /dev/null @@ -1,131 +0,0 @@ - -# GitNexus — Code Intelligence - -This project is indexed by GitNexus as **TornTools** (2096 symbols, 6153 relationships, 147 execution flows). Use the GitNexus MCP tools to understand code, assess impact, and navigate safely. - -> If any GitNexus tool warns the index is stale, run `npx gitnexus analyze` in terminal first. - -## Always Do - -- **MUST run impact analysis before editing any symbol.** Before modifying a function, class, or method, run `gitnexus_impact({target: "symbolName", direction: "upstream"})` and report the blast radius (direct callers, affected processes, risk level) to the user. -- **MUST run `gitnexus_detect_changes()` before committing** to verify your changes only affect expected symbols and execution flows. -- **MUST warn the user** if impact analysis returns HIGH or CRITICAL risk before proceeding with edits. -- When exploring unfamiliar code, use `gitnexus_query({query: "concept"})` to find execution flows instead of grepping. It returns process-grouped results ranked by relevance. -- When you need full context on a specific symbol — callers, callees, which execution flows it participates in — use `gitnexus_context({name: "symbolName"})`. - -## When Debugging - -1. `gitnexus_query({query: ""})` — find execution flows related to the issue -2. `gitnexus_context({name: ""})` — see all callers, callees, and process participation -3. `READ gitnexus://repo/TornTools/process/{processName}` — trace the full execution flow step by step -4. For regressions: `gitnexus_detect_changes({scope: "compare", base_ref: "main"})` — see what your branch changed - -## When Refactoring - -- **Renaming**: MUST use `gitnexus_rename({symbol_name: "old", new_name: "new", dry_run: true})` first. Review the preview — graph edits are safe, text_search edits need manual review. Then run with `dry_run: false`. -- **Extracting/Splitting**: MUST run `gitnexus_context({name: "target"})` to see all incoming/outgoing refs, then `gitnexus_impact({target: "target", direction: "upstream"})` to find all external callers before moving code. -- After any refactor: run `gitnexus_detect_changes({scope: "all"})` to verify only expected files changed. - -## Never Do - -- NEVER edit a function, class, or method without first running `gitnexus_impact` on it. -- NEVER ignore HIGH or CRITICAL risk warnings from impact analysis. -- NEVER rename symbols with find-and-replace — use `gitnexus_rename` which understands the call graph. -- NEVER commit changes without running `gitnexus_detect_changes()` to check affected scope. - -## Tools Quick Reference - -| Tool | When to use | Command | -|------|-------------|---------| -| `query` | Find code by concept | `gitnexus_query({query: "auth validation"})` | -| `context` | 360-degree view of one symbol | `gitnexus_context({name: "validateUser"})` | -| `impact` | Blast radius before editing | `gitnexus_impact({target: "X", direction: "upstream"})` | -| `detect_changes` | Pre-commit scope check | `gitnexus_detect_changes({scope: "staged"})` | -| `rename` | Safe multi-file rename | `gitnexus_rename({symbol_name: "old", new_name: "new", dry_run: true})` | -| `cypher` | Custom graph queries | `gitnexus_cypher({query: "MATCH ..."})` | - -## Impact Risk Levels - -| Depth | Meaning | Action | -|-------|---------|--------| -| d=1 | WILL BREAK — direct callers/importers | MUST update these | -| d=2 | LIKELY AFFECTED — indirect deps | Should test | -| d=3 | MAY NEED TESTING — transitive | Test if critical path | - -## Resources - -| Resource | Use for | -|----------|---------| -| `gitnexus://repo/TornTools/context` | Codebase overview, check index freshness | -| `gitnexus://repo/TornTools/clusters` | All functional areas | -| `gitnexus://repo/TornTools/processes` | All execution flows | -| `gitnexus://repo/TornTools/process/{name}` | Step-by-step execution trace | - -## Self-Check Before Finishing - -Before completing any code modification task, verify: -1. `gitnexus_impact` was run for all modified symbols -2. No HIGH/CRITICAL risk warnings were ignored -3. `gitnexus_detect_changes()` confirms changes match expected scope -4. All d=1 (WILL BREAK) dependents were updated - -## Keeping the Index Fresh - -After committing code changes, the GitNexus index becomes stale. Re-run analyze to update it: - -```bash -npx gitnexus analyze -``` - -If the index previously included embeddings, preserve them by adding `--embeddings`: - -```bash -npx gitnexus analyze --embeddings -``` - -To check whether embeddings exist, inspect `.gitnexus/meta.json` — the `stats.embeddings` field shows the count (0 means no embeddings). **Running analyze without `--embeddings` will delete any previously generated embeddings.** - -> Claude Code users: A PostToolUse hook handles this automatically after `git commit` and `git merge`. - -## CLI - -| Task | Read this skill file | -|------|---------------------| -| Understand architecture / "How does X work?" | `.claude/skills/gitnexus/gitnexus-exploring/SKILL.md` | -| Blast radius / "What breaks if I change X?" | `.claude/skills/gitnexus/gitnexus-impact-analysis/SKILL.md` | -| Trace bugs / "Why is X failing?" | `.claude/skills/gitnexus/gitnexus-debugging/SKILL.md` | -| Rename / extract / split / refactor | `.claude/skills/gitnexus/gitnexus-refactoring/SKILL.md` | -| Tools, resources, schema reference | `.claude/skills/gitnexus/gitnexus-guide/SKILL.md` | -| Index, status, clean, wiki CLI commands | `.claude/skills/gitnexus/gitnexus-cli/SKILL.md` | - - - -## External APIs - -- **Torn API quick reference**: see `context/torn-api.md` for endpoints we use, access levels, the - bazaar category label→cat map, and rate limits. Full Swagger UI is at - . - -## `TODO.md` vs. `context/` — two kinds of notes, two owners - -`TODO.md` and `context/` serve different purposes. Don't conflate them. - -- **`TODO.md` is the durable backlog.** Drew owns it, tracks it, and picks work from it. Any - new piece of work surfaced by a discussion, a session, a code review, or a bug report - belongs here. A handoff alone is not enough — handoffs get archived into `context/sessions/` - and fall out of the default reading path; `TODO.md` outlives them and is the index of what - still needs doing. -- **`context/` is your (Claude's) working memory across sessions.** That includes - `session-handoff.md`, `note-to-next-instance.md`, `next-prompt.txt`, and everything under - `context/sessions/` and `context/plans/`. Drew does not own these files and will not tell - you what to include, exclude, rewrite, or prune. You decide what the next session of you - needs. Don't ask permission to reorganise your own notes. - -**Rule when a discussion surfaces new work**: write it in **both** `TODO.md` and -`context/session-handoff.md`. TODO.md captures *that it needs doing* (one line, a pointer if -needed). The handoff captures *why and with what context* — the reasoning, the artifacts -observed, the proposed approach — so the next session can pick it up intelligently rather -than re-deriving it from a one-line TODO entry. - -If you find yourself writing something substantive in a handoff and *not* also recording it -in `TODO.md`, that's the signal to stop and update `TODO.md` too. diff --git a/api/TornTools.Api/Weav3rPython/bazaar_server.py b/api/TornTools.Api/Weav3rPython/bazaar_server.py index 1d014e3..9a342ba 100644 --- a/api/TornTools.Api/Weav3rPython/bazaar_server.py +++ b/api/TornTools.Api/Weav3rPython/bazaar_server.py @@ -3,14 +3,17 @@ Protocol (line-delimited JSON over stdin/stdout): Request → {"url": "...", "headers": {...}} - Response ← {"ok": true, "body": "..."} - ← {"ok": false, "error": "..."} + Response ← {"ok": true, "status": 200, "body": "..."} + ← {"ok": false, "status": 429, "error": "...", "retry_after_seconds": 12.0} + ← {"ok": false, "error": "..."} (transport failure, no status) One request at a time; the C# side serialises concurrent calls with a SemaphoreSlim. """ +import datetime import json import os import sys +from email.utils import parsedate_to_datetime sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "libs")) @@ -20,6 +23,26 @@ # PYTHONUNBUFFERED=1 is set by the host, but be explicit about line-buffering. sys.stdout.reconfigure(line_buffering=True) + +def parse_retry_after(value): + """RFC 7231: Retry-After is either delta-seconds or an HTTP-date.""" + if not value: + return None + value = value.strip() + try: + return max(0.0, float(value)) + except ValueError: + pass + try: + when = parsedate_to_datetime(value) + if when.tzinfo is None: + when = when.replace(tzinfo=datetime.timezone.utc) + delta = (when - datetime.datetime.now(datetime.timezone.utc)).total_seconds() + return max(0.0, delta) + except (TypeError, ValueError): + return None + + while True: line = sys.stdin.readline() if not line: # EOF - C# closed stdin, exit cleanly @@ -34,11 +57,22 @@ url = req["url"] headers = req.get("headers", {}) - response = cffi_requests.get(url, headers=headers, impersonate="chrome124") - response.raise_for_status() + try: + response = cffi_requests.get(url, headers=headers, impersonate="chrome124") + except RequestException as e: + print(json.dumps({"ok": False, "error": f"curl_cffi error: {e}"}), flush=True) + continue - print(json.dumps({"ok": True, "body": response.text}), flush=True) - except RequestException as e: - print(json.dumps({"ok": False, "error": f"curl_cffi error: {e}"}), flush=True) + status = response.status_code + if response.ok: + print(json.dumps({"ok": True, "status": status, "body": response.text}), flush=True) + else: + retry_after = parse_retry_after(response.headers.get("Retry-After")) + print(json.dumps({ + "ok": False, + "status": status, + "error": f"HTTP {status}", + "retry_after_seconds": retry_after, + }), flush=True) except Exception as e: print(json.dumps({"ok": False, "error": f"Unexpected error: {e}"}), flush=True) diff --git a/api/TornTools.Application/Services/Weav3rPythonServer.cs b/api/TornTools.Application/Services/Weav3rPythonServer.cs index ced922d..56d3b9d 100644 --- a/api/TornTools.Application/Services/Weav3rPythonServer.cs +++ b/api/TornTools.Application/Services/Weav3rPythonServer.cs @@ -14,12 +14,18 @@ namespace TornTools.Application.Services; /// public sealed class Weav3rPythonServer : IDisposable { + // Default pause when upstream returns 429 with no Retry-After header. + private static readonly TimeSpan DefaultCooldown = TimeSpan.FromSeconds(30); + // Sanity cap so a malformed or hostile Retry-After can't park us indefinitely. + private static readonly TimeSpan MaxCooldown = TimeSpan.FromMinutes(5); + private readonly ILogger _logger; private readonly string _compiledPath; private readonly string _scriptPath; private readonly string _pythonExe; private Process? _process; private readonly SemaphoreSlim _requestLock = new(1, 1); + private DateTime _cooldownUntilUtc = DateTime.MinValue; public Weav3rPythonServer(ILogger logger) { @@ -35,6 +41,15 @@ public Weav3rPythonServer(ILogger logger) await _requestLock.WaitAsync(ct); try { + var remaining = _cooldownUntilUtc - DateTime.UtcNow; + if (remaining > TimeSpan.Zero) + { + _logger.LogInformation( + "bazaar_server in cooldown; waiting {Seconds:F1}s before next request.", + remaining.TotalSeconds); + await Task.Delay(remaining, ct); + } + EnsureProcessRunning(); var requestLine = JsonSerializer.Serialize(new RequestPayload(url, headers ?? [])); @@ -56,7 +71,25 @@ public Weav3rPythonServer(ILogger logger) var response = JsonSerializer.Deserialize(responseLine); if (response?.Ok == true) return response.Body; - _logger.LogError("bazaar_server returned error: {Error}", response?.Error); + if (response?.Status == 429) + { + var cooldown = DefaultCooldown; + if (response.RetryAfterSeconds is double seconds && seconds > 0) + { + cooldown = TimeSpan.FromSeconds(seconds); + } + if (cooldown > MaxCooldown) cooldown = MaxCooldown; + + _cooldownUntilUtc = DateTime.UtcNow + cooldown; + _logger.LogWarning( + "bazaar_server got HTTP 429; pausing all Weav3r calls for {Seconds:F1}s (Retry-After: {RetryAfter}).", + cooldown.TotalSeconds, + response.RetryAfterSeconds?.ToString("F1") ?? "absent"); + } + else + { + _logger.LogError("bazaar_server returned error: {Error}", response?.Error); + } return null; } catch (OperationCanceledException) when (!ct.IsCancellationRequested) @@ -132,6 +165,8 @@ private record RequestPayload( private record ResponsePayload( [property: JsonPropertyName("ok")] bool Ok, [property: JsonPropertyName("body")] string? Body, - [property: JsonPropertyName("error")] string? Error + [property: JsonPropertyName("error")] string? Error, + [property: JsonPropertyName("status")] int? Status, + [property: JsonPropertyName("retry_after_seconds")] double? RetryAfterSeconds ); }