diff --git a/apps/webapp/app/v3/services/worker/workerGroupTokenService.server.ts b/apps/webapp/app/v3/services/worker/workerGroupTokenService.server.ts index e66d5429c18..f9226cbc923 100644 --- a/apps/webapp/app/v3/services/worker/workerGroupTokenService.server.ts +++ b/apps/webapp/app/v3/services/worker/workerGroupTokenService.server.ts @@ -13,6 +13,7 @@ import type { StartRunAttemptResult, TaskRunExecutionResult, } from "@trigger.dev/core/v3"; +import { getMeter } from "@internal/tracing"; import { SemanticInternalAttributes } from "@trigger.dev/core/v3"; import { fromFriendlyId } from "@trigger.dev/core/v3/isomorphic"; import { WORKER_HEADERS, type WorkerQueueClass } from "@trigger.dev/core/v3/workers"; @@ -21,11 +22,9 @@ import { Prisma, WorkerInstanceGroupType } from "@trigger.dev/database"; import { json } from "@remix-run/server-runtime"; import { createHash, timingSafeEqual } from "crypto"; import { customAlphabet } from "nanoid"; -import { Counter } from "prom-client"; import { z } from "zod"; import { env } from "~/env.server"; -import { metricsRegister } from "~/metrics.server"; -import { evaluateCreatedAtGate } from "./workloadTokenAuthorization.server"; +import { evaluateCreatedAtGate, runAgeBucket } from "./workloadTokenAuthorization.server"; import { isWorkerQueueDequeueDisabled, recordBlockedDequeue, @@ -62,17 +61,11 @@ if (workloadCreatedAtGateEnabled && !workloadTokenCutoff) { type WorkloadGateAction = "start" | "complete" | "continue" | "snapshots_since"; -// singleton: module-scope registration double-registers under dev HMR -const workloadAuthGateCounter = singleton( - "workloadAuthGateCounter", - () => - new Counter({ - name: "workload_auth_gate_total", - help: "Deployment token authorization outcomes on worker actions", - labelNames: ["outcome", "action"] as const, - registers: [metricsRegister], - }) -); +const meter = getMeter("workload-auth-gate"); + +const workloadAuthGateCounter = meter.createCounter("workload_auth_gate_total", { + description: "Deployment token authorization outcomes on worker actions", +}); function createAuthenticatedWorkerInstanceCache() { return createCache({ @@ -456,7 +449,7 @@ export class AuthenticatedWorkerInstance extends WithRunEngine { if (environmentId) { // Scoping is delegated to the engine snapshot read; no run-row read here. Recorded so the // platform can see how much traffic is env-scoped as enforcement rolls out. - workloadAuthGateCounter.inc({ outcome: "env_scoped", action }); + workloadAuthGateCounter.add(1, { outcome: "env_scoped", action }); return; } @@ -464,7 +457,10 @@ export class AuthenticatedWorkerInstance extends WithRunEngine { return; } - const run = await this._engine.runStore.findRun({ id: runId }, { select: { createdAt: true } }); + const run = await this._engine.runStore.findRun( + { id: runId }, + { select: { createdAt: true, environmentType: true } } + ); if (!run) { // Let the engine method surface the canonical not-found error. @@ -476,7 +472,12 @@ export class AuthenticatedWorkerInstance extends WithRunEngine { cutoff: workloadTokenCutoff, }); - workloadAuthGateCounter.inc({ outcome, action }); + workloadAuthGateCounter.add(1, { + outcome, + action, + env_type: run.environmentType ?? "unknown", + run_age_bucket: runAgeBucket(run.createdAt, new Date()), + }); if (!allow) { logger.warn("[workload-auth] rejecting untokened worker action created after cutoff", { diff --git a/apps/webapp/app/v3/services/worker/workloadTokenAuthorization.server.ts b/apps/webapp/app/v3/services/worker/workloadTokenAuthorization.server.ts index f26a359d0a7..179be37ccbc 100644 --- a/apps/webapp/app/v3/services/worker/workloadTokenAuthorization.server.ts +++ b/apps/webapp/app/v3/services/worker/workloadTokenAuthorization.server.ts @@ -25,3 +25,22 @@ export function evaluateCreatedAtGate(params: { ? { outcome: "suppressed", allow: false } : { outcome: "grandfathered", allow: true }; } + +const HOUR_MS = 60 * 60 * 1000; +const DAY_MS = 24 * HOUR_MS; + +const RUN_AGE_BUCKETS = [ + { under: HOUR_MS, label: "lt_1h" }, + { under: DAY_MS, label: "1h_1d" }, + { under: 7 * DAY_MS, label: "1d_7d" }, + { under: 30 * DAY_MS, label: "7d_30d" }, +] as const; + +/** + * Coarse age of the run behind an untokened worker action. Reading this while the cutoff is set far + * in the future answers "what would a cutoff of X reject" without rejecting anything. + */ +export function runAgeBucket(runCreatedAt: Date, now: Date): string { + const ageMs = now.getTime() - runCreatedAt.getTime(); + return RUN_AGE_BUCKETS.find((bucket) => ageMs < bucket.under)?.label ?? "gt_30d"; +} diff --git a/apps/webapp/test/workloadTokenAuthorization.test.ts b/apps/webapp/test/workloadTokenAuthorization.test.ts index 205db263255..78fc985d684 100644 --- a/apps/webapp/test/workloadTokenAuthorization.test.ts +++ b/apps/webapp/test/workloadTokenAuthorization.test.ts @@ -1,5 +1,8 @@ import { describe, expect, it } from "vitest"; -import { evaluateCreatedAtGate } from "~/v3/services/worker/workloadTokenAuthorization.server"; +import { + evaluateCreatedAtGate, + runAgeBucket, +} from "~/v3/services/worker/workloadTokenAuthorization.server"; const cutoff = new Date("2026-07-09T00:00:00.000Z"); const before = new Date("2026-07-01T00:00:00.000Z"); @@ -24,3 +27,30 @@ describe("evaluateCreatedAtGate", () => { expect(result.allow).toBe(true); }); }); + +describe("runAgeBucket", () => { + const now = new Date("2026-08-28T12:00:00.000Z"); + const agoMs = (ms: number) => new Date(now.getTime() - ms); + + const HOUR = 60 * 60 * 1000; + const DAY = 24 * HOUR; + + it.each([ + [agoMs(0), "lt_1h"], + [agoMs(HOUR - 1), "lt_1h"], + [agoMs(HOUR), "1h_1d"], + [agoMs(DAY - 1), "1h_1d"], + [agoMs(DAY), "1d_7d"], + [agoMs(7 * DAY - 1), "1d_7d"], + [agoMs(7 * DAY), "7d_30d"], + [agoMs(30 * DAY - 1), "7d_30d"], + [agoMs(30 * DAY), "gt_30d"], + [agoMs(365 * DAY), "gt_30d"], + ])("buckets %s as %s", (createdAt, expected) => { + expect(runAgeBucket(createdAt, now)).toBe(expected); + }); + + it("puts a future createdAt in the youngest bucket rather than throwing", () => { + expect(runAgeBucket(new Date(now.getTime() + DAY), now)).toBe("lt_1h"); + }); +});