diff --git a/components/control-plane/cmd/hypershell-controller/main.go b/components/control-plane/cmd/hypershell-controller/main.go index cffc2e1b..e92e340c 100644 --- a/components/control-plane/cmd/hypershell-controller/main.go +++ b/components/control-plane/cmd/hypershell-controller/main.go @@ -260,43 +260,6 @@ func main() { } var gatewayReconciler watcher.Handler[*pb.Gateway] - - if clientset != nil && dynamicClient != nil { - gr, grErr := reconciler.NewGatewayReconciler( - dynamicClient, - clientset, - conn, - helmClient, - cfg.Namespace, - keycloakConfig, - exposurePort, - cfg.ExternalCAIssuerName, - cfg.ExternalCAIssuerKind, - databaseConfig, - ) - if grErr != nil { - log.Printf("WARN gateway reconciler disabled: %v", grErr) - gatewayReconciler = reconciler.NewStubGatewayReconciler() - } else { - gatewayReconciler = gr - } - } else { - log.Printf("WARN no kubernetes client available, using stub gateway reconciler") - gatewayReconciler = reconciler.NewStubGatewayReconciler() - } - - // The gateway reconcile queue is shared: the gateway watch stream drives it, - // and the GatewayRelease reconciler enqueues referencing gateways into it when - // a release image changes. It is created here (not inside WatchGateways) so the - // release reconciler can hold the same instance. - gatewayQueue := watcher.NewGatewayReconcileQueue(ctx, gatewayReconciler, cfg.GatewayReconcileWorkers) - defer gatewayQueue.Stop() - - watchCount := 4 // managed clusters, gateway releases, gateways, networks - if kcClient != nil { - watchCount++ // role bindings - } - // Each background component below runs under supervisor.Run on its own // goroutine: a failure in one (a dropped watch stream, a reconciler's Run // loop returning, the service-account provisioner's listener dying) is @@ -341,6 +304,43 @@ func main() { } log.Printf("INFO registered as cluster_id=%s (name=%s); scoping gateway watch, seed, health, sandbox counts, backfill, release fan-out, and namespace GC to it", clusterID, cfg.ManagedClusterName) + if clientset != nil && dynamicClient != nil { + gr, grErr := reconciler.NewGatewayReconciler( + dynamicClient, + clientset, + conn, + helmClient, + cfg.Namespace, + keycloakConfig, + exposurePort, + cfg.ExternalCAIssuerName, + cfg.ExternalCAIssuerKind, + databaseConfig, + clusterID, + ) + if grErr != nil { + log.Printf("WARN gateway reconciler disabled: %v", grErr) + gatewayReconciler = reconciler.NewStubGatewayReconciler() + } else { + gatewayReconciler = gr + } + } else { + log.Printf("WARN no kubernetes client available, using stub gateway reconciler") + gatewayReconciler = reconciler.NewStubGatewayReconciler() + } + + // The gateway reconcile queue is shared: the gateway watch stream drives it, + // and the GatewayRelease reconciler enqueues referencing gateways into it when + // a release image changes. It is created after registration so the reconciler + // and all queued work use the registered cluster identity. + gatewayQueue := watcher.NewGatewayReconcileQueue(ctx, gatewayReconciler, cfg.GatewayReconcileWorkers) + defer gatewayQueue.Stop() + + watchCount := 4 // managed clusters, gateway releases, gateways, networks + if kcClient != nil { + watchCount++ // role bindings + } + // Heartbeat: re-register every 60s to update last_seen_at on the hub. go func() { ticker := time.NewTicker(60 * time.Second) diff --git a/components/control-plane/internal/otel/metrics.go b/components/control-plane/internal/otel/metrics.go index 8609747a..f6c034dc 100644 --- a/components/control-plane/internal/otel/metrics.go +++ b/components/control-plane/internal/otel/metrics.go @@ -3,6 +3,7 @@ package otel import ( "context" "fmt" + "sync" "time" "go.opentelemetry.io/otel" @@ -20,9 +21,16 @@ var ( gatewaySandboxExpiring metric.Int64Gauge gatewaySandboxIdle metric.Int64Gauge reconcileErrors metric.Int64Counter + reconciliationFailures metric.Int64Counter + reconciliationRetries metric.Int64Counter + reconciliationLag metric.Float64Histogram + staleResourceStatusCount metric.Int64Gauge watchReconnects metric.Int64Counter ) +var staleResources sync.Map +var staleResourcesMu sync.Mutex + func registerMetrics() error { meter := otel.Meter(TracerName) var err error @@ -35,7 +43,6 @@ func registerMetrics() error { if err != nil { return err } - reconcileQueueDepth, err = meter.Int64ObservableGauge( "reconcile.queue.depth", metric.WithUnit("{item}"), @@ -110,6 +117,48 @@ func registerMetrics() error { return err } + reconciliationFailures, err = meter.Int64Counter( + "hypershell.reconciliation.failures", + metric.WithUnit("{failure}"), + metric.WithDescription("Count of failed control-plane reconciliation attempts"), + ) + if err != nil { + return err + } + reconciliationFailures.Add(context.Background(), 0) + + reconciliationRetries, err = meter.Int64Counter( + "hypershell.reconciliation.retries", + metric.WithUnit("{retry}"), + metric.WithDescription("Count of control-plane reconciliation retries"), + ) + if err != nil { + return err + } + reconciliationRetries.Add(context.Background(), 0) + + reconciliationLag, err = meter.Float64Histogram( + "hypershell.reconciliation.lag", + metric.WithUnit("s"), + metric.WithDescription("Time spent processing a control-plane reconciliation"), + metric.WithExplicitBucketBoundaries(0.001, 0.01, 0.1, 1, 5, 10, 30, 60, 300), + ) + if err != nil { + return err + } + + staleResourceStatusCount, err = meter.Int64Gauge( + "hypershell.stale.resource.status.count", + metric.WithUnit("{resource}"), + metric.WithDescription("Resources whose observed status has not caught up with desired state"), + ) + if err != nil { + return err + } + staleResourceStatusCount.Record(context.Background(), 0, metric.WithAttributes( + attribute.String("hypershell.cluster_id", DefaultAttentionClusterID), + )) + watchReconnects, err = meter.Int64Counter( "watch.reconnects", metric.WithUnit("{reconnect}"), @@ -221,6 +270,53 @@ func RecordReconcileError(ctx context.Context, kind string) { reconcileErrors.Add(ctx, 1, metric.WithAttributes( attribute.String("resource.kind", kind), )) + if reconciliationFailures != nil { + reconciliationFailures.Add(ctx, 1, metric.WithAttributes( + attribute.String("resource.kind", kind), + )) + } +} + +// RecordReconciliationRetry records a retry scheduled for a reconciliation. +func RecordReconciliationRetry(ctx context.Context, kind string) { + if reconciliationRetries == nil { + return + } + reconciliationRetries.Add(ctx, 1, metric.WithAttributes( + attribute.String("resource.kind", kind), + )) +} + +// RecordReconciliationLag records the elapsed reconciliation processing time. +func RecordReconciliationLag(ctx context.Context, kind string, duration time.Duration) { + if reconciliationLag == nil || duration < 0 { + return + } + reconciliationLag.Record(ctx, duration.Seconds(), metric.WithAttributes( + attribute.String("resource.kind", kind), + )) +} + +// SetResourceStatusStale records whether a resource's observed status is +// behind its desired state. Resource IDs remain in process memory only and are +// never exported as metric labels. +func SetResourceStatusStale(ctx context.Context, clusterID, kind, resourceID string, stale bool) { + if staleResourceStatusCount == nil || kind == "" || resourceID == "" { + return + } + staleResourcesMu.Lock() + defer staleResourcesMu.Unlock() + key := kind + "\x00" + resourceID + if stale { + staleResources.Store(key, struct{}{}) + } else { + staleResources.Delete(key) + } + count := int64(0) + staleResources.Range(func(_, _ any) bool { count++; return true }) + staleResourceStatusCount.Record(ctx, count, metric.WithAttributes( + attribute.String("hypershell.cluster_id", AttentionClusterID(clusterID)), + )) } // RecordWatchReconnect increments the watch reconnect counter. diff --git a/components/control-plane/internal/otel/metrics_test.go b/components/control-plane/internal/otel/metrics_test.go index fba2d27f..6780841f 100644 --- a/components/control-plane/internal/otel/metrics_test.go +++ b/components/control-plane/internal/otel/metrics_test.go @@ -24,6 +24,10 @@ func testMetricsReader(t *testing.T) *sdkmetric.ManualReader { previousGatewaySandboxExpiring := gatewaySandboxExpiring previousGatewaySandboxIdle := gatewaySandboxIdle previousReconcileErrors := reconcileErrors + previousReconciliationFailures := reconciliationFailures + previousReconciliationRetries := reconciliationRetries + previousReconciliationLag := reconciliationLag + previousStaleResourceStatusCount := staleResourceStatusCount previousWatchReconnects := watchReconnects reader := sdkmetric.NewManualReader() @@ -38,6 +42,10 @@ func testMetricsReader(t *testing.T) *sdkmetric.ManualReader { gatewaySandboxExpiring = nil gatewaySandboxIdle = nil reconcileErrors = nil + reconciliationFailures = nil + reconciliationRetries = nil + reconciliationLag = nil + staleResourceStatusCount = nil watchReconnects = nil t.Cleanup(func() { otel.SetMeterProvider(previousProvider) @@ -50,6 +58,10 @@ func testMetricsReader(t *testing.T) *sdkmetric.ManualReader { gatewaySandboxExpiring = previousGatewaySandboxExpiring gatewaySandboxIdle = previousGatewaySandboxIdle reconcileErrors = previousReconcileErrors + reconciliationFailures = previousReconciliationFailures + reconciliationRetries = previousReconciliationRetries + reconciliationLag = previousReconciliationLag + staleResourceStatusCount = previousStaleResourceStatusCount watchReconnects = previousWatchReconnects _ = provider.Shutdown(context.Background()) }) @@ -60,6 +72,57 @@ func testMetricsReader(t *testing.T) *sdkmetric.ManualReader { return reader } +func TestReconciliationMetrics(t *testing.T) { + reader := testMetricsReader(t) + RecordReconcileError(context.Background(), "Gateway") + RecordReconciliationRetry(context.Background(), "Gateway") + RecordReconciliationLag(context.Background(), "Gateway", 2*time.Second) + SetResourceStatusStale(context.Background(), "cluster-a", "Gateway", "one", true) + SetResourceStatusStale(context.Background(), "cluster-a", "Gateway", "one", true) + SetResourceStatusStale(context.Background(), "cluster-a", "Gateway", "two", true) + SetResourceStatusStale(context.Background(), "cluster-a", "Gateway", "one", false) + + var collected metricdata.ResourceMetrics + if err := reader.Collect(context.Background(), &collected); err != nil { + t.Fatalf("Collect() returned an error: %v", err) + } + for _, scope := range collected.ScopeMetrics { + for _, got := range scope.Metrics { + switch got.Name { + case "hypershell.reconciliation.failures": + if _, ok := got.Data.(metricdata.Sum[int64]); !ok { + t.Fatalf("failures type = %T", got.Data) + } + case "hypershell.reconciliation.retries": + if _, ok := got.Data.(metricdata.Sum[int64]); !ok { + t.Fatalf("retries type = %T", got.Data) + } + case "hypershell.reconciliation.lag": + h := got.Data.(metricdata.Histogram[float64]) + want := []float64{0.001, 0.01, 0.1, 1, 5, 10, 30, 60, 300} + if !reflect.DeepEqual(h.DataPoints[0].Bounds, want) { + t.Fatalf("lag bounds = %v", h.DataPoints[0].Bounds) + } + case "hypershell.stale.resource.status.count": + g := got.Data.(metricdata.Gauge[int64]) + found := false + for _, point := range g.DataPoints { + v, ok := point.Attributes.Value(attribute.Key("hypershell.cluster_id")) + if ok && v.AsString() == "cluster-a" { + found = true + if point.Value != 1 { + t.Fatalf("stale gauge = %d, want 1", point.Value) + } + } + } + if !found { + t.Fatal("stale cluster label was not exported") + } + } + } + } +} + func TestRecordGatewayProvisionDuration(t *testing.T) { reader := testMetricsReader(t) diff --git a/components/control-plane/internal/otel/reconcile.go b/components/control-plane/internal/otel/reconcile.go index ea982305..bb99bcc6 100644 --- a/components/control-plane/internal/otel/reconcile.go +++ b/components/control-plane/internal/otel/reconcile.go @@ -54,6 +54,7 @@ func StartReconcileSpan(ctx context.Context, kind, eventType, traceparent string span.SetStatus(codes.Ok, "") } RecordReconcileDuration(ctx, kind, eventType, start) + RecordReconciliationLag(ctx, kind, time.Since(start)) span.End() } } diff --git a/components/control-plane/internal/reconciler/reconciler.go b/components/control-plane/internal/reconciler/reconciler.go index d9895ef1..efb069d2 100644 --- a/components/control-plane/internal/reconciler/reconciler.go +++ b/components/control-plane/internal/reconciler/reconciler.go @@ -342,7 +342,8 @@ type GatewayReconciler struct { ingressBaseDomain string // database locates the mounted admin credentials every gateway database is // provisioned with; validated once at controller startup. - database gateway.DatabaseConfig + database gateway.DatabaseConfig + clusterID string } func NewGatewayReconciler( @@ -356,6 +357,7 @@ func NewGatewayReconciler( externalCAIssuerName string, externalCAIssuerKind string, database gateway.DatabaseConfig, + clusterID string, ) (*GatewayReconciler, error) { if database.AdminCredentialsDir == "" { return nil, fmt.Errorf("gateway database admin credentials directory is required") @@ -410,6 +412,7 @@ func NewGatewayReconciler( externalCAIssuerKind: externalCAIssuerKind, ingressBaseDomain: ingressBaseDomain, database: database, + clusterID: clusterID, }, nil } @@ -437,8 +440,9 @@ func (r *GatewayReconciler) Handle(ctx context.Context, event watcher.Event[*pb. previousPhase = event.PhaseBeforeRetry } suppressGatewayProvisionObservation(event.ResourceID, previousPhase) - ctx, endSpan := cpotel.StartReconcileSpan(ctx, "Gateway", event.Type.String(), gw.GetMetadata().GetTraceparent()) + cpotel.SetResourceStatusStale(ctx, r.clusterID, "Gateway", event.ResourceID, + event.Type != watcher.EventDeleted && gw.GetReleaseId() != gw.GetObservedReleaseId()) span := trace.SpanFromContext(ctx) span.SetAttributes(attribute.String("hypershell.resource_id", event.ResourceID)) var reconcileErr error diff --git a/components/control-plane/internal/watcher/requeue.go b/components/control-plane/internal/watcher/requeue.go index a8b3369f..19ddd9c3 100644 --- a/components/control-plane/internal/watcher/requeue.go +++ b/components/control-plane/internal/watcher/requeue.go @@ -589,6 +589,7 @@ func (q *reconcileQueue[T]) processNext() bool { } q.mu.Unlock() q.queue.AddAfter(id, delay) + cpotel.RecordReconciliationRetry(q.baseCtx, q.kind) log.Printf("WARN %s %s reconcile failed; retrying in %s: %v", q.kind, id, delay, err) return true } diff --git a/components/web-console/app/adapters/api/dashboard-control-plane.test.ts b/components/web-console/app/adapters/api/dashboard-control-plane.test.ts index e97f5e64..97631d5d 100644 --- a/components/web-console/app/adapters/api/dashboard-control-plane.test.ts +++ b/components/web-console/app/adapters/api/dashboard-control-plane.test.ts @@ -1,7 +1,10 @@ import type { SDKClient } from "@openshift-online/hypershell-sdk"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { createDashboardControlPlaneAdapter } from "./dashboard-control-plane"; +import { + createDashboardControlPlaneAdapter, + mapControlPlaneReconciliationResponse, +} from "./dashboard-control-plane"; import type { PlatformInventoryMetricsResponse } from "./platform-inventory-aggregation"; const fetchMock = vi.fn(); @@ -24,6 +27,32 @@ const context = { correlationId: "11111111-1111-4111-8111-111111111111", }; +describe("mapControlPlaneReconciliationResponse", () => { + it("maps values and hourly trends", () => { + expect( + mapControlPlaneReconciliationResponse({ + reconciliation_failures_count: 2, + reconciliation_retries_count: 3, + reconciliation_lag_p50_seconds: 0.125, + stale_resource_status_count: 1, + hourly_reconciliation_failures_count: [ + { hour: "2026-01-01T00:00", value: 2 }, + ], + }), + ).toEqual([ + { + id: "reconciliation-failures", + unit: "count", + value: "2", + hourlyTrend: { points: [{ label: "2026-01-01T00:00", value: 2 }] }, + }, + { id: "reconciliation-retries", unit: "count", value: "3" }, + { id: "reconciliation-lag", unit: "sec", value: "0.125" }, + { id: "stale-resource-status-count", unit: "count", value: "1" }, + ]); + }); +}); + const mockClusterPodsResponse = { available_pods: 1452, capacity_pods: 2000, @@ -1534,8 +1563,8 @@ describe("createDashboardControlPlaneAdapter", () => { ], }, id: "api-request-rate", - unit: "req/s", - value: "12.50", + unit: "requests/sec", + value: "12.500", }, { hourlyTrend: { @@ -1546,7 +1575,7 @@ describe("createDashboardControlPlaneAdapter", () => { }, id: "api-error-rate", unit: "%", - value: "1.25", + value: "1.250", }, { hourlyTrend: { @@ -1604,7 +1633,10 @@ describe("createDashboardControlPlaneAdapter", () => { const metrics = await adapter.getReliabilityMetrics(context); - expect(metrics.failedSources).toEqual(["api-reliability"]); + expect(metrics.failedSources).toEqual([ + "api-reliability", + "control-plane-reconciliation", + ]); expect(metrics.metrics).toEqual([]); expect(metrics.lastSuccessfulRefresh).toBeInstanceOf(Date); }); @@ -1621,7 +1653,10 @@ describe("createDashboardControlPlaneAdapter", () => { const metrics = await adapter.getReliabilityMetrics(context); - expect(metrics.failedSources).toEqual(["api-reliability"]); + expect(metrics.failedSources).toEqual([ + "api-reliability", + "control-plane-reconciliation", + ]); expect(metrics.metrics).toEqual([]); expect(metrics.lastSuccessfulRefresh).toBeInstanceOf(Date); }); diff --git a/components/web-console/app/adapters/api/dashboard-control-plane.ts b/components/web-console/app/adapters/api/dashboard-control-plane.ts index 0bfeea00..864dea54 100644 --- a/components/web-console/app/adapters/api/dashboard-control-plane.ts +++ b/components/web-console/app/adapters/api/dashboard-control-plane.ts @@ -578,6 +578,22 @@ interface ApiReliabilityResponse { request_rate: number; } +interface ControlPlaneReconciliationPoint { + hour: string; + value: number; +} + +interface ControlPlaneReconciliationResponse { + reconciliation_failures_count: number; + reconciliation_retries_count: number; + reconciliation_lag_p50_seconds?: number; + stale_resource_status_count: number; + hourly_reconciliation_failures_count?: ControlPlaneReconciliationPoint[]; + hourly_reconciliation_retries_count?: ControlPlaneReconciliationPoint[]; + hourly_reconciliation_lag_p50_seconds?: ControlPlaneReconciliationPoint[]; + hourly_stale_resource_status_count?: ControlPlaneReconciliationPoint[]; +} + function mapHourlyTrend( series: readonly ApiReliabilityHourlyPoint[] | undefined, ): OperationalMetric["hourlyTrend"] { @@ -617,14 +633,14 @@ function mapApiReliabilityResponse( return [ { id: "api-request-rate", - unit: "req/s", - value: body.request_rate.toFixed(2), + unit: "requests/sec", + value: body.request_rate.toFixed(3), ...(requestRateTrend ? { hourlyTrend: requestRateTrend } : {}), }, { id: "api-error-rate", unit: "%", - value: body.error_rate_percent.toFixed(2), + value: body.error_rate_percent.toFixed(3), ...(errorRateTrend ? { hourlyTrend: errorRateTrend } : {}), }, { @@ -653,6 +669,76 @@ async function fetchApiReliabilityMetrics( return mapApiReliabilityResponse(body); } +export function mapControlPlaneReconciliationResponse( + body: ControlPlaneReconciliationResponse, +): OperationalMetric[] { + const values = [ + body.reconciliation_failures_count, + body.reconciliation_retries_count, + body.stale_resource_status_count, + ]; + if (values.some((value) => !isFiniteNumber(value))) { + throw new Error( + "Control-plane reconciliation response is missing required fields", + ); + } + + const failuresTrend = mapHourlyTrend( + body.hourly_reconciliation_failures_count, + ); + const retriesTrend = mapHourlyTrend(body.hourly_reconciliation_retries_count); + const lagTrend = mapHourlyTrend(body.hourly_reconciliation_lag_p50_seconds); + const staleTrend = mapHourlyTrend(body.hourly_stale_resource_status_count); + const metrics: OperationalMetric[] = [ + { + id: "reconciliation-failures", + unit: "count", + value: body.reconciliation_failures_count.toFixed(0), + ...(failuresTrend ? { hourlyTrend: failuresTrend } : {}), + }, + { + id: "reconciliation-retries", + unit: "count", + value: body.reconciliation_retries_count.toFixed(0), + ...(retriesTrend ? { hourlyTrend: retriesTrend } : {}), + }, + ...(isFiniteNumber(body.reconciliation_lag_p50_seconds) + ? [ + { + id: "reconciliation-lag", + unit: "sec", + value: body.reconciliation_lag_p50_seconds.toFixed(3), + ...(lagTrend ? { hourlyTrend: lagTrend } : {}), + }, + ] + : []), + { + id: "stale-resource-status-count", + unit: "count", + value: String(Math.round(body.stale_resource_status_count)), + ...(staleTrend ? { hourlyTrend: staleTrend } : {}), + }, + ]; + return metrics; +} + +async function fetchControlPlaneReconciliationMetrics( + context: DashboardInvocationContext, +): Promise { + const response = await fetch("/api/metrics/control-plane-reconciliation", { + credentials: "same-origin", + signal: context.signal, + }); + if (!response.ok) { + throw new Error( + `Failed to fetch control-plane reconciliation metrics: ${String(response.status)}`, + ); + } + return mapControlPlaneReconciliationResponse( + (await response.json()) as ControlPlaneReconciliationResponse, + ); +} + interface MetricSourceDefinition { fetch: ( context: DashboardInvocationContext, @@ -753,8 +839,25 @@ export function createDashboardControlPlaneAdapter( context.signal?.throwIfAborted(); try { - const metrics = await fetchApiReliabilityMetrics(context); + const results = await Promise.allSettled([ + fetchApiReliabilityMetrics(context), + fetchControlPlaneReconciliationMetrics(context), + ]); + const metrics: OperationalMetric[] = []; + const failedSources: ( + "api-reliability" | "control-plane-reconciliation" + )[] = []; + for (const [index, result] of results.entries()) { + const source = + index === 0 ? "api-reliability" : "control-plane-reconciliation"; + if (result.status === "fulfilled") metrics.push(...result.value); + else if (isAbortError(result.reason)) throw result.reason; + else failedSources.push(source); + } + if (metrics.length === 0) + throw new Error("All reliability dashboard metric sources failed"); return { + ...(failedSources.length > 0 ? { failedSources } : {}), lastSuccessfulRefresh: new Date(), metrics, }; @@ -767,7 +870,7 @@ export function createDashboardControlPlaneAdapter( // mergeReliabilityDashboardMetrics can keep stale widgets on // refresh and the page can show the partial-load warning. return { - failedSources: ["api-reliability"], + failedSources: ["api-reliability", "control-plane-reconciliation"], lastSuccessfulRefresh: new Date(), metrics: [], }; diff --git a/components/web-console/bff/src/app.ts b/components/web-console/bff/src/app.ts index 93fc52ae..87b9d99a 100644 --- a/components/web-console/bff/src/app.ts +++ b/components/web-console/bff/src/app.ts @@ -25,6 +25,7 @@ import { type ServerConfig, } from "./config.js"; import { queryApiReliability } from "./metrics-api-reliability.js"; +import { queryControlPlaneReconciliation } from "./metrics-control-plane-reconciliation.js"; import { queryGatewayMetrics } from "./metrics-gateways.js"; import { queryClusterCpu } from "./metrics-cluster-cpu.js"; import { queryClusterMemory } from "./metrics-cluster-memory.js"; @@ -705,6 +706,26 @@ export async function buildApp( }, ); + app.get( + "/api/metrics/control-plane-reconciliation", + { preHandler: requireDashboardMetricsAccess }, + async (request, reply) => { + try { + return await queryControlPlaneReconciliation( + config.prometheusUrl, + config.prometheusQueryTimeoutMs, + ); + } catch (error) { + request.log.warn( + { err: error }, + "control-plane reconciliation metrics query failed", + ); + reply.code(502); + return { error: "Metrics unavailable", statusCode: 502 }; + } + }, + ); + app.all("/api/*", async (request, reply) => { // Start one BFF server span per proxied request. It continues a valid // inbound W3C context and yields the validated upstream context to set on diff --git a/components/web-console/bff/src/metrics-control-plane-reconciliation.ts b/components/web-console/bff/src/metrics-control-plane-reconciliation.ts new file mode 100644 index 00000000..9a76cd79 --- /dev/null +++ b/components/web-console/bff/src/metrics-control-plane-reconciliation.ts @@ -0,0 +1,134 @@ +import { fetchMetrics, type MetricsSource } from "./metrics-source.js"; +import { + formatHourLabel, + hourlyRangeStepSeconds, + queryPrometheusRangeScalarSamples, + rollingHourlyRange, +} from "./prometheus-range-query.js"; + +export const reconciliationFailureCountPromql = + "sum(increase(hypershell_reconciliation_failures_total[24h]))"; +export const reconciliationFailureHourlyCountPromql = + "sum(increase(hypershell_reconciliation_failures_total[1h]))"; +export const reconciliationRetryCountPromql = + "sum(increase(hypershell_reconciliation_retries_total[24h]))"; +export const reconciliationRetryHourlyCountPromql = + "sum(increase(hypershell_reconciliation_retries_total[1h]))"; +export const reconciliationLagP50SecondsPromql = + "histogram_quantile(0.50, sum(rate(hypershell_reconciliation_lag_seconds_bucket[5m])) by (le))"; +export const staleResourceStatusCountPromql = + "sum(max by (hypershell_cluster_id) (hypershell_stale_resource_status_count))"; + +interface PrometheusQueryResponse { + status: string; + data?: { result: { value: [string, string] }[] }; +} + +export interface ReconciliationHourlyPoint { + hour: string; + value: number; +} + +export interface ControlPlaneReconciliationMetrics { + reconciliation_failures_count: number; + reconciliation_retries_count: number; + reconciliation_lag_p50_seconds?: number; + stale_resource_status_count: number; + hourly_reconciliation_failures_count?: ReconciliationHourlyPoint[]; + hourly_reconciliation_retries_count?: ReconciliationHourlyPoint[]; + hourly_reconciliation_lag_p50_seconds?: ReconciliationHourlyPoint[]; + hourly_stale_resource_status_count?: ReconciliationHourlyPoint[]; +} + +async function queryInstant( + source: MetricsSource, + query: string, + timeoutMs: number, + optional = false, +): Promise { + const controller = new AbortController(); + const timeout = setTimeout(() => { + controller.abort(); + }, timeoutMs); + try { + const response = await fetchMetrics(source, query, controller.signal); + if (!response.ok) throw new Error("Prometheus query failed"); + const body = (await response.json()) as PrometheusQueryResponse; + const raw = body.data?.result[0]?.value[1]; + const value = Number(raw); + if (body.status !== "success" || !Number.isFinite(value) || value < 0) { + if (optional) return undefined; + throw new Error("Prometheus query returned an invalid sample"); + } + return value; + } finally { + clearTimeout(timeout); + } +} + +function mapHourly(samples: Map): ReconciliationHourlyPoint[] { + return [...samples.entries()] + .sort(([left], [right]) => left - right) + .filter(([, value]) => Number.isFinite(value) && value >= 0) + .map(([timestamp, value]) => ({ hour: formatHourLabel(timestamp), value })); +} + +async function optionalHourly( + source: MetricsSource, + query: string, + timeoutMs: number, +): Promise { + try { + const { start, end } = rollingHourlyRange(); + const samples = await queryPrometheusRangeScalarSamples( + source, + query, + start, + end, + String(hourlyRangeStepSeconds), + timeoutMs, + ); + const result = mapHourly(samples); + return result.length > 0 ? result : undefined; + } catch { + return undefined; + } +} + +export async function queryControlPlaneReconciliation( + source: MetricsSource, + timeoutMs: number, +): Promise { + const [failures, retries, lag, stale] = await Promise.all([ + queryInstant(source, reconciliationFailureCountPromql, timeoutMs), + queryInstant(source, reconciliationRetryCountPromql, timeoutMs), + queryInstant(source, reconciliationLagP50SecondsPromql, timeoutMs, true), + queryInstant(source, staleResourceStatusCountPromql, timeoutMs), + ]); + if (failures === undefined || retries === undefined || stale === undefined) { + throw new Error("Required reconciliation metric is unavailable"); + } + + const [hourlyFailures, hourlyRetries, hourlyLag, hourlyStale] = + await Promise.all([ + optionalHourly(source, reconciliationFailureHourlyCountPromql, timeoutMs), + optionalHourly(source, reconciliationRetryHourlyCountPromql, timeoutMs), + optionalHourly(source, reconciliationLagP50SecondsPromql, timeoutMs), + optionalHourly(source, staleResourceStatusCountPromql, timeoutMs), + ]); + + return { + reconciliation_failures_count: failures, + reconciliation_retries_count: retries, + ...(lag === undefined ? {} : { reconciliation_lag_p50_seconds: lag }), + stale_resource_status_count: stale, + ...(hourlyFailures + ? { hourly_reconciliation_failures_count: hourlyFailures } + : {}), + ...(hourlyRetries + ? { hourly_reconciliation_retries_count: hourlyRetries } + : {}), + ...(hourlyLag ? { hourly_reconciliation_lag_p50_seconds: hourlyLag } : {}), + ...(hourlyStale ? { hourly_stale_resource_status_count: hourlyStale } : {}), + }; +} diff --git a/components/web-console/bff/test/metrics-control-plane-reconciliation-route.test.ts b/components/web-console/bff/test/metrics-control-plane-reconciliation-route.test.ts new file mode 100644 index 00000000..8d3ffb48 --- /dev/null +++ b/components/web-console/bff/test/metrics-control-plane-reconciliation-route.test.ts @@ -0,0 +1,77 @@ +import { createServer, type Server } from "node:http"; +import { mkdtemp, mkdir, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; + +import type { FastifyInstance } from "fastify"; +import { afterEach, describe, expect, it } from "vitest"; + +import { buildApp } from "../src/app.js"; +import type { ServerConfig } from "../src/config.js"; + +describe("GET /api/metrics/control-plane-reconciliation", () => { + let app: FastifyInstance | undefined; + let prometheus: Server | undefined; + let staticRoot: string; + + afterEach(async () => { + await app?.close(); + await new Promise((resolve) => { + prometheus?.close(() => { + resolve(); + }); + }); + }); + + async function startPrometheus(statusCode: number): Promise { + prometheus = createServer((_request, response) => { + response.statusCode = statusCode; + response.setHeader("content-type", "application/json"); + response.end(JSON.stringify({ status: "error", data: { result: [] } })); + }); + await new Promise((resolve) => + prometheus?.listen(0, "127.0.0.1", resolve), + ); + const address = prometheus.address(); + if (address === null || typeof address === "string") + throw new Error("expected listener"); + return `http://127.0.0.1:${String(address.port)}`; + } + + async function makeApp(prometheusUrl: string): Promise { + staticRoot = await mkdtemp( + path.join(tmpdir(), "hypershell-reconciliation-route-"), + ); + await mkdir(path.join(staticRoot, "assets")); + await writeFile( + path.join(staticRoot, "index.html"), + "App", + ); + const config: ServerConfig = { + apiOrigin: "http://127.0.0.1:1", + apiTimeoutMs: 5_000, + host: "127.0.0.1", + logLevel: "silent", + nodeEnv: "test", + port: 8080, + prometheusQueryTimeoutMs: 1_000, + prometheusUrl, + sessionTtlSeconds: 28_800, + staticRoot, + }; + return buildApp(config); + } + + it("returns 502 when a required reconciliation query fails", async () => { + app = await makeApp(await startPrometheus(500)); + const response = await app.inject({ + method: "GET", + url: "/api/metrics/control-plane-reconciliation", + }); + expect(response.statusCode).toBe(502); + expect(response.json()).toEqual({ + error: "Metrics unavailable", + statusCode: 502, + }); + }); +}); diff --git a/components/web-console/bff/test/metrics-control-plane-reconciliation.test.ts b/components/web-console/bff/test/metrics-control-plane-reconciliation.test.ts new file mode 100644 index 00000000..9275fed7 --- /dev/null +++ b/components/web-console/bff/test/metrics-control-plane-reconciliation.test.ts @@ -0,0 +1,62 @@ +import { createServer } from "node:http"; + +import { afterEach, describe, expect, it } from "vitest"; + +import { + queryControlPlaneReconciliation, + reconciliationFailureCountPromql, + reconciliationLagP50SecondsPromql, + reconciliationRetryCountPromql, + staleResourceStatusCountPromql, +} from "../src/metrics-control-plane-reconciliation.js"; + +describe("queryControlPlaneReconciliation", () => { + let server: ReturnType | undefined; + + afterEach(() => server?.close()); + + it("keeps valid zero counts when an idle lag histogram returns NaN or empty", async () => { + server = createServer((request, response) => { + const query = new URL( + request.url ?? "", + "http://127.0.0.1", + ).searchParams.get("query"); + response.setHeader("content-type", "application/json"); + if (request.url?.startsWith("/api/v1/query_range")) { + response.end( + JSON.stringify({ status: "success", data: { result: [] } }), + ); + return; + } + const value = query === reconciliationLagP50SecondsPromql ? "NaN" : "0"; + response.end( + JSON.stringify({ + status: "success", + data: { + result: query === "not-used" ? [] : [{ value: ["1", value] }], + }, + }), + ); + }); + await new Promise((resolve) => + server?.listen(0, "127.0.0.1", resolve), + ); + const address = server.address(); + if (address === null || typeof address === "string") + throw new Error("expected listener"); + + const metrics = await queryControlPlaneReconciliation( + `http://127.0.0.1:${String(address.port)}`, + 5_000, + ); + expect(metrics).toMatchObject({ + reconciliation_failures_count: 0, + reconciliation_retries_count: 0, + stale_resource_status_count: 0, + }); + expect(metrics.reconciliation_lag_p50_seconds).toBeUndefined(); + expect(reconciliationFailureCountPromql).toContain("increase"); + expect(reconciliationRetryCountPromql).toContain("increase"); + expect(staleResourceStatusCountPromql).toContain("stale"); + }); +}); diff --git a/components/web-console/locales/en.json b/components/web-console/locales/en.json index f577baac..68984a1b 100644 --- a/components/web-console/locales/en.json +++ b/components/web-console/locales/en.json @@ -31,6 +31,10 @@ "defaultMessage": "Add widgets", "description": "Label for the button that opens the widget drawer." }, + "app.dashboard.apiReliability.last5Minutes": { + "defaultMessage": "(last 5 minutes)", + "description": "Lookback period shown with the current API reliability metric value." + }, "app.dashboard.description": { "defaultMessage": "Live view of gateway fleet health, hub cluster capacity, and platform adoption. Metrics refresh every 15 minutes.", "description": "Supporting text on the operational dashboard page." @@ -255,6 +259,10 @@ "defaultMessage": "Gateway provision duration statistics", "description": "Accessible label for the provision time statistics description list." }, + "app.dashboard.reconciliationFailures.last24Hours": { + "defaultMessage": "(last 24 hours)", + "description": "Lookback period shown with the current reconciliation failure count." + }, "app.dashboard.refresh": { "defaultMessage": "Refresh dashboard metrics", "description": "Accessible label for refreshing operational dashboard metrics." @@ -319,10 +327,26 @@ "defaultMessage": "Median latency", "description": "Summary row label for API median latency." }, + "app.dashboard.reliability.summary.reconciliationFailures": { + "defaultMessage": "Reconciliation failures", + "description": "Summary row label for reconciliation failures." + }, + "app.dashboard.reliability.summary.reconciliationLag": { + "defaultMessage": "Reconciliation lag", + "description": "Summary row label for reconciliation lag." + }, + "app.dashboard.reliability.summary.reconciliationRetries": { + "defaultMessage": "Reconciliation retries", + "description": "Summary row label for reconciliation retries." + }, "app.dashboard.reliability.summary.requestRate": { "defaultMessage": "Request rate", "description": "Summary row label for API request rate." }, + "app.dashboard.reliability.summary.staleResourceStatus": { + "defaultMessage": "Stale resource status", + "description": "Summary row label for resources with stale observed status." + }, "app.dashboard.reliability.summaryAriaLabel": { "defaultMessage": "API reliability summary", "description": "Accessible label for the reliability summary widget." @@ -579,6 +603,18 @@ "defaultMessage": "Provisioned gateways", "description": "Title for the provisioned gateways dashboard widget." }, + "app.dashboard.widget.reconciliationFailures": { + "defaultMessage": "Reconciliation failures", + "description": "Title for the reconciliation failures dashboard widget." + }, + "app.dashboard.widget.reconciliationLag": { + "defaultMessage": "Reconciliation lag", + "description": "Title for the reconciliation lag dashboard widget." + }, + "app.dashboard.widget.reconciliationRetries": { + "defaultMessage": "Reconciliation retries", + "description": "Title for the reconciliation retries dashboard widget." + }, "app.dashboard.widget.registeredUsers": { "defaultMessage": "Users", "description": "Title for the registered users dashboard widget." @@ -595,6 +631,10 @@ "defaultMessage": "Section title", "description": "Default label for a section title widget in the add-widgets drawer." }, + "app.dashboard.widget.staleResourceStatus": { + "defaultMessage": "Stale resource status", + "description": "Title for the stale resource status dashboard widget." + }, "app.dashboard.widget.summary": { "defaultMessage": "Summary", "description": "Title for the operational dashboard summary widget." diff --git a/packages/operational-dashboard-ui/DATA_SOURCES.md b/packages/operational-dashboard-ui/DATA_SOURCES.md index dba621d5..728b690b 100644 --- a/packages/operational-dashboard-ui/DATA_SOURCES.md +++ b/packages/operational-dashboard-ui/DATA_SOURCES.md @@ -193,9 +193,35 @@ Reliability dashboard (`/dashboard/reliability`) loads through `createDashboardControlPlaneAdapter.getReliabilityMetrics`. This source is **not** part of the operational `metricSources` list (ARM-05). -| Source ID | BFF route | Metrics emitted | -| ----------------- | ---------------------------------- | --------------------------------------------------- | -| `api-reliability` | `GET /api/metrics/api-reliability` | `api-request-rate`, `api-error-rate`, `api-latency` | +| Source ID | BFF route | Metrics emitted | +| ------------------------------ | ----------------------------------------------- | -------------------------------------------------------------------------------------------------------- | +| `api-reliability` | `GET /api/metrics/api-reliability` | `api-request-rate`, `api-error-rate`, `api-latency` | +| `control-plane-reconciliation` | `GET /api/metrics/control-plane-reconciliation` | `reconciliation-failures`, `reconciliation-retries`, `reconciliation-lag`, `stale-resource-status-count` | + +The reliability dashboard SHALL read metric data through the BFF routes above. +The UI SHALL NOT read or write metric data directly to a database. Historical +reconciliation data uses a rolling 24-hour Prometheus range query with +hourly samples. + +The Reliability summary lists all three API metrics and the three +reconciliation metrics. On wide layouts, API metrics occupy the first column +and reconciliation metrics occupy the second; the columns stack on narrow +layouts. Dedicated reliability widgets expose the same values and historical +trends. Current API metrics use a 5-minute lookback; reconciliation failure +and retry counts use a rolling 24-hour lookback, with one-hour count samples +for their 24-hour trends. Reconciliation lag is a 5-minute P50 when a usable +histogram observation is available; otherwise the lag metric is omitted while +the other metrics remain available. + +| Metric ID | Prometheus type | Current value | Unit | +| ----------------------------- | --------------- | ------------------------------ | ------- | +| `reconciliation-failures` | Counter | Previous 24-hour failure count | `count` | +| `reconciliation-retries` | Counter | Previous 24-hour retry count | `count` | +| `reconciliation-lag` | Histogram | P50 | `sec` | +| `stale-resource-status-count` | Gauge | Current count | `count` | + +Failure and retry counts are displayed as whole numbers; reconciliation lag +is displayed to three fractional digits. Fetch failures soft-return `failedSources: ["api-reliability"]` with an empty metrics list (abort still throws). `mergeReliabilityDashboardMetrics` then @@ -209,11 +235,11 @@ ServiceMonitor (`job="hypershell-api-server"`). The HTTP status label is `http_server_request_duration_seconds` is not required for this dashboard; Kind does not currently set API-server OTLP export.) -| Instant field (BFF) | Metric ID | Unit | Display rounding | Trend field (BFF) | -| --------------------- | ------------------ | ------- | ---------------- | ---------------------------- | -| `request_rate` | `api-request-rate` | `req/s` | 2 fractional | `hourly_request_rate` | -| `error_rate_percent` | `api-error-rate` | `%` | 2 fractional | `hourly_error_rate_percent` | -| `latency_p50_seconds` | `api-latency` | `sec` | 3 fractional | `hourly_latency_p50_seconds` | +| Instant field (BFF) | Metric ID | Unit | Display rounding | Trend field (BFF) | +| --------------------- | ------------------ | -------------- | ---------------- | ---------------------------- | +| `request_rate` | `api-request-rate` | `requests/sec` | 3 fractional | `hourly_request_rate` | +| `error_rate_percent` | `api-error-rate` | `%` | 3 fractional | `hourly_error_rate_percent` | +| `latency_p50_seconds` | `api-latency` | `sec` | 3 fractional | `hourly_latency_p50_seconds` | PromQL shapes (5-minute rate window; see `components/web-console/bff/src/metrics-api-reliability.ts`): diff --git a/packages/operational-dashboard-ui/src/application/dashboard-types.ts b/packages/operational-dashboard-ui/src/application/dashboard-types.ts index fbdf76f9..994ff2fa 100644 --- a/packages/operational-dashboard-ui/src/application/dashboard-types.ts +++ b/packages/operational-dashboard-ui/src/application/dashboard-types.ts @@ -79,7 +79,8 @@ export type DashboardMetricSourceId = | "registered-users"; /** Metric source id used only on reliability-dashboard payloads (REL-DASH / ARM). */ -export type ReliabilityMetricSourceId = "api-reliability"; +export type ReliabilityMetricSourceId = + "api-reliability" | "control-plane-reconciliation"; export type DashboardFailedSourceId = DashboardMetricSourceId | ReliabilityMetricSourceId; diff --git a/packages/operational-dashboard-ui/src/dashboard/dashboard-metric-sources.test.ts b/packages/operational-dashboard-ui/src/dashboard/dashboard-metric-sources.test.ts index 0e8776f4..49bd6d8d 100644 --- a/packages/operational-dashboard-ui/src/dashboard/dashboard-metric-sources.test.ts +++ b/packages/operational-dashboard-ui/src/dashboard/dashboard-metric-sources.test.ts @@ -61,7 +61,7 @@ describe("mergeReliabilityDashboardMetrics", () => { const previousReliability: OperationalDashboardMetrics = { lastSuccessfulRefresh: new Date("2026-09-15T10:00:00.000Z"), metrics: [ - { id: "api-request-rate", unit: "req/s", value: "10.00" }, + { id: "api-request-rate", unit: "requests/sec", value: "10.00" }, { id: "api-error-rate", unit: "%", value: "1.00" }, { id: "api-latency", unit: "sec", value: "0.080" }, ], @@ -87,8 +87,8 @@ describe("mergeReliabilityDashboardMetrics", () => { const next: OperationalDashboardMetrics = { lastSuccessfulRefresh: new Date("2026-09-15T11:00:00.000Z"), metrics: [ - { id: "api-request-rate", unit: "req/s", value: "12.50" }, - { id: "api-error-rate", unit: "%", value: "1.25" }, + { id: "api-request-rate", unit: "requests/sec", value: "12.500" }, + { id: "api-error-rate", unit: "%", value: "1.250" }, { id: "api-latency", unit: "sec", value: "0.084" }, ], }; diff --git a/packages/operational-dashboard-ui/src/dashboard/dashboard-metric-sources.ts b/packages/operational-dashboard-ui/src/dashboard/dashboard-metric-sources.ts index 426add8f..02758330 100644 --- a/packages/operational-dashboard-ui/src/dashboard/dashboard-metric-sources.ts +++ b/packages/operational-dashboard-ui/src/dashboard/dashboard-metric-sources.ts @@ -28,6 +28,12 @@ export const RELIABILITY_METRIC_SOURCE_METRIC_IDS: Readonly< Record > = { "api-reliability": ["api-request-rate", "api-error-rate", "api-latency"], + "control-plane-reconciliation": [ + "reconciliation-failures", + "reconciliation-retries", + "reconciliation-lag", + "stale-resource-status-count", + ], }; function mergeDashboardMetricsBySourceMap( diff --git a/packages/operational-dashboard-ui/src/dashboard/reliability-dashboard-layout-template.ts b/packages/operational-dashboard-ui/src/dashboard/reliability-dashboard-layout-template.ts index 0374d01e..fd67373b 100644 --- a/packages/operational-dashboard-ui/src/dashboard/reliability-dashboard-layout-template.ts +++ b/packages/operational-dashboard-ui/src/dashboard/reliability-dashboard-layout-template.ts @@ -12,6 +12,10 @@ const WIDGET_TITLE_MESSAGES = { "api-request-rate": messages.apiRequestRateWidget, "api-error-rate": messages.apiErrorRateWidget, "api-latency": messages.apiLatencyWidget, + "reconciliation-failures": messages.widgetReconciliationFailures, + "reconciliation-retries": messages.widgetReconciliationRetries, + "reconciliation-lag": messages.widgetReconciliationLag, + "stale-resource-status-count": messages.widgetStaleResourceStatus, } as const; type ReliabilityWidgetType = keyof typeof WIDGET_TITLE_MESSAGES; @@ -53,6 +57,42 @@ const threeColumnLayout = [ x: 2, y: RELIABILITY_SUMMARY_WIDGET_HEIGHT, }, + { + h: RELIABILITY_TREND_WIDGET_HEIGHT, + i: "reconciliation-failures#1", + title: "Reconciliation failures", + w: 1, + widgetType: "reconciliation-failures", + x: 0, + y: RELIABILITY_SUMMARY_WIDGET_HEIGHT + RELIABILITY_TREND_WIDGET_HEIGHT, + }, + { + h: RELIABILITY_TREND_WIDGET_HEIGHT, + i: "reconciliation-retries#1", + title: "Reconciliation retries", + w: 1, + widgetType: "reconciliation-retries", + x: 1, + y: RELIABILITY_SUMMARY_WIDGET_HEIGHT + RELIABILITY_TREND_WIDGET_HEIGHT, + }, + { + h: RELIABILITY_TREND_WIDGET_HEIGHT, + i: "reconciliation-lag#1", + title: "Reconciliation lag", + w: 1, + widgetType: "reconciliation-lag", + x: 2, + y: RELIABILITY_SUMMARY_WIDGET_HEIGHT + RELIABILITY_TREND_WIDGET_HEIGHT, + }, + { + h: RELIABILITY_TREND_WIDGET_HEIGHT, + i: "stale-resource-status-count#1", + title: "Stale resource status", + w: 1, + widgetType: "stale-resource-status-count", + x: 0, + y: RELIABILITY_SUMMARY_WIDGET_HEIGHT + RELIABILITY_TREND_WIDGET_HEIGHT * 2, + }, ] as const; const mobileLayout = [ @@ -92,6 +132,42 @@ const mobileLayout = [ x: 0, y: RELIABILITY_SUMMARY_WIDGET_HEIGHT + RELIABILITY_TREND_WIDGET_HEIGHT * 2, }, + { + h: RELIABILITY_TREND_WIDGET_HEIGHT, + i: "reconciliation-failures#1", + title: "Reconciliation failures", + w: 1, + widgetType: "reconciliation-failures", + x: 0, + y: RELIABILITY_SUMMARY_WIDGET_HEIGHT + RELIABILITY_TREND_WIDGET_HEIGHT * 3, + }, + { + h: RELIABILITY_TREND_WIDGET_HEIGHT, + i: "reconciliation-retries#1", + title: "Reconciliation retries", + w: 1, + widgetType: "reconciliation-retries", + x: 0, + y: RELIABILITY_SUMMARY_WIDGET_HEIGHT + RELIABILITY_TREND_WIDGET_HEIGHT * 4, + }, + { + h: RELIABILITY_TREND_WIDGET_HEIGHT, + i: "reconciliation-lag#1", + title: "Reconciliation lag", + w: 1, + widgetType: "reconciliation-lag", + x: 0, + y: RELIABILITY_SUMMARY_WIDGET_HEIGHT + RELIABILITY_TREND_WIDGET_HEIGHT * 5, + }, + { + h: RELIABILITY_TREND_WIDGET_HEIGHT, + i: "stale-resource-status-count#1", + title: "Stale resource status", + w: 1, + widgetType: "stale-resource-status-count", + x: 0, + y: RELIABILITY_SUMMARY_WIDGET_HEIGHT + RELIABILITY_TREND_WIDGET_HEIGHT * 6, + }, ] as const; export const defaultReliabilityDashboardLayoutTemplate: ExtendedTemplateConfig = diff --git a/packages/operational-dashboard-ui/src/dashboard/reliability-dashboard-widgets.test.tsx b/packages/operational-dashboard-ui/src/dashboard/reliability-dashboard-widgets.test.tsx index b3d63b07..d8e47afe 100644 --- a/packages/operational-dashboard-ui/src/dashboard/reliability-dashboard-widgets.test.tsx +++ b/packages/operational-dashboard-ui/src/dashboard/reliability-dashboard-widgets.test.tsx @@ -33,14 +33,14 @@ const requestRateMetric: OperationalMetric = { ], }, id: "api-request-rate", - unit: "req/s", - value: "12.50", + unit: "requests/sec", + value: "12.500", }; const errorRateMetric: OperationalMetric = { id: "api-error-rate", unit: "%", - value: "1.25", + value: "1.250", }; const latencyMetric: OperationalMetric = { @@ -55,11 +55,49 @@ const latencyMetric: OperationalMetric = { value: "0.084", }; +const reconciliationFailuresMetric: OperationalMetric = { + id: "reconciliation-failures", + unit: "count", + value: "23", +}; + +const reconciliationRetriesMetric: OperationalMetric = { + id: "reconciliation-retries", + unit: "count", + value: "16", +}; + +const reconciliationLagMetric: OperationalMetric = { + id: "reconciliation-lag", + unit: "sec", + value: "0.047", +}; + +const staleResourceStatusMetric: OperationalMetric = { + hourlyTrend: { + points: [ + { label: "2026-09-15T12:00", value: 18 }, + { label: "2026-09-15T13:00", value: 22 }, + ], + }, + id: "stale-resource-status-count", + unit: "count", + value: "22", +}; + describe("ReliabilitySummaryCard", () => { - it("lists current values for all three reliability metrics", () => { + it("lists current values for all reliability metrics", () => { renderWithIntl( , ); @@ -69,6 +107,26 @@ describe("ReliabilitySummaryCard", () => { expect(screen.getByText(/12\.50/u)).toBeTruthy(); expect(screen.getByText(/1\.25/u)).toBeTruthy(); expect(screen.getByText(/0\.084/u)).toBeTruthy(); + expect(screen.getByText("Reconciliation failures")).toBeTruthy(); + expect(screen.getByText("Reconciliation retries")).toBeTruthy(); + expect(screen.getByText("Reconciliation lag")).toBeTruthy(); + expect(screen.getByText("Stale resource status")).toBeTruthy(); + expect(screen.getByText("23 failures")).toBeTruthy(); + expect(screen.getByText("16 retries")).toBeTruthy(); + expect(screen.getByText("0.047 sec")).toBeTruthy(); + expect(screen.getByText("22 count")).toBeTruthy(); + }); + + it("shows the stale resource status trend indicator", () => { + renderWithIntl( + , + ); + + expect( + screen.getByRole("button", { + name: /22% increase in Stale resource status/u, + }), + ).toBeTruthy(); }); it("shows a trend indicator when hourly change meets the threshold", () => { @@ -84,7 +142,7 @@ describe("ReliabilitySummaryCard", () => { it("shows metric unavailable when a summary metric is omitted", () => { renderWithIntl(); - expect(screen.getAllByText("Metric unavailable")).toHaveLength(2); + expect(screen.getAllByText("Metric unavailable")).toHaveLength(6); }); }); @@ -113,7 +171,8 @@ describe("ApiReliabilityTrendCard", () => { />, ); - expect(screen.getAllByText(/API error rate/u).length).toBeGreaterThan(0); + expect(screen.getByText("1.250 %")).toBeTruthy(); + expect(screen.getByText("(last 5 minutes)")).toBeTruthy(); expect(screen.queryByText("Last 24 hours")).toBeNull(); }); diff --git a/packages/operational-dashboard-ui/src/dashboard/reliability-dashboard-widgets.tsx b/packages/operational-dashboard-ui/src/dashboard/reliability-dashboard-widgets.tsx index 044a0f57..d45be4a9 100644 --- a/packages/operational-dashboard-ui/src/dashboard/reliability-dashboard-widgets.tsx +++ b/packages/operational-dashboard-ui/src/dashboard/reliability-dashboard-widgets.tsx @@ -65,16 +65,22 @@ function formatMetricWithUnit( intl: ReturnType, ): string { const displayValue = formatOperationalMetricDisplayValue(metric.value, intl); + const unit = + metric.id === "reconciliation-failures" + ? "failures" + : metric.id === "reconciliation-retries" + ? "retries" + : metric.unit; if ( !isDisplayableOperationalMetricValue(metric.value) || - metric.unit === undefined || - metric.unit === "" + unit === undefined || + unit === "" ) { return displayValue; } return intl.formatMessage(messages.utilizationLabel, { - unit: metric.unit, + unit, value: displayValue, }); } @@ -83,6 +89,10 @@ const RELIABILITY_SUMMARY_METRIC_IDS = [ "api-request-rate", "api-error-rate", "api-latency", + "reconciliation-failures", + "reconciliation-retries", + "reconciliation-lag", + "stale-resource-status-count", ] as const; type ReliabilitySummaryMetricId = @@ -92,8 +102,19 @@ const RELIABILITY_SUMMARY_LABELS = { "api-request-rate": messages.reliabilitySummaryRequestRate, "api-error-rate": messages.reliabilitySummaryErrorRate, "api-latency": messages.reliabilitySummaryLatency, + "reconciliation-failures": messages.reliabilitySummaryReconciliationFailures, + "reconciliation-retries": messages.reliabilitySummaryReconciliationRetries, + "reconciliation-lag": messages.reliabilitySummaryReconciliationLag, + "stale-resource-status-count": messages.reliabilitySummaryStaleResourceStatus, } as const; +const API_RELIABILITY_SUMMARY_METRIC_IDS = RELIABILITY_SUMMARY_METRIC_IDS.slice( + 0, + 3, +); +const RECONCILIATION_SUMMARY_METRIC_IDS = + RELIABILITY_SUMMARY_METRIC_IDS.slice(3); + function reliabilitySummaryTrendSubject( metricId: ReliabilitySummaryMetricId, intl: IntlShape, @@ -169,36 +190,47 @@ export function ReliabilitySummaryCard({ }: Readonly<{ metrics: readonly OperationalMetric[] }>) { const intl = useIntl(); + const renderSummaryColumn = ( + metricIds: readonly ReliabilitySummaryMetricId[], + ) => ( + + {metricIds.map((metricId) => { + const metric = metrics.find((entry) => entry.id === metricId); + + return ( + + + + + + {metric ? ( + + ) : ( + + + + )} + + + ); + })} + + ); + return ( - - {RELIABILITY_SUMMARY_METRIC_IDS.map((metricId) => { - const metric = metrics.find((entry) => entry.id === metricId); - - return ( - - - - - - {metric ? ( - - ) : ( - - - - )} - - - ); - })} - +
+ {renderSummaryColumn(API_RELIABILITY_SUMMARY_METRIC_IDS)} +
+
+ {renderSummaryColumn(RECONCILIATION_SUMMARY_METRIC_IDS)} +
+ ); } @@ -220,13 +252,10 @@ export function ApiReliabilityTrendCard({ ); } + const isCount = + metric.id === "reconciliation-failures" || + metric.id === "reconciliation-retries"; const displayValue = formatMetricWithUnit(metric, intl); - const heading = isDisplayableOperationalMetricValue(metric.value) - ? intl.formatMessage(messages.metricValue, { - label: title, - value: displayValue, - }) - : displayValue; return ( @@ -236,7 +265,17 @@ export function ApiReliabilityTrendCard({ - {heading} + <> + {displayValue} + <br /> + <small> + {intl.formatMessage( + isCount + ? messages.reconciliationFailuresLast24Hours + : messages.apiReliabilityLast5Minutes, + )} + </small> + </> @@ -247,6 +286,7 @@ export function ApiReliabilityTrendCard({ caption={intl.formatMessage(messages.trendLast24Hours)} title={title} trend={metric.hourlyTrend} + valueFormatter={(value) => value.toFixed(3)} /> ) : null} diff --git a/packages/operational-dashboard-ui/src/dashboard/trend-sparkline-chart.tsx b/packages/operational-dashboard-ui/src/dashboard/trend-sparkline-chart.tsx index ff6d747c..868e4b4f 100644 --- a/packages/operational-dashboard-ui/src/dashboard/trend-sparkline-chart.tsx +++ b/packages/operational-dashboard-ui/src/dashboard/trend-sparkline-chart.tsx @@ -23,10 +23,12 @@ export function TrendSparklineChart({ caption, trend, title, + valueFormatter, }: Readonly<{ caption?: string; trend: OperationalMetricTrend; title: string; + valueFormatter?: (value: number) => string; }>) { const intl = useIntl(); const containerRef = useRef(null); @@ -65,7 +67,7 @@ export function TrendSparklineChart({ intl.formatMessage(messages.trendTooltip, { date: datum.x, metric: title, - value: datum.y, + value: valueFormatter ? valueFormatter(datum.y) : datum.y, }); const trendDayCount = trend.points.length.toString(); diff --git a/packages/operational-dashboard-ui/src/fixtures/mock-reliability-dashboard-metrics.ts b/packages/operational-dashboard-ui/src/fixtures/mock-reliability-dashboard-metrics.ts index e66ca08e..7385d0fd 100644 --- a/packages/operational-dashboard-ui/src/fixtures/mock-reliability-dashboard-metrics.ts +++ b/packages/operational-dashboard-ui/src/fixtures/mock-reliability-dashboard-metrics.ts @@ -28,9 +28,37 @@ function buildHourlyTrendPoints( return points; } +function buildHourlyCountTrendPoints( + counts: readonly number[], +): { label: string; value: number }[] { + const end = new Date(); + const endHour = Date.UTC( + end.getUTCFullYear(), + end.getUTCMonth(), + end.getUTCDate(), + end.getUTCHours(), + ); + + return counts.map((value, index) => { + const hour = new Date( + endHour - (counts.length - 1 - index) * 60 * 60 * 1000, + ); + const year = hour.getUTCFullYear(); + const month = String(hour.getUTCMonth() + 1).padStart(2, "0"); + const day = String(hour.getUTCDate()).padStart(2, "0"); + const hourLabel = String(hour.getUTCHours()).padStart(2, "0"); + + return { + label: `${String(year)}-${month}-${day}T${hourLabel}:00`, + value, + }; + }); +} + /** * Storybook and local-dev fixture shaped like `getReliabilityMetrics` output, - * including 24-hour hourly trends for all three API reliability metrics. + * including 24-hour hourly trends for API reliability and reconciliation + * metrics. */ export const mockReliabilityDashboardMetrics: OperationalDashboardMetrics = Object.freeze({ @@ -41,8 +69,8 @@ export const mockReliabilityDashboardMetrics: OperationalDashboardMetrics = points: Object.freeze(buildHourlyTrendPoints(10.2, 0.1)), }), id: "api-request-rate", - unit: "req/s", - value: "12.50", + unit: "requests/sec", + value: "12.500", }), Object.freeze({ hourlyTrend: Object.freeze({ @@ -50,7 +78,7 @@ export const mockReliabilityDashboardMetrics: OperationalDashboardMetrics = }), id: "api-error-rate", unit: "%", - value: "1.25", + value: "1.250", }), Object.freeze({ hourlyTrend: Object.freeze({ @@ -60,5 +88,47 @@ export const mockReliabilityDashboardMetrics: OperationalDashboardMetrics = unit: "sec", value: "0.084", }), + Object.freeze({ + hourlyTrend: Object.freeze({ + points: Object.freeze( + buildHourlyCountTrendPoints([ + 0, 1, 0, 2, 1, 0, 3, 0, 1, 2, 0, 1, 0, 0, 2, 1, 0, 4, 0, 1, 0, 2, + 1, 1, + ]), + ), + }), + id: "reconciliation-failures", + unit: "count", + value: "23", + }), + Object.freeze({ + hourlyTrend: Object.freeze({ + points: Object.freeze( + buildHourlyCountTrendPoints([ + 0, 1, 0, 1, 2, 0, 1, 0, 2, 1, 0, 1, 0, 0, 1, 1, 0, 2, 0, 1, 0, 1, + 1, 1, + ]), + ), + }), + id: "reconciliation-retries", + unit: "count", + value: "16", + }), + Object.freeze({ + hourlyTrend: Object.freeze({ + points: Object.freeze(buildHourlyTrendPoints(0.03, 0.001)), + }), + id: "reconciliation-lag", + unit: "sec", + value: "0.047", + }), + Object.freeze({ + hourlyTrend: Object.freeze({ + points: Object.freeze(buildHourlyTrendPoints(18, 4 / 23)), + }), + id: "stale-resource-status-count", + unit: "count", + value: "22", + }), ]), }); diff --git a/packages/operational-dashboard-ui/src/messages.ts b/packages/operational-dashboard-ui/src/messages.ts index 18aa44d9..3a3bedd2 100644 --- a/packages/operational-dashboard-ui/src/messages.ts +++ b/packages/operational-dashboard-ui/src/messages.ts @@ -16,6 +16,12 @@ export const messages = defineMessages({ defaultMessage: "API latency", description: "Title for the API median latency reliability widget.", }, + apiReliabilityLast5Minutes: { + id: "app.dashboard.apiReliability.last5Minutes", + defaultMessage: "(last 5 minutes)", + description: + "Lookback period shown with the current API reliability metric value.", + }, apiRequestRateWidget: { id: "app.dashboard.widget.apiRequestRate", defaultMessage: "API request rate", @@ -413,6 +419,12 @@ export const messages = defineMessages({ defaultMessage: "Gateway provision time", description: "Title for the gateway provision time dashboard widget.", }, + reconciliationFailuresLast24Hours: { + id: "app.dashboard.reconciliationFailures.last24Hours", + defaultMessage: "(last 24 hours)", + description: + "Lookback period shown with the current reconciliation failure count.", + }, refresh: { id: "app.dashboard.refresh", defaultMessage: "Refresh dashboard metrics", @@ -535,11 +547,31 @@ export const messages = defineMessages({ defaultMessage: "Median latency", description: "Summary row label for API median latency.", }, + reliabilitySummaryReconciliationFailures: { + id: "app.dashboard.reliability.summary.reconciliationFailures", + defaultMessage: "Reconciliation failures", + description: "Summary row label for reconciliation failures.", + }, + reliabilitySummaryReconciliationLag: { + id: "app.dashboard.reliability.summary.reconciliationLag", + defaultMessage: "Reconciliation lag", + description: "Summary row label for reconciliation lag.", + }, + reliabilitySummaryReconciliationRetries: { + id: "app.dashboard.reliability.summary.reconciliationRetries", + defaultMessage: "Reconciliation retries", + description: "Summary row label for reconciliation retries.", + }, reliabilitySummaryRequestRate: { id: "app.dashboard.reliability.summary.requestRate", defaultMessage: "Request rate", description: "Summary row label for API request rate.", }, + reliabilitySummaryStaleResourceStatus: { + id: "app.dashboard.reliability.summary.staleResourceStatus", + defaultMessage: "Stale resource status", + description: "Summary row label for resources with stale observed status.", + }, reliabilitySummaryWidget: { id: "app.dashboard.widget.reliabilitySummary", defaultMessage: "Reliability summary", @@ -779,10 +811,30 @@ export const messages = defineMessages({ defaultMessage: "Pods", description: "Title for the pods utilization dashboard widget.", }, + widgetReconciliationFailures: { + id: "app.dashboard.widget.reconciliationFailures", + defaultMessage: "Reconciliation failures", + description: "Title for the reconciliation failures dashboard widget.", + }, + widgetReconciliationLag: { + id: "app.dashboard.widget.reconciliationLag", + defaultMessage: "Reconciliation lag", + description: "Title for the reconciliation lag dashboard widget.", + }, + widgetReconciliationRetries: { + id: "app.dashboard.widget.reconciliationRetries", + defaultMessage: "Reconciliation retries", + description: "Title for the reconciliation retries dashboard widget.", + }, widgetSandboxes: { id: "app.dashboard.summary.sandboxes", defaultMessage: "Sandboxes", description: "Label for provisioned sandboxes on the operational dashboard.", }, + widgetStaleResourceStatus: { + id: "app.dashboard.widget.staleResourceStatus", + defaultMessage: "Stale resource status", + description: "Title for the stale resource status dashboard widget.", + }, }); diff --git a/packages/operational-dashboard-ui/src/pages/dashboard-widget.css b/packages/operational-dashboard-ui/src/pages/dashboard-widget.css index 32fe2885..740002f0 100644 --- a/packages/operational-dashboard-ui/src/pages/dashboard-widget.css +++ b/packages/operational-dashboard-ui/src/pages/dashboard-widget.css @@ -316,6 +316,37 @@ ); } +.hypershell-dashboard-reliability-summary { + display: grid; + gap: var(--pf-t--global--spacer--lg); + grid-template-columns: repeat(2, minmax(0, 1fr)); +} + +.hypershell-dashboard-reliability-summary .pf-v6-c-description-list { + display: block; +} + +.hypershell-dashboard-reliability-summary .pf-v6-c-description-list__group { + display: flex; + align-items: baseline; +} + +.hypershell-dashboard-reliability-summary__term { + flex: 0 0 14rem; +} + +.hypershell-dashboard-reliability-summary + .pf-v6-c-description-list__description { + flex: 1 1 auto; + min-width: 0; +} + +@media (max-width: 768px) { + .hypershell-dashboard-reliability-summary { + grid-template-columns: 1fr; + } +} + .hypershell-dashboard-provision-reliability__donut--danger { --pf-v6-chart-donut--label--title--Fill: var( --pf-t--global--icon--color--status--danger--default diff --git a/packages/operational-dashboard-ui/src/pages/reliability-dashboard-page.tsx b/packages/operational-dashboard-ui/src/pages/reliability-dashboard-page.tsx index 25cf2cf0..fd9977a4 100644 --- a/packages/operational-dashboard-ui/src/pages/reliability-dashboard-page.tsx +++ b/packages/operational-dashboard-ui/src/pages/reliability-dashboard-page.tsx @@ -17,6 +17,7 @@ import { ChartLineIcon, ExclamationCircleIcon, OutlinedClockIcon, + SyncAltIcon, TachometerAltIcon, } from "@patternfly/react-icons"; import { @@ -61,7 +62,7 @@ import { useGetReliabilityMetricsData } from "./get-reliability-metrics-data"; const baseTemplate = defaultReliabilityDashboardLayoutTemplate; -const LAYOUT_STORAGE_KEY = "hypershell.reliability-dashboard.layout.v1"; +const LAYOUT_STORAGE_KEY = "hypershell.reliability-dashboard.layout.v2"; const CUSTOM_COLUMNS: Record = { xl: RELIABILITY_DASHBOARD_COLUMN_COUNT, lg: RELIABILITY_DASHBOARD_COLUMN_COUNT, @@ -206,6 +207,71 @@ function createWidgetMapping( }, renderWidget: () => renderTrend("api-latency", messages.apiLatencyWidget), }, + "reconciliation-failures": { + defaults: { + h: RELIABILITY_TREND_WIDGET_HEIGHT, + maxH: RELIABILITY_TREND_WIDGET_HEIGHT + 2, + minH: METRIC_WIDGET_DEFAULTS.minH, + w: 1, + }, + config: { + icon: , + title: intl.formatMessage(messages.widgetReconciliationFailures), + }, + renderWidget: () => + renderTrend( + "reconciliation-failures", + messages.widgetReconciliationFailures, + ), + }, + "reconciliation-retries": { + defaults: { + h: RELIABILITY_TREND_WIDGET_HEIGHT, + maxH: RELIABILITY_TREND_WIDGET_HEIGHT + 2, + minH: METRIC_WIDGET_DEFAULTS.minH, + w: 1, + }, + config: { + icon: , + title: intl.formatMessage(messages.widgetReconciliationRetries), + }, + renderWidget: () => + renderTrend( + "reconciliation-retries", + messages.widgetReconciliationRetries, + ), + }, + "reconciliation-lag": { + defaults: { + h: RELIABILITY_TREND_WIDGET_HEIGHT, + maxH: RELIABILITY_TREND_WIDGET_HEIGHT + 2, + minH: METRIC_WIDGET_DEFAULTS.minH, + w: 1, + }, + config: { + icon: , + title: intl.formatMessage(messages.widgetReconciliationLag), + }, + renderWidget: () => + renderTrend("reconciliation-lag", messages.widgetReconciliationLag), + }, + "stale-resource-status-count": { + defaults: { + h: RELIABILITY_TREND_WIDGET_HEIGHT, + maxH: RELIABILITY_TREND_WIDGET_HEIGHT + 2, + minH: METRIC_WIDGET_DEFAULTS.minH, + w: 1, + }, + config: { + icon: , + title: intl.formatMessage(messages.widgetStaleResourceStatus), + }, + renderWidget: () => + renderTrend( + "stale-resource-status-count", + messages.widgetStaleResourceStatus, + ), + }, }; } diff --git a/specs/platform/api-reliability-metrics.spec.md b/specs/platform/api-reliability-metrics.spec.md index 259b1f54..ab3da456 100644 --- a/specs/platform/api-reliability-metrics.spec.md +++ b/specs/platform/api-reliability-metrics.spec.md @@ -214,7 +214,7 @@ The host dashboard adapter SHALL map `GET /api/metrics/api-reliability` into thr | Metric ID | `value` | `unit` | Instant source | Trend source | | --- | --- | --- | --- | --- | -| `api-request-rate` | Decimal string of `request_rate` | `"req/s"` | `request_rate` | `hourly_request_rate` → `hourlyTrend.points` | +| `api-request-rate` | Decimal string of `request_rate` | `"requests/sec"` | `request_rate` | `hourly_request_rate` → `hourlyTrend.points` | | `api-error-rate` | Decimal string of `error_rate_percent` | `"%"` | `error_rate_percent` | `hourly_error_rate_percent` → `hourlyTrend.points` | | `api-latency` | Decimal string of `latency_p50_seconds` | `"sec"` | `latency_p50_seconds` | `hourly_latency_p50_seconds` → `hourlyTrend.points` | @@ -229,8 +229,8 @@ Rounding: | Metric | Display rounding | | --- | --- | -| `api-request-rate` | Two fractional digits | -| `api-error-rate` | Two fractional digits | +| `api-request-rate` | Three fractional digits | +| `api-error-rate` | Three fractional digits | | `api-latency` | Three fractional digits | When a BFF historical field is absent, the adapter SHALL omit `hourlyTrend` for that metric only and SHALL still emit the metric with the instant `value`. @@ -241,8 +241,8 @@ The three metrics SHALL load as one independent metric source (`api-reliability` - GIVEN the BFF returns the ARM-03 example JSON - WHEN the adapter builds reliability metrics -- THEN `api-request-rate` SHALL have `value: "12.50"`, `unit: "req/s"`, and `hourlyTrend` populated -- AND `api-error-rate` SHALL have `value: "1.25"`, `unit: "%"` +- THEN `api-request-rate` SHALL have `value: "12.500"`, `unit: "requests/sec"`, and `hourlyTrend` populated +- AND `api-error-rate` SHALL have `value: "1.250"`, `unit: "%"` - AND `api-latency` SHALL have `value: "0.084"`, `unit: "sec"` #### Scenario: Missing hourly latency omits only latency trend @@ -279,7 +279,7 @@ Instant PromQL failure, timeout, empty result set, or non-finite values for requ The adapter SHALL treat a non-success BFF response as failure of the entire `api-reliability` source. -The dashboard SHALL NOT display `0 req/s`, `0%`, or `0 sec` as a fallback when the source failed. +The dashboard SHALL NOT display `0 requests/sec`, `0%`, or `0 sec` as a fallback when the source failed. #### Scenario: Prometheus timeout yields 502 diff --git a/specs/platform/control-plane-reconciliation-metrics.spec.md b/specs/platform/control-plane-reconciliation-metrics.spec.md new file mode 100644 index 00000000..e29adcc3 --- /dev/null +++ b/specs/platform/control-plane-reconciliation-metrics.spec.md @@ -0,0 +1,79 @@ +# Control-Plane Reconciliation Metrics + +**Status:** Draft +**Version:** 0.1.0 +**Created:** 2026-09-23 +**Tracks:** [HYPERSHELL-286](https://redhat.atlassian.net/browse/HYPERSHELL-286) + +## Purpose + +Expose control-plane reconciliation health to the reliability dashboard through +Prometheus and the web-console BFF without introducing a dashboard database. + +## Requirements + +### Requirement: CRM-001 -- Prometheus Metric Types + +The control plane SHALL expose the following fleet-wide Prometheus metrics: + +| Metric | Type | Meaning | Unit | +| --- | --- | --- | --- | +| `hypershell_reconciliation_failures_total` | Counter | Failed reconciliation attempts | count | +| `hypershell_reconciliation_retries_total` | Counter | Reconciliation retries | count | +| `hypershell_reconciliation_lag_seconds` | Histogram | Time from reconciliation eligibility to completion | seconds | +| `hypershell_stale_resource_status_count` | Gauge | Resources whose observed status is stale | count | + +The control plane SHALL expose the metrics through its Prometheus scrape +endpoint. The dashboard SHALL NOT read reconciliation data from a database. + +### Requirement: CRM-002 -- Prometheus Labels + +The counters and histogram SHALL use only bounded labels approved by the +control-plane observability conventions. The metrics SHALL NOT include resource +IDs, customer IDs, names, or other unbounded-cardinality labels. + +The failure and retry counters MAY include a bounded reconciliation-kind label. +The lag histogram MAY include a bounded reconciliation-kind label. The stale +resource gauge MAY include a bounded resource-kind label. + +### Requirement: CRM-003 -- Historical Query Contract + +The BFF SHALL query the metrics through Prometheus `query_range` over the most +recent **24 hours** with a **3600-second** step. + +The BFF SHALL calculate: + +- failure counts from `increase(hypershell_reconciliation_failures_total[24h])` for current values and `increase(hypershell_reconciliation_failures_total[1h])` for hourly samples; +- retry counts from `increase(hypershell_reconciliation_retries_total[24h])` for current values and `increase(hypershell_reconciliation_retries_total[1h])` for hourly samples; +- median lag from `histogram_quantile(0.50, ...)` over the histogram buckets when + a valid sample exists; and +- stale resource status count from the gauge value. + +The BFF SHALL use the same aggregation dimensions for instant and range +queries. The BFF SHALL omit a historical series when Prometheus returns an +invalid or unavailable sample instead of converting the sample to zero. + +### Requirement: CRM-004 -- BFF Response + +The BFF SHALL expose `GET /api/metrics/control-plane-reconciliation` and SHALL +return current values plus optional historical arrays with UTC hourly labels. +The failure, retry, and stale-resource current queries are required and SHALL +return HTTP `502` when they fail. The current lag query is optional: when it +returns no valid sample, the response SHALL omit the lag field while preserving +the other current values. Failure of one historical query SHALL omit only that +historical array while preserving successful current values and sibling +historical arrays. + +### Requirement: CRM-005 -- Verification + +Tests SHALL verify metric registration, histogram bucket export, bounded label +sets, PromQL aggregation, 24-hour range parameters, response mapping, and +failure behavior. + +## Assumptions + +- Prometheus retention is at least 24 hours. +- HYPERSHELL-285 API reliability metrics are available to the dashboard. This + specification intentionally reuses and standardizes their shared dashboard + presentation as `requests/sec` with three fractional digits; these are + compatibility changes to the existing reliability-dashboard contract. diff --git a/specs/web-console/reliability-dashboard.spec.md b/specs/web-console/reliability-dashboard.spec.md index 33dd52cf..20abde2f 100644 --- a/specs/web-console/reliability-dashboard.spec.md +++ b/specs/web-console/reliability-dashboard.spec.md @@ -1,16 +1,20 @@ # Reliability Dashboard **Status:** Draft +**Version:** 0.2.0 +**Created:** 2026-09-23 **Applies to:** `packages/operational-dashboard-ui`, `components/web-console` SPA and BFF -**Tracks:** [HYPERSHELL-285](https://redhat.atlassian.net/browse/HYPERSHELL-285) +**Tracks:** [HYPERSHELL-286](https://redhat.atlassian.net/browse/HYPERSHELL-286) ## Purpose -Provide a **widgetized reliability dashboard** in the HyperShell web console where administrators can assess API reliability at a glance. Version 1 surfaces API request rate, 5xx error rate, and median latency with current values and 24-hour hourly trend sparklines. +Provide a **widgetized reliability dashboard** in the HyperShell web console where administrators can assess API and control-plane reliability at a glance. This specification adds control-plane reconciliation health to the existing API reliability dashboard. The reliability dashboard lives in the same reusable package as the operational dashboard (`@openshift-online/hypershell-operational-dashboard-ui`) to share hexagonal ports, widgetized-dashboard infrastructure, localization patterns, and presentation helpers - while remaining a **separate route and layout** so reliability metrics do not clutter the operational overview. -Metric collection, PromQL, and BFF contracts are defined in `platform/api-reliability-metrics.spec.md`. Cross-dashboard navigation uses PatternFly **secondary horizontal navigation** so each dashboard keeps a distinct URL ([PatternFly Navigation - secondary horizontal](https://www.patternfly.org/components/navigation/design-guidelines#secondary-horizontal-navigation)). +Metric collection, PromQL, and BFF contracts are defined in `platform/api-reliability-metrics.spec.md` and `platform/control-plane-reconciliation-metrics.spec.md`. The browser SHALL obtain metrics only through same-origin BFF routes backed by Prometheus. The dashboard SHALL NOT read from or write to a database for metric collection or historical data. + +Historical reconciliation data SHALL cover the rolling previous **24 hours** and SHALL use hourly samples. Prometheus metric types SHALL match the measurement: reconciliation lag SHALL use a histogram, reconciliation failures and retries SHALL use counters, and stale resource status SHALL use a gauge. Authentication failure rate and login failure rate are out of scope for version 1. @@ -69,7 +73,7 @@ The reliability dashboard SHALL follow the narrow hexagonal UI boundary defined ### Requirement: REL-DASH-03 -- Host Composition -The web console host SHALL extend dashboard composition so `ReliabilityDashboardPage` receives a control-plane adapter that implements `getReliabilityMetrics` by calling BFF `GET /api/metrics/api-reliability` (ARM-03 / ARM-04). +The web console host SHALL extend dashboard composition so `ReliabilityDashboardPage` receives a control-plane adapter that implements `getReliabilityMetrics` by calling BFF `GET /api/metrics/api-reliability` and `GET /api/metrics/control-plane-reconciliation` (ARM-03 / ARM-04 and REL-DASH-12 / REL-DASH-13). #### Scenario: Production host uses the API adapter for reliability @@ -181,20 +185,28 @@ Version 1 SHALL connect the following metrics from the `api-reliability` source | Widget type | Metric ID | Presentation | | --- | --- | --- | -| `reliability-summary` | `api-request-rate`, `api-error-rate`, `api-latency` | Summary card listing current values for all three. When a metric's `hourlyTrend` has at least two points and `getMetricTrendChange` reports a change of at least 5%, the row SHALL show the same up/down trend indicator used on the operational dashboard summary cards (OP-DASH-13) | +| `reliability-summary` | API metrics plus `reconciliation-failures`, `reconciliation-retries`, `reconciliation-lag` | Two-column summary card listing all six current values. When a metric's `hourlyTrend` has at least two points and `getMetricTrendChange` reports a change of at least 5%, the row SHALL show the same up/down trend indicator used on the operational dashboard summary cards (OP-DASH-13) | | `api-request-rate` | `api-request-rate` | Current rate plus 24-hour hourly usage trend graph when `hourlyTrend` has at least two points | | `api-error-rate` | `api-error-rate` | Current 5xx error percent plus 24-hour hourly usage trend graph when `hourlyTrend` has at least two points | | `api-latency` | `api-latency` | Current median latency plus 24-hour hourly usage trend graph when `hourlyTrend` has at least two points | +The `reliability-summary` widget SHALL also list `reconciliation-failures`, +`reconciliation-retries`, and `reconciliation-lag` alongside the three API +metrics. The summary SHALL use two columns on sufficiently wide layouts, with +API metrics in the first column and reconciliation metrics in the second. The +columns SHALL stack when the summary becomes narrow. Reconciliation count +values SHALL display `failures` or `retries` as their unit labels, and lag +values SHALL display `sec`. + Widgets without a connected metric SHALL remain on the grid and render the localized metric-unavailable empty state (same pattern as OP-DASH unavailable widgets). -Trend graphs SHALL reuse the shared sparkline presentation used by the operational dashboard where practical (`TrendSparklineChart` or equivalent). Units on tooltips and summary rows SHALL match ARM-04 (`req/s`, `%`, `sec`). +Trend graphs SHALL reuse the shared sparkline presentation used by the operational dashboard where practical (`TrendSparklineChart` or equivalent). Units on tooltips and summary rows SHALL use `requests/sec`, `%`, and `sec`. -#### Scenario: Summary lists all three current values +#### Scenario: Summary lists all six current values -- GIVEN `api-request-rate`, `api-error-rate`, and `api-latency` are present +- GIVEN the three API metrics and the three reconciliation metrics are present - WHEN the `reliability-summary` widget renders -- THEN it SHALL show the current request rate, error rate, and median latency with their units +- THEN it SHALL show all six current values with their units #### Scenario: Summary shows trend arrows when hourly change meets threshold @@ -231,8 +243,11 @@ The default layout SHALL include: | `api-request-rate` | Trend graph tile | | `api-error-rate` | Trend graph tile | | `api-latency` | Trend graph tile | +| `reconciliation-failures` | Trend graph tile | +| `reconciliation-retries` | Trend graph tile | +| `reconciliation-lag` | Trend graph tile | -Exact column coordinates MAY be chosen during implementation; the default SHALL place the summary above or beside the three trend widgets so all four are visible without opening the drawer. +Exact column coordinates MAY be chosen during implementation; the default SHALL place the summary above or beside the six trend widgets so all seven are visible without opening the drawer. Users SHALL be able to add widgets from the drawer, drag to rearrange, remove widgets, and reset to the reliability default template. Layout persistence SHALL use a **separate** `localStorage` key from the operational dashboard (for example `hypershell.reliability-dashboard.layout.v1`) so customizing one dashboard does not rewrite the other. @@ -257,7 +272,7 @@ The page header SHALL include last-refreshed timestamp, manual refresh, reset-to The reliability dashboard SHALL load metrics through TanStack Query with `reliabilityDashboardRefreshMilliseconds` equal to `operationalDashboardRefreshMilliseconds` (15 minutes). -Manual refresh SHALL re-fetch the `api-reliability` source immediately. Refresh-in-flight, partial-failure, and total-failure presentation SHALL follow the operational dashboard patterns (OP-DASH-09): spinner on initial load only; warning alert on partial failure; danger empty state on total initial-load failure; keep last successful values for failed sources on refresh when applicable. +Manual refresh SHALL re-fetch the `api-reliability` and `control-plane-reconciliation` sources immediately. Refresh-in-flight, partial-failure, and total-failure presentation SHALL follow the operational dashboard patterns (OP-DASH-09): spinner on initial load only; warning alert on partial failure; danger empty state on total initial-load failure; keep last successful values for failed sources on refresh when applicable. #### Scenario: Manual refresh re-queries Prometheus via BFF @@ -295,12 +310,47 @@ BFF and SPA access-control tests SHALL cover `/dashboard/reliability` and `GET / #### Scenario: Storybook shows sparklines with mock hourly trends -- GIVEN `mockReliabilityDashboardMetrics` includes `hourlyTrend` points on all three metrics +- GIVEN `mockReliabilityDashboardMetrics` includes `hourlyTrend` points on the reliability metrics - WHEN the default reliability Storybook story renders -- THEN the three trend widgets SHALL show sparklines +- THEN the trend widgets with at least two points SHALL show sparklines --- +### Requirement: REL-DASH-12 -- Control-Plane Reconciliation Data Flow + +The control-plane reconciliation metrics SHALL flow through the following boundary: + +`control-plane Prometheus metrics -> Prometheus query API -> web-console BFF -> dashboard adapter -> reliability dashboard`. + +The BFF SHALL be the only browser-facing integration point for reconciliation metrics. The dashboard SHALL NOT connect directly to Prometheus, the control plane, or a database. + +The BFF SHALL expose `GET /api/metrics/control-plane-reconciliation` and SHALL return current values plus independently optional 24-hour hourly trend arrays for failures, retries, lag, and stale resource status count. + +The BFF SHALL return HTTP `502` when a required current query fails. A failed historical query SHALL omit only that historical array while preserving successful current values and sibling historical arrays. + +### Requirement: REL-DASH-13 -- 24-Hour Historical Series + +The BFF SHALL query Prometheus over a rolling **24-hour** window ending at the current UTC time with a `3600s` step. + +Historical arrays SHALL be ordered oldest to newest, use UTC hour labels in `YYYY-MM-DDTHH:00` format, and contain no synthetic zero values for missing or invalid samples. + +The BFF SHALL derive the current failure and retry counts from counter increases over the previous 24 hours, derive hourly failure and retry counts from one-hour counter increases, derive reconciliation lag from the histogram, and SHALL query stale resource status count from the gauge series. + +### Requirement: REL-DASH-14 -- Reconciliation Metric Presentation + +The dashboard SHALL render these reconciliation metrics in the Reliability summary and dedicated widgets: + +| Metric | Prometheus type | Current value | Unit | +| --- | --- | --- | --- | +| Reconciliation failures | Counter | Previous 24-hour failure count | `count` | +| Reconciliation retries | Counter | Previous 24-hour retry count | `count` | +| Reconciliation lag | Histogram | P50 lag | `sec` | +| Stale resource status count | Gauge | Current stale-resource count | `count` | + +The dashboard SHALL show a 24-hour hourly trend graph when the corresponding historical array contains at least two valid points. The dashboard SHALL omit the graph when the current value is unavailable or the historical array contains fewer than two points. + +The dashboard SHALL NOT display zero as a substitute for an unavailable current value. When current reconciliation lag is unavailable, the lag metric and its trend graph SHALL be omitted while the other reconciliation metrics remain visible. + ## Non-Goals - Authentication failure rate and login failure rate @@ -309,3 +359,5 @@ BFF and SPA access-control tests SHALL cover `/dashboard/reliability` and `GET / - Changing the dashboard-host root (`/`) to the reliability page - Per-route latency or error drill-down pages - Using PatternFly Tabs as the cross-dashboard primary switcher +- Reading or writing metric data directly from a database +- Per-resource reconciliation drill-down or reconciliation-history export