From e21a4a3362472b723dc282612db0253feb76c48c Mon Sep 17 00:00:00 2001 From: James McDermott Date: Tue, 13 Oct 2026 17:06:20 +0100 Subject: [PATCH 1/4] Docs: update docs for PR#1198; add dashboard for in-flight operations Signed-off-by: James McDermott --- .../grafana-porch-server-dashboard.json | 70 +++++++++++++++++++ .../configurations/opentelemetry.md | 39 +++++++++-- 2 files changed, 105 insertions(+), 4 deletions(-) diff --git a/deployments/metrics-resources/grafana-porch-server-dashboard.json b/deployments/metrics-resources/grafana-porch-server-dashboard.json index 139ae542f..099377800 100644 --- a/deployments/metrics-resources/grafana-porch-server-dashboard.json +++ b/deployments/metrics-resources/grafana-porch-server-dashboard.json @@ -1151,6 +1151,76 @@ "title": "All Resources \u2014 Total Request Count by Resource, Operation & User", "type": "timeseries" }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Counts of in-flight (currently in progress) Porch API operations, broken down by resource, operation, and user.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisLabel": "", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 60 + }, + "id": 503, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(porch_in_flight_api_operations) by (resource, operation, user)", + "legendFormat": "{{resource}} / {{operation}} / {{user}}", + "refId": "A" + } + ], + "title": "All Resources \u2014 In-Flight Operation Count by Resource, Operation & User", + "type": "timeseries" + }, { "collapsed": false, "gridPos": { diff --git a/docs/content/en/docs/6_configuration_and_deployments/configurations/opentelemetry.md b/docs/content/en/docs/6_configuration_and_deployments/configurations/opentelemetry.md index 230641956..4d244a6a8 100644 --- a/docs/content/en/docs/6_configuration_and_deployments/configurations/opentelemetry.md +++ b/docs/content/en/docs/6_configuration_and_deployments/configurations/opentelemetry.md @@ -717,17 +717,43 @@ Load-test process metrics (`porch_perf_*`) are documented separately in [Perform | Metric Name | Type | Unit | Description | |-------------|------|------|-------------| -| `porch_api_call_duration_seconds` | Histogram | seconds | Duration of Porch API calls and related operations | +| `porch_api_call_duration_seconds` | Histogram | seconds | Duration of Porch API calls and related operations, especially package revision lifecycle-management operations | | Attribute | Description | |-----------|-------------| | `resource` | Resource kind (see table below) | -| `verb` | Operation name (see table below) | +| `verb` | HTTP-style operation name (see [table below](#resources-and-operations)) | | `api_version` | `v1alpha1` or `v1alpha2`. Omitted for `ExternalRepo` operations, which are shared infrastructure | +| `operation` | Porch-specific operation name (e.g. `ListPackageRevisions`, `ClonePackageRevision`, `UpdatePackageRevisionResources`, `Subpackage UpgradePackageRevision`) | +| `operation_outcome` | `error` if the operation returned an error; `success` otherwise | +| `lifecycle_after` | Lifecycle state of the affected package revision after the operation. Omitted for list operations and cases where no lifecycle exists (e.g. failed Create, successful Delete, or any Not Found error) | +| `namespace` | Kubernetes namespace of the package revision | +| `repository` | Name of the Porch repository containing the affected package revision | +| `package` | Path and name of the package | +| `workspace_name` | WorkspaceName of the package revision | + Histogram buckets start at 0.001 seconds and double for 16 boundaries (up to 32.768 seconds). -### Request Counts +### Request Counts - In Flight + +| Metric Name | Type | Unit | Description | +|-------------|------|------|-------------| +| `porch_in_flight_api_operations` | Up/Down Counter | — | Total number of requests in progress at a point in time. Incremented at the start of an operation and decremented at the end. | + +| Attribute | Description | +|-----------|-------------| +| `resource` | Resource kind (see table below) | +| `verb` | HTTP-style operation name (see [table below](#resources-and-operations)) | +| `api_version` | `v1alpha1` or `v1alpha2`. Omitted for `ExternalRepo` operations, which are shared infrastructure | +| `operation` | Porch-specific operation name (e.g. `ListPackageRevisions`, `ClonePackageRevision`, `UpdatePackageRevisionResources`, `Subpackage UpgradePackageRevision`) | +| `desired_lifecycle` | Expected lifecycle state of the affected package revision after the operation. Omitted for list operations and cases where no lifecycle is expected to exist (e.g. successful Delete) | +| `namespace` | Kubernetes namespace of the package revision | +| `repository` | Name of the Porch repository containing the affected package revision | +| `package` | Path and name of the package | +| `workspace_name` | WorkspaceName of the package revision | + +### Request Counts - Total | Metric Name | Type | Unit | Description | |-------------|------|------|-------------| @@ -745,10 +771,11 @@ Histogram buckets start at 0.001 seconds and double for 16 boundaries (up to 32. | `resource` | Component | `verb` / `op` values | `api_version` | |------------|-----------|----------------------|---------------| | `PackageRevision` | porch-server | `LIST`, `GET`, `CREATE`, `UPDATE`, `DELETE` | `v1alpha1` | -| `PackageRevision` | porch-controllers | `CREATE`, `UPDATE`, `DELETE` | `v1alpha2` | +| `PackageRevision` | porch-controllers | `GET`, `CREATE`, `UPDATE`, `DELETE` | `v1alpha2` | | `PackageRevisionResources` | porch-server | `LIST`, `GET`, `UPDATE` | `v1alpha1` | | `PackageRevisionResources` | porch-controllers | `UPDATE` | `v1alpha2` | | `PackageRevisionApproval` | porch-server | `GET`, `UPDATE` | `v1alpha1` | +| `PackageRevisionApproval` | porch-controllers | `UPDATE` | `v1alpha2` | | `ExternalRepo` | porch-server (git) | `FETCH`, `PUSH` | omitted | ### Package Size Metrics @@ -787,6 +814,10 @@ When using the Prometheus exporter, these are made available under the metric na - `porch_api_call_duration_seconds_count` - `porch_api_call_duration_seconds_sum` +**In-flight API operations** + +- `porch_in_flight_api_operations` + **Request counts** - `porch_api_requests_by_user_total` From c0bb38f0b190205eb86c8cb0bd01433fb4a1b7d4 Mon Sep 17 00:00:00 2001 From: James McDermott Date: Tue, 13 Oct 2026 17:48:56 +0100 Subject: [PATCH 2/4] change new dashboard panel to group by namespace instead of (non-emitted) user - nitpick in PackageRevision/PackageRevisions distinction in doc table Signed-off-by: James McDermott --- .../grafana-porch-server-dashboard.json | 10 +++++----- .../configurations/opentelemetry.md | 3 ++- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/deployments/metrics-resources/grafana-porch-server-dashboard.json b/deployments/metrics-resources/grafana-porch-server-dashboard.json index 099377800..159af1e40 100644 --- a/deployments/metrics-resources/grafana-porch-server-dashboard.json +++ b/deployments/metrics-resources/grafana-porch-server-dashboard.json @@ -1156,7 +1156,7 @@ "type": "prometheus", "uid": "prometheus" }, - "description": "Counts of in-flight (currently in progress) Porch API operations, broken down by resource, operation, and user.", + "description": "Counts of in-flight (currently in progress) Porch API operations, broken down by resource, operation, and namespace.", "fieldConfig": { "defaults": { "color": { @@ -1195,7 +1195,7 @@ "x": 0, "y": 60 }, - "id": 503, + "id": 504, "options": { "legend": { "displayMode": "list", @@ -1213,12 +1213,12 @@ "type": "prometheus", "uid": "prometheus" }, - "expr": "sum(porch_in_flight_api_operations) by (resource, operation, user)", - "legendFormat": "{{resource}} / {{operation}} / {{user}}", + "expr": "sum(porch_in_flight_api_operations) by (resource, operation, namespace)", + "legendFormat": "{{resource}} / {{operation}} / {{namespace}}", "refId": "A" } ], - "title": "All Resources \u2014 In-Flight Operation Count by Resource, Operation & User", + "title": "All Resources \u2014 In-Flight Operation Count by Resource, Operation & Namespace", "type": "timeseries" }, { diff --git a/docs/content/en/docs/6_configuration_and_deployments/configurations/opentelemetry.md b/docs/content/en/docs/6_configuration_and_deployments/configurations/opentelemetry.md index 4d244a6a8..1e77c7855 100644 --- a/docs/content/en/docs/6_configuration_and_deployments/configurations/opentelemetry.md +++ b/docs/content/en/docs/6_configuration_and_deployments/configurations/opentelemetry.md @@ -770,7 +770,8 @@ Histogram buckets start at 0.001 seconds and double for 16 boundaries (up to 32. | `resource` | Component | `verb` / `op` values | `api_version` | |------------|-----------|----------------------|---------------| -| `PackageRevision` | porch-server | `LIST`, `GET`, `CREATE`, `UPDATE`, `DELETE` | `v1alpha1` | +| `PackageRevision` | porch-server | `GET`, `CREATE`, `UPDATE`, `DELETE` | `v1alpha1` | +| `PackageRevisions` (plural form only for `ListPackageRevisions`) | porch-server | `LIST` | `v1alpha1` | | `PackageRevision` | porch-controllers | `GET`, `CREATE`, `UPDATE`, `DELETE` | `v1alpha2` | | `PackageRevisionResources` | porch-server | `LIST`, `GET`, `UPDATE` | `v1alpha1` | | `PackageRevisionResources` | porch-controllers | `UPDATE` | `v1alpha2` | From 6aa49cbf634118baed39970103638e4f228c4cdc Mon Sep 17 00:00:00 2001 From: James McDermott Date: Tue, 13 Oct 2026 18:29:20 +0100 Subject: [PATCH 3/4] Address Copilot comments Signed-off-by: James McDermott --- .../grafana-porch-server-dashboard.json | 10 +++++----- .../configurations/opentelemetry.md | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/deployments/metrics-resources/grafana-porch-server-dashboard.json b/deployments/metrics-resources/grafana-porch-server-dashboard.json index 159af1e40..5d79740cb 100644 --- a/deployments/metrics-resources/grafana-porch-server-dashboard.json +++ b/deployments/metrics-resources/grafana-porch-server-dashboard.json @@ -938,7 +938,7 @@ }, "id": 500, "panels": [], - "title": "Request Counts by User", + "title": "Request & Operation Counts", "type": "row" }, { @@ -1193,7 +1193,7 @@ "h": 8, "w": 24, "x": 0, - "y": 60 + "y": 68 }, "id": 504, "options": { @@ -1227,7 +1227,7 @@ "h": 1, "w": 24, "x": 0, - "y": 68 + "y": 76 }, "id": 600, "panels": [], @@ -1276,7 +1276,7 @@ "h": 8, "w": 12, "x": 0, - "y": 69 + "y": 77 }, "id": 601, "options": { @@ -1364,7 +1364,7 @@ "h": 8, "w": 12, "x": 12, - "y": 69 + "y": 77 }, "id": 602, "options": { diff --git a/docs/content/en/docs/6_configuration_and_deployments/configurations/opentelemetry.md b/docs/content/en/docs/6_configuration_and_deployments/configurations/opentelemetry.md index 1e77c7855..79cb50120 100644 --- a/docs/content/en/docs/6_configuration_and_deployments/configurations/opentelemetry.md +++ b/docs/content/en/docs/6_configuration_and_deployments/configurations/opentelemetry.md @@ -735,11 +735,11 @@ Load-test process metrics (`porch_perf_*`) are documented separately in [Perform Histogram buckets start at 0.001 seconds and double for 16 boundaries (up to 32.768 seconds). -### Request Counts - In Flight +### In-flight API Operation Counts | Metric Name | Type | Unit | Description | |-------------|------|------|-------------| -| `porch_in_flight_api_operations` | Up/Down Counter | — | Total number of requests in progress at a point in time. Incremented at the start of an operation and decremented at the end. | +| `porch_in_flight_api_operations` | Up/Down Counter | — | Total number of Porch API operations in progress at a point in time. Incremented at the start of an operation and decremented at the end. | | Attribute | Description | |-----------|-------------| From b0faff27c400b7225cd3931e2e9da76efb439f5e Mon Sep 17 00:00:00 2001 From: James McDermott Date: Fri, 16 Oct 2026 12:56:16 +0100 Subject: [PATCH 4/4] Address review comments Signed-off-by: James McDermott --- .../functionality/subpackage-operations.md | 2 +- .../configurations/opentelemetry.md | 58 +++++++------------ 2 files changed, 22 insertions(+), 38 deletions(-) diff --git a/docs/content/en/docs/5_architecture_and_components/controllers/packagerevision-controller/functionality/subpackage-operations.md b/docs/content/en/docs/5_architecture_and_components/controllers/packagerevision-controller/functionality/subpackage-operations.md index be2df3553..0ee368729 100644 --- a/docs/content/en/docs/5_architecture_and_components/controllers/packagerevision-controller/functionality/subpackage-operations.md +++ b/docs/content/en/docs/5_architecture_and_components/controllers/packagerevision-controller/functionality/subpackage-operations.md @@ -67,7 +67,7 @@ The controller: 6. Creates a draft, writes the result, and closes the draft 7. Writes `status.lastSubpackageOperationHash` -The same merge strategies available for regular package upgrades apply here. See [source-execution.md]({{< relref "source-execution.md" >}}) for strategy descriptions. +The same merge strategies available for regular package upgrades apply here. See [source-execution.md]({{% relref "source-execution.md" %}}) for strategy descriptions. ## Error Handling diff --git a/docs/content/en/docs/6_configuration_and_deployments/configurations/opentelemetry.md b/docs/content/en/docs/6_configuration_and_deployments/configurations/opentelemetry.md index 79cb50120..0cf95b6a3 100644 --- a/docs/content/en/docs/6_configuration_and_deployments/configurations/opentelemetry.md +++ b/docs/content/en/docs/6_configuration_and_deployments/configurations/opentelemetry.md @@ -713,47 +713,31 @@ Porch records the following metrics via OpenTelemetry. They are emitted by porch Load-test process metrics (`porch_perf_*`) are documented separately in [Performance Tests]({{% relref "/docs/12_contributing/code-contribution/performance-tests#performance-test-metrics" %}}). -### API Call Duration - -| Metric Name | Type | Unit | Description | -|-------------|------|------|-------------| -| `porch_api_call_duration_seconds` | Histogram | seconds | Duration of Porch API calls and related operations, especially package revision lifecycle-management operations | - -| Attribute | Description | -|-----------|-------------| -| `resource` | Resource kind (see table below) | -| `verb` | HTTP-style operation name (see [table below](#resources-and-operations)) | -| `api_version` | `v1alpha1` or `v1alpha2`. Omitted for `ExternalRepo` operations, which are shared infrastructure | -| `operation` | Porch-specific operation name (e.g. `ListPackageRevisions`, `ClonePackageRevision`, `UpdatePackageRevisionResources`, `Subpackage UpgradePackageRevision`) | -| `operation_outcome` | `error` if the operation returned an error; `success` otherwise | -| `lifecycle_after` | Lifecycle state of the affected package revision after the operation. Omitted for list operations and cases where no lifecycle exists (e.g. failed Create, successful Delete, or any Not Found error) | -| `namespace` | Kubernetes namespace of the package revision | -| `repository` | Name of the Porch repository containing the affected package revision | -| `package` | Path and name of the package | -| `workspace_name` | WorkspaceName of the package revision | +### Porch API Operations + +| Metric Name | Type | Unit | Description | +|-----------------------------------|-----------------|---------|-------------| +| `porch_api_call_duration_seconds` | Histogram | seconds | Duration of Porch API calls and related operations, especially package revision lifecycle-management operations | +| `porch_in_flight_api_operations` | Up/Down Counter | - | Total number of Porch API operations in progress at a point in time. Incremented at the start of an operation and decremented at the end. | + +| Attribute | Recorded for metric(s) | Description | +|---------------------|-----------------------------------|-------------| +| `resource` | both | Resource kind (see [table below](#resources-and-operations)) | +| `verb` | both | HTTP-style operation name (see [table below](#resources-and-operations)) | +| `api_version` | both | `v1alpha1` or `v1alpha2`. Omitted for `ExternalRepo` operations, which are shared infrastructure | +| `operation` | both | Porch-specific operation name (e.g. `ListPackageRevisions`, `ClonePackageRevision`, `UpdatePackageRevisionResources`, `Subpackage UpgradePackageRevision`) | +| `operation_outcome` | `porch_api_call_duration_seconds` | `error` if the operation returned an error; `success` otherwise | +| `lifecycle_after` | `porch_api_call_duration_seconds` | Lifecycle state of the affected package revision after the operation. Omitted for list operations and cases where no lifecycle exists (e.g. failed Create, successful Delete, or any Not Found error) | +| `desired_lifecycle` | `porch_in_flight_api_operations` | Expected lifecycle state of the affected package revision after the operation. Omitted for list operations and cases where no lifecycle is expected to exist (e.g. successful Delete) | +| `namespace` | both | Kubernetes namespace of the package revision | +| `repository` | both | Name of the Porch repository containing the affected package revision | +| `package` | both | Path and name of the package | +| `workspace_name` | both | WorkspaceName of the package revision | Histogram buckets start at 0.001 seconds and double for 16 boundaries (up to 32.768 seconds). -### In-flight API Operation Counts - -| Metric Name | Type | Unit | Description | -|-------------|------|------|-------------| -| `porch_in_flight_api_operations` | Up/Down Counter | — | Total number of Porch API operations in progress at a point in time. Incremented at the start of an operation and decremented at the end. | - -| Attribute | Description | -|-----------|-------------| -| `resource` | Resource kind (see table below) | -| `verb` | HTTP-style operation name (see [table below](#resources-and-operations)) | -| `api_version` | `v1alpha1` or `v1alpha2`. Omitted for `ExternalRepo` operations, which are shared infrastructure | -| `operation` | Porch-specific operation name (e.g. `ListPackageRevisions`, `ClonePackageRevision`, `UpdatePackageRevisionResources`, `Subpackage UpgradePackageRevision`) | -| `desired_lifecycle` | Expected lifecycle state of the affected package revision after the operation. Omitted for list operations and cases where no lifecycle is expected to exist (e.g. successful Delete) | -| `namespace` | Kubernetes namespace of the package revision | -| `repository` | Name of the Porch repository containing the affected package revision | -| `package` | Path and name of the package | -| `workspace_name` | WorkspaceName of the package revision | - -### Request Counts - Total +### Request Counts | Metric Name | Type | Unit | Description | |-------------|------|------|-------------|