diff --git a/deployments/metrics-resources/grafana-porch-server-dashboard.json b/deployments/metrics-resources/grafana-porch-server-dashboard.json index 139ae542f..5d79740cb 100644 --- a/deployments/metrics-resources/grafana-porch-server-dashboard.json +++ b/deployments/metrics-resources/grafana-porch-server-dashboard.json @@ -938,7 +938,7 @@ }, "id": 500, "panels": [], - "title": "Request Counts by User", + "title": "Request & Operation Counts", "type": "row" }, { @@ -1151,13 +1151,83 @@ "title": "All Resources \u2014 Total Request Count by Resource, Operation & User", "type": "timeseries" }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Counts of in-flight (currently in progress) Porch API operations, broken down by resource, operation, and namespace.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisLabel": "", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 68 + }, + "id": 504, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(porch_in_flight_api_operations) by (resource, operation, namespace)", + "legendFormat": "{{resource}} / {{operation}} / {{namespace}}", + "refId": "A" + } + ], + "title": "All Resources \u2014 In-Flight Operation Count by Resource, Operation & Namespace", + "type": "timeseries" + }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 68 + "y": 76 }, "id": 600, "panels": [], @@ -1206,7 +1276,7 @@ "h": 8, "w": 12, "x": 0, - "y": 69 + "y": 77 }, "id": 601, "options": { @@ -1294,7 +1364,7 @@ "h": 8, "w": 12, "x": 12, - "y": 69 + "y": 77 }, "id": 602, "options": { diff --git a/docs/content/en/docs/5_architecture_and_components/controllers/packagerevision-controller/functionality/subpackage-operations.md b/docs/content/en/docs/5_architecture_and_components/controllers/packagerevision-controller/functionality/subpackage-operations.md index be2df3553..0ee368729 100644 --- a/docs/content/en/docs/5_architecture_and_components/controllers/packagerevision-controller/functionality/subpackage-operations.md +++ b/docs/content/en/docs/5_architecture_and_components/controllers/packagerevision-controller/functionality/subpackage-operations.md @@ -67,7 +67,7 @@ The controller: 6. Creates a draft, writes the result, and closes the draft 7. Writes `status.lastSubpackageOperationHash` -The same merge strategies available for regular package upgrades apply here. See [source-execution.md]({{< relref "source-execution.md" >}}) for strategy descriptions. +The same merge strategies available for regular package upgrades apply here. See [source-execution.md]({{% relref "source-execution.md" %}}) for strategy descriptions. ## Error Handling diff --git a/docs/content/en/docs/6_configuration_and_deployments/configurations/opentelemetry.md b/docs/content/en/docs/6_configuration_and_deployments/configurations/opentelemetry.md index 1142ca24c..3a0ba778d 100644 --- a/docs/content/en/docs/6_configuration_and_deployments/configurations/opentelemetry.md +++ b/docs/content/en/docs/6_configuration_and_deployments/configurations/opentelemetry.md @@ -713,17 +713,27 @@ Porch records the following metrics via OpenTelemetry. They are emitted by porch Load-test process metrics (`porch_perf_*`) are documented separately in [Performance Tests]({{% relref "/docs/12_contributing/code-contribution/performance-tests#performance-test-metrics" %}}). -### API Call Duration +### Porch API Operations + +| Metric Name | Type | Unit | Description | +|-----------------------------------|-----------------|---------|-------------| +| `porch_api_call_duration_seconds` | Histogram | seconds | Duration of Porch API calls and related operations, especially package revision lifecycle-management operations | +| `porch_in_flight_api_operations` | Up/Down Counter | - | Total number of Porch API operations in progress at a point in time. Incremented at the start of an operation and decremented at the end. | + +| Attribute | Recorded for metric(s) | Description | +|---------------------|-----------------------------------|-------------| +| `resource` | both | Resource kind (see [table below](#resources-and-operations)) | +| `verb` | both | HTTP-style operation name (see [table below](#resources-and-operations)) | +| `api_version` | both | `v1alpha1` or `v1alpha2`. Omitted for `ExternalRepo` operations, which are shared infrastructure | +| `operation` | both | Porch-specific operation name (e.g. `ListPackageRevisions`, `ClonePackageRevision`, `UpdatePackageRevisionResources`, `Subpackage UpgradePackageRevision`) | +| `operation_outcome` | `porch_api_call_duration_seconds` | `error` if the operation returned an error; `success` otherwise | +| `lifecycle_after` | `porch_api_call_duration_seconds` | Lifecycle state of the affected package revision after the operation. Omitted for list operations and cases where no lifecycle exists (e.g. failed Create, successful Delete, or any Not Found error) | +| `desired_lifecycle` | `porch_in_flight_api_operations` | Expected lifecycle state of the affected package revision after the operation. Omitted for list operations and cases where no lifecycle is expected to exist (e.g. successful Delete) | +| `namespace` | both | Kubernetes namespace of the package revision | +| `repository` | both | Name of the Porch repository containing the affected package revision | +| `package` | both | Path and name of the package | +| `workspace_name` | both | WorkspaceName of the package revision | -| Metric Name | Type | Unit | Description | -|-------------|------|------|-------------| -| `porch_api_call_duration_seconds` | Histogram | seconds | Duration of Porch API calls and related operations | - -| Attribute | Description | -|-----------|-------------| -| `resource` | Resource kind (see table below) | -| `verb` | Operation name (see table below) | -| `api_version` | `v1alpha1` or `v1alpha2`. Omitted for `ExternalRepo` operations, which are shared infrastructure | Histogram buckets start at 0.001 seconds and double for 16 boundaries (up to 32.768 seconds). @@ -744,11 +754,13 @@ Histogram buckets start at 0.001 seconds and double for 16 boundaries (up to 32. | `resource` | Component | `verb` / `op` values | `api_version` | |------------|-----------|----------------------|---------------| -| `PackageRevision` | porch-server | `LIST`, `GET`, `CREATE`, `UPDATE`, `DELETE` | `v1alpha1` | -| `PackageRevision` | porch-controllers | `CREATE`, `UPDATE`, `DELETE` | `v1alpha2` | +| `PackageRevision` | porch-server | `GET`, `CREATE`, `UPDATE`, `DELETE` | `v1alpha1` | +| `PackageRevisions` (plural form only for `ListPackageRevisions`) | porch-server | `LIST` | `v1alpha1` | +| `PackageRevision` | porch-controllers | `GET`, `CREATE`, `UPDATE`, `DELETE` | `v1alpha2` | | `PackageRevisionResources` | porch-server | `LIST`, `GET`, `UPDATE` | `v1alpha1` | | `PackageRevisionResources` | porch-controllers | `UPDATE` | `v1alpha2` | | `PackageRevisionApproval` | porch-server | `GET`, `UPDATE` | `v1alpha1` | +| `PackageRevisionApproval` | porch-controllers | `UPDATE` | `v1alpha2` | | `ExternalRepo` | porch-server (git) | `FETCH`, `PUSH` | omitted | ### Package Size Metrics @@ -813,6 +825,10 @@ When using the Prometheus exporter, these are made available under the metric na - `porch_api_call_duration_seconds_count` - `porch_api_call_duration_seconds_sum` +**In-flight API operations** + +- `porch_in_flight_api_operations` + **Request counts** - `porch_api_requests_by_user_total`