USHIFT-7407: Wire prometheus receiver scrape.d drop-ins for metrics exporters

assets/optional/observability/02-cluster-role.yaml

@@ -7,10 +7,11 @@ rules:
   - apiGroups: [""]
     resources: ["nodes/stats"]
     verbs: ["get"]
-  # OpenTelemetry Component: k8s_event receiver
+  # OpenTelemetry Component: k8s_event receiver, prometheus receiver (endpoints SD)
   - apiGroups:
     - ""
     resources:
+    - endpoints
     - events
     - namespaces
     - namespaces/status
@@ -63,4 +64,9 @@ rules:
     verbs:
       - get
       - list
-      - watch
+      - watch
+  # OpenTelemetry Component: prometheus receiver (kube-rbac-proxy authorization)
+  - nonResourceURLs:
+      - /metrics
+    verbs:
+      - get

packaging/observability/opentelemetry-collector-large.yaml

@@ -4,6 +4,7 @@
 # - Host's CPU, memory, disk, and network usage metrics
 # - System journals for selected MicroShift services and dependencies, priority < Info
 # - Metrics exposed by Pods that have "prometheus.io/scrape": "true" annotation
+# - Metrics from optional component scrape configs (scrape.d drop-ins)
 
 receivers:
   kubeletstats:
@@ -40,6 +41,8 @@ receivers:
     priority: info
   prometheus:
     config:
+      scrape_config_files:
+        - /etc/microshift/observability/scrape.d/*.yaml
       scrape_configs:
         - job_name: k8s
           scrape_interval: 10s
@@ -109,7 +112,7 @@ service:
       receivers: [ journald ]
       processors: [ resourcedetection/system ]
       exporters: [ otlp ]
-    metrics/pods:
+    metrics/scrape:
       receivers: [ prometheus ]
       processors: [ batch ]
       exporters: [ otlp ]

packaging/observability/opentelemetry-collector-medium.yaml

@@ -2,8 +2,13 @@
 # - Resource usage metrics of MicroShift's Containers, Pods, Volumes, and Node
 # - Kubernetes Events
 # - System journal for MicroShift service, priority < Info
+# - Metrics from optional component scrape configs (scrape.d drop-ins)
 
 receivers:
+  prometheus:
+    config:
+      scrape_config_files:
+        - /etc/microshift/observability/scrape.d/*.yaml
   kubeletstats:
     auth_type: tls
     ca_file: /var/lib/microshift/certs/ca-bundle/client-ca.crt
@@ -48,6 +53,10 @@ extensions:
 service:
   extensions: [ file_storage ]
   pipelines:
+    metrics/scrape:
+      receivers: [ prometheus ]
+      processors: [ batch ]
+      exporters: [ otlp ]
     metrics/kubeletstats:
       receivers: [ kubeletstats ]
       processors: [ batch ]

packaging/observability/opentelemetry-collector-small.yaml

@@ -1,8 +1,13 @@
 # Following file contains OpenTelemetry Collector configuration that exports:
 # - Resource usage metrics of MicroShift's Containers, Pods, Volumes, and Node
 # - Kubernetes Events
+# - Metrics from optional component scrape configs (scrape.d drop-ins)
 
 receivers:
+  prometheus:
+    config:
+      scrape_config_files:
+        - /etc/microshift/observability/scrape.d/*.yaml
   kubeletstats:
     auth_type: tls
     ca_file: /var/lib/microshift/certs/ca-bundle/client-ca.crt
@@ -43,6 +48,10 @@ extensions:
 service:
   extensions: [ file_storage ]
   pipelines:
+    metrics/scrape:
+      receivers: [ prometheus ]
+      processors: [ batch ]
+      exporters: [ otlp ]
     metrics/kubeletstats:
       receivers: [ kubeletstats ]
       processors: [ batch ]

packaging/observability/scrape.d/kube-state-metrics.yaml

@@ -0,0 +1,37 @@
+scrape_configs:
+  - job_name: kube-state-metrics
+    scrape_interval: 10s
+    scheme: https
+    tls_config:
+      ca_file: /var/lib/microshift/certs/service-ca/ca.crt
+      cert_file: /var/lib/microshift/certs/admin-kubeconfig-signer/openshift-observability-client/client.crt
+      key_file: /var/lib/microshift/certs/admin-kubeconfig-signer/openshift-observability-client/client.key
+      server_name: kube-state-metrics.openshift-monitoring.svc
+    kubernetes_sd_configs:
+      - kubeconfig_file: /var/lib/microshift/resources/observability-client/kubeconfig
+        role: endpoints
+        namespaces:
+          names:
+            - openshift-monitoring
+    relabel_configs:
+      - source_labels: [__meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]
+        action: keep
+        regex: kube-state-metrics;https-main
+  - job_name: kube-state-metrics-self
+    scrape_interval: 10s
+    scheme: https
+    tls_config:
+      ca_file: /var/lib/microshift/certs/service-ca/ca.crt
+      cert_file: /var/lib/microshift/certs/admin-kubeconfig-signer/openshift-observability-client/client.crt
+      key_file: /var/lib/microshift/certs/admin-kubeconfig-signer/openshift-observability-client/client.key
+      server_name: kube-state-metrics.openshift-monitoring.svc
+    kubernetes_sd_configs:
+      - kubeconfig_file: /var/lib/microshift/resources/observability-client/kubeconfig
+        role: endpoints
+        namespaces:
+          names:
+            - openshift-monitoring
+    relabel_configs:
+      - source_labels: [__meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]
+        action: keep
+        regex: kube-state-metrics;https-self

packaging/observability/scrape.d/metrics-server.yaml

@@ -0,0 +1,19 @@
+scrape_configs:
+  - job_name: metrics-server
+    scrape_interval: 10s
+    scheme: https
+    tls_config:
+      ca_file: /var/lib/microshift/certs/service-ca/ca.crt
+      cert_file: /var/lib/microshift/certs/admin-kubeconfig-signer/openshift-observability-client/client.crt
+      key_file: /var/lib/microshift/certs/admin-kubeconfig-signer/openshift-observability-client/client.key
+      server_name: metrics-server.openshift-monitoring.svc
+    kubernetes_sd_configs:
+      - kubeconfig_file: /var/lib/microshift/resources/observability-client/kubeconfig
+        role: endpoints
+        namespaces:
+          names:
+            - openshift-monitoring
+    relabel_configs:
+      - source_labels: [__meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]
+        action: keep
+        regex: metrics-server;https

packaging/observability/scrape.d/node-exporter.yaml

@@ -0,0 +1,19 @@
+scrape_configs:
+  - job_name: node-exporter
+    scrape_interval: 10s
+    scheme: https
+    tls_config:
+      ca_file: /var/lib/microshift/certs/service-ca/ca.crt
+      cert_file: /var/lib/microshift/certs/admin-kubeconfig-signer/openshift-observability-client/client.crt
+      key_file: /var/lib/microshift/certs/admin-kubeconfig-signer/openshift-observability-client/client.key
+      server_name: node-exporter.openshift-monitoring.svc
+    kubernetes_sd_configs:
+      - kubeconfig_file: /var/lib/microshift/resources/observability-client/kubeconfig
+        role: endpoints
+        namespaces:
+          names:
+            - openshift-monitoring
+    relabel_configs:
+      - source_labels: [__meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]
+        action: keep
+        regex: node-exporter;https

packaging/rpm/microshift.spec

@@ -303,6 +303,7 @@ Requires: microshift-release-info = %{version}
 The microshift-metrics-node-exporter-release-info package provides release information files for this
 release. These files contain the list of container image references used by node-exporter
 and can be used to embed those images into osbuilder blueprints or bootc containerfiles.
+
 %package metrics-kube-state
 Summary: Kubernetes kube-state-metrics for MicroShift
 ExclusiveArch: x86_64 aarch64
@@ -623,6 +624,7 @@ install -p -m644 assets/optional/ai-model-serving/release-ai-model-serving-x86_6
 
 # observability
 install -d -m755 %{buildroot}/%{_sysconfdir}/microshift/observability
+install -d -m755 %{buildroot}/%{_sysconfdir}/microshift/observability/scrape.d
 install -p -m644 packaging/observability/*.yaml -D %{buildroot}%{_sysconfdir}/microshift/observability/
 # Explicit copy of large config as default. Not using symlink to avoid accidental package upgrade overwriting user config if the user edits the config without copying (i.e. edits the target of symlink).
 install -p -m644 packaging/observability/opentelemetry-collector-large.yaml -D %{buildroot}%{_sysconfdir}/microshift/observability/opentelemetry-collector.yaml
@@ -681,6 +683,9 @@ cat assets/optional/metrics-server/kustomization.aarch64.yaml >> %{buildroot}/%{
 cat assets/optional/metrics-server/kustomization.x86_64.yaml >> %{buildroot}/%{_prefix}/lib/microshift/manifests.d/080-microshift-metrics-server/kustomization.yaml
 %endif
 
+install -d -m755 %{buildroot}/%{_sysconfdir}/microshift/observability/scrape.d
+install -p -m644 packaging/observability/scrape.d/metrics-server.yaml %{buildroot}/%{_sysconfdir}/microshift/observability/scrape.d/
+
 # metrics-server-release-info
 mkdir -p -m755 %{buildroot}%{_datadir}/microshift/release
 install -p -m644 assets/optional/metrics-server/release-metrics-server-{x86_64,aarch64}.json %{buildroot}%{_datadir}/microshift/release/
@@ -705,6 +710,9 @@ cat assets/optional/node-exporter/kustomization.aarch64.yaml >> %{buildroot}/%{_
 cat assets/optional/node-exporter/kustomization.x86_64.yaml >> %{buildroot}/%{_prefix}/lib/microshift/manifests.d/082-microshift-node-exporter/kustomization.yaml
 %endif
 
+install -d -m755 %{buildroot}/%{_sysconfdir}/microshift/observability/scrape.d
+install -p -m644 packaging/observability/scrape.d/node-exporter.yaml %{buildroot}/%{_sysconfdir}/microshift/observability/scrape.d/
+
 # node-exporter-release-info
 mkdir -p -m755 %{buildroot}%{_datadir}/microshift/release
 install -p -m644 assets/optional/node-exporter/release-node-exporter-{x86_64,aarch64}.json %{buildroot}%{_datadir}/microshift/release/
@@ -728,6 +736,9 @@ cat assets/optional/kube-state-metrics/kustomization.aarch64.yaml >> %{buildroot
 cat assets/optional/kube-state-metrics/kustomization.x86_64.yaml >> %{buildroot}/%{_prefix}/lib/microshift/manifests.d/081-microshift-kube-state-metrics/kustomization.yaml
 %endif
 
+install -d -m755 %{buildroot}/%{_sysconfdir}/microshift/observability/scrape.d
+install -p -m644 packaging/observability/scrape.d/kube-state-metrics.yaml %{buildroot}/%{_sysconfdir}/microshift/observability/scrape.d/
+
 # kube-state-metrics-release-info
 mkdir -p -m755 %{buildroot}%{_datadir}/microshift/release
 install -p -m644 assets/optional/kube-state-metrics/release-kube-state-metrics-{x86_64,aarch64}.json %{buildroot}%{_datadir}/microshift/release/
@@ -923,6 +934,7 @@ fi
 %files observability
 %dir %{_prefix}/lib/microshift/manifests.d/003-microshift-observability
 %dir %{_sysconfdir}/microshift/observability/
+%dir %{_sysconfdir}/microshift/observability/scrape.d/
 %{_unitdir}/microshift-observability.service
 %config(noreplace) %{_sysconfdir}/microshift/observability/opentelemetry-collector.yaml
 %{_sysconfdir}/microshift/observability/opentelemetry-collector-*.yaml
@@ -938,19 +950,29 @@ fi
 %files metrics-server
 %dir %{_prefix}/lib/microshift/manifests.d/080-microshift-metrics-server
 %{_prefix}/lib/microshift/manifests.d/080-microshift-metrics-server/*
+%dir %{_sysconfdir}/microshift/observability
+%dir %{_sysconfdir}/microshift/observability/scrape.d
+%config(noreplace) %{_sysconfdir}/microshift/observability/scrape.d/metrics-server.yaml
 
 %files metrics-server-release-info
 %{_datadir}/microshift/release/release-metrics-server-{x86_64,aarch64}.json
 
 %files metrics-node-exporter
 %dir %{_prefix}/lib/microshift/manifests.d/082-microshift-node-exporter
 %{_prefix}/lib/microshift/manifests.d/082-microshift-node-exporter/*
+%dir %{_sysconfdir}/microshift/observability
+%dir %{_sysconfdir}/microshift/observability/scrape.d
+%config(noreplace) %{_sysconfdir}/microshift/observability/scrape.d/node-exporter.yaml
 
 %files metrics-node-exporter-release-info
 %{_datadir}/microshift/release/release-node-exporter-{x86_64,aarch64}.json
+
 %files metrics-kube-state
 %dir %{_prefix}/lib/microshift/manifests.d/081-microshift-kube-state-metrics
 %{_prefix}/lib/microshift/manifests.d/081-microshift-kube-state-metrics/*
+%dir %{_sysconfdir}/microshift/observability
+%dir %{_sysconfdir}/microshift/observability/scrape.d
+%config(noreplace) %{_sysconfdir}/microshift/observability/scrape.d/kube-state-metrics.yaml
 
 %files metrics-kube-state-release-info
 %{_datadir}/microshift/release/release-kube-state-metrics-{x86_64,aarch64}.json

test/assets/observability/otel_config.yaml

@@ -1,5 +1,9 @@
 # This OTEL config is only for testing purposes
 receivers:
+  prometheus:
+    config:
+      scrape_config_files:
+        - /etc/microshift/observability/scrape.d/*.yaml
   hostmetrics:
     root_path: /
     collection_interval: 10s
@@ -67,6 +71,10 @@ extensions:
 service:
   extensions: [ file_storage ]
   pipelines:
+    metrics/scrape:
+      receivers: [ prometheus ]
+      processors: [ resourcedetection/system ]
+      exporters: [ prometheus, prometheusremotewrite ]
     metrics/hostmetrics:
       receivers: [ hostmetrics ]
       processors: [ resourcedetection/system ]

test/suites/optional/metrics.robot

@@ -109,14 +109,10 @@ Scrape Metrics From
     ...    with mTLS client certs. Resolves the IP from the Endpoints object
     ...    to validate that the headless Service selector matches the pod.
     [Arguments]    ${metrics_service}    ${port}
-    ${ep_ip}=    Oc Get JsonPath
-    ...    endpoints
-    ...    ${METRICS_NS}
-    ...    ${metrics_service}
-    ...    .subsets[0].addresses[0].ip
-    ${ep_ip}=    Get Regexp Matches    ${ep_ip}    \\d+\\.\\d+\\.\\d+\\.\\d+
-    Length Should Be    ${ep_ip}    1
-    VAR    ${ep_ip}=    ${ep_ip}[0]
+    # Oc Get JsonPath uses Run With Kubeconfig which merges stderr into stdout;
+    # the v1 Endpoints deprecation warning from K8s 1.33+ pollutes the IP value.
+    ${ep_ip}=    Command Should Work
+    ...    oc get endpoints -n ${METRICS_NS} ${metrics_service} -o jsonpath\='{.subsets[0].addresses[0].ip}' --kubeconfig\=/var/lib/microshift/resources/kubeadmin/kubeconfig
     VAR    ${svc_host}=    ${metrics_service}.${METRICS_NS}.svc
     ${stdout}=    Command Should Work
     ...    curl -s --resolve ${svc_host}:${port}:${ep_ip} --cacert ${SERVICE_CA} --cert ${CLIENT_CERT} --key ${CLIENT_KEY} https://${svc_host}:${port}/metrics

test/suites/optional/observability.robot

@@ -57,6 +57,30 @@ Kube Events Logs Are Exported
     Wait Until Keyword Succeeds    10x    5s
     ...    Check Loki Query    ${LOKI_HOST}    ${LOKI_PORT}    {service_name="kube_events"}
 
+KSM Metrics Are Exported Via Scrape Drop-In
+    [Documentation]    The prometheus receiver should scrape kube-state-metrics via the
+    ...    scrape.d drop-in and export kube_node_info to the prometheus exporter.
+
+    VAR    ${METRIC}    kube_node_info    scope=TEST
+    Check Prometheus Query    ${PROMETHEUS_HOST}    ${PROMETHEUS_PORT}    ${METRIC}
+    Check Prometheus Exporter    ${USHIFT_HOST}    ${PROM_EXPORTER_PORT}    ${METRIC}
+
+Node Exporter Metrics Are Exported Via Scrape Drop-In
+    [Documentation]    The prometheus receiver should scrape node-exporter via the
+    ...    scrape.d drop-in and export node_cpu_seconds_total to the prometheus exporter.
+
+    VAR    ${METRIC}    node_cpu_seconds_total    scope=TEST
+    Check Prometheus Query    ${PROMETHEUS_HOST}    ${PROMETHEUS_PORT}    ${METRIC}
+    Check Prometheus Exporter    ${USHIFT_HOST}    ${PROM_EXPORTER_PORT}    ${METRIC}
+
+Metrics Server Metrics Are Exported Via Scrape Drop-In
+    [Documentation]    The prometheus receiver should scrape metrics-server via the
+    ...    scrape.d drop-in and export metrics to the prometheus exporter.
+
+    VAR    ${METRIC}    metrics_server_kubelet_request_total    scope=TEST
+    Check Prometheus Query    ${PROMETHEUS_HOST}    ${PROMETHEUS_PORT}    ${METRIC}
+    Check Prometheus Exporter    ${USHIFT_HOST}    ${PROM_EXPORTER_PORT}    ${METRIC}
+
 Logs Should Not Contain Receiver Errors
     [Documentation]    Internal receiver errors are not treated as fatal. Typically these are due to a misconfiguration
     ...    and thus indicate the provided default config should be reviewed.
@@ -71,7 +95,11 @@ Setup Suite And Prepare Test Host
     [Documentation]    The service starts after MicroShift starts and thus will start generating pertinent log data
     ...    right away. When the suite is executed, immediately get the cursor for the microshift-observability unit.
     Setup Suite
-    Setup MicroShift With Optionals    003-microshift-observability
+    Setup MicroShift With Optionals
+    ...    003-microshift-observability
+    ...    080-microshift-metrics-server
+    ...    081-microshift-kube-state-metrics
+    ...    082-microshift-node-exporter
     ${ns}    Create Unique Namespace
     VAR    ${NAMESPACE}    ${ns}    scope=SUITE
     Configure Firewall And Observability