[shiftstack] Bound soft-transient oc exec retries during pod polling

roles/shiftstack/README.md

@@ -16,7 +16,9 @@ Role for triggering Openshift on Openstack QA automation (installation and tests
 * `cifmw_shiftstack_hypervisor`: (*string*) The hypervisor where RHOSO is deployed. Defaults to `"{{ hostvars[hostvars['controller-0']['cifmw_hypervisor_host'] | default ('')]['ansible_host'] | default('') }}"`.
 * `cifmw_shiftstack_exec_delay`: (*integer*) Seconds between polling attempts when waiting for a command to complete in the pod. Defaults to `5`.
 * `cifmw_shiftstack_exec_retries`: (*integer*) Maximum polling attempts when waiting for a command to complete in the pod. With the default delay of 5s, 5760 retries gives an 8-hour timeout. Defaults to `5760`.
-* `cifmw_shiftstack_exec_transient_patterns`: (*list*) `oc exec` stderr patterns treated as transient during polling (matched as a case-insensitive regex alternation). Defaults to admission controller timeouts and short-lived connection errors only.
+* `cifmw_shiftstack_exec_transient_patterns`: (*list*) `oc exec` stderr patterns treated as hard-transient during polling (matched as a case-insensitive regex alternation). These retry for the full `cifmw_shiftstack_exec_retries` window. Defaults to admission controller timeouts and short-lived stream errors.
+* `cifmw_shiftstack_exec_soft_transient_patterns`: (*list*) `oc exec` stderr patterns treated as soft-transient (for example `connection refused`, `i/o timeout`). These only retry for `cifmw_shiftstack_exec_soft_transient_max` consecutive failures so a brief API VIP blip does not fail the job, while a dead API does not burn the full 8-hour poll window. Defaults to `connection refused` and `i/o timeout`.
+* `cifmw_shiftstack_exec_soft_transient_max`: (*integer*) Maximum consecutive soft-transient `oc exec` failures before failing the poll. With the default delay of 5s, `12` allows about 60s of API unavailability. Defaults to `12`.
 * `cifmw_shiftstack_exclude_artifacts_regex`: (*string*) Regex that will be passed on `oc rsync` command as `--exclude` param, so the role does not gather the artifacts matching it.
 * `cifmw_shiftstack_installation_dir`: (*string*) Directory to place installation files. Defaults to `"{{ cifmw_shiftstack_shiftstackclient_artifacts_dir }}/installation"`.
 * `cifmw_shiftstack_manifests_dir`: (*string*) Directory name for the role generated Openshift manifests. Defaults to `"{{ cifmw_shiftstack_basedir }}/manifests"`.

roles/shiftstack/defaults/main.yml

@@ -30,11 +30,18 @@ cifmw_shiftstack_client_pvc_manifest: "{{ cifmw_shiftstack_client_pod_name }}_pv
 cifmw_shiftstack_cluster_name: "ostest"
 cifmw_shiftstack_exec_delay: 5
 cifmw_shiftstack_exec_retries: 5760
+# Hard-transient: retry for the full poll window (API reachable, flaky admission/stream).
 cifmw_shiftstack_exec_transient_patterns:
   - SCCExecRestrictions
   - Internal error occurred
   - unable to upgrade connection
   - connection reset by peer
+# Soft-transient: short consecutive budget only. Avoids spinning ~8h when the API is dead,
+# while still riding through brief VIP/DNS blips (e.g. connection refused).
+cifmw_shiftstack_exec_soft_transient_patterns:
+  - connection refused
+  - i/o timeout
+cifmw_shiftstack_exec_soft_transient_max: 12
 cifmw_shiftstack_exclude_artifacts_regex: "openshift-install"
 cifmw_shiftstack_hypervisor: "{{ hostvars[hostvars['controller-0']['cifmw_hypervisor_host'] | default ('')]['ansible_host'] | default('') }}"
 cifmw_shiftstack_installation_dir: "{{ cifmw_shiftstack_basedir }}/installation"

roles/shiftstack/files/shiftstack_cmd_poll.sh

@@ -0,0 +1,114 @@
+#!/bin/bash
+# Copyright Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# Poll for a detached command running in a shiftstackclient pod.
+#
+# Soft-transient oc exec errors (connection refused / i/o timeout) use a
+# consecutive failure budget so brief API VIP blips do not fail the job, while
+# a dead API does not burn the full poll window. Hard-transient patterns retry
+# for the full max-retries window.
+#
+# Required environment:
+#   CIFMW_SHIFTSTACK_POLL_NAMESPACE
+#   CIFMW_SHIFTSTACK_POLL_POD
+#   CIFMW_SHIFTSTACK_POLL_MAX_RETRIES
+#   CIFMW_SHIFTSTACK_POLL_DELAY
+#   CIFMW_SHIFTSTACK_POLL_SOFT_MAX
+#   CIFMW_SHIFTSTACK_POLL_HARD_RE   (grep -E pattern, may be empty)
+#   CIFMW_SHIFTSTACK_POLL_SOFT_RE   (grep -E pattern, may be empty)
+#
+# Exit codes:
+#   0  - background command finished (stdout holds /tmp/cifmw_cmd_rc)
+#   2  - background process gone without writing a marker
+#   3  - soft-transient budget exhausted
+#  10  - still running after max retries
+# other - non-retryable oc exec failure
+#
+# On completion, prints CIFMW_SHIFTSTACK_POLL_ATTEMPTS=<n> on stderr.
+
+set -euo pipefail
+
+: "${CIFMW_SHIFTSTACK_POLL_NAMESPACE:?}"
+: "${CIFMW_SHIFTSTACK_POLL_POD:?}"
+: "${CIFMW_SHIFTSTACK_POLL_MAX_RETRIES:?}"
+: "${CIFMW_SHIFTSTACK_POLL_DELAY:?}"
+: "${CIFMW_SHIFTSTACK_POLL_SOFT_MAX:?}"
+: "${CIFMW_SHIFTSTACK_POLL_HARD_RE?}"
+: "${CIFMW_SHIFTSTACK_POLL_SOFT_RE?}"
+
+soft=0
+attempts=0
+max_retries="${CIFMW_SHIFTSTACK_POLL_MAX_RETRIES}"
+delay="${CIFMW_SHIFTSTACK_POLL_DELAY}"
+soft_max="${CIFMW_SHIFTSTACK_POLL_SOFT_MAX}"
+hard_re="${CIFMW_SHIFTSTACK_POLL_HARD_RE}"
+soft_re="${CIFMW_SHIFTSTACK_POLL_SOFT_RE}"
+ns="${CIFMW_SHIFTSTACK_POLL_NAMESPACE}"
+pod="${CIFMW_SHIFTSTACK_POLL_POD}"
+poll_cmd='if cat /tmp/cifmw_cmd_rc 2>/dev/null; then exit 0; elif [ -f /tmp/cifmw_cmd_pid ] && kill -0 $(cat /tmp/cifmw_cmd_pid) 2>/dev/null; then exit 10; else exit 2; fi'
+
+emit_attempts() {
+  echo "CIFMW_SHIFTSTACK_POLL_ATTEMPTS=${attempts}" >&2
+}
+
+while [ "${attempts}" -lt "${max_retries}" ]; do
+  attempts=$((attempts + 1))
+  stderr_file="$(mktemp)"
+  set +e
+  stdout="$(oc exec -n "${ns}" "${pod}" -- bash -c "${poll_cmd}" 2>"${stderr_file}")"
+  rc=$?
+  set -e
+  stderr="$(cat "${stderr_file}" 2>/dev/null || true)"
+  rm -f "${stderr_file}"
+
+  if [ "${rc}" -eq 0 ] || [ "${rc}" -eq 2 ]; then
+    printf '%s\n' "${stdout}"
+    printf '%s\n' "${stderr}" >&2
+    emit_attempts
+    exit "${rc}"
+  fi
+
+  if [ "${rc}" -eq 10 ]; then
+    soft=0
+    sleep "${delay}"
+    continue
+  fi
+
+  # oc exec itself failed (typically rc=1). Classify stderr.
+  if [ -n "${hard_re}" ] && printf '%s\n' "${stderr}" | grep -Eiq -- "${hard_re}"; then
+    soft=0
+    sleep "${delay}"
+    continue
+  fi
+
+  if [ -n "${soft_re}" ] && printf '%s\n' "${stderr}" | grep -Eiq -- "${soft_re}"; then
+    soft=$((soft + 1))
+    if [ "${soft}" -ge "${soft_max}" ]; then
+      printf '%s\n' "${stderr}" >&2
+      emit_attempts
+      exit 3
+    fi
+    sleep "${delay}"
+    continue
+  fi
+
+  printf '%s\n' "${stderr}" >&2
+  emit_attempts
+  exit "${rc}"
+done
+
+emit_attempts
+exit 10

roles/shiftstack/tasks/exec_command_in_pod.yml

@@ -54,34 +54,69 @@
           echo $! > /tmp/cifmw_cmd_pid && cat /tmp/cifmw_cmd_pid'
       register: pod_command_result
 
+    # Soft-transient oc exec errors use a consecutive failure budget in
+    # files/shiftstack_cmd_poll.sh so brief API VIP blips do not fail the job,
+    # while a dead API does not burn the full poll window. Hard-transient
+    # patterns still retry for the full cifmw_shiftstack_exec_retries window.
+    # Exit codes from the poller:
+    #   0  - background command finished (stdout holds /tmp/cifmw_cmd_rc)
+    #   2  - background process gone without writing a marker
+    #   3  - soft-transient budget exhausted
+    #  10  - still running after cifmw_shiftstack_exec_retries attempts
+    # other - non-retryable oc exec failure
     - name: Wait for the command to complete in the pod
       environment:
         PATH: "{{ cifmw_path }}"
         KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}"
+        CIFMW_SHIFTSTACK_POLL_NAMESPACE: "{{ namespace }}"
+        CIFMW_SHIFTSTACK_POLL_POD: "{{ pod_name }}"
+        CIFMW_SHIFTSTACK_POLL_MAX_RETRIES: "{{ cifmw_shiftstack_exec_retries }}"
+        CIFMW_SHIFTSTACK_POLL_DELAY: "{{ cifmw_shiftstack_exec_delay }}"
+        CIFMW_SHIFTSTACK_POLL_SOFT_MAX: "{{ cifmw_shiftstack_exec_soft_transient_max }}"
+        CIFMW_SHIFTSTACK_POLL_HARD_RE: "{{ cifmw_shiftstack_exec_transient_patterns | join('|') }}"
+        CIFMW_SHIFTSTACK_POLL_SOFT_RE: "{{ cifmw_shiftstack_exec_soft_transient_patterns | join('|') }}"
       ansible.builtin.command:
-        cmd: >-
-          oc exec -n {{ namespace }} {{ pod_name }} --
-          bash -c 'if cat /tmp/cifmw_cmd_rc 2>/dev/null; then exit 0;
-          elif [ -f /tmp/cifmw_cmd_pid ] && kill -0 $(cat /tmp/cifmw_cmd_pid) 2>/dev/null; then exit 10;
-          else exit 2; fi'
+        cmd: "{{ role_path }}/files/shiftstack_cmd_poll.sh"
       register: pod_exit_code
       changed_when: false
       failed_when: false
-      # Keep polling while the background command is running (rc=10) or while
-      # oc exec fails with transient API/admission errors during long-running
-      # shiftstack tests (for example SCCExecRestrictions webhook timeouts).
-      until: >-
-        pod_exit_code.rc == 0 or
-        pod_exit_code.rc == 2 or
-        (
-          pod_exit_code.rc != 10 and
-          not (
-            pod_exit_code.stderr | default('') is
-            search(cifmw_shiftstack_exec_transient_patterns | join('|'), ignorecase=True)
-          )
-        )
-      retries: "{{ cifmw_shiftstack_exec_retries }}"
-      delay: "{{ cifmw_shiftstack_exec_delay }}"
+
+    - name: Normalize poll result metadata
+      vars:
+        _attempt_line: >-
+          {{
+            pod_exit_code.stderr_lines | default([])
+            | select('match', '^CIFMW_SHIFTSTACK_POLL_ATTEMPTS=')
+            | list | first | default('CIFMW_SHIFTSTACK_POLL_ATTEMPTS=1')
+          }}
+        _stderr_without_attempts: >-
+          {{
+            pod_exit_code.stderr_lines | default([])
+            | reject('match', '^CIFMW_SHIFTSTACK_POLL_ATTEMPTS=')
+            | list | join('\n')
+          }}
+      ansible.builtin.set_fact:
+        pod_exit_code: >-
+          {{
+            pod_exit_code | combine({
+              'attempts': (_attempt_line | regex_replace('^CIFMW_SHIFTSTACK_POLL_ATTEMPTS=', '') | int),
+              'stderr': _stderr_without_attempts,
+              'stderr_lines': (
+                _stderr_without_attempts | split('\n')
+                if (_stderr_without_attempts | length) > 0 else []
+              )
+            })
+          }}
+
+    - name: Fail if soft-transient API errors exhausted
+      ansible.builtin.fail:
+        msg: >-
+          Lost connection to pod {{ pod_name }} after
+          {{ cifmw_shiftstack_exec_soft_transient_max }} consecutive soft-transient
+          oc exec errors (for example connection refused or i/o timeout).
+          Failed after {{ pod_exit_code.attempts | default(1) }} polling attempt(s).
+          oc exec stderr: {{ pod_exit_code.stderr | default('(none)') | trim }}
+      when: pod_exit_code.rc == 3
 
     - name: Fail if the pod is no longer reachable
       ansible.builtin.fail:
@@ -93,6 +128,7 @@
       when:
         - pod_exit_code.rc != 0
         - pod_exit_code.rc != 2
+        - pod_exit_code.rc != 3
         - pod_exit_code.rc != 10
 
     - name: Fail if the background process died without writing the marker