Merge pull request #42 from shiftstack/fix/osp-verification-bogus-lb-hygiene

osp_verification: purge bogus Machines and harden LB suite cleanup

Author: openshift-merge-bot[bot]

collection/stages/roles/lb_tests/tasks/cleanup_lb_test_leftovers.yml

@@ -67,3 +67,23 @@
     OS_CLOUD: "{{ user_cloud }}"
   changed_when: false
   failed_when: false
+
+# Orphan lb-sg-* Neutron SGs left on node ports trip ProviderSpec SG matching
+# when lb_tests runs before openstack_test (seen on 4.21 warm re-runs).
+- name: Best-effort remove orphan lb-sg-* security group attachments and SGs
+  ansible.builtin.shell: |
+    set -o pipefail
+    openstack security group list -f value -c ID -c Name 2>/dev/null | while read -r sg_id sg_name; do
+      case "$sg_name" in
+        lb-sg-*)
+          for port_id in $(openstack port list --security-group "$sg_id" -f value -c ID 2>/dev/null); do
+            openstack port unset --security-group "$sg_id" "$port_id" 2>/dev/null || true
+          done
+          openstack security group delete "$sg_id" 2>/dev/null || true
+          ;;
+      esac
+    done
+  environment:
+    OS_CLOUD: "{{ user_cloud }}"
+  changed_when: false
+  failed_when: false

collection/stages/roles/lb_tests/tasks/main.yml

@@ -101,3 +101,7 @@
     openstack_test_ote_run_serial: true
     openstack_testsuite_name: openstack_tests_lb_ovn
     openstack_reset_result_dir: no # As we want to keep the logs generated in the previous step
+
+- name: Best-effort cleanup after LB tests (before openstack_test)
+  ansible.builtin.include_tasks: cleanup_lb_test_leftovers.yml
+  when: lb_tests_guest_cleanup | bool

collection/stages/roles/openstack_test/files/ote_resolve_results.py

@@ -10,7 +10,7 @@
 or in ginkgo summary lines (``SUCCESS!`` / ``FAIL!``).
 
 Usage:
-  ote_resolve_results.py count <log> passed|failed|skipped
+  ote_resolve_results.py count <log> passed|failed|skipped|unknown
   ote_resolve_results.py junit <log> <junit_xml_path>
 """
 
@@ -23,26 +23,47 @@
 from typing import Any
 
 
+def _is_ote_shaped(entry: Any) -> bool:
+    return isinstance(entry, dict) and "result" in entry
+
+
 def _load_outer_results(raw: str) -> list[dict[str, Any]]:
     raw = raw.strip()
     if not raw:
         return []
 
-    # Skip leading klog / noise before the JSON array/object.
-    start_candidates = [i for i, ch in enumerate(raw) if ch in "[{"]
-    for start in start_candidates:
-        chunk = raw[start:]
+    # Scan top-level JSON values. Skip empty lists / non-OTE-shaped JSON
+    # (e.g. bare ``[]`` from ginkgo text like ``map[]``) so we reach the real
+    # result array(s) at the end of the log. Advance past each decoded value
+    # to avoid re-parsing nested JSON inside ``output`` fields. Serial
+    # run-test appends multiple arrays — collect all OTE-shaped entries.
+    results: list[dict[str, Any]] = []
+    decoder = json.JSONDecoder()
+    i = 0
+    n = len(raw)
+    while i < n:
+        while i < n and raw[i] not in "[{":
+            i += 1
+        if i >= n:
+            break
         try:
-            data, _ = json.JSONDecoder().raw_decode(chunk)
+            data, end = decoder.raw_decode(raw[i:])
         except json.JSONDecodeError:
+            i += 1
             continue
+        i = i + end
         if isinstance(data, list):
-            return [r for r in data if isinstance(r, dict)]
-        if isinstance(data, dict):
-            return [data]
+            ote = [r for r in data if _is_ote_shaped(r)]
+            if ote:
+                results.extend(ote)
+            continue
+        if _is_ote_shaped(data):
+            results.append(data)
+
+    if results:
+        return results
 
     # NDJSON fallback
-    results: list[dict[str, Any]] = []
     for line in raw.splitlines():
         line = line.strip()
         if not line or not line.startswith("{"):
@@ -51,7 +72,7 @@ def _load_outer_results(raw: str) -> list[dict[str, Any]]:
             obj = json.loads(line)
         except json.JSONDecodeError:
             continue
-        if isinstance(obj, dict):
+        if _is_ote_shaped(obj):
             results.append(obj)
     return results
 
@@ -206,7 +227,7 @@ def main(argv: list[str]) -> int:
     cmd = argv[1]
     if cmd == "count":
         if len(argv) != 4:
-            print("usage: count <log> passed|failed|skipped", file=sys.stderr)
+            print("usage: count <log> passed|failed|skipped|unknown", file=sys.stderr)
             return 2
         print(cmd_count(argv[2], argv[3]))
         return 0

collection/stages/roles/openstack_test/tasks/cleanup_bogus_machines.yml

@@ -0,0 +1,90 @@
+---
+# Purge leftover bogus-* MachineSets/Machines before openstack_test.
+# Pairs with openstack-test bz_2073398 cleanup; suite-level hygiene if that It left
+# stuck objects that poison later Machine / MachineSet / ProviderSpec specs.
+# After delete, wait until gone; fail if leftovers remain so openstack_test does not start dirty.
+- name: List bogus MachineSets in openshift-machine-api
+  ansible.builtin.shell: |
+    set -o pipefail
+    oc get machineset -n openshift-machine-api -o name 2>/dev/null \
+      | grep -E 'bogus-' || true
+  environment:
+    KUBECONFIG: "{{ kubeconfig }}"
+  register: bogus_machinesets_cmd
+  changed_when: false
+  failed_when: false
+
+- name: List bogus Machines in openshift-machine-api
+  ansible.builtin.shell: |
+    set -o pipefail
+    oc get machine -n openshift-machine-api -o name 2>/dev/null \
+      | grep -E 'bogus-' || true
+  environment:
+    KUBECONFIG: "{{ kubeconfig }}"
+  register: bogus_machines_cmd
+  changed_when: false
+  failed_when: false
+
+- name: Delete leftover bogus MachineSets
+  ansible.builtin.command:
+    argv:
+      - oc
+      - delete
+      - "{{ item }}"
+      - -n
+      - openshift-machine-api
+      - --wait=false
+  environment:
+    KUBECONFIG: "{{ kubeconfig }}"
+  loop: "{{ bogus_machinesets_cmd.stdout_lines | default([]) }}"
+  loop_control:
+    label: "{{ item }}"
+  register: bogus_ms_delete
+  changed_when: bogus_ms_delete.rc == 0
+  failed_when: false
+  when: bogus_machinesets_cmd.stdout_lines | default([]) | length > 0
+
+- name: Delete leftover bogus Machines
+  ansible.builtin.command:
+    argv:
+      - oc
+      - delete
+      - "{{ item }}"
+      - -n
+      - openshift-machine-api
+      - --wait=false
+  environment:
+    KUBECONFIG: "{{ kubeconfig }}"
+  loop: "{{ bogus_machines_cmd.stdout_lines | default([]) }}"
+  loop_control:
+    label: "{{ item }}"
+  register: bogus_machine_delete
+  changed_when: bogus_machine_delete.rc == 0
+  failed_when: false
+  when: bogus_machines_cmd.stdout_lines | default([]) | length > 0
+
+# Deletes used --wait=false; poll until matching objects are gone so openstack_test
+# does not start while bogus-* resources are still terminating. Bounded timeout.
+# Fail when leftovers remain after the timeout so openstack_test cannot start dirty.
+- name: Wait until bogus Machines and MachineSets are gone
+  ansible.builtin.shell: |
+    set -o pipefail
+    ms=$(oc get machineset -n openshift-machine-api -o name 2>/dev/null | grep -E 'bogus-' || true)
+    m=$(oc get machine -n openshift-machine-api -o name 2>/dev/null | grep -E 'bogus-' || true)
+    if [ -n "$ms" ] || [ -n "$m" ]; then
+      echo "bogus leftovers still present:"
+      printf '%s\n' "$ms" "$m" | sed '/^$/d'
+      exit 1
+    fi
+    echo "no bogus Machines or MachineSets remain"
+    exit 0
+  environment:
+    KUBECONFIG: "{{ kubeconfig }}"
+  register: bogus_gone
+  until: bogus_gone.rc == 0
+  retries: 30
+  delay: 10
+  changed_when: false
+  when: >-
+    (bogus_machinesets_cmd.stdout_lines | default([]) | length > 0)
+    or (bogus_machines_cmd.stdout_lines | default([]) | length > 0)

collection/stages/roles/openstack_test/tasks/main.yml

@@ -32,6 +32,9 @@
       delegate_to: "{{ hypervisor }}"
       remote_user: root
 
+    - name: Best-effort purge leftover bogus Machines before openstack-test
+      ansible.builtin.include_tasks: cleanup_bogus_machines.yml
+
     - name: Include Openstack-Test tasks
       ansible.builtin.include_tasks: run_openstack_test.yml
 

collection/stages/roles/openstack_test/tasks/run_openstack_test.yml

@@ -188,18 +188,57 @@
       ansible.builtin.shell: |
         set -o pipefail
         : > {{ openstack_test_log_path }}
-        run_exit=0
+        suite_exit=0
         py_exit=0
+        expected=0
+        resolve="{{ openstack_test_ote_resolve_script }}"
         while IFS= read -r test || [[ -n "$test" ]]; do
           [[ -z "${test// }" ]] && continue
-          if ! {{ openstack_test_executable }} run-test --output=json "$test" \
-            >> {{ openstack_test_log_path }} 2>&1; then
-            run_exit=1
+          expected=$((expected + 1))
+          tmp=$(mktemp)
+          rc=0
+          {{ openstack_test_executable }} run-test --output=json "$test" \
+            > "$tmp" 2>&1 || rc=$?
+          cat "$tmp" >> {{ openstack_test_log_path }}
+          # Skip-only (and pass) often exit non-zero from run-test; ignore that
+          # only when the resolver confirms skipped/passed. Failed/unknown keep
+          # the suite failed. Empty resolve on this chunk alone is deferred to
+          # the end-of-suite resolvable==expected check (OTE often exits
+          # non-zero before JSON is parseable as a standalone result).
+          if [ "$rc" -ne 0 ]; then
+            failed=$(python3 "$resolve" count "$tmp" failed || echo 1)
+            skipped=$(python3 "$resolve" count "$tmp" skipped || echo 0)
+            passed=$(python3 "$resolve" count "$tmp" passed || echo 0)
+            unknown=$(python3 "$resolve" count "$tmp" unknown || echo 0)
+            if [ "$failed" -gt 0 ] || [ "$unknown" -gt 0 ]; then
+              suite_exit=1
+            elif [ $((failed + skipped + passed + unknown)) -eq 0 ]; then
+              if grep -q 'FAIL!' "$tmp" 2>/dev/null; then
+                suite_exit=1
+              elif ! grep -qE 'SUCCESS!|SKIP' "$tmp" 2>/dev/null; then
+                # No resolvable outcome and no ginkgo success/skip signal.
+                suite_exit=1
+              fi
+            fi
           fi
+          rm -f "$tmp"
         done < {{ tests_to_run_path }}
-        python3 "{{ openstack_test_ote_resolve_script }}" junit \
+        python3 "$resolve" junit \
           "{{ openstack_test_log_path }}" "{{ openstack_test_junit_path }}" || py_exit=$?
-        if [ "$run_exit" -ne 0 ] || [ "$py_exit" -ne 0 ]; then
+        failed_count=$(python3 "$resolve" count \
+          "{{ openstack_test_log_path }}" failed || echo 0)
+        skipped_count=$(python3 "$resolve" count \
+          "{{ openstack_test_log_path }}" skipped || echo 0)
+        passed_count=$(python3 "$resolve" count \
+          "{{ openstack_test_log_path }}" passed || echo 0)
+        unknown_count=$(python3 "$resolve" count \
+          "{{ openstack_test_log_path }}" unknown || echo 0)
+        # Require every requested test to resolve as pass/skip/fail (no unknown,
+        # no missing entries). Exit 0 with empty/unresolvable output must fail.
+        resolvable=$(( ${passed_count:-0} + ${skipped_count:-0} + ${failed_count:-0} ))
+        if [ "$suite_exit" -ne 0 ] || [ "$py_exit" -ne 0 ] \
+          || [ "${failed_count:-0}" -gt 0 ] || [ "${unknown_count:-0}" -gt 0 ] \
+          || [ "$resolvable" -ne "$expected" ]; then
           exit 1
         fi
         exit 0

jobs_definitions/osp_verification_4.21_nightly.yaml

@@ -13,6 +13,7 @@
 openshift_release: "4.21"
 openshift_build_name: "" # Empty resolves to latest nightly via 4.21.0-0.nightly/latest
 installation_type: ipi
+lb_tests_guest_cleanup: true
 stages:
   - prepare
   - install

jobs_definitions/osp_verification_4.22_nightly.yaml

@@ -13,6 +13,7 @@
 openshift_release: "4.22"
 openshift_build_name: "" # Empty resolves to latest nightly via 4.22.0-0.nightly/latest
 installation_type: ipi
+lb_tests_guest_cleanup: true
 stages:
   - prepare
   - install

jobs_definitions/osp_verification_4.23_nightly.yaml

@@ -13,6 +13,7 @@
 openshift_release: "4.23"
 openshift_build_name: "" # Empty resolves to latest nightly via 4.23.0-0.nightly/latest
 installation_type: ipi
+lb_tests_guest_cleanup: true
 stages:
   - prepare
   - install

jobs_definitions/osp_verification_5.0_nightly.yaml

@@ -16,6 +16,7 @@
 openshift_release: "5.0"
 openshift_build_name: "" # Empty resolves to latest nightly via 5.0.0-0.nightly/latest
 installation_type: ipi
+lb_tests_guest_cleanup: true
 stages:
   - prepare
   - install