From ec46f4ea9264007f30de91ace3226d2e346cd513 Mon Sep 17 00:00:00 2001 From: tusharjadhav3302 Date: Sun, 9 Aug 2026 21:34:28 +0530 Subject: [PATCH 1/4] Adapt openstack_test stage to OTE after legacy Makefile drop Upstream openstack-test removed make openstack-tests on release-4.20+ (PR #303 and backports). Use make extension / run-suite for those releases, keep the legacy path for older branches, and preserve run-test/junit exit codes for filtered runs. Co-authored-by: Cursor --- .../roles/openstack_test/defaults/main.yml | 4 + .../tasks/run_openstack_test.yml | 236 +++++++++++++++++- 2 files changed, 230 insertions(+), 10 deletions(-) diff --git a/collection/stages/roles/openstack_test/defaults/main.yml b/collection/stages/roles/openstack_test/defaults/main.yml index b95948f1..e7754b15 100644 --- a/collection/stages/roles/openstack_test/defaults/main.yml +++ b/collection/stages/roles/openstack_test/defaults/main.yml @@ -3,8 +3,12 @@ openstack_test_name: openstack-test openstack_testsuite_name: openstack openstack_test_dir: "{{ artifacts_dir }}/{{ openstack_test_name }}" +# Installed binary path is set in run_openstack_test.yml (OTE vs legacy). openstack_test_executable: "{{ home_dir }}/openstack-tests" openstack_test_results_dir: "{{ artifacts_dir }}/{{ openstack_test_name }}-results" openstack_tests_allowlist_file: "" openstack_tests_blocklist_file: "" openstack_tests_go_version: "{{ tests.default_go_version_target }}" +# OTE (OpenShift Tests Extension) — used when discovered_openshift_release >= 4.20 +openstack_test_suite: openstack-test/all +openstack_test_ote_binary_relpath: bin/openstack-test-tests-ext diff --git a/collection/stages/roles/openstack_test/tasks/run_openstack_test.yml b/collection/stages/roles/openstack_test/tasks/run_openstack_test.yml index 4de88db9..055e805e 100644 --- a/collection/stages/roles/openstack_test/tasks/run_openstack_test.yml +++ b/collection/stages/roles/openstack_test/tasks/run_openstack_test.yml @@ -1,5 +1,5 @@ --- -- name: Set artifact paths +- name: Set artifact paths and OTE vs legacy mode ansible.builtin.set_fact: all_tests_path: >- {{ openstack_test_results_dir }}/openstack_tests.txt @@ -9,28 +9,79 @@ {{ openstack_tests_blocklist_file | ternary(openstack_test_results_dir + '/blocklist.txt', '') }} tests_to_run_path: >- {{ openstack_test_results_dir }}/list_of_tests_to_run.txt + openstack_test_log_path: >- + {{ openstack_test_results_dir }}/{{ openstack_test_name }}.log + openstack_test_junit_path: >- + {{ openstack_test_results_dir }}/junit_e2e_openstack.xml + # Upstream dropped make openstack-tests on release-4.20+ (OTE migration, + # openshift/openstack-test#303 / OSASINFRA-4375; backports #306/#307/#308). + # Older release branches still ship the legacy target. + openstack_test_use_ote: "{{ discovered_openshift_release is version('4.20', '>=') }}" -- name: Build the openstack-tests for tests in the "openstack-test" repository +- name: Set openstack-test executable path for OTE + ansible.builtin.set_fact: + openstack_test_executable: "{{ home_dir }}/openstack-test-tests-ext" + when: openstack_test_use_ote | bool + +- name: Set openstack-test executable path for legacy openshift-tests + ansible.builtin.set_fact: + openstack_test_executable: "{{ home_dir }}/openstack-tests" + when: not (openstack_test_use_ote | bool) + +# --- Build --- +# OTE: make extension → bin/openstack-test-tests-ext (openshift/openstack-test#303) +- name: Build the OTE openstack-test extension binary + ansible.builtin.shell: | + source {{ home_dir }}/.bashrc + make extension + args: + chdir: "{{ openstack_test_dir }}" + changed_when: true + when: openstack_test_use_ote | bool + +- name: Build the legacy openstack-tests binary ansible.builtin.shell: | source {{ home_dir }}/.bashrc make openstack-tests args: chdir: "{{ openstack_test_dir }}" changed_when: true + when: not (openstack_test_use_ote | bool) -- name: Copy openstack-test executable +- name: Copy OTE openstack-test executable + ansible.builtin.copy: + src: "{{ openstack_test_dir }}/{{ openstack_test_ote_binary_relpath }}" + dest: "{{ openstack_test_executable }}" + mode: u=rwx,g=rwx,o=rwx + remote_src: yes + when: openstack_test_use_ote | bool + +- name: Copy legacy openstack-test executable ansible.builtin.copy: src: "{{ openstack_test_dir }}/openstack-tests" dest: "{{ openstack_test_executable }}" mode: u=rwx,g=rwx,o=rwx remote_src: yes + when: not (openstack_test_use_ote | bool) - name: Remove source directory once compilation succeeded ansible.builtin.file: path: "{{ openstack_test_dir }}" state: absent -- name: Prepare {{ all_tests_path }} file with all the tests that can be run +# --- List tests --- +- name: Prepare {{ all_tests_path }} with OTE list (names) + ansible.builtin.shell: > + {{ openstack_test_executable }} list + --suite {{ openstack_test_suite }} + -o names > {{ all_tests_path }} + environment: + OS_CLOUD: "{{ user_cloud }}" + KUBECONFIG: "{{ kubeconfig }}" + changed_when: true + when: openstack_test_use_ote | bool + +- name: Prepare {{ all_tests_path }} with legacy dry-run ansible.builtin.shell: > {{ openstack_test_executable }} run openshift/openstack --dry-run > {{ all_tests_path }} @@ -38,6 +89,7 @@ OS_CLOUD: "{{ user_cloud }}" KUBECONFIG: "{{ kubeconfig }}" changed_when: true + when: not (openstack_test_use_ote | bool) - name: Convert the allowlist YAML files to TXT ansible.builtin.include_role: @@ -67,13 +119,104 @@ blocklist_file: "{{ blocklist_path }}" output_file: "{{ tests_to_run_path }}" +- name: Detect whether allow or block list filtering is active + ansible.builtin.set_fact: + openstack_test_filtering: "{{ (openstack_tests_allowlist_file | length > 0) or (openstack_tests_blocklist_file | length > 0) }}" + - name: Run the openshift tests block: - - name: Run openstack tests + - name: Run OTE suite (no allow/block filter) + ansible.builtin.shell: > + {{ openstack_test_executable }} run-suite {{ openstack_test_suite }} + --junit-path {{ openstack_test_junit_path }} + > {{ openstack_test_log_path }} + environment: + # RHOSO_KUBECONFIG and SHIFTSTACK_PASS_FILE are required for the observability test + OS_CLOUD: "{{ user_cloud }}" + KUBECONFIG: "{{ kubeconfig }}" + RHOSO_KUBECONFIG: "{{ rhoso_kubeconfig }}" + SHIFTSTACK_PASS_FILE: "{{ kubeadmin_password }}" + changed_when: true + when: + - openstack_test_use_ote | bool + - not (openstack_test_filtering | bool) + + # OTE run-test has no --junit-path; pipe filtered names via stdin and + # synthesize junit_e2e_*.xml from JSON results for post_openshift_tests. + - name: Run OTE filtered tests via run-test and write junit + ansible.builtin.shell: | + set -o pipefail + cat {{ tests_to_run_path }} | {{ openstack_test_executable }} run-test \ + > {{ openstack_test_log_path }} || run_exit=$? + python3 - {{ openstack_test_log_path }} {{ openstack_test_junit_path }} <<'PY' + import json + import sys + import xml.etree.ElementTree as ET + + log_path, junit_path = sys.argv[1], sys.argv[2] + with open(log_path, encoding="utf-8") as f: + raw = f.read().strip() + if not raw: + results = [] + else: + try: + data = json.loads(raw) + results = data if isinstance(data, list) else [data] + except json.JSONDecodeError: + results = [] + for line in raw.splitlines(): + line = line.strip() + if not line: + continue + try: + results.append(json.loads(line)) + except json.JSONDecodeError: + continue + + suite = ET.Element("testsuite", name="openstack-test") + for r in results: + name = r.get("name", "unknown") + duration_ms = r.get("duration") or 0 + try: + time_s = f"{float(duration_ms) / 1000.0:.3f}" + except (TypeError, ValueError): + time_s = "0" + case = ET.SubElement(suite, "testcase", name=name, time=time_s) + result = (r.get("result") or "").lower() + if result == "failed": + fail = ET.SubElement(case, "failure") + fail.text = r.get("error") or r.get("output") or "failed" + elif result == "skipped": + skip = ET.SubElement(case, "skipped") + skip.text = r.get("output") or "skipped" + suite.set("tests", str(len(results))) + suite.set( + "failures", + str(sum(1 for r in results if (r.get("result") or "").lower() == "failed")), + ) + suite.set( + "skipped", + str(sum(1 for r in results if (r.get("result") or "").lower() == "skipped")), + ) + ET.ElementTree(suite).write(junit_path, encoding="utf-8", xml_declaration=True) + PY + py_exit=$? + exit ${run_exit:-${py_exit:-0}} + environment: + OS_CLOUD: "{{ user_cloud }}" + KUBECONFIG: "{{ kubeconfig }}" + RHOSO_KUBECONFIG: "{{ rhoso_kubeconfig }}" + SHIFTSTACK_PASS_FILE: "{{ kubeadmin_password }}" + changed_when: true + when: + - openstack_test_use_ote | bool + - openstack_test_filtering | bool + + - name: Run legacy openstack tests ansible.builtin.shell: > {{ openstack_test_executable }} run openshift/openstack -f {{ tests_to_run_path }} - --output-file {{ openstack_test_results_dir }}/{{ openstack_test_name }}.log + --output-file {{ openstack_test_log_path }} --junit-dir={{ openstack_test_results_dir }} > /dev/null environment: # RHOSO_KUBECONFIG and SHIFTSTACK_PASS_FILE are required for the observability test @@ -82,6 +225,7 @@ RHOSO_KUBECONFIG: "{{ rhoso_kubeconfig }}" SHIFTSTACK_PASS_FILE: "{{ kubeadmin_password }}" changed_when: true + when: not (openstack_test_use_ote | bool) rescue: - name: Mark the openshift tests as UNSTABLE @@ -92,21 +236,93 @@ unstable_msg: >- The openshift test Suite failed. - - name: Get the number of failed openstack tests + - name: Get the number of failed openstack tests (OTE JSON log) + ansible.builtin.shell: | + set -o pipefail + python3 - {{ openstack_test_log_path }} failed <<'PY' || true + import json + import sys + path, want = sys.argv[1], sys.argv[2] + try: + with open(path, encoding="utf-8") as f: + raw = f.read().strip() + except OSError: + print(0) + raise SystemExit + if not raw: + print(0) + raise SystemExit + results = [] + try: + data = json.loads(raw) + results = data if isinstance(data, list) else [data] + except json.JSONDecodeError: + for line in raw.splitlines(): + line = line.strip() + if not line: + continue + try: + results.append(json.loads(line)) + except json.JSONDecodeError: + continue + print(sum(1 for r in results if (r.get("result") or "").lower() == want)) + PY + register: number_of_failed_tests + changed_when: false + when: openstack_test_use_ote | bool + + - name: Get the number of passed openstack tests (OTE JSON log) + ansible.builtin.shell: | + set -o pipefail + python3 - {{ openstack_test_log_path }} passed <<'PY' || true + import json + import sys + path, want = sys.argv[1], sys.argv[2] + try: + with open(path, encoding="utf-8") as f: + raw = f.read().strip() + except OSError: + print(0) + raise SystemExit + if not raw: + print(0) + raise SystemExit + results = [] + try: + data = json.loads(raw) + results = data if isinstance(data, list) else [data] + except json.JSONDecodeError: + for line in raw.splitlines(): + line = line.strip() + if not line: + continue + try: + results.append(json.loads(line)) + except json.JSONDecodeError: + continue + print(sum(1 for r in results if (r.get("result") or "").lower() == want)) + PY + register: number_of_passed_tests + changed_when: false + when: openstack_test_use_ote | bool + + - name: Get the number of failed openstack tests (legacy log) ansible.builtin.shell: > set -o pipefail && - grep -e "^failed:" {{ openstack_test_results_dir }}/{{ openstack_test_name }}.log | wc -l + grep -e "^failed:" {{ openstack_test_log_path }} | wc -l || true register: number_of_failed_tests changed_when: false + when: not (openstack_test_use_ote | bool) - - name: Get the number of passed openstack tests + - name: Get the number of passed openstack tests (legacy log) ansible.builtin.shell: > set -o pipefail && - grep -e "^passed:" {{ openstack_test_results_dir }}/{{ openstack_test_name }}.log | wc -l + grep -e "^passed:" {{ openstack_test_log_path }} | wc -l || true register: number_of_passed_tests changed_when: false + when: not (openstack_test_use_ote | bool) # This fail task is added to detect failures in openstack-test execution - name: Fail the playbook in case there are no failed and passed tests From 5abe8ef477fdd1ff825a58102f791efea25845a7 Mon Sep 17 00:00:00 2001 From: tusharjadhav3302 Date: Mon, 10 Aug 2026 12:06:54 +0530 Subject: [PATCH 2/4] Resolve OTE run-suite false failures from klog deserialize errors Outer JSON marks every result failed when stderr starts with klog "I..." lines. Count and rewrite junit from nested STDOUT / ginkgo SUCCESS|FAIL so real suite failures stay UNSTABLE instead of "no tests run". Co-authored-by: Cursor --- .../files/ote_resolve_results.py | 224 ++++++++++++++++++ .../tasks/run_openstack_test.yml | 181 +++++--------- 2 files changed, 278 insertions(+), 127 deletions(-) create mode 100644 collection/stages/roles/openstack_test/files/ote_resolve_results.py diff --git a/collection/stages/roles/openstack_test/files/ote_resolve_results.py b/collection/stages/roles/openstack_test/files/ote_resolve_results.py new file mode 100644 index 00000000..aa561a84 --- /dev/null +++ b/collection/stages/roles/openstack_test/files/ote_resolve_results.py @@ -0,0 +1,224 @@ +#!/usr/bin/env python3 +"""Resolve true OTE openstack-test outcomes from run-suite / run-test logs. + +OTE often marks every outer JSON entry as failed when stderr begins with klog +lines (``I0809 ...``), producing:: + + Deserializaion Error: invalid character 'I' looking for beginning of value + +The real outcome is in nested STDOUT JSON (``"result": "passed"|"failed"|...``) +or in ginkgo summary lines (``SUCCESS!`` / ``FAIL!``). + +Usage: + ote_resolve_results.py count passed|failed|skipped + ote_resolve_results.py junit +""" + +from __future__ import annotations + +import json +import re +import sys +import xml.etree.ElementTree as ET +from typing import Any + + +def _load_outer_results(raw: str) -> list[dict[str, Any]]: + raw = raw.strip() + if not raw: + return [] + + # Skip leading klog / noise before the JSON array/object. + start_candidates = [i for i, ch in enumerate(raw) if ch in "[{"] + for start in start_candidates: + chunk = raw[start:] + try: + data, _ = json.JSONDecoder().raw_decode(chunk) + except json.JSONDecodeError: + continue + if isinstance(data, list): + return [r for r in data if isinstance(r, dict)] + if isinstance(data, dict): + return [data] + + # NDJSON fallback + results: list[dict[str, Any]] = [] + for line in raw.splitlines(): + line = line.strip() + if not line or not line.startswith("{"): + continue + try: + obj = json.loads(line) + except json.JSONDecodeError: + continue + if isinstance(obj, dict): + results.append(obj) + return results + + +def _extract_nested_results(output: str) -> list[dict[str, Any]] | None: + if not output: + return None + + m = re.search(r"STDOUT:\n(.*?)(?:\n\nSTDERR:|\nSTDERR:|\Z)", output, re.S) + stdout = m.group(1) if m else output + + for match in re.finditer(r"\[", stdout): + chunk = stdout[match.start() :] + try: + obj, _ = json.JSONDecoder().raw_decode(chunk) + except json.JSONDecodeError: + continue + if ( + isinstance(obj, list) + and obj + and isinstance(obj[0], dict) + and "result" in obj[0] + ): + return obj + return None + + +def _result_from_ginkgo(output: str) -> str | None: + if not output: + return None + # Prefer FAIL over SUCCESS when both somehow appear. + if re.search(r"FAIL! -- .*\| 1 Failed", output) or re.search( + r"FAIL! -- 0 Passed \| [1-9]", output + ): + return "failed" + if re.search(r"SUCCESS! -- 0 Passed \| 0 Failed \| 0 Pending \| [1-9]+ Skipped", output): + return "skipped" + if re.search(r"SUCCESS! -- [1-9]\d* Passed \| 0 Failed", output): + return "passed" + if "[FAILED]" in output and "SUCCESS! -- 1 Passed" not in output: + return "failed" + return None + + +def _has_deser_error(output: str) -> bool: + return "Deserializaion Error" in output or "Deserialization Error" in output + + +def resolve_result(entry: dict[str, Any]) -> str: + """Return passed|failed|skipped|unknown for one outer OTE log entry.""" + outer = (entry.get("result") or "").lower() + output = entry.get("output") or "" + + nested = _extract_nested_results(output) + if nested: + # One outer entry maps to one (or few) nested specs; use first, or + # failed if any nested failed. + nested_results = [(r.get("result") or "").lower() for r in nested] + if "failed" in nested_results: + return "failed" + if nested_results and all(r == "skipped" for r in nested_results): + return "skipped" + if nested_results and all(r == "passed" for r in nested_results): + return "passed" + if "passed" in nested_results and "failed" not in nested_results: + return "passed" + + ginkgo = _result_from_ginkgo(output) + if ginkgo: + return ginkgo + + # Outer "failed" with only deserialize noise and no other signal is unknown; + # callers should not treat unknown as a real failure count source of truth. + if outer == "failed" and _has_deser_error(output) and not nested and not ginkgo: + return "unknown" + + if outer in ("passed", "failed", "skipped"): + return outer + return "unknown" + + +def resolve_all(log_path: str) -> list[dict[str, Any]]: + try: + with open(log_path, encoding="utf-8") as f: + raw = f.read() + except OSError: + return [] + + resolved: list[dict[str, Any]] = [] + for entry in _load_outer_results(raw): + result = resolve_result(entry) + resolved.append( + { + "name": entry.get("name") or "unknown", + "result": result, + "duration": entry.get("duration") or 0, + "output": entry.get("output") or entry.get("error") or "", + "error": entry.get("error") or "", + } + ) + return resolved + + +def cmd_count(log_path: str, want: str) -> int: + want = want.lower() + return sum(1 for r in resolve_all(log_path) if r["result"] == want) + + +def cmd_junit(log_path: str, junit_path: str) -> None: + results = resolve_all(log_path) + suite = ET.Element("testsuite", name="openstack-test") + failures = 0 + skipped = 0 + for r in results: + duration = r.get("duration") or 0 + try: + # OTE durations are often nanoseconds; keep seconds for junit. + dur = float(duration) + if dur > 10_000: + # ns or ms — treat large values as ns + time_s = f"{dur / 1_000_000_000.0:.3f}" if dur > 1_000_000 else f"{dur / 1000.0:.3f}" + else: + time_s = f"{dur:.3f}" + except (TypeError, ValueError): + time_s = "0" + + case = ET.SubElement(suite, "testcase", name=r["name"], time=time_s) + result = r["result"] + if result == "failed": + failures += 1 + fail = ET.SubElement(case, "failure") + fail.text = r.get("error") or r.get("output") or "failed" + elif result == "skipped": + skipped += 1 + skip = ET.SubElement(case, "skipped") + skip.text = r.get("output") or "skipped" + elif result == "unknown": + # Do not inflate failure counts for unparsable deserialize-only noise. + skip = ET.SubElement(case, "skipped") + skip.text = "unresolved OTE outcome" + + suite.set("tests", str(len(results))) + suite.set("failures", str(failures)) + suite.set("skipped", str(skipped)) + ET.ElementTree(suite).write(junit_path, encoding="utf-8", xml_declaration=True) + + +def main(argv: list[str]) -> int: + if len(argv) < 2: + print(__doc__, file=sys.stderr) + return 2 + cmd = argv[1] + if cmd == "count": + if len(argv) != 4: + print("usage: count passed|failed|skipped", file=sys.stderr) + return 2 + print(cmd_count(argv[2], argv[3])) + return 0 + if cmd == "junit": + if len(argv) != 4: + print("usage: junit ", file=sys.stderr) + return 2 + cmd_junit(argv[2], argv[3]) + return 0 + print(__doc__, file=sys.stderr) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv)) diff --git a/collection/stages/roles/openstack_test/tasks/run_openstack_test.yml b/collection/stages/roles/openstack_test/tasks/run_openstack_test.yml index 055e805e..69763f1d 100644 --- a/collection/stages/roles/openstack_test/tasks/run_openstack_test.yml +++ b/collection/stages/roles/openstack_test/tasks/run_openstack_test.yml @@ -123,84 +123,55 @@ ansible.builtin.set_fact: openstack_test_filtering: "{{ (openstack_tests_allowlist_file | length > 0) or (openstack_tests_blocklist_file | length > 0) }}" +# OTE run-suite often marks every outer JSON result as failed when stderr has +# klog lines (Deserializaion Error: invalid character 'I'...). True outcomes +# live in nested STDOUT JSON / ginkgo SUCCESS!|FAIL! lines — see +# files/ote_resolve_results.py. +- name: Set path to OTE result resolver helper + ansible.builtin.set_fact: + openstack_test_ote_resolve_script: "{{ role_path }}/files/ote_resolve_results.py" + when: openstack_test_use_ote | bool + - name: Run the openshift tests block: - name: Run OTE suite (no allow/block filter) - ansible.builtin.shell: > - {{ openstack_test_executable }} run-suite {{ openstack_test_suite }} - --junit-path {{ openstack_test_junit_path }} - > {{ openstack_test_log_path }} - environment: - # RHOSO_KUBECONFIG and SHIFTSTACK_PASS_FILE are required for the observability test - OS_CLOUD: "{{ user_cloud }}" - KUBECONFIG: "{{ kubeconfig }}" - RHOSO_KUBECONFIG: "{{ rhoso_kubeconfig }}" - SHIFTSTACK_PASS_FILE: "{{ kubeadmin_password }}" - changed_when: true when: - openstack_test_use_ote | bool - not (openstack_test_filtering | bool) + block: + - name: Execute OTE run-suite + ansible.builtin.shell: > + {{ openstack_test_executable }} run-suite {{ openstack_test_suite }} + --junit-path {{ openstack_test_junit_path }} + > {{ openstack_test_log_path }} + environment: + # RHOSO_KUBECONFIG and SHIFTSTACK_PASS_FILE are required for the observability test + OS_CLOUD: "{{ user_cloud }}" + KUBECONFIG: "{{ kubeconfig }}" + RHOSO_KUBECONFIG: "{{ rhoso_kubeconfig }}" + SHIFTSTACK_PASS_FILE: "{{ kubeadmin_password }}" + changed_when: true + always: + # Rewrite junit from resolved outcomes (OTE's own junit inherits false fails). + - name: Rewrite junit from resolved OTE run-suite outcomes + ansible.builtin.command: + argv: + - python3 + - "{{ openstack_test_ote_resolve_script }}" + - junit + - "{{ openstack_test_log_path }}" + - "{{ openstack_test_junit_path }}" + changed_when: true # OTE run-test has no --junit-path; pipe filtered names via stdin and - # synthesize junit_e2e_*.xml from JSON results for post_openshift_tests. + # synthesize junit from resolved log outcomes for post_openshift_tests. - name: Run OTE filtered tests via run-test and write junit ansible.builtin.shell: | set -o pipefail cat {{ tests_to_run_path }} | {{ openstack_test_executable }} run-test \ > {{ openstack_test_log_path }} || run_exit=$? - python3 - {{ openstack_test_log_path }} {{ openstack_test_junit_path }} <<'PY' - import json - import sys - import xml.etree.ElementTree as ET - - log_path, junit_path = sys.argv[1], sys.argv[2] - with open(log_path, encoding="utf-8") as f: - raw = f.read().strip() - if not raw: - results = [] - else: - try: - data = json.loads(raw) - results = data if isinstance(data, list) else [data] - except json.JSONDecodeError: - results = [] - for line in raw.splitlines(): - line = line.strip() - if not line: - continue - try: - results.append(json.loads(line)) - except json.JSONDecodeError: - continue - - suite = ET.Element("testsuite", name="openstack-test") - for r in results: - name = r.get("name", "unknown") - duration_ms = r.get("duration") or 0 - try: - time_s = f"{float(duration_ms) / 1000.0:.3f}" - except (TypeError, ValueError): - time_s = "0" - case = ET.SubElement(suite, "testcase", name=name, time=time_s) - result = (r.get("result") or "").lower() - if result == "failed": - fail = ET.SubElement(case, "failure") - fail.text = r.get("error") or r.get("output") or "failed" - elif result == "skipped": - skip = ET.SubElement(case, "skipped") - skip.text = r.get("output") or "skipped" - suite.set("tests", str(len(results))) - suite.set( - "failures", - str(sum(1 for r in results if (r.get("result") or "").lower() == "failed")), - ) - suite.set( - "skipped", - str(sum(1 for r in results if (r.get("result") or "").lower() == "skipped")), - ) - ET.ElementTree(suite).write(junit_path, encoding="utf-8", xml_declaration=True) - PY - py_exit=$? + python3 "{{ openstack_test_ote_resolve_script }}" junit \ + "{{ openstack_test_log_path }}" "{{ openstack_test_junit_path }}" || py_exit=$? exit ${run_exit:-${py_exit:-0}} environment: OS_CLOUD: "{{ user_cloud }}" @@ -236,74 +207,30 @@ unstable_msg: >- The openshift test Suite failed. - - name: Get the number of failed openstack tests (OTE JSON log) - ansible.builtin.shell: | - set -o pipefail - python3 - {{ openstack_test_log_path }} failed <<'PY' || true - import json - import sys - path, want = sys.argv[1], sys.argv[2] - try: - with open(path, encoding="utf-8") as f: - raw = f.read().strip() - except OSError: - print(0) - raise SystemExit - if not raw: - print(0) - raise SystemExit - results = [] - try: - data = json.loads(raw) - results = data if isinstance(data, list) else [data] - except json.JSONDecodeError: - for line in raw.splitlines(): - line = line.strip() - if not line: - continue - try: - results.append(json.loads(line)) - except json.JSONDecodeError: - continue - print(sum(1 for r in results if (r.get("result") or "").lower() == want)) - PY + - name: Get the number of failed openstack tests (OTE resolved log) + ansible.builtin.command: + argv: + - python3 + - "{{ openstack_test_ote_resolve_script }}" + - count + - "{{ openstack_test_log_path }}" + - failed register: number_of_failed_tests changed_when: false + failed_when: false when: openstack_test_use_ote | bool - - name: Get the number of passed openstack tests (OTE JSON log) - ansible.builtin.shell: | - set -o pipefail - python3 - {{ openstack_test_log_path }} passed <<'PY' || true - import json - import sys - path, want = sys.argv[1], sys.argv[2] - try: - with open(path, encoding="utf-8") as f: - raw = f.read().strip() - except OSError: - print(0) - raise SystemExit - if not raw: - print(0) - raise SystemExit - results = [] - try: - data = json.loads(raw) - results = data if isinstance(data, list) else [data] - except json.JSONDecodeError: - for line in raw.splitlines(): - line = line.strip() - if not line: - continue - try: - results.append(json.loads(line)) - except json.JSONDecodeError: - continue - print(sum(1 for r in results if (r.get("result") or "").lower() == want)) - PY + - name: Get the number of passed openstack tests (OTE resolved log) + ansible.builtin.command: + argv: + - python3 + - "{{ openstack_test_ote_resolve_script }}" + - count + - "{{ openstack_test_log_path }}" + - passed register: number_of_passed_tests changed_when: false + failed_when: false when: openstack_test_use_ote | bool - name: Get the number of failed openstack tests (legacy log) From 5781579a1c901ab867215aa657be9919c27b4622 Mon Sep 17 00:00:00 2001 From: tusharjadhav3302 Date: Mon, 10 Aug 2026 12:42:41 +0530 Subject: [PATCH 3/4] Keep OTE pass/fail counts when legacy count tasks are skipped Skipped Ansible tasks still overwrite a shared register, which cleared OTE resolved counts and falsely triggered "no tests where run!". Co-authored-by: Cursor --- .../tasks/run_openstack_test.yml | 36 +++++++++++++++---- 1 file changed, 30 insertions(+), 6 deletions(-) diff --git a/collection/stages/roles/openstack_test/tasks/run_openstack_test.yml b/collection/stages/roles/openstack_test/tasks/run_openstack_test.yml index 69763f1d..ff1af866 100644 --- a/collection/stages/roles/openstack_test/tasks/run_openstack_test.yml +++ b/collection/stages/roles/openstack_test/tasks/run_openstack_test.yml @@ -207,6 +207,9 @@ unstable_msg: >- The openshift test Suite failed. + # Use distinct registers for OTE vs legacy. Skipped tasks still overwrite + # a shared register in Ansible, which wiped OTE counts and falsely hit + # "no tests where run!" after a real suite run. - name: Get the number of failed openstack tests (OTE resolved log) ansible.builtin.command: argv: @@ -215,7 +218,7 @@ - count - "{{ openstack_test_log_path }}" - failed - register: number_of_failed_tests + register: openstack_test_ote_failed_count changed_when: false failed_when: false when: openstack_test_use_ote | bool @@ -228,7 +231,7 @@ - count - "{{ openstack_test_log_path }}" - passed - register: number_of_passed_tests + register: openstack_test_ote_passed_count changed_when: false failed_when: false when: openstack_test_use_ote | bool @@ -238,7 +241,7 @@ set -o pipefail && grep -e "^failed:" {{ openstack_test_log_path }} | wc -l || true - register: number_of_failed_tests + register: openstack_test_legacy_failed_count changed_when: false when: not (openstack_test_use_ote | bool) @@ -247,18 +250,39 @@ set -o pipefail && grep -e "^passed:" {{ openstack_test_log_path }} | wc -l || true - register: number_of_passed_tests + register: openstack_test_legacy_passed_count changed_when: false when: not (openstack_test_use_ote | bool) + - name: Normalize openstack-test pass/fail counts + ansible.builtin.set_fact: + number_of_failed_tests: >- + {{ ( + openstack_test_ote_failed_count.stdout | default('0') + if (openstack_test_use_ote | bool) + else openstack_test_legacy_failed_count.stdout | default('0') + ) | trim }} + number_of_passed_tests: >- + {{ ( + openstack_test_ote_passed_count.stdout | default('0') + if (openstack_test_use_ote | bool) + else openstack_test_legacy_passed_count.stdout | default('0') + ) | trim }} + + - name: Print resolved openstack-test pass/fail counts + ansible.builtin.debug: + msg: >- + openstack-test counts: passed={{ number_of_passed_tests }}, + failed={{ number_of_failed_tests }} + # This fail task is added to detect failures in openstack-test execution - name: Fail the playbook in case there are no failed and passed tests ansible.builtin.fail: msg: > openstack-test execution failed or no tests where run! when: - - (number_of_failed_tests.stdout | default("0")) | int == 0 - - (number_of_passed_tests.stdout | default("0")) | int == 0 + - (number_of_failed_tests | default("0")) | int == 0 + - (number_of_passed_tests | default("0")) | int == 0 # must-gather is retrieved if openstack-test is successfully executed and there are failing # tests From 3259e02b5c7fef8d2ed1cd70be224e0e2a279606 Mon Sep 17 00:00:00 2001 From: tusharjadhav3302 Date: Mon, 10 Aug 2026 13:33:15 +0530 Subject: [PATCH 4/4] Fix OTE allowlist matching for lb/egressip filtered stages OTE list -o names emits unquoted names plus klog on stdout, while allowlists are converted as quoted patterns for legacy dry-run. Strip optional quotes, drop klog lines, and filter list stdout so .*[lb].* matches again. Co-authored-by: Cursor --- .../tasks/run_openstack_test.yml | 13 +++- .../plugins/modules/filter_tests_list.py | 76 ++++++++++++++----- 2 files changed, 68 insertions(+), 21 deletions(-) diff --git a/collection/stages/roles/openstack_test/tasks/run_openstack_test.yml b/collection/stages/roles/openstack_test/tasks/run_openstack_test.yml index ff1af866..5896bf9f 100644 --- a/collection/stages/roles/openstack_test/tasks/run_openstack_test.yml +++ b/collection/stages/roles/openstack_test/tasks/run_openstack_test.yml @@ -71,10 +71,15 @@ # --- List tests --- - name: Prepare {{ all_tests_path }} with OTE list (names) - ansible.builtin.shell: > - {{ openstack_test_executable }} list - --suite {{ openstack_test_suite }} - -o names > {{ all_tests_path }} + # OTE prints klog to stdout (I0810 ...); strip those so allow/block filters + # match real test names. stderr discarded for the same reason. + ansible.builtin.shell: | + set -o pipefail + {{ openstack_test_executable }} list \ + --suite {{ openstack_test_suite }} \ + -o names 2>/dev/null \ + | grep -v -E '^I[0-9]{4} ' \ + > {{ all_tests_path }} environment: OS_CLOUD: "{{ user_cloud }}" KUBECONFIG: "{{ kubeconfig }}" diff --git a/collection/tools/plugins/modules/filter_tests_list.py b/collection/tools/plugins/modules/filter_tests_list.py index ce6f7774..ff79f9bc 100644 --- a/collection/tools/plugins/modules/filter_tests_list.py +++ b/collection/tools/plugins/modules/filter_tests_list.py @@ -98,6 +98,25 @@ def escape_special_characters(string): return string.translate(translation_table) +def normalize_test_line(line): + """Normalize a tests-list / allow|block list line for matching. + + - Strip whitespace/newlines + - Strip optional surrounding double quotes (legacy dry-run / convert_yaml + wrap patterns as ``".*[lb].*"`` while OTE ``list -o names`` is unquoted) + - Drop OTE/klog noise lines that land in list output (``I0810 ...``) + """ + s = line.strip() + if not s: + return None + # klog-style lines mixed into OTE list stdout + if re.match(r'I\d{4}\s', s) or 'test_context.go:' in s: + return None + if len(s) >= 2 and s[0] == '"' and s[-1] == '"': + s = s[1:-1].strip() + return s or None + + def run_module(): # define the AnsibleModule object with the available # arguments/parameters a user can pass to the module @@ -128,10 +147,17 @@ def run_module(): try: with open(input_tests_file, 'r') as f: - input_tests = set([line for line in f]) + # Preserve original lines for output, keyed by normalized name. + input_tests_raw = [line for line in f] except IOError: module.fail_json(msg="Error opening the input tests file") + input_tests = [] + for line in input_tests_raw: + normalized = normalize_test_line(line) + if normalized is not None: + input_tests.append((normalized, line if line.endswith('\n') else line + '\n')) + if allowlist_file and blocklist_file: module.fail_json(msg="parameters are mutually exclusive: " "allowlist_file|blocklist_file", **result) @@ -146,12 +172,15 @@ def run_module(): module.fail_json(msg="Error opening the allowlist file") for allowlist_test in allowlist: + allow_norm = normalize_test_line(allowlist_test) + if allow_norm is None: + continue allowlist_test_in_input_tests = False + escaped_allow_test = escape_special_characters(allow_norm) - for test in input_tests: - escaped_allow_test = escape_special_characters(allowlist_test) - if re.fullmatch(escaped_allow_test, test): - tests_to_run.append(test) + for test_norm, test_raw in input_tests: + if re.fullmatch(escaped_allow_test, test_norm): + tests_to_run.append(test_raw) allowlist_test_in_input_tests = True if not allowlist_test_in_input_tests: @@ -173,35 +202,42 @@ def run_module(): elif blocklist_file: try: with open(blocklist_file, 'r') as f: - blocklist = set([line for line in f]) + blocklist = [line for line in f] except IOError: module.fail_json(msg="Error opening the blocklist file") + blocklist_norms = [] + for blocklist_test in blocklist: + block_norm = normalize_test_line(blocklist_test) + if block_norm is not None: + blocklist_norms.append((block_norm, blocklist_test)) + # initialize lists for tests tests_to_run = [] blocked_tests = [] unused_blocklist_tests = [] # iterate over the list of tests and set the tests to run - for test in input_tests: + for test_norm, test_raw in input_tests: test_in_blocklist = False - for blocklist_test in blocklist: - escaped_block_test = escape_special_characters(blocklist_test) - if re.fullmatch(escaped_block_test, test): + for block_norm, _block_raw in blocklist_norms: + escaped_block_test = escape_special_characters(block_norm) + if re.fullmatch(escaped_block_test, test_norm): test_in_blocklist = True break if test_in_blocklist: - blocked_tests.append(test) + blocked_tests.append(test_raw) else: - tests_to_run.append(test) + tests_to_run.append(test_raw) # set the unused blocklist tests - for blocklist_test in blocklist: - escaped_block_test = escape_special_characters(blocklist_test) - if not any(re.fullmatch(escaped_block_test, test) for test in input_tests): - unused_blocklist_tests.append(blocklist_test) + for block_norm, block_raw in blocklist_norms: + escaped_block_test = escape_special_characters(block_norm) + if not any(re.fullmatch(escaped_block_test, test_norm) + for test_norm, _ in input_tests): + unused_blocklist_tests.append(block_raw) if unused_blocklist_tests: module.warn("Warning! Some tests in the blocklist were not used") result['unused_blocklist_tests'] = unused_blocklist_tests @@ -227,7 +263,13 @@ def run_module(): result['changed'] = True else: - shutil.copyfile(input_tests_file, output_file) + # Drop klog noise even when no allow/block filter is applied (OTE list). + try: + with open(output_file, 'w') as f: + for _test_norm, test_raw in input_tests: + f.write(test_raw) + except IOError: + module.fail_json(msg="Error writing to output file") result['filter_type'] = 'no filter applied' result['changed'] = True