diff --git a/collection/stages/roles/openstack_test/defaults/main.yml b/collection/stages/roles/openstack_test/defaults/main.yml index b95948f1..e7754b15 100644 --- a/collection/stages/roles/openstack_test/defaults/main.yml +++ b/collection/stages/roles/openstack_test/defaults/main.yml @@ -3,8 +3,12 @@ openstack_test_name: openstack-test openstack_testsuite_name: openstack openstack_test_dir: "{{ artifacts_dir }}/{{ openstack_test_name }}" +# Installed binary path is set in run_openstack_test.yml (OTE vs legacy). openstack_test_executable: "{{ home_dir }}/openstack-tests" openstack_test_results_dir: "{{ artifacts_dir }}/{{ openstack_test_name }}-results" openstack_tests_allowlist_file: "" openstack_tests_blocklist_file: "" openstack_tests_go_version: "{{ tests.default_go_version_target }}" +# OTE (OpenShift Tests Extension) — used when discovered_openshift_release >= 4.20 +openstack_test_suite: openstack-test/all +openstack_test_ote_binary_relpath: bin/openstack-test-tests-ext diff --git a/collection/stages/roles/openstack_test/files/ote_resolve_results.py b/collection/stages/roles/openstack_test/files/ote_resolve_results.py new file mode 100644 index 00000000..aa561a84 --- /dev/null +++ b/collection/stages/roles/openstack_test/files/ote_resolve_results.py @@ -0,0 +1,224 @@ +#!/usr/bin/env python3 +"""Resolve true OTE openstack-test outcomes from run-suite / run-test logs. + +OTE often marks every outer JSON entry as failed when stderr begins with klog +lines (``I0809 ...``), producing:: + + Deserializaion Error: invalid character 'I' looking for beginning of value + +The real outcome is in nested STDOUT JSON (``"result": "passed"|"failed"|...``) +or in ginkgo summary lines (``SUCCESS!`` / ``FAIL!``). + +Usage: + ote_resolve_results.py count passed|failed|skipped + ote_resolve_results.py junit +""" + +from __future__ import annotations + +import json +import re +import sys +import xml.etree.ElementTree as ET +from typing import Any + + +def _load_outer_results(raw: str) -> list[dict[str, Any]]: + raw = raw.strip() + if not raw: + return [] + + # Skip leading klog / noise before the JSON array/object. + start_candidates = [i for i, ch in enumerate(raw) if ch in "[{"] + for start in start_candidates: + chunk = raw[start:] + try: + data, _ = json.JSONDecoder().raw_decode(chunk) + except json.JSONDecodeError: + continue + if isinstance(data, list): + return [r for r in data if isinstance(r, dict)] + if isinstance(data, dict): + return [data] + + # NDJSON fallback + results: list[dict[str, Any]] = [] + for line in raw.splitlines(): + line = line.strip() + if not line or not line.startswith("{"): + continue + try: + obj = json.loads(line) + except json.JSONDecodeError: + continue + if isinstance(obj, dict): + results.append(obj) + return results + + +def _extract_nested_results(output: str) -> list[dict[str, Any]] | None: + if not output: + return None + + m = re.search(r"STDOUT:\n(.*?)(?:\n\nSTDERR:|\nSTDERR:|\Z)", output, re.S) + stdout = m.group(1) if m else output + + for match in re.finditer(r"\[", stdout): + chunk = stdout[match.start() :] + try: + obj, _ = json.JSONDecoder().raw_decode(chunk) + except json.JSONDecodeError: + continue + if ( + isinstance(obj, list) + and obj + and isinstance(obj[0], dict) + and "result" in obj[0] + ): + return obj + return None + + +def _result_from_ginkgo(output: str) -> str | None: + if not output: + return None + # Prefer FAIL over SUCCESS when both somehow appear. + if re.search(r"FAIL! -- .*\| 1 Failed", output) or re.search( + r"FAIL! -- 0 Passed \| [1-9]", output + ): + return "failed" + if re.search(r"SUCCESS! -- 0 Passed \| 0 Failed \| 0 Pending \| [1-9]+ Skipped", output): + return "skipped" + if re.search(r"SUCCESS! -- [1-9]\d* Passed \| 0 Failed", output): + return "passed" + if "[FAILED]" in output and "SUCCESS! -- 1 Passed" not in output: + return "failed" + return None + + +def _has_deser_error(output: str) -> bool: + return "Deserializaion Error" in output or "Deserialization Error" in output + + +def resolve_result(entry: dict[str, Any]) -> str: + """Return passed|failed|skipped|unknown for one outer OTE log entry.""" + outer = (entry.get("result") or "").lower() + output = entry.get("output") or "" + + nested = _extract_nested_results(output) + if nested: + # One outer entry maps to one (or few) nested specs; use first, or + # failed if any nested failed. + nested_results = [(r.get("result") or "").lower() for r in nested] + if "failed" in nested_results: + return "failed" + if nested_results and all(r == "skipped" for r in nested_results): + return "skipped" + if nested_results and all(r == "passed" for r in nested_results): + return "passed" + if "passed" in nested_results and "failed" not in nested_results: + return "passed" + + ginkgo = _result_from_ginkgo(output) + if ginkgo: + return ginkgo + + # Outer "failed" with only deserialize noise and no other signal is unknown; + # callers should not treat unknown as a real failure count source of truth. + if outer == "failed" and _has_deser_error(output) and not nested and not ginkgo: + return "unknown" + + if outer in ("passed", "failed", "skipped"): + return outer + return "unknown" + + +def resolve_all(log_path: str) -> list[dict[str, Any]]: + try: + with open(log_path, encoding="utf-8") as f: + raw = f.read() + except OSError: + return [] + + resolved: list[dict[str, Any]] = [] + for entry in _load_outer_results(raw): + result = resolve_result(entry) + resolved.append( + { + "name": entry.get("name") or "unknown", + "result": result, + "duration": entry.get("duration") or 0, + "output": entry.get("output") or entry.get("error") or "", + "error": entry.get("error") or "", + } + ) + return resolved + + +def cmd_count(log_path: str, want: str) -> int: + want = want.lower() + return sum(1 for r in resolve_all(log_path) if r["result"] == want) + + +def cmd_junit(log_path: str, junit_path: str) -> None: + results = resolve_all(log_path) + suite = ET.Element("testsuite", name="openstack-test") + failures = 0 + skipped = 0 + for r in results: + duration = r.get("duration") or 0 + try: + # OTE durations are often nanoseconds; keep seconds for junit. + dur = float(duration) + if dur > 10_000: + # ns or ms — treat large values as ns + time_s = f"{dur / 1_000_000_000.0:.3f}" if dur > 1_000_000 else f"{dur / 1000.0:.3f}" + else: + time_s = f"{dur:.3f}" + except (TypeError, ValueError): + time_s = "0" + + case = ET.SubElement(suite, "testcase", name=r["name"], time=time_s) + result = r["result"] + if result == "failed": + failures += 1 + fail = ET.SubElement(case, "failure") + fail.text = r.get("error") or r.get("output") or "failed" + elif result == "skipped": + skipped += 1 + skip = ET.SubElement(case, "skipped") + skip.text = r.get("output") or "skipped" + elif result == "unknown": + # Do not inflate failure counts for unparsable deserialize-only noise. + skip = ET.SubElement(case, "skipped") + skip.text = "unresolved OTE outcome" + + suite.set("tests", str(len(results))) + suite.set("failures", str(failures)) + suite.set("skipped", str(skipped)) + ET.ElementTree(suite).write(junit_path, encoding="utf-8", xml_declaration=True) + + +def main(argv: list[str]) -> int: + if len(argv) < 2: + print(__doc__, file=sys.stderr) + return 2 + cmd = argv[1] + if cmd == "count": + if len(argv) != 4: + print("usage: count passed|failed|skipped", file=sys.stderr) + return 2 + print(cmd_count(argv[2], argv[3])) + return 0 + if cmd == "junit": + if len(argv) != 4: + print("usage: junit ", file=sys.stderr) + return 2 + cmd_junit(argv[2], argv[3]) + return 0 + print(__doc__, file=sys.stderr) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv)) diff --git a/collection/stages/roles/openstack_test/tasks/run_openstack_test.yml b/collection/stages/roles/openstack_test/tasks/run_openstack_test.yml index 4de88db9..ff1af866 100644 --- a/collection/stages/roles/openstack_test/tasks/run_openstack_test.yml +++ b/collection/stages/roles/openstack_test/tasks/run_openstack_test.yml @@ -1,5 +1,5 @@ --- -- name: Set artifact paths +- name: Set artifact paths and OTE vs legacy mode ansible.builtin.set_fact: all_tests_path: >- {{ openstack_test_results_dir }}/openstack_tests.txt @@ -9,28 +9,79 @@ {{ openstack_tests_blocklist_file | ternary(openstack_test_results_dir + '/blocklist.txt', '') }} tests_to_run_path: >- {{ openstack_test_results_dir }}/list_of_tests_to_run.txt + openstack_test_log_path: >- + {{ openstack_test_results_dir }}/{{ openstack_test_name }}.log + openstack_test_junit_path: >- + {{ openstack_test_results_dir }}/junit_e2e_openstack.xml + # Upstream dropped make openstack-tests on release-4.20+ (OTE migration, + # openshift/openstack-test#303 / OSASINFRA-4375; backports #306/#307/#308). + # Older release branches still ship the legacy target. + openstack_test_use_ote: "{{ discovered_openshift_release is version('4.20', '>=') }}" -- name: Build the openstack-tests for tests in the "openstack-test" repository +- name: Set openstack-test executable path for OTE + ansible.builtin.set_fact: + openstack_test_executable: "{{ home_dir }}/openstack-test-tests-ext" + when: openstack_test_use_ote | bool + +- name: Set openstack-test executable path for legacy openshift-tests + ansible.builtin.set_fact: + openstack_test_executable: "{{ home_dir }}/openstack-tests" + when: not (openstack_test_use_ote | bool) + +# --- Build --- +# OTE: make extension → bin/openstack-test-tests-ext (openshift/openstack-test#303) +- name: Build the OTE openstack-test extension binary + ansible.builtin.shell: | + source {{ home_dir }}/.bashrc + make extension + args: + chdir: "{{ openstack_test_dir }}" + changed_when: true + when: openstack_test_use_ote | bool + +- name: Build the legacy openstack-tests binary ansible.builtin.shell: | source {{ home_dir }}/.bashrc make openstack-tests args: chdir: "{{ openstack_test_dir }}" changed_when: true + when: not (openstack_test_use_ote | bool) -- name: Copy openstack-test executable +- name: Copy OTE openstack-test executable + ansible.builtin.copy: + src: "{{ openstack_test_dir }}/{{ openstack_test_ote_binary_relpath }}" + dest: "{{ openstack_test_executable }}" + mode: u=rwx,g=rwx,o=rwx + remote_src: yes + when: openstack_test_use_ote | bool + +- name: Copy legacy openstack-test executable ansible.builtin.copy: src: "{{ openstack_test_dir }}/openstack-tests" dest: "{{ openstack_test_executable }}" mode: u=rwx,g=rwx,o=rwx remote_src: yes + when: not (openstack_test_use_ote | bool) - name: Remove source directory once compilation succeeded ansible.builtin.file: path: "{{ openstack_test_dir }}" state: absent -- name: Prepare {{ all_tests_path }} file with all the tests that can be run +# --- List tests --- +- name: Prepare {{ all_tests_path }} with OTE list (names) + ansible.builtin.shell: > + {{ openstack_test_executable }} list + --suite {{ openstack_test_suite }} + -o names > {{ all_tests_path }} + environment: + OS_CLOUD: "{{ user_cloud }}" + KUBECONFIG: "{{ kubeconfig }}" + changed_when: true + when: openstack_test_use_ote | bool + +- name: Prepare {{ all_tests_path }} with legacy dry-run ansible.builtin.shell: > {{ openstack_test_executable }} run openshift/openstack --dry-run > {{ all_tests_path }} @@ -38,6 +89,7 @@ OS_CLOUD: "{{ user_cloud }}" KUBECONFIG: "{{ kubeconfig }}" changed_when: true + when: not (openstack_test_use_ote | bool) - name: Convert the allowlist YAML files to TXT ansible.builtin.include_role: @@ -67,13 +119,75 @@ blocklist_file: "{{ blocklist_path }}" output_file: "{{ tests_to_run_path }}" +- name: Detect whether allow or block list filtering is active + ansible.builtin.set_fact: + openstack_test_filtering: "{{ (openstack_tests_allowlist_file | length > 0) or (openstack_tests_blocklist_file | length > 0) }}" + +# OTE run-suite often marks every outer JSON result as failed when stderr has +# klog lines (Deserializaion Error: invalid character 'I'...). True outcomes +# live in nested STDOUT JSON / ginkgo SUCCESS!|FAIL! lines — see +# files/ote_resolve_results.py. +- name: Set path to OTE result resolver helper + ansible.builtin.set_fact: + openstack_test_ote_resolve_script: "{{ role_path }}/files/ote_resolve_results.py" + when: openstack_test_use_ote | bool + - name: Run the openshift tests block: - - name: Run openstack tests + - name: Run OTE suite (no allow/block filter) + when: + - openstack_test_use_ote | bool + - not (openstack_test_filtering | bool) + block: + - name: Execute OTE run-suite + ansible.builtin.shell: > + {{ openstack_test_executable }} run-suite {{ openstack_test_suite }} + --junit-path {{ openstack_test_junit_path }} + > {{ openstack_test_log_path }} + environment: + # RHOSO_KUBECONFIG and SHIFTSTACK_PASS_FILE are required for the observability test + OS_CLOUD: "{{ user_cloud }}" + KUBECONFIG: "{{ kubeconfig }}" + RHOSO_KUBECONFIG: "{{ rhoso_kubeconfig }}" + SHIFTSTACK_PASS_FILE: "{{ kubeadmin_password }}" + changed_when: true + always: + # Rewrite junit from resolved outcomes (OTE's own junit inherits false fails). + - name: Rewrite junit from resolved OTE run-suite outcomes + ansible.builtin.command: + argv: + - python3 + - "{{ openstack_test_ote_resolve_script }}" + - junit + - "{{ openstack_test_log_path }}" + - "{{ openstack_test_junit_path }}" + changed_when: true + + # OTE run-test has no --junit-path; pipe filtered names via stdin and + # synthesize junit from resolved log outcomes for post_openshift_tests. + - name: Run OTE filtered tests via run-test and write junit + ansible.builtin.shell: | + set -o pipefail + cat {{ tests_to_run_path }} | {{ openstack_test_executable }} run-test \ + > {{ openstack_test_log_path }} || run_exit=$? + python3 "{{ openstack_test_ote_resolve_script }}" junit \ + "{{ openstack_test_log_path }}" "{{ openstack_test_junit_path }}" || py_exit=$? + exit ${run_exit:-${py_exit:-0}} + environment: + OS_CLOUD: "{{ user_cloud }}" + KUBECONFIG: "{{ kubeconfig }}" + RHOSO_KUBECONFIG: "{{ rhoso_kubeconfig }}" + SHIFTSTACK_PASS_FILE: "{{ kubeadmin_password }}" + changed_when: true + when: + - openstack_test_use_ote | bool + - openstack_test_filtering | bool + + - name: Run legacy openstack tests ansible.builtin.shell: > {{ openstack_test_executable }} run openshift/openstack -f {{ tests_to_run_path }} - --output-file {{ openstack_test_results_dir }}/{{ openstack_test_name }}.log + --output-file {{ openstack_test_log_path }} --junit-dir={{ openstack_test_results_dir }} > /dev/null environment: # RHOSO_KUBECONFIG and SHIFTSTACK_PASS_FILE are required for the observability test @@ -82,6 +196,7 @@ RHOSO_KUBECONFIG: "{{ rhoso_kubeconfig }}" SHIFTSTACK_PASS_FILE: "{{ kubeadmin_password }}" changed_when: true + when: not (openstack_test_use_ote | bool) rescue: - name: Mark the openshift tests as UNSTABLE @@ -92,21 +207,73 @@ unstable_msg: >- The openshift test Suite failed. - - name: Get the number of failed openstack tests + # Use distinct registers for OTE vs legacy. Skipped tasks still overwrite + # a shared register in Ansible, which wiped OTE counts and falsely hit + # "no tests where run!" after a real suite run. + - name: Get the number of failed openstack tests (OTE resolved log) + ansible.builtin.command: + argv: + - python3 + - "{{ openstack_test_ote_resolve_script }}" + - count + - "{{ openstack_test_log_path }}" + - failed + register: openstack_test_ote_failed_count + changed_when: false + failed_when: false + when: openstack_test_use_ote | bool + + - name: Get the number of passed openstack tests (OTE resolved log) + ansible.builtin.command: + argv: + - python3 + - "{{ openstack_test_ote_resolve_script }}" + - count + - "{{ openstack_test_log_path }}" + - passed + register: openstack_test_ote_passed_count + changed_when: false + failed_when: false + when: openstack_test_use_ote | bool + + - name: Get the number of failed openstack tests (legacy log) ansible.builtin.shell: > set -o pipefail && - grep -e "^failed:" {{ openstack_test_results_dir }}/{{ openstack_test_name }}.log | wc -l + grep -e "^failed:" {{ openstack_test_log_path }} | wc -l || true - register: number_of_failed_tests + register: openstack_test_legacy_failed_count changed_when: false + when: not (openstack_test_use_ote | bool) - - name: Get the number of passed openstack tests + - name: Get the number of passed openstack tests (legacy log) ansible.builtin.shell: > set -o pipefail && - grep -e "^passed:" {{ openstack_test_results_dir }}/{{ openstack_test_name }}.log | wc -l + grep -e "^passed:" {{ openstack_test_log_path }} | wc -l || true - register: number_of_passed_tests + register: openstack_test_legacy_passed_count changed_when: false + when: not (openstack_test_use_ote | bool) + + - name: Normalize openstack-test pass/fail counts + ansible.builtin.set_fact: + number_of_failed_tests: >- + {{ ( + openstack_test_ote_failed_count.stdout | default('0') + if (openstack_test_use_ote | bool) + else openstack_test_legacy_failed_count.stdout | default('0') + ) | trim }} + number_of_passed_tests: >- + {{ ( + openstack_test_ote_passed_count.stdout | default('0') + if (openstack_test_use_ote | bool) + else openstack_test_legacy_passed_count.stdout | default('0') + ) | trim }} + + - name: Print resolved openstack-test pass/fail counts + ansible.builtin.debug: + msg: >- + openstack-test counts: passed={{ number_of_passed_tests }}, + failed={{ number_of_failed_tests }} # This fail task is added to detect failures in openstack-test execution - name: Fail the playbook in case there are no failed and passed tests @@ -114,8 +281,8 @@ msg: > openstack-test execution failed or no tests where run! when: - - (number_of_failed_tests.stdout | default("0")) | int == 0 - - (number_of_passed_tests.stdout | default("0")) | int == 0 + - (number_of_failed_tests | default("0")) | int == 0 + - (number_of_passed_tests | default("0")) | int == 0 # must-gather is retrieved if openstack-test is successfully executed and there are failing # tests