Skip to content

Commit db20cfe

Browse files
Merge pull request #44 from shiftstack/fix/openstack-test-machines-leak-cluster-serial
openstack_test: serial coda for Machine leak, suite flakes, and topology
2 parents f01bb8a + d900268 commit db20cfe

4 files changed

Lines changed: 291 additions & 0 deletions

File tree

‎collection/stages/roles/openstack_test/defaults/main.yml‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,3 +14,26 @@ openstack_test_suite: openstack-test/all
1414
openstack_test_ote_binary_relpath: bin/openstack-test-tests-ext
1515
# Run filtered OTE tests one-by-one (required for [lb][Serial] under Octavia limits).
1616
openstack_test_ote_run_serial: false
17+
# Serial coda: after the main filtered batch suite, run selected tests
18+
# serially in fixed order. Starts with the bogus Machine leak cluster
19+
# (bz_2073398 cleanup, then Machine trio), then known suite-load flakes
20+
# that pass alone but time out in the randomized batch (egressIP FIP
21+
# failover AAP wait, prometheus PVC resize pod-ready wait), then topology
22+
# pair ordered so the AZ check runs while enable_topology is still true
23+
# before the mutator ([Serial] set enable_topology=false) can race it in
24+
# batch. Keeps the bulk suite in batch to limit wall time. No-ops when
25+
# the matchers are absent (e.g. lb_tests allowlists). Requires OTE
26+
# filtered run path.
27+
openstack_test_machines_leak_cluster_serial: true
28+
# Ordered substrings matched against list_of_tests_to_run.txt (one phrase
29+
# each; written to a matchers file so shell never splits them). For the
30+
# MachineSet replica matcher, ControlPlane variants are skipped.
31+
openstack_test_machines_leak_cluster_matchers:
32+
- "Bugfix bz_2073398: [Serial] MachineSet scale-in does not leak OpenStack ports"
33+
- "Machine are in phase Running"
34+
- "MachineSet replica number corresponds to the number of Machines"
35+
- "Machine ProviderSpec is correctly applied to OpenStack instances"
36+
- "egressIP attached to a floating IP should be kept after EgressIP node failover"
37+
- "should follow PVC specs during resizing for prometheus"
38+
- "enables topology aware scheduling when compute and volume AZs are identical"
39+
- "should allow the manual setting of enable_topology to false"
Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,111 @@
1+
#!/usr/bin/env python3
2+
"""Split serial-coda tests out of an OTE test list.
3+
4+
Reads the filtered list_of_tests_to_run.txt and an ordered matchers file
5+
(one substring per line), writes:
6+
- leak_cluster_serial.txt: ordered matches for serial coda
7+
(Machine leak cluster, suite-load flakes, topology pair)
8+
- list_of_tests_to_run.txt: remaining tests for the batch suite
9+
10+
Matchers are plain substrings (one full phrase per line). The MachineSet
11+
replica matcher skips any line that also contains "ControlPlane".
12+
13+
When any matcher hits, every matcher must resolve to exactly one test;
14+
missing or non-unique matches fail hard. Zero hits keep the allowlist
15+
no-op (empty serial file, batch unchanged).
16+
"""
17+
from __future__ import annotations
18+
19+
import argparse
20+
import sys
21+
from pathlib import Path
22+
23+
24+
def load_matchers(path: str) -> list[str]:
25+
with open(path, "r", encoding="utf-8") as f:
26+
return [ln.strip() for ln in f if ln.strip() and not ln.strip().startswith("#")]
27+
28+
29+
def pick(remaining: list[str], matcher: str) -> list[str]:
30+
"""Return all candidate lines for matcher (ControlPlane skipped for MachineSet replica)."""
31+
candidates: list[str] = []
32+
for ln in remaining:
33+
if not ln.strip():
34+
continue
35+
if matcher not in ln:
36+
continue
37+
# Avoid ControlPlane MachineSet when matching worker MachineSet replica.
38+
if "MachineSet replica number corresponds to the number of Machines" in matcher:
39+
if "ControlPlane" in ln:
40+
continue
41+
candidates.append(ln)
42+
return candidates
43+
44+
45+
def main() -> int:
46+
parser = argparse.ArgumentParser()
47+
parser.add_argument("tests_to_run_path")
48+
parser.add_argument("leak_cluster_path")
49+
parser.add_argument("matchers_file")
50+
args = parser.parse_args()
51+
52+
matchers = load_matchers(args.matchers_file)
53+
if not matchers:
54+
print("leak_cluster=0 batch_remaining=unchanged (empty matchers file)")
55+
Path(args.leak_cluster_path).write_text("", encoding="utf-8")
56+
return 0
57+
58+
with open(args.tests_to_run_path, "r", encoding="utf-8") as f:
59+
lines = [ln.rstrip("\n") for ln in f]
60+
61+
remaining = list(lines)
62+
resolved: list[tuple[str, list[str]]] = []
63+
any_hit = False
64+
65+
for matcher in matchers:
66+
candidates = pick(remaining, matcher)
67+
if candidates:
68+
any_hit = True
69+
resolved.append((matcher, candidates))
70+
71+
if not any_hit:
72+
# Allowlist no-op: none of the coda tests are in this filtered list.
73+
print("leak_cluster=0 batch_remaining=unchanged (no matchers hit)")
74+
Path(args.leak_cluster_path).write_text("", encoding="utf-8")
75+
return 0
76+
77+
selected: list[str] = []
78+
for matcher, candidates in resolved:
79+
if len(candidates) == 0:
80+
print(
81+
f"error: matcher matched no tests: {matcher!r}",
82+
file=sys.stderr,
83+
)
84+
return 1
85+
if len(candidates) > 1:
86+
print(
87+
f"error: matcher matched {len(candidates)} tests (expected 1): {matcher!r}",
88+
file=sys.stderr,
89+
)
90+
for c in candidates:
91+
print(f" candidate: {c}", file=sys.stderr)
92+
return 1
93+
selected.append(candidates[0])
94+
remaining = [ln for ln in remaining if ln != candidates[0]]
95+
96+
with open(args.leak_cluster_path, "w", encoding="utf-8") as f:
97+
for ln in selected:
98+
f.write(ln + "\n")
99+
100+
with open(args.tests_to_run_path, "w", encoding="utf-8") as f:
101+
for ln in remaining:
102+
f.write(ln + "\n")
103+
104+
print(
105+
f"leak_cluster={len(selected)} batch_remaining={len([x for x in remaining if x.strip()])}"
106+
)
107+
return 0
108+
109+
110+
if __name__ == "__main__":
111+
sys.exit(main())
Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
---
2+
# Serial coda: serially run the ordered coda list after the main batch suite
3+
# (Machine leak cluster, egressIP / prometheus, then topology AZ check
4+
# before enable_topology=false mutator).
5+
# Expects openstack_test_leak_cluster_path to already list the ordered tests.
6+
- name: Stat machines leak-cluster serial list
7+
ansible.builtin.stat:
8+
path: "{{ openstack_test_leak_cluster_path }}"
9+
register: openstack_test_leak_cluster_stat
10+
11+
- name: Count machines leak-cluster serial tests
12+
ansible.builtin.command:
13+
argv:
14+
- bash
15+
- -c
16+
- "grep -cve '^[[:space:]]*$' '{{ openstack_test_leak_cluster_path }}' || true"
17+
register: openstack_test_leak_cluster_count
18+
changed_when: false
19+
when: openstack_test_leak_cluster_stat.stat.exists | default(false)
20+
21+
- name: Skip machines leak-cluster serial coda when empty
22+
ansible.builtin.debug:
23+
msg: >-
24+
No machines leak-cluster tests matched the filtered list; skipping serial coda.
25+
when: >
26+
(not (openstack_test_leak_cluster_stat.stat.exists | default(false)))
27+
or ((openstack_test_leak_cluster_count.stdout | default('0') | trim | int) == 0)
28+
29+
- name: Run machines leak-cluster tests serially and merge into suite log/junit
30+
when:
31+
- openstack_test_leak_cluster_stat.stat.exists | default(false)
32+
- (openstack_test_leak_cluster_count.stdout | default('0') | trim | int) > 0
33+
block:
34+
- name: Execute machines leak-cluster run-test loop
35+
ansible.builtin.shell: |
36+
set -o pipefail
37+
suite_exit=0
38+
py_exit=0
39+
resolve="{{ openstack_test_ote_resolve_script }}"
40+
leak_log="{{ openstack_test_results_dir }}/machines_leak_cluster.log"
41+
leak_list="{{ openstack_test_leak_cluster_path }}"
42+
# Pre-count from the list file (do not increment in-loop; avoids
43+
# false UNSTABLE when resolvable!=expected after a successful coda).
44+
expected="$(grep -cve '^[[:space:]]*$' "$leak_list" || true)"
45+
: > "$leak_log"
46+
while IFS= read -r tname || [ -n "${tname:-}" ]; do
47+
[ -z "${tname// }" ] && continue
48+
tmp=$(mktemp)
49+
rc=0
50+
{{ openstack_test_executable }} run-test --output=json -c 1 "$tname" \
51+
> "$tmp" 2>&1 || rc=$?
52+
cat "$tmp" >> "$leak_log"
53+
cat "$tmp" >> {{ openstack_test_log_path }}
54+
if [ "$rc" -ne 0 ]; then
55+
failed=$(python3 "$resolve" count "$tmp" failed || echo 1)
56+
skipped=$(python3 "$resolve" count "$tmp" skipped || echo 0)
57+
passed=$(python3 "$resolve" count "$tmp" passed || echo 0)
58+
unknown=$(python3 "$resolve" count "$tmp" unknown || echo 0)
59+
if [ "$failed" -gt 0 ] || [ "$unknown" -gt 0 ]; then
60+
suite_exit=1
61+
elif [ $((failed + skipped + passed + unknown)) -eq 0 ]; then
62+
if grep -q 'FAIL!' "$tmp" 2>/dev/null; then
63+
suite_exit=1
64+
elif ! grep -qE 'SUCCESS!|SKIP' "$tmp" 2>/dev/null; then
65+
suite_exit=1
66+
fi
67+
fi
68+
fi
69+
rm -f "$tmp"
70+
done < "$leak_list"
71+
python3 "$resolve" junit \
72+
"{{ openstack_test_log_path }}" "{{ openstack_test_junit_path }}" || py_exit=$?
73+
failed_count=$(python3 "$resolve" count "$leak_log" failed || echo 0)
74+
skipped_count=$(python3 "$resolve" count "$leak_log" skipped || echo 0)
75+
passed_count=$(python3 "$resolve" count "$leak_log" passed || echo 0)
76+
unknown_count=$(python3 "$resolve" count "$leak_log" unknown || echo 0)
77+
resolvable=$(( ${passed_count:-0} + ${skipped_count:-0} + ${failed_count:-0} ))
78+
echo "machines_leak_cluster counts: passed=${passed_count} failed=${failed_count} skipped=${skipped_count} unknown=${unknown_count} expected=${expected} resolvable=${resolvable}"
79+
if [ "$suite_exit" -ne 0 ] || [ "$py_exit" -ne 0 ] \
80+
|| [ "${failed_count:-0}" -gt 0 ] || [ "${unknown_count:-0}" -gt 0 ] \
81+
|| [ "$resolvable" -ne "$expected" ]; then
82+
exit 1
83+
fi
84+
exit 0
85+
environment:
86+
OS_CLOUD: "{{ user_cloud }}"
87+
KUBECONFIG: "{{ kubeconfig }}"
88+
RHOSO_KUBECONFIG: "{{ rhoso_kubeconfig }}"
89+
SHIFTSTACK_PASS_FILE: "{{ kubeadmin_password }}"
90+
changed_when: true
91+
register: openstack_test_leak_cluster_run
92+
93+
rescue:
94+
- name: Mark openstack-test UNSTABLE after machines leak-cluster serial failure
95+
ansible.builtin.include_role:
96+
name: tools_stage_results
97+
tasks_from: mark_test_stage_unstable.yml
98+
vars:
99+
unstable_msg: >-
100+
openstack_test serial coda failed (Machine leak cluster, egressIP /
101+
prometheus resize, and/or topology enable_topology pair).

‎collection/stages/roles/openstack_test/tasks/run_openstack_test.yml‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -141,6 +141,53 @@
141141
ansible.builtin.set_fact:
142142
openstack_test_filtering: "{{ (openstack_tests_allowlist_file | length > 0) or (openstack_tests_blocklist_file | length > 0) }}"
143143

144+
# Serial coda: pull matchers (Machine leak cluster + suite-load flakes)
145+
# out of the batch list for an ordered serial run after the batch.
146+
- name: Set machines leak-cluster serial list path
147+
ansible.builtin.set_fact:
148+
openstack_test_leak_cluster_path: >-
149+
{{ openstack_test_results_dir }}/machines_leak_cluster_serial.txt
150+
openstack_test_leak_cluster_matchers_path: >-
151+
{{ openstack_test_results_dir }}/machines_leak_cluster_matchers.txt
152+
when:
153+
- openstack_test_use_ote | bool
154+
- openstack_test_machines_leak_cluster_serial | bool
155+
- openstack_test_filtering | bool
156+
157+
# Write matchers one-per-line so multi-word phrases are never shell-split.
158+
- name: Write machines leak-cluster matchers file
159+
ansible.builtin.copy:
160+
dest: "{{ openstack_test_leak_cluster_matchers_path }}"
161+
mode: "0644"
162+
content: "{{ openstack_test_machines_leak_cluster_matchers | map('trim') | join('\n') }}\n"
163+
when:
164+
- openstack_test_use_ote | bool
165+
- openstack_test_machines_leak_cluster_serial | bool
166+
- openstack_test_filtering | bool
167+
168+
- name: Split machines leak-cluster tests from batch list for serial coda
169+
ansible.builtin.command:
170+
argv:
171+
- python3
172+
- "{{ role_path }}/files/split_machines_leak_cluster.py"
173+
- "{{ tests_to_run_path }}"
174+
- "{{ openstack_test_leak_cluster_path }}"
175+
- "{{ openstack_test_leak_cluster_matchers_path }}"
176+
register: openstack_test_leak_cluster_split
177+
changed_when: true
178+
when:
179+
- openstack_test_use_ote | bool
180+
- openstack_test_machines_leak_cluster_serial | bool
181+
- openstack_test_filtering | bool
182+
183+
- name: Print machines leak-cluster split summary
184+
ansible.builtin.debug:
185+
msg: "{{ openstack_test_leak_cluster_split.stdout | default('leak cluster split skipped') }}"
186+
when:
187+
- openstack_test_use_ote | bool
188+
- openstack_test_machines_leak_cluster_serial | bool
189+
- openstack_test_filtering | bool
190+
144191
# OTE run-suite often marks every outer JSON result as failed when stderr has
145192
# klog lines (Deserializaion Error: invalid character 'I'...). True outcomes
146193
# live in nested STDOUT JSON / ginkgo SUCCESS!|FAIL! lines — see
@@ -385,3 +432,12 @@
385432
name: tools_must-gather
386433
vars:
387434
must_gather_suffix: "openstack-tests"
435+
436+
# Serial coda after the main batch (or filtered serial) suite.
437+
- name: Run serial-coda tests after main suite
438+
ansible.builtin.include_tasks: run_machines_leak_cluster_serial.yml
439+
when:
440+
- openstack_test_use_ote | bool
441+
- openstack_test_machines_leak_cluster_serial | bool
442+
- openstack_test_filtering | bool
443+
- openstack_test_leak_cluster_path is defined

0 commit comments

Comments
 (0)