From 3eeb4db946ff089d9e222423e9dd11d27183b44b Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Mon, 5 Oct 2026 22:39:22 -0700 Subject: [PATCH 01/34] gh-158895: Teach librertest to output github job summaries Goal is to make it faster to triage individual test failures while keeping the full existing regrtest fidelity. Fairly minimal integration to hopefully save some human eyeball time. Setup so only the test summary process should output the summary, subprocesses have the env var which will cause output unset. --- Lib/test/libregrtest/main.py | 23 ++++++++++++++++++- Lib/test/libregrtest/worker.py | 2 ++ ...-10-05-22-36-06.gh-issue-158895.3Dmc4J.rst | 2 ++ 3 files changed, 26 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Tests/2026-10-05-22-36-06.gh-issue-158895.3Dmc4J.rst diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 2e8397a8a91d324..acaebf96e9ce533 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -1,3 +1,5 @@ +import contextlib +import io import os import random import re @@ -6,7 +8,7 @@ import sysconfig import time import trace -from _colorize import get_colors # type: ignore[import-not-found] +from _colorize import decolor, get_colors # type: ignore[import-not-found] from typing import NoReturn from test.support import os_helper, MS_WINDOWS, flush_std_streams @@ -480,6 +482,25 @@ def finalize_tests(self, coverage: trace.CoverageResults | None) -> None: if self.junit_filename: self.results.write_junit(self.junit_filename) + self.write_github_summary() + + def write_github_summary(self) -> None: + filename = os.environ.get("GITHUB_STEP_SUMMARY") + if not filename: + return + with contextlib.redirect_stdout(io.StringIO()) as summary: + self.display_summary() + lines = ["## Test results", + f"```\n{decolor(summary.getvalue()).strip()}\n```"] + # Tests which failed in the last run (the re-run, if any) + for result in self.results.rerun_results: + lines.append(f"### {decolor(str(result))}") + for name, traceback in (result.errors or []) + (result.failures or []): + lines.append(f"
{name}\n\n" + f"```\n{traceback}\n```\n
") + with open(filename, "a", encoding="utf-8") as fp: + fp.write("\n\n".join(lines) + "\n") + def display_summary(self) -> None: if self.first_runtests is None: raise ValueError( diff --git a/Lib/test/libregrtest/worker.py b/Lib/test/libregrtest/worker.py index 4e69ab9d8fad05f..24ae95ed23914f6 100644 --- a/Lib/test/libregrtest/worker.py +++ b/Lib/test/libregrtest/worker.py @@ -28,6 +28,8 @@ def create_worker_process(runtests: WorkerRunTests, output_fd: int, cmd.extend(['-m', 'test.libregrtest.worker', worker_json]) env = dict(os.environ) + # Only the main process writes the GitHub Actions job summary + env.pop('GITHUB_STEP_SUMMARY', None) if tmp_dir is not None: env['TMPDIR'] = tmp_dir env['TEMP'] = tmp_dir diff --git a/Misc/NEWS.d/next/Tests/2026-10-05-22-36-06.gh-issue-158895.3Dmc4J.rst b/Misc/NEWS.d/next/Tests/2026-10-05-22-36-06.gh-issue-158895.3Dmc4J.rst new file mode 100644 index 000000000000000..ebeca7174e5903a --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2026-10-05-22-36-06.gh-issue-158895.3Dmc4J.rst @@ -0,0 +1,2 @@ +Individual test failures and an overall summary is now written by +:mod:`test.regrtest` to the Github Action Summary. From 8fcfa5d0f2f8a68424594d4e22824cf1817af2cf Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Mon, 5 Oct 2026 23:09:38 -0700 Subject: [PATCH 02/34] Only output errors, too many jobs. Try coloring a bit more --- Lib/test/libregrtest/main.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index acaebf96e9ce533..40fcfe5f5c577f0 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -486,7 +486,10 @@ def finalize_tests(self, coverage: trace.CoverageResults | None) -> None: def write_github_summary(self) -> None: filename = os.environ.get("GITHUB_STEP_SUMMARY") - if not filename: + # Only write a summary on failure: a CI run has many test jobs + exitcode = self.results.get_exitcode(self.fail_env_changed, + self.fail_rerun) + if not filename or not exitcode: return with contextlib.redirect_stdout(io.StringIO()) as summary: self.display_summary() @@ -497,7 +500,7 @@ def write_github_summary(self) -> None: lines.append(f"### {decolor(str(result))}") for name, traceback in (result.errors or []) + (result.failures or []): lines.append(f"
{name}\n\n" - f"```\n{traceback}\n```\n
") + f"```pytb\n{decolor(traceback)}\n```\n") with open(filename, "a", encoding="utf-8") as fp: fp.write("\n\n".join(lines) + "\n") From e6e09bd77e81ba8def2a1d2b399a11521d995e90 Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Mon, 5 Oct 2026 23:10:30 -0700 Subject: [PATCH 03/34] Fail CI on linux for experimenting with output formatting --- Lib/test/test_os/test_os.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_os/test_os.py b/Lib/test/test_os/test_os.py index 81b3043eb7e75bc..04321264e6f121d 100644 --- a/Lib/test/test_os/test_os.py +++ b/Lib/test/test_os/test_os.py @@ -4080,7 +4080,8 @@ def test_memfd_create(self): self.assertFalse(os.get_inheritable(fd)) with open(fd, "wb", closefd=False) as f: f.write(b'memfd_create') - self.assertEqual(f.tell(), 12) + # XXX: Intentional failure to test GitHub Actions job summaries + self.assertEqual(f.tell(), 13) fd2 = os.memfd_create("Hi") self.addCleanup(os.close, fd2) From 54d123fd515a812147582e8dd1a8264ac1e13cb2 Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Mon, 5 Oct 2026 23:42:57 -0700 Subject: [PATCH 04/34] Reformat failures to be failure first, easier visibility. Simplify formatting code some --- Lib/test/libregrtest/main.py | 35 +++++++++++++++++++++++------------ 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 40fcfe5f5c577f0..42dbf166837a6fc 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -1,5 +1,3 @@ -import contextlib -import io import os import random import re @@ -491,18 +489,31 @@ def write_github_summary(self) -> None: self.fail_rerun) if not filename or not exitcode: return - with contextlib.redirect_stdout(io.StringIO()) as summary: - self.display_summary() - lines = ["## Test results", - f"```\n{decolor(summary.getvalue()).strip()}\n```"] # Tests which failed in the last run (the re-run, if any) - for result in self.results.rerun_results: - lines.append(f"### {decolor(str(result))}") - for name, traceback in (result.errors or []) + (result.failures or []): - lines.append(f"
{name}\n\n" - f"```pytb\n{decolor(traceback)}\n```\n
") + failed = self.results.rerun_results + cases = [(result.errors or []) + (result.failures or []) + for result in failed] + ncase = sum(map(len, cases)) with open(filename, "a", encoding="utf-8") as fp: - fp.write("\n\n".join(lines) + "\n") + def write(text: str) -> None: + # Separate Markdown blocks with an empty line + print(text, end="\n\n", file=fp) + + write(f"## {decolor(self.get_state())}: " + f"{count(len(failed), 'test file')} and " + f"{count(ncase, 'test case')} failed") + for result, result_cases in zip(failed, cases): + write(f"### {decolor(str(result))}") + if result.env_changed_reasons: + write("\n".join(f"- {reason}" + for reason in result.env_changed_reasons)) + for name, traceback in result_cases: + # Expand short tracebacks when there are only a few + is_open = ncase <= 5 and traceback.count("\n") < 30 + write(f"" + f"{name}") + write(f"```pytb\n{decolor(traceback).rstrip()}\n```") + write("") def display_summary(self) -> None: if self.first_runtests is None: From 3468742f5b333d261a0544c1c807283a2111842d Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Mon, 5 Oct 2026 23:44:54 -0700 Subject: [PATCH 05/34] add error annotations --- Lib/test/libregrtest/main.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 42dbf166837a6fc..f191dd123ed8961 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -30,6 +30,15 @@ get_process_memory_usage, EXIT_TIMEOUT) +def github_error(title: str, message: str) -> None: + # Emit a GitHub Actions error annotation (workflow command) + def escape(text: str) -> str: + return (text.replace("%", "%25").replace("\r", "%0D") + .replace("\n", "%0A")) + title = escape(title).replace(":", "%3A").replace(",", "%2C") + print(f"::error title={title}::{escape(message)}", flush=True) + + class Regrtest: """Execute a test suite. @@ -504,10 +513,14 @@ def write(text: str) -> None: f"{count(ncase, 'test case')} failed") for result, result_cases in zip(failed, cases): write(f"### {decolor(str(result))}") + if not result_cases: + github_error(result.test_name, decolor(str(result))) if result.env_changed_reasons: write("\n".join(f"- {reason}" for reason in result.env_changed_reasons)) for name, traceback in result_cases: + github_error(f"{result.test_name}: {name}", + decolor(traceback)) # Expand short tracebacks when there are only a few is_open = ncase <= 5 and traceback.count("\n") < 30 write(f"" From ff7b75a45b6302ee2ab25d99265262882ddee709 Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Tue, 6 Oct 2026 00:14:59 -0700 Subject: [PATCH 06/34] Add groups more of the time --- Lib/test/libregrtest/main.py | 13 ------------- Lib/test/libregrtest/results.py | 20 ++++++++++++++++++-- Lib/test/libregrtest/run_workers.py | 26 ++++++++++++++++++++++++++ Lib/test/libregrtest/utils.py | 23 +++++++++++++++++++++++ 4 files changed, 67 insertions(+), 15 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index f191dd123ed8961..42dbf166837a6fc 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -30,15 +30,6 @@ get_process_memory_usage, EXIT_TIMEOUT) -def github_error(title: str, message: str) -> None: - # Emit a GitHub Actions error annotation (workflow command) - def escape(text: str) -> str: - return (text.replace("%", "%25").replace("\r", "%0D") - .replace("\n", "%0A")) - title = escape(title).replace(":", "%3A").replace(",", "%2C") - print(f"::error title={title}::{escape(message)}", flush=True) - - class Regrtest: """Execute a test suite. @@ -513,14 +504,10 @@ def write(text: str) -> None: f"{count(ncase, 'test case')} failed") for result, result_cases in zip(failed, cases): write(f"### {decolor(str(result))}") - if not result_cases: - github_error(result.test_name, decolor(str(result))) if result.env_changed_reasons: write("\n".join(f"- {reason}" for reason in result.env_changed_reasons)) for name, traceback in result_cases: - github_error(f"{result.test_name}: {name}", - decolor(traceback)) # Expand short tracebacks when there are only a few is_open = ncase <= 5 and traceback.count("\n") < 30 write(f"" diff --git a/Lib/test/libregrtest/results.py b/Lib/test/libregrtest/results.py index 3475f645729182c..9015cc1fe5f8a13 100644 --- a/Lib/test/libregrtest/results.py +++ b/Lib/test/libregrtest/results.py @@ -1,13 +1,14 @@ +import os import sys import trace -from _colorize import get_colors # type: ignore[import-not-found] +from _colorize import decolor, get_colors # type: ignore[import-not-found] lazy from xml.etree.ElementTree import Element from .runtests import RunTests from .result import State, TestResult, TestStats, Location from .utils import ( StrPath, TestName, TestTuple, TestList, FilterDict, - printlist, count, format_duration) + printlist, count, format_duration, github_annotation) # Python uses exit code 1 when an exception is not caught @@ -19,6 +20,16 @@ EXITCODE_INTERRUPTED = 130 # 128 + signal.SIGINT=2 +def annotate_github(result: TestResult, level: str) -> None: + cases = (result.errors or []) + (result.failures or []) + for name, traceback in cases: + github_annotation(level, f"{result.test_name}: {name}", + decolor(traceback)) + if not cases: + message = "\n".join([str(result), *(result.env_changed_reasons or ())]) + github_annotation(level, result.test_name, decolor(message)) + + class TestResults: def __init__(self) -> None: self.bad: TestList = [] @@ -128,6 +139,11 @@ def accumulate_result(self, result: TestResult, runtests: RunTests) -> None: if result.state == State.WORKER_BUG: self.worker_bug = True + if (os.environ.get("GITHUB_STEP_SUMMARY") + and result.is_failed(fail_env_changed)): + # A failure is only an error if it fails again when re-run + annotate_github(result, "error" if rerun else "warning") + if result.has_meaningful_duration() and not rerun: if result.duration is None: raise ValueError("result.duration is None") diff --git a/Lib/test/libregrtest/run_workers.py b/Lib/test/libregrtest/run_workers.py index c6db78e0882ba31..69ff5edb2607da9 100644 --- a/Lib/test/libregrtest/run_workers.py +++ b/Lib/test/libregrtest/run_workers.py @@ -10,6 +10,7 @@ import threading import time import traceback +from _colorize import decolor # type: ignore[import-not-found] from typing import Any, Literal, TextIO from test import support @@ -519,6 +520,11 @@ def __init__(self, num_workers: int, runtests: RunTests, else: self.worker_timeout = None self.workers: list[WorkerThread] = [] + # On GitHub Actions, fold the first run's log into groups: ranges of + # passing tests, and one group per failed test. Re-runs aren't folded. + self.github_groups = (not runtests.rerun + and bool(os.environ.get("GITHUB_STEP_SUMMARY"))) + self.in_github_group = False jobs = self.runtests.get_jobs() if jobs is not None: @@ -611,7 +617,16 @@ def _process_result(self, item: QueueOutput) -> TestResult: self.test_index += 1 mp_result = item[1] result = mp_result.result + failed = result.is_failed(self.runtests.fail_env_changed) + if self.github_groups and failed: + # Keep the failure annotation out of collapsed groups + self.github_group(None) self.results.accumulate_result(result, self.runtests) + if self.github_groups: + if failed: + self.github_group(decolor(str(result))) + elif not self.in_github_group: + self.github_group(f"Tests from #{self.test_index}") self.display_result(mp_result) # Display worker stdout @@ -624,9 +639,19 @@ def _process_result(self, item: QueueOutput) -> TestResult: stdout = mp_result.worker_stdout if stdout: print(stdout, flush=True) + if self.github_groups and failed: + self.github_group(None) return result + def github_group(self, title: str | None) -> None: + """End the current GitHub Actions log group, start a new one if title.""" + if self.in_github_group: + print("::endgroup::", flush=True) + self.in_github_group = title is not None + if title is not None: + print(f"::group::{title}", flush=True) + def get_mem_usage(self): usage = 0 main_mem = get_process_memory_usage(os.getpid()) @@ -659,6 +684,7 @@ def run(self) -> None: print() self.results.interrupted = True finally: + self.github_group(None) if self.timeout is not None: faulthandler.cancel_dump_traceback_later() diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index 83e0575619c99a0..c31022ea044ece6 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -140,6 +140,29 @@ def print_warning(msg: str) -> None: orig_unraisablehook: Callable[..., None] | None = None +def github_annotation(level: str, title: str, message: str) -> None: + """Emit a GitHub Actions annotation (workflow command). + + Point it at the last frame of the traceback in message, if any. + """ + def escape(text: str) -> str: + return (text.replace("%", "%25").replace("\r", "%0D") + .replace("\n", "%0A")) + + def escape_property(text: str) -> str: + return escape(text).replace(":", "%3A").replace(",", "%2C") + + props = f"title={escape_property(title)}" + frames = re.findall(r'^ File "(.+)", line (\d+)', message, re.MULTILINE) + # Source checkout which contains Lib/ + srcdir = os.path.dirname(os.path.dirname(os.__file__)) + os.sep + if frames and frames[-1][0].startswith(srcdir): + filename, line = frames[-1] + filename = filename.removeprefix(srcdir).replace(os.sep, "/") + props = f"file={escape_property(filename)},line={line},{props}" + print(f"::{level} {props}::{escape(message)}", flush=True) + + def regrtest_unraisable_hook(unraisable) -> None: global orig_unraisablehook support.set_environment_altered( From 946c801af026df3da4514eaa4cbca2e9bb9b18bf Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Tue, 6 Oct 2026 00:50:16 -0700 Subject: [PATCH 07/34] drop groups, error only on final failure --- Lib/test/libregrtest/main.py | 6 +++++- Lib/test/libregrtest/results.py | 12 ++++++------ Lib/test/libregrtest/run_workers.py | 26 -------------------------- Lib/test/libregrtest/utils.py | 11 ++++++----- 4 files changed, 17 insertions(+), 38 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 42dbf166837a6fc..1120dacf3b5a6a1 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -16,7 +16,7 @@ from .logger import Logger from .pgo import setup_pgo_tests from .result import TestResult -from .results import TestResults, EXITCODE_INTERRUPTED +from .results import TestResults, EXITCODE_INTERRUPTED, annotate_github from .runtests import RunTests, HuntRefleak from .setup import setup_process, setup_test_dir from .single import run_single_test, PROGRESS_MIN_TIME @@ -491,6 +491,10 @@ def write_github_summary(self) -> None: return # Tests which failed in the last run (the re-run, if any) failed = self.results.rerun_results + if not self.results.rerun: + # Failed tests were not re-run (ex: --python): annotate them now + for result in failed: + annotate_github(result) cases = [(result.errors or []) + (result.failures or []) for result in failed] ncase = sum(map(len, cases)) diff --git a/Lib/test/libregrtest/results.py b/Lib/test/libregrtest/results.py index 9015cc1fe5f8a13..a860ad0f9c8549e 100644 --- a/Lib/test/libregrtest/results.py +++ b/Lib/test/libregrtest/results.py @@ -20,14 +20,14 @@ EXITCODE_INTERRUPTED = 130 # 128 + signal.SIGINT=2 -def annotate_github(result: TestResult, level: str) -> None: +def annotate_github(result: TestResult) -> None: cases = (result.errors or []) + (result.failures or []) for name, traceback in cases: - github_annotation(level, f"{result.test_name}: {name}", + github_annotation("error", f"{result.test_name}: {name}", decolor(traceback)) if not cases: message = "\n".join([str(result), *(result.env_changed_reasons or ())]) - github_annotation(level, result.test_name, decolor(message)) + github_annotation("error", result.test_name, decolor(message)) class TestResults: @@ -139,10 +139,10 @@ def accumulate_result(self, result: TestResult, runtests: RunTests) -> None: if result.state == State.WORKER_BUG: self.worker_bug = True - if (os.environ.get("GITHUB_STEP_SUMMARY") + if (os.environ.get("GITHUB_STEP_SUMMARY") and rerun and result.is_failed(fail_env_changed)): - # A failure is only an error if it fails again when re-run - annotate_github(result, "error" if rerun else "warning") + # Only annotate tests which fail again when re-run + annotate_github(result) if result.has_meaningful_duration() and not rerun: if result.duration is None: diff --git a/Lib/test/libregrtest/run_workers.py b/Lib/test/libregrtest/run_workers.py index 69ff5edb2607da9..c6db78e0882ba31 100644 --- a/Lib/test/libregrtest/run_workers.py +++ b/Lib/test/libregrtest/run_workers.py @@ -10,7 +10,6 @@ import threading import time import traceback -from _colorize import decolor # type: ignore[import-not-found] from typing import Any, Literal, TextIO from test import support @@ -520,11 +519,6 @@ def __init__(self, num_workers: int, runtests: RunTests, else: self.worker_timeout = None self.workers: list[WorkerThread] = [] - # On GitHub Actions, fold the first run's log into groups: ranges of - # passing tests, and one group per failed test. Re-runs aren't folded. - self.github_groups = (not runtests.rerun - and bool(os.environ.get("GITHUB_STEP_SUMMARY"))) - self.in_github_group = False jobs = self.runtests.get_jobs() if jobs is not None: @@ -617,16 +611,7 @@ def _process_result(self, item: QueueOutput) -> TestResult: self.test_index += 1 mp_result = item[1] result = mp_result.result - failed = result.is_failed(self.runtests.fail_env_changed) - if self.github_groups and failed: - # Keep the failure annotation out of collapsed groups - self.github_group(None) self.results.accumulate_result(result, self.runtests) - if self.github_groups: - if failed: - self.github_group(decolor(str(result))) - elif not self.in_github_group: - self.github_group(f"Tests from #{self.test_index}") self.display_result(mp_result) # Display worker stdout @@ -639,19 +624,9 @@ def _process_result(self, item: QueueOutput) -> TestResult: stdout = mp_result.worker_stdout if stdout: print(stdout, flush=True) - if self.github_groups and failed: - self.github_group(None) return result - def github_group(self, title: str | None) -> None: - """End the current GitHub Actions log group, start a new one if title.""" - if self.in_github_group: - print("::endgroup::", flush=True) - self.in_github_group = title is not None - if title is not None: - print(f"::group::{title}", flush=True) - def get_mem_usage(self): usage = 0 main_mem = get_process_memory_usage(os.getpid()) @@ -684,7 +659,6 @@ def run(self) -> None: print() self.results.interrupted = True finally: - self.github_group(None) if self.timeout is not None: faulthandler.cancel_dump_traceback_later() diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index c31022ea044ece6..0f21c7981c5141b 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -154,12 +154,13 @@ def escape_property(text: str) -> str: props = f"title={escape_property(title)}" frames = re.findall(r'^ File "(.+)", line (\d+)', message, re.MULTILINE) - # Source checkout which contains Lib/ - srcdir = os.path.dirname(os.path.dirname(os.__file__)) + os.sep - if frames and frames[-1][0].startswith(srcdir): + # Map the stdlib directory (Lib/ in a source checkout or an installed + # lib/python3.X/) to Lib/ in the repository + stdlib_dir = os.path.dirname(os.__file__) + os.sep + if frames and frames[-1][0].startswith(stdlib_dir): filename, line = frames[-1] - filename = filename.removeprefix(srcdir).replace(os.sep, "/") - props = f"file={escape_property(filename)},line={line},{props}" + filename = filename.removeprefix(stdlib_dir).replace(os.sep, "/") + props = f"file=Lib/{escape_property(filename)},line={line},{props}" print(f"::{level} {props}::{escape(message)}", flush=True) From 14394da14e30c9bda7aaef14c950d692c1415fcc Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Tue, 6 Oct 2026 10:27:47 -0700 Subject: [PATCH 08/34] More structured error blocks --- Lib/test/libregrtest/utils.py | 55 ++++++++++++++++++++++++++++------- 1 file changed, 45 insertions(+), 10 deletions(-) diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index 0f21c7981c5141b..ee241844098c406 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -1,5 +1,6 @@ import contextlib import faulthandler +import linecache import locale import math import os.path @@ -140,6 +141,39 @@ def print_warning(msg: str) -> None: orig_unraisablehook: Callable[..., None] | None = None +def traceback_location(traceback: str) -> dict[str, str | int] | None: + """Location of the last frame of a traceback, relative to the repository. + + Return a dict with file and line, plus col and endColumn when the + traceback underlines the failing expression (~~~^^^). + """ + lines = traceback.splitlines() + for index in reversed(range(len(lines))): + if match := re.match(r' File "(.+)", line (\d+)', lines[index]): + break + else: + return None + filename, lineno = match.group(1), int(match.group(2)) + # Map the stdlib directory (Lib/ in a source checkout or an installed + # lib/python3.X/) to Lib/ in the repository + stdlib_dir = os.path.dirname(os.__file__) + os.sep + if not filename.startswith(stdlib_dir): + return None + relpath = filename.removeprefix(stdlib_dir).replace(os.sep, "/") + location: dict[str, str | int] = {"file": f"Lib/{relpath}", "line": lineno} + + # The frame is followed by the dedented source line (" code") and, + # for a single-line expression, by a line of ~ and ^ markers. + markers = lines[index + 2] if index + 2 < len(lines) else "" + source = linecache.getline(filename, lineno) + if markers.strip() and not markers.strip(" ~^") and source: + indent = len(source) - len(source.lstrip()) + start = len(markers) - len(markers.lstrip()) + location["col"] = indent + start - 4 + 1 + location["endColumn"] = indent + len(markers.rstrip()) - 4 + return location + + def github_annotation(level: str, title: str, message: str) -> None: """Emit a GitHub Actions annotation (workflow command). @@ -152,16 +186,17 @@ def escape(text: str) -> str: def escape_property(text: str) -> str: return escape(text).replace(":", "%3A").replace(",", "%2C") - props = f"title={escape_property(title)}" - frames = re.findall(r'^ File "(.+)", line (\d+)', message, re.MULTILINE) - # Map the stdlib directory (Lib/ in a source checkout or an installed - # lib/python3.X/) to Lib/ in the repository - stdlib_dir = os.path.dirname(os.__file__) + os.sep - if frames and frames[-1][0].startswith(stdlib_dir): - filename, line = frames[-1] - filename = filename.removeprefix(stdlib_dir).replace(os.sep, "/") - props = f"file=Lib/{escape_property(filename)},line={line},{props}" - print(f"::{level} {props}::{escape(message)}", flush=True) + props: dict[str, str | int] = {} + if location := traceback_location(message): + props |= location + # The job log only shows the message: start it with the location + position = ":".join(str(location[key]) + for key in ("file", "line", "col") if key in location) + message = f"{position}\n{message}" + props["title"] = title + props_text = ",".join(f"{key}={escape_property(str(value))}" + for key, value in props.items()) + print(f"::{level} {props_text}::{escape(message)}", flush=True) def regrtest_unraisable_hook(unraisable) -> None: From 63598766247dd7c00eb73a02990113730b936e8a Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Tue, 6 Oct 2026 10:39:56 -0700 Subject: [PATCH 09/34] Fail more tests --- Lib/test/test_bool.py | 6 ++++++ Lib/test/test_json/test_decode.py | 4 ++++ Lib/test/test_math.py | 5 +++++ Lib/test/test_textwrap.py | 5 +++++ 4 files changed, 20 insertions(+) diff --git a/Lib/test/test_bool.py b/Lib/test/test_bool.py index dcdf7bdce03b800..2214e7408aa5603 100644 --- a/Lib/test/test_bool.py +++ b/Lib/test/test_bool.py @@ -4,6 +4,7 @@ from test.support import os_helper import os +import sys class BoolTest(unittest.TestCase): @@ -25,6 +26,11 @@ def test_repr(self): self.assertIs(eval(repr(True)), True) def test_str(self): + if sys.platform == "linux": + # XXX: Intentional failure to test GitHub Actions annotations + self.assertEqual( + str(True), + 'Yes') self.assertEqual(str(False), 'False') self.assertEqual(str(True), 'True') diff --git a/Lib/test/test_json/test_decode.py b/Lib/test/test_json/test_decode.py index 7bff16ddf2155e9..d01368254f6701e 100644 --- a/Lib/test/test_json/test_decode.py +++ b/Lib/test/test_json/test_decode.py @@ -1,4 +1,5 @@ import decimal +import sys import unittest.mock from io import StringIO from collections import OrderedDict @@ -9,6 +10,9 @@ class TestDecode: def test_decimal(self): rval = self.loads('1.1', parse_float=decimal.Decimal) + if sys.platform == "linux": + # XXX: Intentional failure to test GitHub Actions annotations + self.loads('{"bad": }') self.assertIsInstance(rval, decimal.Decimal) self.assertEqual(rval, decimal.Decimal('1.1')) diff --git a/Lib/test/test_math.py b/Lib/test/test_math.py index a1250c668223e7f..40dc0d9acc75fff 100644 --- a/Lib/test/test_math.py +++ b/Lib/test/test_math.py @@ -513,6 +513,11 @@ def testFabs(self): def testFloor(self): self.assertRaises(TypeError, math.floor) + if sys.platform == "linux": + # XXX: Intentional failure to test GitHub Actions annotations + for x in (0.5, 1.5): + with self.subTest(x=x): + self.assertEqual(math.floor(x), x) self.assertEqual(int, type(math.floor(0.5))) self.assertEqual(math.floor(0.5), 0) self.assertEqual(math.floor(1.0), 1) diff --git a/Lib/test/test_textwrap.py b/Lib/test/test_textwrap.py index aca1f427656bb50..c1d670c978c9702 100644 --- a/Lib/test/test_textwrap.py +++ b/Lib/test/test_textwrap.py @@ -8,6 +8,8 @@ # $Id$ # +import os +import sys import unittest from textwrap import TextWrapper, wrap, fill, dedent, indent, shorten @@ -66,6 +68,9 @@ def test_simple(self): self.check_wrap(text, 80, [text]) def test_empty_string(self): + if sys.platform == "linux": + # XXX: Intentional failure to test GitHub Actions annotations + os.environ["GHA_ANNOTATION_TEST"] = "1" # Check that wrapping the empty string returns an empty list. self.check_wrap("", 6, []) self.check_wrap("", 6, [], drop_whitespace=False) From 041e57792d78978a97ff19dde573506b894a9e2a Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Tue, 6 Oct 2026 10:44:04 -0700 Subject: [PATCH 10/34] Adjust and simplify a little --- Lib/test/libregrtest/main.py | 15 +++++++++------ Lib/test/libregrtest/results.py | 20 ++------------------ 2 files changed, 11 insertions(+), 24 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 1120dacf3b5a6a1..dc6e8ebc232fccd 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -16,7 +16,7 @@ from .logger import Logger from .pgo import setup_pgo_tests from .result import TestResult -from .results import TestResults, EXITCODE_INTERRUPTED, annotate_github +from .results import TestResults, EXITCODE_INTERRUPTED from .runtests import RunTests, HuntRefleak from .setup import setup_process, setup_test_dir from .single import run_single_test, PROGRESS_MIN_TIME @@ -27,7 +27,7 @@ printlist, get_temp_dir, get_work_dir, exit_timeout, display_header, cleanup_temp_dir, print_warning, is_cross_compiled, get_host_runner, display_title, - get_process_memory_usage, EXIT_TIMEOUT) + get_process_memory_usage, github_annotation, EXIT_TIMEOUT) class Regrtest: @@ -491,10 +491,6 @@ def write_github_summary(self) -> None: return # Tests which failed in the last run (the re-run, if any) failed = self.results.rerun_results - if not self.results.rerun: - # Failed tests were not re-run (ex: --python): annotate them now - for result in failed: - annotate_github(result) cases = [(result.errors or []) + (result.failures or []) for result in failed] ncase = sum(map(len, cases)) @@ -511,7 +507,14 @@ def write(text: str) -> None: if result.env_changed_reasons: write("\n".join(f"- {reason}" for reason in result.env_changed_reasons)) + if not result_cases: + message = "\n".join([str(result), + *(result.env_changed_reasons or ())]) + github_annotation("error", result.test_name, + decolor(message)) for name, traceback in result_cases: + github_annotation("error", f"{result.test_name}: {name}", + decolor(traceback)) # Expand short tracebacks when there are only a few is_open = ncase <= 5 and traceback.count("\n") < 30 write(f"" diff --git a/Lib/test/libregrtest/results.py b/Lib/test/libregrtest/results.py index a860ad0f9c8549e..3475f645729182c 100644 --- a/Lib/test/libregrtest/results.py +++ b/Lib/test/libregrtest/results.py @@ -1,14 +1,13 @@ -import os import sys import trace -from _colorize import decolor, get_colors # type: ignore[import-not-found] +from _colorize import get_colors # type: ignore[import-not-found] lazy from xml.etree.ElementTree import Element from .runtests import RunTests from .result import State, TestResult, TestStats, Location from .utils import ( StrPath, TestName, TestTuple, TestList, FilterDict, - printlist, count, format_duration, github_annotation) + printlist, count, format_duration) # Python uses exit code 1 when an exception is not caught @@ -20,16 +19,6 @@ EXITCODE_INTERRUPTED = 130 # 128 + signal.SIGINT=2 -def annotate_github(result: TestResult) -> None: - cases = (result.errors or []) + (result.failures or []) - for name, traceback in cases: - github_annotation("error", f"{result.test_name}: {name}", - decolor(traceback)) - if not cases: - message = "\n".join([str(result), *(result.env_changed_reasons or ())]) - github_annotation("error", result.test_name, decolor(message)) - - class TestResults: def __init__(self) -> None: self.bad: TestList = [] @@ -139,11 +128,6 @@ def accumulate_result(self, result: TestResult, runtests: RunTests) -> None: if result.state == State.WORKER_BUG: self.worker_bug = True - if (os.environ.get("GITHUB_STEP_SUMMARY") and rerun - and result.is_failed(fail_env_changed)): - # Only annotate tests which fail again when re-run - annotate_github(result) - if result.has_meaningful_duration() and not rerun: if result.duration is None: raise ValueError("result.duration is None") From ba449b42c06f60b25d70a5a2d2f12d559bc16e7f Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Tue, 6 Oct 2026 11:01:32 -0700 Subject: [PATCH 11/34] Annotate inline with test output --- Lib/test/libregrtest/main.py | 16 ++++++------ Lib/test/libregrtest/result.py | 39 +++++++++++++++++++++++++++-- Lib/test/libregrtest/run_workers.py | 2 ++ Lib/test/libregrtest/utils.py | 25 ------------------ 4 files changed, 47 insertions(+), 35 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index dc6e8ebc232fccd..9df71bbcf009b9c 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -27,7 +27,7 @@ printlist, get_temp_dir, get_work_dir, exit_timeout, display_header, cleanup_temp_dir, print_warning, is_cross_compiled, get_host_runner, display_title, - get_process_memory_usage, github_annotation, EXIT_TIMEOUT) + get_process_memory_usage, EXIT_TIMEOUT) class Regrtest: @@ -395,6 +395,8 @@ def run_test( result = run_single_test(test_name, runtests) self.results.accumulate_result(result, runtests) + if runtests.rerun and result.is_failed(runtests.fail_env_changed): + result.print_github_annotations() return result @@ -491,6 +493,11 @@ def write_github_summary(self) -> None: return # Tests which failed in the last run (the re-run, if any) failed = self.results.rerun_results + if not self.results.rerun: + # Failed tests were not re-run (ex: --python): annotate them now. + # Otherwise, they were annotated when they failed again. + for result in failed: + result.print_github_annotations() cases = [(result.errors or []) + (result.failures or []) for result in failed] ncase = sum(map(len, cases)) @@ -507,14 +514,7 @@ def write(text: str) -> None: if result.env_changed_reasons: write("\n".join(f"- {reason}" for reason in result.env_changed_reasons)) - if not result_cases: - message = "\n".join([str(result), - *(result.env_changed_reasons or ())]) - github_annotation("error", result.test_name, - decolor(message)) for name, traceback in result_cases: - github_annotation("error", f"{result.test_name}: {name}", - decolor(traceback)) # Expand short tracebacks when there are only a few is_open = ncase <= 5 and traceback.count("\n") < 30 write(f"" diff --git a/Lib/test/libregrtest/result.py b/Lib/test/libregrtest/result.py index 5324a46bb8cb1c5..e806de29139a98e 100644 --- a/Lib/test/libregrtest/result.py +++ b/Lib/test/libregrtest/result.py @@ -1,11 +1,12 @@ import dataclasses import json -from _colorize import get_colors # type: ignore[import-not-found] +import os +from _colorize import decolor, get_colors # type: ignore[import-not-found] from typing import Any from .utils import ( StrJSON, TestName, FilterTuple, - format_duration, normalize_test_name, print_warning) + format_duration, normalize_test_name, print_warning, traceback_location) @dataclasses.dataclass(slots=True) @@ -178,6 +179,40 @@ def __str__(self) -> str: def has_meaningful_duration(self): return State.has_meaningful_duration(self.state) + def print_github_annotations(self) -> None: + """Print a GitHub Actions error annotation per failed test case.""" + if not os.environ.get("GITHUB_STEP_SUMMARY"): + return + + def escape(text: str) -> str: + return (text.replace("%", "%25").replace("\r", "%0D") + .replace("\n", "%0A")) + + def escape_property(text: str) -> str: + return escape(text).replace(":", "%3A").replace(",", "%2C") + + annotations = [(f"{self.test_name}: {name}", traceback) + for name, traceback in (self.errors or []) + + (self.failures or [])] + if not annotations: + # No test case details: crash, timeout, env changed, etc. + message = "\n".join([str(self), *(self.env_changed_reasons or ())]) + annotations = [(self.test_name, message)] + for title, message in annotations: + message = decolor(message) + props: dict[str, str | int] = {} + if location := traceback_location(message): + props |= location + # The job log only shows the message: start it with the location + position = ":".join(str(location[key]) + for key in ("file", "line", "col") + if key in location) + message = f"{position}\n{message}" + props["title"] = title + props_text = ",".join(f"{key}={escape_property(str(value))}" + for key, value in props.items()) + print(f"::error {props_text}::{escape(message)}", flush=True) + def set_env_changed(self, *reasons): if self.state is None or self.state == State.PASSED: self.state = State.ENV_CHANGED diff --git a/Lib/test/libregrtest/run_workers.py b/Lib/test/libregrtest/run_workers.py index c6db78e0882ba31..d88787fb30613e7 100644 --- a/Lib/test/libregrtest/run_workers.py +++ b/Lib/test/libregrtest/run_workers.py @@ -624,6 +624,8 @@ def _process_result(self, item: QueueOutput) -> TestResult: stdout = mp_result.worker_stdout if stdout: print(stdout, flush=True) + if self.runtests.rerun and result.is_failed(self.runtests.fail_env_changed): + result.print_github_annotations() return result diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index ee241844098c406..e39a995662bc91f 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -174,31 +174,6 @@ def traceback_location(traceback: str) -> dict[str, str | int] | None: return location -def github_annotation(level: str, title: str, message: str) -> None: - """Emit a GitHub Actions annotation (workflow command). - - Point it at the last frame of the traceback in message, if any. - """ - def escape(text: str) -> str: - return (text.replace("%", "%25").replace("\r", "%0D") - .replace("\n", "%0A")) - - def escape_property(text: str) -> str: - return escape(text).replace(":", "%3A").replace(",", "%2C") - - props: dict[str, str | int] = {} - if location := traceback_location(message): - props |= location - # The job log only shows the message: start it with the location - position = ":".join(str(location[key]) - for key in ("file", "line", "col") if key in location) - message = f"{position}\n{message}" - props["title"] = title - props_text = ",".join(f"{key}={escape_property(str(value))}" - for key, value in props.items()) - print(f"::{level} {props_text}::{escape(message)}", flush=True) - - def regrtest_unraisable_hook(unraisable) -> None: global orig_unraisablehook support.set_environment_altered( From 59b1608fc0b40afcad22e75576d990ea8e4c64b4 Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Tue, 6 Oct 2026 11:18:53 -0700 Subject: [PATCH 12/34] Move annotations before test output --- Lib/test/libregrtest/result.py | 4 +--- Lib/test/libregrtest/run_workers.py | 5 +++-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/Lib/test/libregrtest/result.py b/Lib/test/libregrtest/result.py index e806de29139a98e..f2fe90cab9a8712 100644 --- a/Lib/test/libregrtest/result.py +++ b/Lib/test/libregrtest/result.py @@ -191,9 +191,7 @@ def escape(text: str) -> str: def escape_property(text: str) -> str: return escape(text).replace(":", "%3A").replace(",", "%2C") - annotations = [(f"{self.test_name}: {name}", traceback) - for name, traceback in (self.errors or []) - + (self.failures or [])] + annotations = (self.errors or []) + (self.failures or []) if not annotations: # No test case details: crash, timeout, env changed, etc. message = "\n".join([str(self), *(self.env_changed_reasons or ())]) diff --git a/Lib/test/libregrtest/run_workers.py b/Lib/test/libregrtest/run_workers.py index d88787fb30613e7..2000b0b469e254f 100644 --- a/Lib/test/libregrtest/run_workers.py +++ b/Lib/test/libregrtest/run_workers.py @@ -613,6 +613,9 @@ def _process_result(self, item: QueueOutput) -> TestResult: result = mp_result.result self.results.accumulate_result(result, self.runtests) self.display_result(mp_result) + if self.runtests.rerun and result.is_failed(self.runtests.fail_env_changed): + # Annotate before the test output + result.print_github_annotations() # Display worker stdout if not self.runtests.output_on_failure: @@ -624,8 +627,6 @@ def _process_result(self, item: QueueOutput) -> TestResult: stdout = mp_result.worker_stdout if stdout: print(stdout, flush=True) - if self.runtests.rerun and result.is_failed(self.runtests.fail_env_changed): - result.print_github_annotations() return result From 75c44b00d81e346f3ec56338a4ae6402dc9cad45 Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Tue, 6 Oct 2026 12:05:54 -0700 Subject: [PATCH 13/34] Have regrtest workers print annotations in better place --- Lib/test/libregrtest/main.py | 9 +++-- Lib/test/libregrtest/result.py | 53 ++++++++++++++--------------- Lib/test/libregrtest/run_workers.py | 5 ++- Lib/test/libregrtest/runtests.py | 2 ++ Lib/test/libregrtest/setup.py | 3 ++ Lib/test/libregrtest/testresult.py | 22 +++++++++++- Lib/test/libregrtest/utils.py | 23 +++++++++++++ 7 files changed, 82 insertions(+), 35 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 9df71bbcf009b9c..2df99a0fa19b123 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -284,7 +284,9 @@ def _rerun_failed_tests(self, runtests: RunTests) -> RunTests: forever=False, fail_fast=False, match_tests_dict=match_tests_dict, - output_on_failure=False) + output_on_failure=False, + # Annotate tests which fail again + github_annotations=bool(os.environ.get("GITHUB_STEP_SUMMARY"))) self.logger.set_tests(runtests) msg = f"Re-running {len(tests)} failed tests in verbose mode" @@ -395,8 +397,8 @@ def run_test( result = run_single_test(test_name, runtests) self.results.accumulate_result(result, runtests) - if runtests.rerun and result.is_failed(runtests.fail_env_changed): - result.print_github_annotations() + if runtests.github_annotations: + result.print_github_crash_annotation(runtests.fail_env_changed) return result @@ -559,6 +561,7 @@ def create_run_tests(self, tests: TestTuple) -> RunTests: hunt_refleak=self.hunt_refleak, test_dir=self.test_dir, use_junit=(self.junit_filename is not None), + github_annotations=False, coverage=self.coverage, memory_limit=self.memory_limit, gc_threshold=self.gc_threshold, diff --git a/Lib/test/libregrtest/result.py b/Lib/test/libregrtest/result.py index f2fe90cab9a8712..6ed97a2148b5b4a 100644 --- a/Lib/test/libregrtest/result.py +++ b/Lib/test/libregrtest/result.py @@ -1,12 +1,12 @@ import dataclasses import json import os -from _colorize import decolor, get_colors # type: ignore[import-not-found] +from _colorize import get_colors # type: ignore[import-not-found] from typing import Any from .utils import ( StrJSON, TestName, FilterTuple, - format_duration, normalize_test_name, print_warning, traceback_location) + format_duration, normalize_test_name, print_warning, github_annotation) @dataclasses.dataclass(slots=True) @@ -179,37 +179,34 @@ def __str__(self) -> str: def has_meaningful_duration(self): return State.has_meaningful_duration(self.state) - def print_github_annotations(self) -> None: - """Print a GitHub Actions error annotation per failed test case.""" - if not os.environ.get("GITHUB_STEP_SUMMARY"): - return - - def escape(text: str) -> str: - return (text.replace("%", "%25").replace("\r", "%0D") - .replace("\n", "%0A")) - - def escape_property(text: str) -> str: - return escape(text).replace(":", "%3A").replace(",", "%2C") + def get_github_annotations(self) -> list[tuple[str, str]]: + """(test case name, message) of each failed test case. + Without test case details (crash, timeout, env changed, etc.), return + the test name and the test result. + """ + if not os.environ.get("GITHUB_STEP_SUMMARY"): + return [] annotations = (self.errors or []) + (self.failures or []) if not annotations: - # No test case details: crash, timeout, env changed, etc. message = "\n".join([str(self), *(self.env_changed_reasons or ())]) annotations = [(self.test_name, message)] - for title, message in annotations: - message = decolor(message) - props: dict[str, str | int] = {} - if location := traceback_location(message): - props |= location - # The job log only shows the message: start it with the location - position = ":".join(str(location[key]) - for key in ("file", "line", "col") - if key in location) - message = f"{position}\n{message}" - props["title"] = title - props_text = ",".join(f"{key}={escape_property(str(value))}" - for key, value in props.items()) - print(f"::error {props_text}::{escape(message)}", flush=True) + return annotations + + def print_github_annotations(self) -> None: + """Print a GitHub Actions error annotation per failed test case.""" + for title, message in self.get_github_annotations(): + print(github_annotation(title, message), flush=True) + + def print_github_crash_annotation(self, fail_env_changed: bool) -> None: + """Annotate a failed test without test case failures. + + Test case failures are annotated by the test runner, where they are + reported (see RegressionTestResult.printErrorList()). + """ + if (self.is_failed(fail_env_changed) + and not self.errors and not self.failures): + self.print_github_annotations() def set_env_changed(self, *reasons): if self.state is None or self.state == State.PASSED: diff --git a/Lib/test/libregrtest/run_workers.py b/Lib/test/libregrtest/run_workers.py index 2000b0b469e254f..45bf9d9327fa143 100644 --- a/Lib/test/libregrtest/run_workers.py +++ b/Lib/test/libregrtest/run_workers.py @@ -613,9 +613,8 @@ def _process_result(self, item: QueueOutput) -> TestResult: result = mp_result.result self.results.accumulate_result(result, self.runtests) self.display_result(mp_result) - if self.runtests.rerun and result.is_failed(self.runtests.fail_env_changed): - # Annotate before the test output - result.print_github_annotations() + if self.runtests.github_annotations: + result.print_github_crash_annotation(self.runtests.fail_env_changed) # Display worker stdout if not self.runtests.output_on_failure: diff --git a/Lib/test/libregrtest/runtests.py b/Lib/test/libregrtest/runtests.py index d86b082fb34a604..a116b6e4166fc63 100644 --- a/Lib/test/libregrtest/runtests.py +++ b/Lib/test/libregrtest/runtests.py @@ -93,6 +93,8 @@ class RunTests: hunt_refleak: HuntRefleak | None test_dir: StrPath | None use_junit: bool + # Print a GitHub Actions error annotation before each failure report + github_annotations: bool coverage: bool memory_limit: str | None gc_threshold: int | None diff --git a/Lib/test/libregrtest/setup.py b/Lib/test/libregrtest/setup.py index d62194acd9c29e5..7531e69d0988e64 100644 --- a/Lib/test/libregrtest/setup.py +++ b/Lib/test/libregrtest/setup.py @@ -119,6 +119,9 @@ def setup_tests(runtests: RunTests) -> None: else: support.junit_xml_list = None + from .testresult import RegressionTestResult + RegressionTestResult.GITHUB_ANNOTATIONS = runtests.github_annotations + if runtests.memory_limit is not None: support.set_memlimit(runtests.memory_limit) diff --git a/Lib/test/libregrtest/testresult.py b/Lib/test/libregrtest/testresult.py index 605f1f4e6a89fb6..cfd024a000e8a3f 100644 --- a/Lib/test/libregrtest/testresult.py +++ b/Lib/test/libregrtest/testresult.py @@ -9,10 +9,11 @@ import traceback import unittest from test import support -from test.libregrtest.utils import sanitize_xml +from test.libregrtest.utils import github_annotation, sanitize_xml class RegressionTestResult(unittest.TextTestResult): USE_XML = False + GITHUB_ANNOTATIONS = False def __init__(self, stream, descriptions, verbosity): super().__init__(stream=stream, descriptions=descriptions, @@ -130,6 +131,25 @@ def addUnexpectedSuccess(self, test): self._add_result(test, outcome='UNEXPECTED_SUCCESS') super().addUnexpectedSuccess(test) + def printErrorList(self, flavour, errors): + if not self.GITHUB_ANNOTATIONS: + super().printErrorList(flavour, errors) + return + stream = self.stream + for test, err in errors: + # Precede each failure report with a GitHub Actions annotation + # which has the report as message, so that the annotation links + # to the report in the job log. + self.stream = unittest.runner._WritelnDecorator(io.StringIO()) + try: + super().printErrorList(flavour, [(test, err)]) + report = self.stream.getvalue() + finally: + self.stream = stream + stream.writeln(github_annotation(str(test), report)) + stream.write(report) + stream.flush() + def get_xml_element(self): if not self.USE_XML: raise ValueError("USE_XML is false") diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index e39a995662bc91f..d4218fffbf64aea 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -14,6 +14,7 @@ import tempfile import textwrap import types +from _colorize import decolor # type: ignore[import-not-found] from collections.abc import Callable _winapi: types.ModuleType | None try: @@ -174,6 +175,28 @@ def traceback_location(traceback: str) -> dict[str, str | int] | None: return location +def github_annotation(title: str, message: str) -> str: + """Format a GitHub Actions error annotation. + + Locate it at the last frame of the traceback in message, if any. + """ + def escape(text: str) -> str: + return (text.replace("%", "%25").replace("\r", "%0D") + .replace("\n", "%0A")) + + def escape_property(text: str) -> str: + return escape(text).replace(":", "%3A").replace(",", "%2C") + + message = decolor(message) + props: dict[str, str | int] = {} + if location := traceback_location(message): + props |= location + props["title"] = title + props_text = ",".join(f"{key}={escape_property(str(value))}" + for key, value in props.items()) + return f"::error {props_text}::{escape(message.rstrip())}" + + def regrtest_unraisable_hook(unraisable) -> None: global orig_unraisablehook support.set_environment_altered( From 1222c34f70a4e66c4ad7c321a7ce7e16df51898e Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Tue, 6 Oct 2026 12:37:38 -0700 Subject: [PATCH 14/34] Full annotations from worker --- Lib/test/libregrtest/utils.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index d4218fffbf64aea..4953a024d355988 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -187,14 +187,22 @@ def escape(text: str) -> str: def escape_property(text: str) -> str: return escape(text).replace(":", "%3A").replace(",", "%2C") - message = decolor(message) + message = decolor(message).rstrip() props: dict[str, str | int] = {} + header = title if location := traceback_location(message): props |= location + position = ":".join(str(location[key]) + for key in ("file", "line", "col") + if key in location) + header = f"{position}: {title}" props["title"] = title + # The job log only shows the message, not the properties: start the + # message with the location and the title + message = f"{header}\n{message}" props_text = ",".join(f"{key}={escape_property(str(value))}" for key, value in props.items()) - return f"::error {props_text}::{escape(message.rstrip())}" + return f"::error {props_text}::{escape(message)}" def regrtest_unraisable_hook(unraisable) -> None: From 287df00e91602e457d2bbb839fa9c4a7ad42f3fa Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Tue, 6 Oct 2026 12:53:04 -0700 Subject: [PATCH 15/34] Remove redundant printing --- Lib/test/libregrtest/testresult.py | 19 ++++++------------- Lib/test/libregrtest/utils.py | 9 ++++++--- 2 files changed, 12 insertions(+), 16 deletions(-) diff --git a/Lib/test/libregrtest/testresult.py b/Lib/test/libregrtest/testresult.py index cfd024a000e8a3f..6ad6c2acdc272db 100644 --- a/Lib/test/libregrtest/testresult.py +++ b/Lib/test/libregrtest/testresult.py @@ -135,20 +135,13 @@ def printErrorList(self, flavour, errors): if not self.GITHUB_ANNOTATIONS: super().printErrorList(flavour, errors) return - stream = self.stream for test, err in errors: - # Precede each failure report with a GitHub Actions annotation - # which has the report as message, so that the annotation links - # to the report in the job log. - self.stream = unittest.runner._WritelnDecorator(io.StringIO()) - try: - super().printErrorList(flavour, [(test, err)]) - report = self.stream.getvalue() - finally: - self.stream = stream - stream.writeln(github_annotation(str(test), report)) - stream.write(report) - stream.flush() + # Precede each failure report with a GitHub Actions annotation, + # so that the annotation links to the report in the job log. The + # report follows: don't repeat it in the annotation. + self.stream.writeln(github_annotation(str(test), err, + include_message=False)) + super().printErrorList(flavour, [(test, err)]) def get_xml_element(self): if not self.USE_XML: diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index 4953a024d355988..8e04ba960be45f5 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -175,10 +175,13 @@ def traceback_location(traceback: str) -> dict[str, str | int] | None: return location -def github_annotation(title: str, message: str) -> str: +def github_annotation(title: str, message: str, *, + include_message: bool = True) -> str: """Format a GitHub Actions error annotation. - Locate it at the last frame of the traceback in message, if any. + Locate it at the last frame of the traceback in message, if any. If + include_message is false, the annotation only has the location and the + title. """ def escape(text: str) -> str: return (text.replace("%", "%25").replace("\r", "%0D") @@ -199,7 +202,7 @@ def escape_property(text: str) -> str: props["title"] = title # The job log only shows the message, not the properties: start the # message with the location and the title - message = f"{header}\n{message}" + message = f"{header}\n{message}" if include_message else header props_text = ",".join(f"{key}={escape_property(str(value))}" for key, value in props.items()) return f"::error {props_text}::{escape(message)}" From 53024ed83c9815984b7f1caba450f018aebb4985 Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Tue, 6 Oct 2026 13:32:22 -0700 Subject: [PATCH 16/34] Remove special summary case, more full output everywhere --- Lib/test/libregrtest/main.py | 11 ++++------- Lib/test/libregrtest/result.py | 28 +++++----------------------- Lib/test/libregrtest/testresult.py | 5 +++++ 3 files changed, 14 insertions(+), 30 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 2df99a0fa19b123..58e011ac752ec8c 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -285,7 +285,6 @@ def _rerun_failed_tests(self, runtests: RunTests) -> RunTests: fail_fast=False, match_tests_dict=match_tests_dict, output_on_failure=False, - # Annotate tests which fail again github_annotations=bool(os.environ.get("GITHUB_STEP_SUMMARY"))) self.logger.set_tests(runtests) @@ -495,11 +494,6 @@ def write_github_summary(self) -> None: return # Tests which failed in the last run (the re-run, if any) failed = self.results.rerun_results - if not self.results.rerun: - # Failed tests were not re-run (ex: --python): annotate them now. - # Otherwise, they were annotated when they failed again. - for result in failed: - result.print_github_annotations() cases = [(result.errors or []) + (result.failures or []) for result in failed] ncase = sum(map(len, cases)) @@ -561,7 +555,10 @@ def create_run_tests(self, tests: TestTuple) -> RunTests: hunt_refleak=self.hunt_refleak, test_dir=self.test_dir, use_junit=(self.junit_filename is not None), - github_annotations=False, + # Only annotate failures of the last run: the re-run, if any + github_annotations=(bool(os.environ.get("GITHUB_STEP_SUMMARY")) + and not (self.want_rerun + and not self.python_cmd)), coverage=self.coverage, memory_limit=self.memory_limit, gc_threshold=self.gc_threshold, diff --git a/Lib/test/libregrtest/result.py b/Lib/test/libregrtest/result.py index 6ed97a2148b5b4a..1bb9f3a9a48dd31 100644 --- a/Lib/test/libregrtest/result.py +++ b/Lib/test/libregrtest/result.py @@ -1,6 +1,5 @@ import dataclasses import json -import os from _colorize import get_colors # type: ignore[import-not-found] from typing import Any @@ -179,34 +178,17 @@ def __str__(self) -> str: def has_meaningful_duration(self): return State.has_meaningful_duration(self.state) - def get_github_annotations(self) -> list[tuple[str, str]]: - """(test case name, message) of each failed test case. - - Without test case details (crash, timeout, env changed, etc.), return - the test name and the test result. - """ - if not os.environ.get("GITHUB_STEP_SUMMARY"): - return [] - annotations = (self.errors or []) + (self.failures or []) - if not annotations: - message = "\n".join([str(self), *(self.env_changed_reasons or ())]) - annotations = [(self.test_name, message)] - return annotations - - def print_github_annotations(self) -> None: - """Print a GitHub Actions error annotation per failed test case.""" - for title, message in self.get_github_annotations(): - print(github_annotation(title, message), flush=True) - def print_github_crash_annotation(self, fail_env_changed: bool) -> None: """Annotate a failed test without test case failures. - Test case failures are annotated by the test runner, where they are - reported (see RegressionTestResult.printErrorList()). + For example: crash, timeout, env changed. Test case failures are + annotated by the test runner, where they are reported (see + RegressionTestResult.printErrorList()). """ if (self.is_failed(fail_env_changed) and not self.errors and not self.failures): - self.print_github_annotations() + message = "\n".join([str(self), *(self.env_changed_reasons or ())]) + print(github_annotation(self.test_name, message), flush=True) def set_env_changed(self, *reasons): if self.state is None or self.state == State.PASSED: diff --git a/Lib/test/libregrtest/testresult.py b/Lib/test/libregrtest/testresult.py index 6ad6c2acdc272db..dabd0f5870c01ee 100644 --- a/Lib/test/libregrtest/testresult.py +++ b/Lib/test/libregrtest/testresult.py @@ -154,11 +154,16 @@ def get_xml_element(self): class QuietRegressionTestRunner: def __init__(self, stream, buffer=False): + # Wrap the stream as TextTestRunner does, for printErrors() + stream = unittest.runner._WritelnDecorator(stream) self.result = RegressionTestResult(stream, None, 0) self.result.buffer = buffer def run(self, test): test(self.result) + if self.result.GITHUB_ANNOTATIONS: + # Report annotated failures, as in verbose mode + self.result.printErrors() return self.result def get_test_runner_class(verbosity, buffer=False): From c2d1d79dbc2f8247041baf525c0a45ec9f79ba32 Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Tue, 6 Oct 2026 13:58:11 -0700 Subject: [PATCH 17/34] Better message in the annotations --- Lib/test/libregrtest/testresult.py | 6 ++---- Lib/test/libregrtest/utils.py | 29 +++++++++++++++++++++++------ 2 files changed, 25 insertions(+), 10 deletions(-) diff --git a/Lib/test/libregrtest/testresult.py b/Lib/test/libregrtest/testresult.py index dabd0f5870c01ee..36ee03fc8ed9f74 100644 --- a/Lib/test/libregrtest/testresult.py +++ b/Lib/test/libregrtest/testresult.py @@ -137,10 +137,8 @@ def printErrorList(self, flavour, errors): return for test, err in errors: # Precede each failure report with a GitHub Actions annotation, - # so that the annotation links to the report in the job log. The - # report follows: don't repeat it in the annotation. - self.stream.writeln(github_annotation(str(test), err, - include_message=False)) + # so that the annotation links to the report in the job log + self.stream.writeln(github_annotation(str(test), err)) super().printErrorList(flavour, [(test, err)]) def get_xml_element(self): diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index 8e04ba960be45f5..95e69454b90a5ec 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -175,13 +175,30 @@ def traceback_location(traceback: str) -> dict[str, str | int] | None: return location -def github_annotation(title: str, message: str, *, - include_message: bool = True) -> str: +def traceback_exception(traceback: str) -> str: + """The exception which ends a traceback: type, message and notes. + + Return traceback unchanged if it has no frame. + """ + lines = traceback.splitlines() + for index in reversed(range(len(lines))): + if lines[index].startswith(' File "'): + break + else: + return traceback + # Skip the frame's indented source line and ~^ markers + index += 1 + while index < len(lines) and lines[index].startswith(" "): + index += 1 + return "\n".join(lines[index:]) + + +def github_annotation(title: str, message: str) -> str: """Format a GitHub Actions error annotation. - Locate it at the last frame of the traceback in message, if any. If - include_message is false, the annotation only has the location and the - title. + message is a traceback or a failure description. Locate the annotation at + the last frame of the traceback, if any, and only keep the exception which + ends the traceback: the job log has the full traceback. """ def escape(text: str) -> str: return (text.replace("%", "%25").replace("\r", "%0D") @@ -202,7 +219,7 @@ def escape_property(text: str) -> str: props["title"] = title # The job log only shows the message, not the properties: start the # message with the location and the title - message = f"{header}\n{message}" if include_message else header + message = f"{header}\n{traceback_exception(message).strip()}" props_text = ",".join(f"{key}={escape_property(str(value))}" for key, value in props.items()) return f"::error {props_text}::{escape(message)}" From c3cca61c6863cac97df8f0dfc87d7c618f79c80f Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Tue, 6 Oct 2026 14:25:57 -0700 Subject: [PATCH 18/34] Tweak message --- Lib/test/libregrtest/utils.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index 95e69454b90a5ec..489acc56a2491ff 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -207,19 +207,12 @@ def escape(text: str) -> str: def escape_property(text: str) -> str: return escape(text).replace(":", "%3A").replace(",", "%2C") - message = decolor(message).rstrip() + message = decolor(message) props: dict[str, str | int] = {} - header = title if location := traceback_location(message): props |= location - position = ":".join(str(location[key]) - for key in ("file", "line", "col") - if key in location) - header = f"{position}: {title}" props["title"] = title - # The job log only shows the message, not the properties: start the - # message with the location and the title - message = f"{header}\n{traceback_exception(message).strip()}" + message = traceback_exception(message).strip() props_text = ",".join(f"{key}={escape_property(str(value))}" for key, value in props.items()) return f"::error {props_text}::{escape(message)}" From b125f6fe7b184cc95ab363a6696552dbebb8b650 Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Tue, 6 Oct 2026 15:14:34 -0700 Subject: [PATCH 19/34] Move to simpler regex traceback info extraction --- Lib/test/libregrtest/testresult.py | 6 ++- Lib/test/libregrtest/utils.py | 80 +++++++----------------------- 2 files changed, 24 insertions(+), 62 deletions(-) diff --git a/Lib/test/libregrtest/testresult.py b/Lib/test/libregrtest/testresult.py index 36ee03fc8ed9f74..7011120936242f8 100644 --- a/Lib/test/libregrtest/testresult.py +++ b/Lib/test/libregrtest/testresult.py @@ -138,7 +138,11 @@ def printErrorList(self, flavour, errors): for test, err in errors: # Precede each failure report with a GitHub Actions annotation, # so that the annotation links to the report in the job log - self.stream.writeln(github_annotation(str(test), err)) + # Locate the annotation in the test file + case = getattr(test, "test_case", test) # subTest() + module = sys.modules.get(type(case).__module__) + self.stream.writeln(github_annotation( + str(test), err, getattr(module, "__file__", None))) super().printErrorList(flavour, [(test, err)]) def get_xml_element(self): diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index 489acc56a2491ff..1eb31fd3bc7b541 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -1,6 +1,5 @@ import contextlib import faulthandler -import linecache import locale import math import os.path @@ -142,63 +141,13 @@ def print_warning(msg: str) -> None: orig_unraisablehook: Callable[..., None] | None = None -def traceback_location(traceback: str) -> dict[str, str | int] | None: - """Location of the last frame of a traceback, relative to the repository. - - Return a dict with file and line, plus col and endColumn when the - traceback underlines the failing expression (~~~^^^). - """ - lines = traceback.splitlines() - for index in reversed(range(len(lines))): - if match := re.match(r' File "(.+)", line (\d+)', lines[index]): - break - else: - return None - filename, lineno = match.group(1), int(match.group(2)) - # Map the stdlib directory (Lib/ in a source checkout or an installed - # lib/python3.X/) to Lib/ in the repository - stdlib_dir = os.path.dirname(os.__file__) + os.sep - if not filename.startswith(stdlib_dir): - return None - relpath = filename.removeprefix(stdlib_dir).replace(os.sep, "/") - location: dict[str, str | int] = {"file": f"Lib/{relpath}", "line": lineno} - - # The frame is followed by the dedented source line (" code") and, - # for a single-line expression, by a line of ~ and ^ markers. - markers = lines[index + 2] if index + 2 < len(lines) else "" - source = linecache.getline(filename, lineno) - if markers.strip() and not markers.strip(" ~^") and source: - indent = len(source) - len(source.lstrip()) - start = len(markers) - len(markers.lstrip()) - location["col"] = indent + start - 4 + 1 - location["endColumn"] = indent + len(markers.rstrip()) - 4 - return location - - -def traceback_exception(traceback: str) -> str: - """The exception which ends a traceback: type, message and notes. - - Return traceback unchanged if it has no frame. - """ - lines = traceback.splitlines() - for index in reversed(range(len(lines))): - if lines[index].startswith(' File "'): - break - else: - return traceback - # Skip the frame's indented source line and ~^ markers - index += 1 - while index < len(lines) and lines[index].startswith(" "): - index += 1 - return "\n".join(lines[index:]) - - -def github_annotation(title: str, message: str) -> str: +def github_annotation(title: str, message: str, + filename: str | None = None) -> str: """Format a GitHub Actions error annotation. - message is a traceback or a failure description. Locate the annotation at - the last frame of the traceback, if any, and only keep the exception which - ends the traceback: the job log has the full traceback. + message is a traceback or a failure description. Only keep the exception + which ends the traceback: the job log has the full traceback. Locate the + annotation at the last frame of the traceback in filename, if any. """ def escape(text: str) -> str: return (text.replace("%", "%25").replace("\r", "%0D") @@ -208,12 +157,21 @@ def escape_property(text: str) -> str: return escape(text).replace(":", "%3A").replace(",", "%2C") message = decolor(message) - props: dict[str, str | int] = {} - if location := traceback_location(message): - props |= location + props: dict[str, str] = {} + # Map the stdlib directory (Lib/ in a source checkout or an installed + # lib/python3.X/) to Lib/ in the repository + stdlib_dir = os.path.dirname(os.__file__) + os.sep + if filename and filename.startswith(stdlib_dir): + lines = re.findall(rf'^ File "{re.escape(filename)}", line (\d+)', + message, re.MULTILINE) + if lines: + relpath = filename.removeprefix(stdlib_dir).replace(os.sep, "/") + props |= {"file": f"Lib/{relpath}", "line": lines[-1]} props["title"] = title - message = traceback_exception(message).strip() - props_text = ",".join(f"{key}={escape_property(str(value))}" + # Strip the frames: each " File" line and its indented source lines + message = re.split(r'^ File .*\n(?: .*\n)*', message, + flags=re.MULTILINE)[-1].strip() + props_text = ",".join(f"{key}={escape_property(value)}" for key, value in props.items()) return f"::error {props_text}::{escape(message)}" From 3f320dd94958495a1d8e2edb2d958465a3e40e3b Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Tue, 6 Oct 2026 20:16:37 -0700 Subject: [PATCH 20/34] lots of review changes --- Lib/test/libregrtest/main.py | 31 ++-- Lib/test/libregrtest/result.py | 20 +- Lib/test/libregrtest/run_workers.py | 2 +- Lib/test/libregrtest/runtests.py | 2 +- Lib/test/libregrtest/setup.py | 4 +- Lib/test/libregrtest/testresult.py | 32 ++-- Lib/test/libregrtest/utils.py | 39 ++-- Lib/test/test_regrtest.py | 173 ++++++++++++++++++ ...-10-05-22-36-06.gh-issue-158895.3Dmc4J.rst | 5 +- 9 files changed, 253 insertions(+), 55 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 58e011ac752ec8c..ccac65aed1e25ea 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -1,3 +1,4 @@ +import html import os import random import re @@ -397,7 +398,7 @@ def run_test( self.results.accumulate_result(result, runtests) if runtests.github_annotations: - result.print_github_crash_annotation(runtests.fail_env_changed) + result.print_github_annotation(runtests) return result @@ -487,16 +488,16 @@ def finalize_tests(self, coverage: trace.CoverageResults | None) -> None: def write_github_summary(self) -> None: filename = os.environ.get("GITHUB_STEP_SUMMARY") + # Tests which failed in the last run (the re-run, if any) + failed = self.results.rerun_results # Only write a summary on failure: a CI run has many test jobs exitcode = self.results.get_exitcode(self.fail_env_changed, self.fail_rerun) - if not filename or not exitcode: + if not filename or not failed or not exitcode: return - # Tests which failed in the last run (the re-run, if any) - failed = self.results.rerun_results cases = [(result.errors or []) + (result.failures or []) for result in failed] - ncase = sum(map(len, cases)) + ncases = sum(map(len, cases)) with open(filename, "a", encoding="utf-8") as fp: def write(text: str) -> None: # Separate Markdown blocks with an empty line @@ -504,17 +505,18 @@ def write(text: str) -> None: write(f"## {decolor(self.get_state())}: " f"{count(len(failed), 'test file')} and " - f"{count(ncase, 'test case')} failed") + f"{count(ncases, 'test case')} failed") for result, result_cases in zip(failed, cases): write(f"### {decolor(str(result))}") if result.env_changed_reasons: write("\n".join(f"- {reason}" for reason in result.env_changed_reasons)) for name, traceback in result_cases: - # Expand short tracebacks when there are only a few - is_open = ncase <= 5 and traceback.count("\n") < 30 + # Expand the tracebacks if they fit on a screen: up to 5 + # test cases with a traceback of less than 30 lines + is_open = ncases <= 5 and traceback.count("\n") < 30 write(f"" - f"{name}") + f"{html.escape(name)}") write(f"```pytb\n{decolor(traceback).rstrip()}\n```") write("") @@ -538,6 +540,12 @@ def display_summary(self) -> None: print(f"Result: {state}") def create_run_tests(self, tests: TestTuple) -> RunTests: + # Annotate test failures in the GitHub Actions job log of the last + # run (the re-run, if any), if it reports failures + will_rerun = self.want_rerun and not self.python_cmd + github_annotations = (bool(os.environ.get("GITHUB_STEP_SUMMARY")) + and not will_rerun + and bool(self.verbose or self.output_on_failure)) return RunTests( tests, fail_fast=self.fail_fast, @@ -555,10 +563,7 @@ def create_run_tests(self, tests: TestTuple) -> RunTests: hunt_refleak=self.hunt_refleak, test_dir=self.test_dir, use_junit=(self.junit_filename is not None), - # Only annotate failures of the last run: the re-run, if any - github_annotations=(bool(os.environ.get("GITHUB_STEP_SUMMARY")) - and not (self.want_rerun - and not self.python_cmd)), + github_annotations=github_annotations, coverage=self.coverage, memory_limit=self.memory_limit, gc_threshold=self.gc_threshold, diff --git a/Lib/test/libregrtest/result.py b/Lib/test/libregrtest/result.py index 1bb9f3a9a48dd31..6cf816fad1438aa 100644 --- a/Lib/test/libregrtest/result.py +++ b/Lib/test/libregrtest/result.py @@ -1,10 +1,12 @@ import dataclasses +import importlib.util import json from _colorize import get_colors # type: ignore[import-not-found] from typing import Any +from .runtests import RunTests from .utils import ( - StrJSON, TestName, FilterTuple, + StrJSON, TestName, FilterTuple, abs_module_name, format_duration, normalize_test_name, print_warning, github_annotation) @@ -178,17 +180,21 @@ def __str__(self) -> str: def has_meaningful_duration(self): return State.has_meaningful_duration(self.state) - def print_github_crash_annotation(self, fail_env_changed: bool) -> None: + def print_github_annotation(self, runtests: RunTests) -> None: """Annotate a failed test without test case failures. - For example: crash, timeout, env changed. Test case failures are - annotated by the test runner, where they are reported (see - RegressionTestResult.printErrorList()). + For example: crash, timeout, env changed. Locate the annotation in the + test file. Test case failures are annotated by the test runner, where + they are reported (see RegressionTestResult.printErrorList()). """ - if (self.is_failed(fail_env_changed) + if (self.is_failed(runtests.fail_env_changed) and not self.errors and not self.failures): message = "\n".join([str(self), *(self.env_changed_reasons or ())]) - print(github_annotation(self.test_name, message), flush=True) + spec = importlib.util.find_spec( + abs_module_name(self.test_name, runtests.test_dir)) + filename = spec.origin if spec is not None else None + print(github_annotation(self.test_name, message, filename), + flush=True) def set_env_changed(self, *reasons): if self.state is None or self.state == State.PASSED: diff --git a/Lib/test/libregrtest/run_workers.py b/Lib/test/libregrtest/run_workers.py index 45bf9d9327fa143..9bc220e6db52094 100644 --- a/Lib/test/libregrtest/run_workers.py +++ b/Lib/test/libregrtest/run_workers.py @@ -614,7 +614,7 @@ def _process_result(self, item: QueueOutput) -> TestResult: self.results.accumulate_result(result, self.runtests) self.display_result(mp_result) if self.runtests.github_annotations: - result.print_github_crash_annotation(self.runtests.fail_env_changed) + result.print_github_annotation(self.runtests) # Display worker stdout if not self.runtests.output_on_failure: diff --git a/Lib/test/libregrtest/runtests.py b/Lib/test/libregrtest/runtests.py index a116b6e4166fc63..06b1ea34a8140d0 100644 --- a/Lib/test/libregrtest/runtests.py +++ b/Lib/test/libregrtest/runtests.py @@ -93,7 +93,7 @@ class RunTests: hunt_refleak: HuntRefleak | None test_dir: StrPath | None use_junit: bool - # Print a GitHub Actions error annotation before each failure report + # Annotate test failures in the GitHub Actions job log github_annotations: bool coverage: bool memory_limit: str | None diff --git a/Lib/test/libregrtest/setup.py b/Lib/test/libregrtest/setup.py index 7531e69d0988e64..21734461d155cc5 100644 --- a/Lib/test/libregrtest/setup.py +++ b/Lib/test/libregrtest/setup.py @@ -112,14 +112,12 @@ def setup_tests(runtests: RunTests) -> None: set_match_tests(runtests.match_tests) + from .testresult import RegressionTestResult if runtests.use_junit: support.junit_xml_list = [] - from .testresult import RegressionTestResult RegressionTestResult.USE_XML = True else: support.junit_xml_list = None - - from .testresult import RegressionTestResult RegressionTestResult.GITHUB_ANNOTATIONS = runtests.github_annotations if runtests.memory_limit is not None: diff --git a/Lib/test/libregrtest/testresult.py b/Lib/test/libregrtest/testresult.py index 7011120936242f8..625a5ea6816aa21 100644 --- a/Lib/test/libregrtest/testresult.py +++ b/Lib/test/libregrtest/testresult.py @@ -4,6 +4,7 @@ import functools import io +import re import sys import time import traceback @@ -132,19 +133,25 @@ def addUnexpectedSuccess(self, test): super().addUnexpectedSuccess(test) def printErrorList(self, flavour, errors): - if not self.GITHUB_ANNOTATIONS: - super().printErrorList(flavour, errors) - return for test, err in errors: - # Precede each failure report with a GitHub Actions annotation, - # so that the annotation links to the report in the job log - # Locate the annotation in the test file - case = getattr(test, "test_case", test) # subTest() - module = sys.modules.get(type(case).__module__) - self.stream.writeln(github_annotation( - str(test), err, getattr(module, "__file__", None))) + if self.GITHUB_ANNOTATIONS: + # Write the annotation just before the failure report, so + # that it links to the report in the job log + self.stream.writeln(github_annotation(str(test), err, + self._test_file(test))) super().printErrorList(flavour, [(test, err)]) + @staticmethod + def _test_file(test): + # Test id: "module.Class.method", "module.function" (doctest), or + # "setUpClass (module.Class)" (error in a class or module fixture) + name = getattr(test, 'test_case', test).id() # subTest() + if match := re.fullmatch(r'\w+ \((.+)\)', name): + name = match[1] + while name and name not in sys.modules: + name = name.rpartition('.')[0] + return getattr(sys.modules.get(name), '__file__', None) + def get_xml_element(self): if not self.USE_XML: raise ValueError("USE_XML is false") @@ -156,16 +163,11 @@ def get_xml_element(self): class QuietRegressionTestRunner: def __init__(self, stream, buffer=False): - # Wrap the stream as TextTestRunner does, for printErrors() - stream = unittest.runner._WritelnDecorator(stream) self.result = RegressionTestResult(stream, None, 0) self.result.buffer = buffer def run(self, test): test(self.result) - if self.result.GITHUB_ANNOTATIONS: - # Report annotated failures, as in verbose mode - self.result.printErrors() return self.result def get_test_runner_class(verbosity, buffer=False): diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index 1eb31fd3bc7b541..ec83a0de0d5bc46 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -141,14 +141,30 @@ def print_warning(msg: str) -> None: orig_unraisablehook: Callable[..., None] | None = None +def repository_path(filename: str) -> str: + """Path of a stdlib file relative to the repository. + + Map the stdlib directory (Lib/ in a source checkout or an installed + lib/python3.X/) to Lib/. Return other paths unchanged. + """ + stdlib_dir = os.path.dirname(os.__file__) + os.sep + if not filename.startswith(stdlib_dir): + return filename + return "Lib/" + filename.removeprefix(stdlib_dir).replace(os.sep, "/") + + def github_annotation(title: str, message: str, - filename: str | None = None) -> str: + filename: str | None) -> str: """Format a GitHub Actions error annotation. message is a traceback or a failure description. Only keep the exception which ends the traceback: the job log has the full traceback. Locate the - annotation at the last frame of the traceback in filename, if any. + annotation in filename, the test file, at the last frame of the traceback + in this file, if any. """ + if not filename: + raise ValueError(f"missing test file of annotation {title!r}") + def escape(text: str) -> str: return (text.replace("%", "%25").replace("\r", "%0D") .replace("\n", "%0A")) @@ -157,18 +173,15 @@ def escape_property(text: str) -> str: return escape(text).replace(":", "%3A").replace(",", "%2C") message = decolor(message) - props: dict[str, str] = {} - # Map the stdlib directory (Lib/ in a source checkout or an installed - # lib/python3.X/) to Lib/ in the repository - stdlib_dir = os.path.dirname(os.__file__) + os.sep - if filename and filename.startswith(stdlib_dir): - lines = re.findall(rf'^ File "{re.escape(filename)}", line (\d+)', - message, re.MULTILINE) - if lines: - relpath = filename.removeprefix(stdlib_dir).replace(os.sep, "/") - props |= {"file": f"Lib/{relpath}", "line": lines[-1]} + props = {"file": repository_path(filename)} + # Line of the last traceback frame (' File "filename", line 123') or + # failed doctest example (same, but not indented) in filename + if lines := re.findall(rf'^ *File "{re.escape(filename)}", line (\d+)', + message, re.MULTILINE): + props["line"] = lines[-1] props["title"] = title - # Strip the frames: each " File" line and its indented source lines + # Only keep the exception: strip the traceback frames, each one a + # ' File' line followed by its indented source lines message = re.split(r'^ File .*\n(?: .*\n)*', message, flags=re.MULTILINE)[-1].strip() props_text = ",".join(f"{key}={escape_property(value)}" diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 53f0a2863dc4e45..ed5533bfe971ed0 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -856,6 +856,8 @@ def run_command(self, args, input=None, exitcode=0, **kw): if env is None: env = dict(os.environ) env.pop('SOURCE_DATE_EPOCH', None) + # Don't annotate the GitHub Actions job of the test suite + env.pop('GITHUB_STEP_SUMMARY', None) proc = subprocess.run(args, text=True, @@ -1639,6 +1641,177 @@ def test_env_changed(self): success=True), stats=2) + def create_marked_test(self, name, code): + # Create a test: return its name and the line number of each of its + # "# marker" comments + code = textwrap.dedent(code) + name = self.create_test(name, code) + markers = {match[1]: str(lineno) + for lineno, text in enumerate(code.splitlines(), 1) + if (match := re.search(r'# (\w+)$', text))} + return name, markers + + def run_tests_github(self, *args, exitcode=0): + # Run tests as in GitHub Actions: return the output and the job + # summary (None if regrtest didn't write it) + filename = os.path.join(self.tmptestdir, 'github_step_summary.md') + self.addCleanup(os_helper.unlink, filename) + os_helper.unlink(filename) + env = dict(os.environ) + env.pop('SOURCE_DATE_EPOCH', None) + env['GITHUB_STEP_SUMMARY'] = filename + output = self.run_tests(*args, env=env, exitcode=exitcode) + try: + with open(filename, encoding='utf-8') as fp: + return output, fp.read() + except FileNotFoundError: + return output, None + + @staticmethod + def parse_github_annotations(output): + # Return the (title, file, line) of each "::error" annotation + def unescape(text): + return (text.replace('%3A', ':').replace('%2C', ',') + .replace('%0A', '\n').replace('%0D', '\r') + .replace('%25', '%')) + + annotations = [] + for match in re.finditer(r'^::error (.*?)::', output, re.MULTILINE): + props = dict(prop.split('=', 1) for prop in match[1].split(',')) + props = {key: unescape(value) for key, value in props.items()} + annotations.append((props['title'], props['file'], + props.get('line'))) + return annotations + + def test_github_annotations(self): + # Every test failure is annotated once in the GitHub Actions job log, + # located in the test file, at the failing line of the test file if + # there is a traceback. The job summary lists the failures. + cases, lines = self.create_marked_test('github_cases', """ + import doctest, json, sys, unittest + + def load_tests(loader, tests, pattern): + tests.addTests(doctest.DocTestSuite(sys.modules[__name__])) + return tests + + def doctest_fail(): + ''' + >>> 1 + 1 # doctest + 3 + ''' + + class Tests(unittest.TestCase): + def test_error(self): + json.loads("{") # error + + def test_fail(self): + self.assertEqual(1, 2) # fail + + def test_subtest(self): + with self.subTest(x=1.5): + self.fail("subtest") # subtest + + class SetUpClassTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + raise ValueError("setUpClass") # setUpClass + + def test_never(self): + pass + """) + setup_module, setup_module_lines = self.create_marked_test( + 'github_setup_module', """ + import unittest + + def setUpModule(): + raise ValueError("setUpModule") # setUpModule + + class Tests(unittest.TestCase): + def test_never(self): + pass + """) + env_changed, _ = self.create_marked_test('github_env_changed', """ + import os, unittest + + class Tests(unittest.TestCase): + def test_env_changed(self): + os.environ["REGRTEST_GITHUB_ENV_CHANGED"] = "1" + """) + crash, _ = self.create_marked_test('github_crash', """ + import os, unittest + + class Tests(unittest.TestCase): + def test_crash(self): + os._exit(1) + """) + + def path(name): + return os.path.join(self.tmptestdir, f'{name}.py') + + # Test cases: (title, file, line) + test_cases = [ + (f'test_error ({cases}.Tests.test_error)', + path(cases), lines['error']), + (f'test_fail ({cases}.Tests.test_fail)', + path(cases), lines['fail']), + (f'test_subtest ({cases}.Tests.test_subtest) (x=1.5)', + path(cases), lines['subtest']), + # Doctest examples are subtests: "[0]" is the example index + (f'doctest_fail ({cases}) [0]', + path(cases), lines['doctest']), + (f'setUpClass ({cases}.SetUpClassTests)', + path(cases), lines['setUpClass']), + (f'setUpModule ({setup_module})', + path(setup_module), setup_module_lines['setUpModule']), + ] + # Test files: heading in the job summary + test_files = { + cases: f'### {cases} failed (2 errors, 3 failures)', + setup_module: f'### {setup_module} failed (1 error)', + env_changed: f'### {env_changed} failed (env changed)', + crash: f'### {crash} worker non-zero exit code', + } + + for args in (['-j2', '-W'], ['-j2', '--rerun'], ['-j0', '-W']): + tests = list(test_files) + if '-j0' in args: + # A crash kills the main process + tests.remove(crash) + with self.subTest(args=args): + output, summary = self.run_tests_github( + '--fail-env-changed', *args, *tests, + exitcode=EXITCODE_BAD_TEST) + + self.assertCountEqual( + self.parse_github_annotations(output), + [*test_cases, + *((name, path(name), None) + for name in (env_changed, crash) if name in tests)], + output) + + # The job summary lists the failed tests in completion order + summary_lines = summary.splitlines() + self.assertEqual(summary_lines[0], + f'## FAILURE: {len(tests)} test files and ' + f'{len(test_cases)} test cases failed') + self.assertCountEqual( + [line for line in summary_lines + if line.startswith('### ')], + [test_files[name] for name in tests]) + self.assertIn('- os.environ was modified', summary_lines) + self.assertCountEqual( + re.findall(r'(.*)', summary), + [title for title, _, _ in test_cases]) + self.assertEqual(summary.count('```pytb\n'), + len(test_cases)) + + def test_github_summary_success(self): + # No annotation and no job summary when all tests pass + testname = self.create_test() + output, summary = self.run_tests_github(testname) + self.assertNotIn('::error', output) + self.assertIsNone(summary) + def test_rerun_fail(self): # FAILURE then FAILURE code = textwrap.dedent(""" diff --git a/Misc/NEWS.d/next/Tests/2026-10-05-22-36-06.gh-issue-158895.3Dmc4J.rst b/Misc/NEWS.d/next/Tests/2026-10-05-22-36-06.gh-issue-158895.3Dmc4J.rst index ebeca7174e5903a..a1c9e648e5c744e 100644 --- a/Misc/NEWS.d/next/Tests/2026-10-05-22-36-06.gh-issue-158895.3Dmc4J.rst +++ b/Misc/NEWS.d/next/Tests/2026-10-05-22-36-06.gh-issue-158895.3Dmc4J.rst @@ -1,2 +1,3 @@ -Individual test failures and an overall summary is now written by -:mod:`test.regrtest` to the Github Action Summary. +When run in GitHub Actions, :mod:`test.regrtest` now annotates each test +failure in the job log and writes a summary of the failures to the job +summary. From 9b1ce485f33570cc756c8ca3424a0d040982de39 Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Tue, 6 Oct 2026 21:20:44 -0700 Subject: [PATCH 21/34] Add comment --- Lib/test/test_regrtest.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index ed5533bfe971ed0..25d9c0364396134 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -1772,7 +1772,10 @@ def path(name): crash: f'### {crash} worker non-zero exit code', } - for args in (['-j2', '-W'], ['-j2', '--rerun'], ['-j0', '-W']): + # Only failures of the last run are annotated: with --rerun and -W, + # failures of the first run are reported, but not annotated + for args in (['-j2', '-W'], ['-j2', '--rerun'], + ['-j2', '-W', '--rerun'], ['-j0', '-W']): tests = list(test_files) if '-j0' in args: # A crash kills the main process From b5610603c83b2ccd5f209b3a61998bd159ff31d1 Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Wed, 7 Oct 2026 09:56:03 -0700 Subject: [PATCH 22/34] Annotations after failure, better test_regrtest coverage --- Lib/test/libregrtest/testresult.py | 4 +-- Lib/test/test_regrtest.py | 55 ++++++++++++++++++++---------- 2 files changed, 39 insertions(+), 20 deletions(-) diff --git a/Lib/test/libregrtest/testresult.py b/Lib/test/libregrtest/testresult.py index 625a5ea6816aa21..45a363b155cb3d9 100644 --- a/Lib/test/libregrtest/testresult.py +++ b/Lib/test/libregrtest/testresult.py @@ -134,12 +134,12 @@ def addUnexpectedSuccess(self, test): def printErrorList(self, flavour, errors): for test, err in errors: + super().printErrorList(flavour, [(test, err)]) if self.GITHUB_ANNOTATIONS: - # Write the annotation just before the failure report, so + # Write the annotation just after the failure report, so # that it links to the report in the job log self.stream.writeln(github_annotation(str(test), err, self._test_file(test))) - super().printErrorList(flavour, [(test, err)]) @staticmethod def _test_file(test): diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 25d9c0364396134..0cecacc4b076d29 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -1772,25 +1772,44 @@ def path(name): crash: f'### {crash} worker non-zero exit code', } - # Only failures of the last run are annotated: with --rerun and -W, - # failures of the first run are reported, but not annotated - for args in (['-j2', '-W'], ['-j2', '--rerun'], - ['-j2', '-W', '--rerun'], ['-j0', '-W']): - tests = list(test_files) - if '-j0' in args: - # A crash kills the main process - tests.remove(crash) - with self.subTest(args=args): + # Representative command lines of the CI jobs + command_lines = { + # "make ci", Windows, macOS, installed Python; the JIT jobs use + # the equivalent "-j0 --verbose2 --verbose3". Only failures of + # the re-run are annotated, not the reported first failures. + 'fast-ci': ['--fast-ci', '-j2'], + # Sanitizers and Hypothesis: env changed is not a failure + 'parallel': ['-j2', '-W'], + # iOS and WASI run tests in the main process: a crash kills it + 'single-process': ['--fast-ci', '--single-process'], + # Profile task of PGO and BOLT builds: no annotations + 'pgo': ['--pgo'], + } + for name, args in command_lines.items(): + with self.subTest(name): + tests = list(test_files) + if '--single-process' in args or '--pgo' in args: + tests.remove(crash) output, summary = self.run_tests_github( - '--fail-env-changed', *args, *tests, - exitcode=EXITCODE_BAD_TEST) - - self.assertCountEqual( - self.parse_github_annotations(output), - [*test_cases, - *((name, path(name), None) - for name in (env_changed, crash) if name in tests)], - output) + *args, *tests, exitcode=EXITCODE_BAD_TEST) + + annotations = [] + if '--pgo' not in args: + annotations += test_cases + if '--fast-ci' in args: + annotations.append((env_changed, path(env_changed), + None)) + if crash in tests: + annotations.append((crash, path(crash), None)) + self.assertCountEqual(self.parse_github_annotations(output), + annotations, output) + # Each test case is annotated right after its failure report + for title, _, _ in annotations[:len(test_cases)]: + title = re.escape(title) + self.assertRegex( + output, + rf'(?m)^(?:ERROR|FAIL): {title}\n' + rf'(?:(?!={{70}}$).*\n)*?::error .*title={title}::') # The job summary lists the failed tests in completion order summary_lines = summary.splitlines() From e506d7d940bbcb119a0d9aec20405197a6adf7f5 Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Wed, 7 Oct 2026 10:31:23 -0700 Subject: [PATCH 23/34] annotate env changed better. pgo runs --- Lib/test/libregrtest/main.py | 5 +++-- Lib/test/libregrtest/run_workers.py | 5 +++-- Lib/test/libregrtest/save_env.py | 19 +++++++++++++++- Lib/test/test_regrtest.py | 35 ++++++++++++++++++++--------- 4 files changed, 49 insertions(+), 15 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index ccac65aed1e25ea..290bc628f2e7356 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -541,11 +541,12 @@ def display_summary(self) -> None: def create_run_tests(self, tests: TestTuple) -> RunTests: # Annotate test failures in the GitHub Actions job log of the last - # run (the re-run, if any), if it reports failures + # run (the re-run, if any), if it reports failures: -v, -W or --pgo will_rerun = self.want_rerun and not self.python_cmd github_annotations = (bool(os.environ.get("GITHUB_STEP_SUMMARY")) and not will_rerun - and bool(self.verbose or self.output_on_failure)) + and bool(self.verbose or self.output_on_failure + or self.pgo)) return RunTests( tests, fail_fast=self.fail_fast, diff --git a/Lib/test/libregrtest/run_workers.py b/Lib/test/libregrtest/run_workers.py index 9bc220e6db52094..4a83f8389870467 100644 --- a/Lib/test/libregrtest/run_workers.py +++ b/Lib/test/libregrtest/run_workers.py @@ -613,8 +613,6 @@ def _process_result(self, item: QueueOutput) -> TestResult: result = mp_result.result self.results.accumulate_result(result, self.runtests) self.display_result(mp_result) - if self.runtests.github_annotations: - result.print_github_annotation(self.runtests) # Display worker stdout if not self.runtests.output_on_failure: @@ -626,6 +624,9 @@ def _process_result(self, item: QueueOutput) -> TestResult: stdout = mp_result.worker_stdout if stdout: print(stdout, flush=True) + # Annotate after the output: env changed warnings, crash traceback + if self.runtests.github_annotations: + result.print_github_annotation(self.runtests) return result diff --git a/Lib/test/libregrtest/save_env.py b/Lib/test/libregrtest/save_env.py index 4149210fa173109..6a902eaf91f5f42 100644 --- a/Lib/test/libregrtest/save_env.py +++ b/Lib/test/libregrtest/save_env.py @@ -139,6 +139,18 @@ def restore_os_environ(self, saved_environ): os.environ = saved_environ[1] os.environ.clear() os.environ.update(saved_environ[2]) + @staticmethod + def describe_os_environ(original, current): + # Only list the keys: values can be secrets + before, after = original[2], current[2] + changes = ( + ('added', after.keys() - before.keys()), + ('removed', before.keys() - after.keys()), + ('changed', {key for key in before.keys() & after.keys() + if before[key] != after[key]}), + ) + return '; '.join(f'{change} {", ".join(sorted(keys))}' + for change, keys in changes if keys) def get_sys_path(self): return id(sys.path), sys.path, sys.path[:] @@ -347,7 +359,12 @@ def __exit__(self, exc_type, exc_val, exc_tb): current = get() # Check for changes to the resource's value if current != original: - support.set_environment_altered(f"{name} was modified") + reason = f"{name} was modified" + if name == 'os.environ': + delta = self.describe_os_environ(original, current) + if delta: + reason = f"{reason}: {delta}" + support.set_environment_altered(reason) restore(original) if not self.quiet and not self.pgo: print_warning( diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 0cecacc4b076d29..a85304262a1d7e8 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -38,6 +38,7 @@ from test.libregrtest import utils from test.libregrtest.filter import get_match_tests, match_test from test.libregrtest.result import TestStats +from test.libregrtest.save_env import saved_test_environment from test.libregrtest.utils import normalize_test_name if not support.has_subprocess_support: @@ -1782,7 +1783,7 @@ def path(name): 'parallel': ['-j2', '-W'], # iOS and WASI run tests in the main process: a crash kills it 'single-process': ['--fast-ci', '--single-process'], - # Profile task of PGO and BOLT builds: no annotations + # Profile task of PGO and BOLT builds: failures stop the build 'pgo': ['--pgo'], } for name, args in command_lines.items(): @@ -1793,14 +1794,11 @@ def path(name): output, summary = self.run_tests_github( *args, *tests, exitcode=EXITCODE_BAD_TEST) - annotations = [] - if '--pgo' not in args: - annotations += test_cases - if '--fast-ci' in args: - annotations.append((env_changed, path(env_changed), - None)) - if crash in tests: - annotations.append((crash, path(crash), None)) + annotations = list(test_cases) + if '--fast-ci' in args: + annotations.append((env_changed, path(env_changed), None)) + if crash in tests: + annotations.append((crash, path(crash), None)) self.assertCountEqual(self.parse_github_annotations(output), annotations, output) # Each test case is annotated right after its failure report @@ -1810,6 +1808,13 @@ def path(name): output, rf'(?m)^(?:ERROR|FAIL): {title}\n' rf'(?:(?!={{70}}$).*\n)*?::error .*title={title}::') + # The env changed annotation comes right after the warnings + if '--fast-ci' in args: + self.assertRegex( + output, + rf'(?m)^Warning -- os\.environ was modified by ' + rf'{env_changed}\n(?:Warning -- .*\n)*' + rf'::error .*title={env_changed}::') # The job summary lists the failed tests in completion order summary_lines = summary.splitlines() @@ -1820,7 +1825,9 @@ def path(name): [line for line in summary_lines if line.startswith('### ')], [test_files[name] for name in tests]) - self.assertIn('- os.environ was modified', summary_lines) + self.assertIn('- os.environ was modified: ' + 'added REGRTEST_GITHUB_ENV_CHANGED', + summary_lines) self.assertCountEqual( re.findall(r'(.*)', summary), [title for title, _, _ in test_cases]) @@ -2950,6 +2957,14 @@ def worker(): class TestUtils(unittest.TestCase): + def test_describe_os_environ(self): + describe = saved_test_environment.describe_os_environ + before = {'A': '1', 'B': '2', 'C': '3'} + after = {'A': '1', 'B': 'secret', 'D': '4'} + self.assertEqual(describe((0, None, before), (0, None, after)), + 'added D; removed C; changed B') + self.assertEqual(describe((0, None, before), (1, None, before)), '') + def test_format_duration(self): self.assertEqual(utils.format_duration(0), '0 ms') From 2ca705b3556086a24fa82cff579a7ec021e41931 Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Wed, 7 Oct 2026 12:20:39 -0700 Subject: [PATCH 24/34] simplify, rely on env var everywhere. Stop using findspec --- Lib/test/libregrtest/main.py | 28 ++++++++----------- Lib/test/libregrtest/result.py | 21 ++++++++------ Lib/test/libregrtest/run_workers.py | 3 +- Lib/test/libregrtest/runtests.py | 2 -- Lib/test/libregrtest/save_env.py | 19 +------------ Lib/test/libregrtest/setup.py | 1 - Lib/test/libregrtest/testresult.py | 12 ++++---- Lib/test/libregrtest/utils.py | 27 +++++++++--------- Lib/test/libregrtest/worker.py | 2 -- .../test_live_collector_ui.py | 6 +++- Lib/test/test_regrtest.py | 18 +++--------- 11 files changed, 55 insertions(+), 84 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 290bc628f2e7356..20b94eff8aa3fdd 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -285,8 +285,7 @@ def _rerun_failed_tests(self, runtests: RunTests) -> RunTests: forever=False, fail_fast=False, match_tests_dict=match_tests_dict, - output_on_failure=False, - github_annotations=bool(os.environ.get("GITHUB_STEP_SUMMARY"))) + output_on_failure=False) self.logger.set_tests(runtests) msg = f"Re-running {len(tests)} failed tests in verbose mode" @@ -397,8 +396,7 @@ def run_test( result = run_single_test(test_name, runtests) self.results.accumulate_result(result, runtests) - if runtests.github_annotations: - result.print_github_annotation(runtests) + result.print_github_annotation(runtests) return result @@ -540,13 +538,6 @@ def display_summary(self) -> None: print(f"Result: {state}") def create_run_tests(self, tests: TestTuple) -> RunTests: - # Annotate test failures in the GitHub Actions job log of the last - # run (the re-run, if any), if it reports failures: -v, -W or --pgo - will_rerun = self.want_rerun and not self.python_cmd - github_annotations = (bool(os.environ.get("GITHUB_STEP_SUMMARY")) - and not will_rerun - and bool(self.verbose or self.output_on_failure - or self.pgo)) return RunTests( tests, fail_fast=self.fail_fast, @@ -564,7 +555,6 @@ def create_run_tests(self, tests: TestTuple) -> RunTests: hunt_refleak=self.hunt_refleak, test_dir=self.test_dir, use_junit=(self.junit_filename is not None), - github_annotations=github_annotations, coverage=self.coverage, memory_limit=self.memory_limit, gc_threshold=self.gc_threshold, @@ -627,10 +617,16 @@ def _run_tests(self, selected: TestTuple, tests: TestList | None) -> int: if use_load_tracker: self.logger.start_load_tracker() try: - if self.num_workers: - self._run_tests_mp(runtests, self.num_workers) - else: - self.run_tests_sequentially(runtests) + with os_helper.EnvironmentVarGuard() as env: + # In GitHub Actions, only annotate failures of the last run: + # not the first run if failed tests will be re-run (with + # --python, they are not: see rerun_failed_tests()) + if self.want_rerun and not self.python_cmd: + env.unset("GITHUB_STEP_SUMMARY") + if self.num_workers: + self._run_tests_mp(runtests, self.num_workers) + else: + self.run_tests_sequentially(runtests) coverage = self.results.get_coverage_results() self.display_result(runtests) diff --git a/Lib/test/libregrtest/result.py b/Lib/test/libregrtest/result.py index 6cf816fad1438aa..e1d05777598f2e9 100644 --- a/Lib/test/libregrtest/result.py +++ b/Lib/test/libregrtest/result.py @@ -1,13 +1,15 @@ import dataclasses -import importlib.util import json +import os from _colorize import get_colors # type: ignore[import-not-found] from typing import Any +from .findtests import findtestdir from .runtests import RunTests from .utils import ( - StrJSON, TestName, FilterTuple, abs_module_name, - format_duration, normalize_test_name, print_warning, github_annotation) + StrJSON, TestName, FilterTuple, + format_duration, normalize_test_name, print_warning, + print_github_annotation) @dataclasses.dataclass(slots=True) @@ -190,11 +192,14 @@ def print_github_annotation(self, runtests: RunTests) -> None: if (self.is_failed(runtests.fail_env_changed) and not self.errors and not self.failures): message = "\n".join([str(self), *(self.env_changed_reasons or ())]) - spec = importlib.util.find_spec( - abs_module_name(self.test_name, runtests.test_dir)) - filename = spec.origin if spec is not None else None - print(github_annotation(self.test_name, message, filename), - flush=True) + # Test file: "test_x" is test_x.py or test_x/__init__.py, + # "test.test_x.test_y" is test_x/test_y.py + name = self.test_name.removeprefix("test.") + path = os.path.join(findtestdir(runtests.test_dir), *name.split(".")) + filename = path + ".py" + if not os.path.exists(filename): + filename = os.path.join(path, "__init__.py") + print_github_annotation(self.test_name, message, filename) def set_env_changed(self, *reasons): if self.state is None or self.state == State.PASSED: diff --git a/Lib/test/libregrtest/run_workers.py b/Lib/test/libregrtest/run_workers.py index 4a83f8389870467..995543d676876cc 100644 --- a/Lib/test/libregrtest/run_workers.py +++ b/Lib/test/libregrtest/run_workers.py @@ -625,8 +625,7 @@ def _process_result(self, item: QueueOutput) -> TestResult: if stdout: print(stdout, flush=True) # Annotate after the output: env changed warnings, crash traceback - if self.runtests.github_annotations: - result.print_github_annotation(self.runtests) + result.print_github_annotation(self.runtests) return result diff --git a/Lib/test/libregrtest/runtests.py b/Lib/test/libregrtest/runtests.py index 06b1ea34a8140d0..d86b082fb34a604 100644 --- a/Lib/test/libregrtest/runtests.py +++ b/Lib/test/libregrtest/runtests.py @@ -93,8 +93,6 @@ class RunTests: hunt_refleak: HuntRefleak | None test_dir: StrPath | None use_junit: bool - # Annotate test failures in the GitHub Actions job log - github_annotations: bool coverage: bool memory_limit: str | None gc_threshold: int | None diff --git a/Lib/test/libregrtest/save_env.py b/Lib/test/libregrtest/save_env.py index 6a902eaf91f5f42..4149210fa173109 100644 --- a/Lib/test/libregrtest/save_env.py +++ b/Lib/test/libregrtest/save_env.py @@ -139,18 +139,6 @@ def restore_os_environ(self, saved_environ): os.environ = saved_environ[1] os.environ.clear() os.environ.update(saved_environ[2]) - @staticmethod - def describe_os_environ(original, current): - # Only list the keys: values can be secrets - before, after = original[2], current[2] - changes = ( - ('added', after.keys() - before.keys()), - ('removed', before.keys() - after.keys()), - ('changed', {key for key in before.keys() & after.keys() - if before[key] != after[key]}), - ) - return '; '.join(f'{change} {", ".join(sorted(keys))}' - for change, keys in changes if keys) def get_sys_path(self): return id(sys.path), sys.path, sys.path[:] @@ -359,12 +347,7 @@ def __exit__(self, exc_type, exc_val, exc_tb): current = get() # Check for changes to the resource's value if current != original: - reason = f"{name} was modified" - if name == 'os.environ': - delta = self.describe_os_environ(original, current) - if delta: - reason = f"{reason}: {delta}" - support.set_environment_altered(reason) + support.set_environment_altered(f"{name} was modified") restore(original) if not self.quiet and not self.pgo: print_warning( diff --git a/Lib/test/libregrtest/setup.py b/Lib/test/libregrtest/setup.py index 21734461d155cc5..0c7fad58fd03bf6 100644 --- a/Lib/test/libregrtest/setup.py +++ b/Lib/test/libregrtest/setup.py @@ -118,7 +118,6 @@ def setup_tests(runtests: RunTests) -> None: RegressionTestResult.USE_XML = True else: support.junit_xml_list = None - RegressionTestResult.GITHUB_ANNOTATIONS = runtests.github_annotations if runtests.memory_limit is not None: support.set_memlimit(runtests.memory_limit) diff --git a/Lib/test/libregrtest/testresult.py b/Lib/test/libregrtest/testresult.py index 45a363b155cb3d9..9c453a9ef585162 100644 --- a/Lib/test/libregrtest/testresult.py +++ b/Lib/test/libregrtest/testresult.py @@ -10,11 +10,10 @@ import traceback import unittest from test import support -from test.libregrtest.utils import github_annotation, sanitize_xml +from test.libregrtest.utils import print_github_annotation, sanitize_xml class RegressionTestResult(unittest.TextTestResult): USE_XML = False - GITHUB_ANNOTATIONS = False def __init__(self, stream, descriptions, verbosity): super().__init__(stream=stream, descriptions=descriptions, @@ -135,11 +134,10 @@ def addUnexpectedSuccess(self, test): def printErrorList(self, flavour, errors): for test, err in errors: super().printErrorList(flavour, [(test, err)]) - if self.GITHUB_ANNOTATIONS: - # Write the annotation just after the failure report, so - # that it links to the report in the job log - self.stream.writeln(github_annotation(str(test), err, - self._test_file(test))) + # Annotate just after the failure report, so that the annotation + # links to the report in the job log + print_github_annotation(str(test), err, self._test_file(test), + file=self.stream) @staticmethod def _test_file(test): diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index ec83a0de0d5bc46..60921d5e018a123 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -153,25 +153,25 @@ def repository_path(filename: str) -> str: return "Lib/" + filename.removeprefix(stdlib_dir).replace(os.sep, "/") -def github_annotation(title: str, message: str, - filename: str | None) -> str: - """Format a GitHub Actions error annotation. +# Escape the data and the properties of GitHub Actions workflow commands +GITHUB_ESCAPE_DATA = str.maketrans({"%": "%25", "\r": "%0D", "\n": "%0A"}) +GITHUB_ESCAPE_PROPERTY = GITHUB_ESCAPE_DATA | str.maketrans({":": "%3A", + ",": "%2C"}) + + +def print_github_annotation(title: str, message: str, filename: str | None, + file=None) -> None: + """Print a GitHub Actions error annotation, if run in GitHub Actions. message is a traceback or a failure description. Only keep the exception which ends the traceback: the job log has the full traceback. Locate the annotation in filename, the test file, at the last frame of the traceback in this file, if any. """ + if not os.environ.get("GITHUB_STEP_SUMMARY"): + return if not filename: raise ValueError(f"missing test file of annotation {title!r}") - - def escape(text: str) -> str: - return (text.replace("%", "%25").replace("\r", "%0D") - .replace("\n", "%0A")) - - def escape_property(text: str) -> str: - return escape(text).replace(":", "%3A").replace(",", "%2C") - message = decolor(message) props = {"file": repository_path(filename)} # Line of the last traceback frame (' File "filename", line 123') or @@ -184,9 +184,10 @@ def escape_property(text: str) -> str: # ' File' line followed by its indented source lines message = re.split(r'^ File .*\n(?: .*\n)*', message, flags=re.MULTILINE)[-1].strip() - props_text = ",".join(f"{key}={escape_property(value)}" + props_text = ",".join(f"{key}={value.translate(GITHUB_ESCAPE_PROPERTY)}" for key, value in props.items()) - return f"::error {props_text}::{escape(message)}" + print(f"::error {props_text}::{message.translate(GITHUB_ESCAPE_DATA)}", + file=file, flush=True) def regrtest_unraisable_hook(unraisable) -> None: diff --git a/Lib/test/libregrtest/worker.py b/Lib/test/libregrtest/worker.py index 24ae95ed23914f6..4e69ab9d8fad05f 100644 --- a/Lib/test/libregrtest/worker.py +++ b/Lib/test/libregrtest/worker.py @@ -28,8 +28,6 @@ def create_worker_process(runtests: WorkerRunTests, output_fd: int, cmd.extend(['-m', 'test.libregrtest.worker', worker_json]) env = dict(os.environ) - # Only the main process writes the GitHub Actions job summary - env.pop('GITHUB_STEP_SUMMARY', None) if tmp_dir is not None: env['TMPDIR'] = tmp_dir env['TEMP'] = tmp_dir diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_live_collector_ui.py b/Lib/test/test_profiling/test_sampling_profiler/test_live_collector_ui.py index ae4ee969f34d8b3..57598715f8b2880 100644 --- a/Lib/test/test_profiling/test_sampling_profiler/test_live_collector_ui.py +++ b/Lib/test/test_profiling/test_sampling_profiler/test_live_collector_ui.py @@ -11,7 +11,8 @@ import time import unittest from unittest import mock -from test.support import requires, requires_remote_subprocess_debugging +from test.support import ( + os_helper, requires, requires_remote_subprocess_debugging) from test.support.import_helper import import_module # Only run these tests if curses is available @@ -856,6 +857,9 @@ def mock_init_curses_side_effect(self, n_times, mock_self, stdscr): def test_run_failed_module_live(self): """Test that running a existing module that fails exits with clean error.""" + # Don't write to the GitHub Actions job summary of the test suite + env = self.enterContext(os_helper.EnvironmentVarGuard()) + env.unset('GITHUB_STEP_SUMMARY') args = [ "profiling.sampling.cli", "run", "--live", "-m", "test", diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index a85304262a1d7e8..0de0e76665baf5a 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -38,7 +38,6 @@ from test.libregrtest import utils from test.libregrtest.filter import get_match_tests, match_test from test.libregrtest.result import TestStats -from test.libregrtest.save_env import saved_test_environment from test.libregrtest.utils import normalize_test_name if not support.has_subprocess_support: @@ -625,6 +624,9 @@ class BaseTestCase(unittest.TestCase): def setUp(self): self.testdir = os.path.realpath(os.path.dirname(__file__)) + # Don't annotate the GitHub Actions job running the test suite + env = self.enterContext(os_helper.EnvironmentVarGuard()) + env.unset('GITHUB_STEP_SUMMARY') self.tmptestdir = tempfile.mkdtemp() self.addCleanup(os_helper.rmtree, self.tmptestdir) @@ -857,8 +859,6 @@ def run_command(self, args, input=None, exitcode=0, **kw): if env is None: env = dict(os.environ) env.pop('SOURCE_DATE_EPOCH', None) - # Don't annotate the GitHub Actions job of the test suite - env.pop('GITHUB_STEP_SUMMARY', None) proc = subprocess.run(args, text=True, @@ -1825,9 +1825,7 @@ def path(name): [line for line in summary_lines if line.startswith('### ')], [test_files[name] for name in tests]) - self.assertIn('- os.environ was modified: ' - 'added REGRTEST_GITHUB_ENV_CHANGED', - summary_lines) + self.assertIn('- os.environ was modified', summary_lines) self.assertCountEqual( re.findall(r'(.*)', summary), [title for title, _, _ in test_cases]) @@ -2957,14 +2955,6 @@ def worker(): class TestUtils(unittest.TestCase): - def test_describe_os_environ(self): - describe = saved_test_environment.describe_os_environ - before = {'A': '1', 'B': '2', 'C': '3'} - after = {'A': '1', 'B': 'secret', 'D': '4'} - self.assertEqual(describe((0, None, before), (0, None, after)), - 'added D; removed C; changed B') - self.assertEqual(describe((0, None, before), (1, None, before)), '') - def test_format_duration(self): self.assertEqual(utils.format_duration(0), '0 ms') From fe082633b9111239bdc45f6da806111c7ede82dc Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Wed, 7 Oct 2026 12:45:15 -0700 Subject: [PATCH 25/34] Simplify filename extraction, use pathlib --- Lib/test/libregrtest/setup.py | 2 +- Lib/test/libregrtest/testresult.py | 27 ++++---- Lib/test/libregrtest/utils.py | 16 +++-- Lib/test/test_regrtest.py | 101 ++++++++++++++--------------- 4 files changed, 73 insertions(+), 73 deletions(-) diff --git a/Lib/test/libregrtest/setup.py b/Lib/test/libregrtest/setup.py index 0c7fad58fd03bf6..8909225d0265bf5 100644 --- a/Lib/test/libregrtest/setup.py +++ b/Lib/test/libregrtest/setup.py @@ -112,8 +112,8 @@ def setup_tests(runtests: RunTests) -> None: set_match_tests(runtests.match_tests) - from .testresult import RegressionTestResult if runtests.use_junit: + from .testresult import RegressionTestResult support.junit_xml_list = [] RegressionTestResult.USE_XML = True else: diff --git a/Lib/test/libregrtest/testresult.py b/Lib/test/libregrtest/testresult.py index 9c453a9ef585162..bdce67226fff3bf 100644 --- a/Lib/test/libregrtest/testresult.py +++ b/Lib/test/libregrtest/testresult.py @@ -4,7 +4,6 @@ import functools import io -import re import sys import time import traceback @@ -12,6 +11,16 @@ from test import support from test.libregrtest.utils import print_github_annotation, sanitize_xml +def test_to_filename(test): + test = getattr(test, 'test_case', test) # subTest() + if dt_test := getattr(test, '_dt_test', None): # doctest.DocTestCase + return dt_test.filename + module = type(test).__module__ + # Fixture errors (setUpClass, setUpModule) are not supported + if module.startswith('unittest.'): + return None + return getattr(sys.modules.get(module), '__file__', None) + class RegressionTestResult(unittest.TextTestResult): USE_XML = False @@ -136,19 +145,9 @@ def printErrorList(self, flavour, errors): super().printErrorList(flavour, [(test, err)]) # Annotate just after the failure report, so that the annotation # links to the report in the job log - print_github_annotation(str(test), err, self._test_file(test), - file=self.stream) - - @staticmethod - def _test_file(test): - # Test id: "module.Class.method", "module.function" (doctest), or - # "setUpClass (module.Class)" (error in a class or module fixture) - name = getattr(test, 'test_case', test).id() # subTest() - if match := re.fullmatch(r'\w+ \((.+)\)', name): - name = match[1] - while name and name not in sys.modules: - name = name.rpartition('.')[0] - return getattr(sys.modules.get(name), '__file__', None) + if filename := test_to_filename(test): + print_github_annotation(str(test), err, filename, + file=self.stream) def get_xml_element(self): if not self.USE_XML: diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index 60921d5e018a123..50de4ae9ab6d674 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -3,6 +3,7 @@ import locale import math import os.path +import pathlib import platform import random import re @@ -141,16 +142,21 @@ def print_warning(msg: str) -> None: orig_unraisablehook: Callable[..., None] | None = None +# Stdlib directory: Lib/ in a source checkout or an installed lib/python3.X/ +STDLIB_DIR = pathlib.Path(os.__file__).parent +# Stdlib directory relative to the repository +REPOSITORY_LIB_DIR = pathlib.Path("Lib") + + def repository_path(filename: str) -> str: """Path of a stdlib file relative to the repository. - Map the stdlib directory (Lib/ in a source checkout or an installed - lib/python3.X/) to Lib/. Return other paths unchanged. + Map STDLIB_DIR to Lib/. Return other paths unchanged. """ - stdlib_dir = os.path.dirname(os.__file__) + os.sep - if not filename.startswith(stdlib_dir): + path = pathlib.Path(filename) + if not path.is_relative_to(STDLIB_DIR): return filename - return "Lib/" + filename.removeprefix(stdlib_dir).replace(os.sep, "/") + return (REPOSITORY_LIB_DIR / path.relative_to(STDLIB_DIR)).as_posix() # Escape the data and the properties of GitHub Actions workflow commands diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 0de0e76665baf5a..fa569207bee3366 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -1668,22 +1668,6 @@ def run_tests_github(self, *args, exitcode=0): except FileNotFoundError: return output, None - @staticmethod - def parse_github_annotations(output): - # Return the (title, file, line) of each "::error" annotation - def unescape(text): - return (text.replace('%3A', ':').replace('%2C', ',') - .replace('%0A', '\n').replace('%0D', '\r') - .replace('%25', '%')) - - annotations = [] - for match in re.finditer(r'^::error (.*?)::', output, re.MULTILINE): - props = dict(prop.split('=', 1) for prop in match[1].split(',')) - props = {key: unescape(value) for key, value in props.items()} - annotations.append((props['title'], props['file'], - props.get('line'))) - return annotations - def test_github_annotations(self): # Every test failure is annotated once in the GitHub Actions job log, # located in the test file, at the failing line of the test file if @@ -1715,17 +1699,17 @@ def test_subtest(self): class SetUpClassTests(unittest.TestCase): @classmethod def setUpClass(cls): - raise ValueError("setUpClass") # setUpClass + raise ValueError("setUpClass") def test_never(self): pass """) - setup_module, setup_module_lines = self.create_marked_test( + setup_module, _ = self.create_marked_test( 'github_setup_module', """ import unittest def setUpModule(): - raise ValueError("setUpModule") # setUpModule + raise ValueError("setUpModule") class Tests(unittest.TestCase): def test_never(self): @@ -1746,24 +1730,23 @@ def test_crash(self): os._exit(1) """) - def path(name): - return os.path.join(self.tmptestdir, f'{name}.py') - - # Test cases: (title, file, line) + # Annotated test cases: (title, file, line) test_cases = [ (f'test_error ({cases}.Tests.test_error)', - path(cases), lines['error']), + f'{cases}.py', lines['error']), (f'test_fail ({cases}.Tests.test_fail)', - path(cases), lines['fail']), + f'{cases}.py', lines['fail']), (f'test_subtest ({cases}.Tests.test_subtest) (x=1.5)', - path(cases), lines['subtest']), + f'{cases}.py', lines['subtest']), # Doctest examples are subtests: "[0]" is the example index (f'doctest_fail ({cases}) [0]', - path(cases), lines['doctest']), - (f'setUpClass ({cases}.SetUpClassTests)', - path(cases), lines['setUpClass']), - (f'setUpModule ({setup_module})', - path(setup_module), setup_module_lines['setUpModule']), + f'{cases}.py', lines['doctest']), + ] + # Test cases only listed in the job summary: fixture errors + # (setUpClass, setUpModule) are not annotated + failed_titles = [title for title, _, _ in test_cases] + [ + f'setUpClass ({cases}.SetUpClassTests)', + f'setUpModule ({setup_module})', ] # Test files: heading in the job summary test_files = { @@ -1794,33 +1777,45 @@ def path(name): output, summary = self.run_tests_github( *args, *tests, exitcode=EXITCODE_BAD_TEST) - annotations = list(test_cases) + expected = list(test_cases) if '--fast-ci' in args: - annotations.append((env_changed, path(env_changed), None)) + expected.append((env_changed, f'{env_changed}.py', None)) if crash in tests: - annotations.append((crash, path(crash), None)) - self.assertCountEqual(self.parse_github_annotations(output), - annotations, output) - # Each test case is annotated right after its failure report - for title, _, _ in annotations[:len(test_cases)]: - title = re.escape(title) - self.assertRegex( - output, - rf'(?m)^(?:ERROR|FAIL): {title}\n' - rf'(?:(?!={{70}}$).*\n)*?::error .*title={title}::') - # The env changed annotation comes right after the warnings - if '--fast-ci' in args: - self.assertRegex( - output, - rf'(?m)^Warning -- os\.environ was modified by ' - rf'{env_changed}\n(?:Warning -- .*\n)*' - rf'::error .*title={env_changed}::') + expected.append((crash, f'{crash}.py', None)) + + # "::error file=...,line=...,title=...::message" + annotations = [] + output_lines = output.splitlines() + for i, line in enumerate(output_lines): + if not line.startswith('::error '): + continue + _, props, message = line.split('::', 2) + props = props.removeprefix('error ').split(',') + props = dict(prop.split('=', 1) for prop in props) + title = props['title'] + annotations.append((title, os.path.basename(props['file']), + props.get('line'))) + before = output_lines[:i] + if title == env_changed: + # Right after the env changed warnings + self.assertStartsWith(before[-1], 'Warning -- ') + elif title != crash: + # Right after the failure report of the test, which + # ends with the exception and an empty line + report = next( + text for text in reversed(before) + if text.startswith(('ERROR: ', 'FAIL: '))) + self.assertEqual(report.split(': ', 1)[1], title) + message = message.replace('%0A', '\n').splitlines() + self.assertEqual(before[-len(message) - 1:], + message + ['']) + self.assertCountEqual(annotations, expected, output) # The job summary lists the failed tests in completion order summary_lines = summary.splitlines() self.assertEqual(summary_lines[0], f'## FAILURE: {len(tests)} test files and ' - f'{len(test_cases)} test cases failed') + f'{len(failed_titles)} test cases failed') self.assertCountEqual( [line for line in summary_lines if line.startswith('### ')], @@ -1828,9 +1823,9 @@ def path(name): self.assertIn('- os.environ was modified', summary_lines) self.assertCountEqual( re.findall(r'(.*)', summary), - [title for title, _, _ in test_cases]) + failed_titles) self.assertEqual(summary.count('```pytb\n'), - len(test_cases)) + len(failed_titles)) def test_github_summary_success(self): # No annotation and no job summary when all tests pass From b5247ede1141cf61367a8a98daecaca4a5b26062 Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Thu, 8 Oct 2026 13:31:15 -0700 Subject: [PATCH 26/34] Simplify test --- Lib/test/test_regrtest.py | 207 +++++++++++++------------------------- 1 file changed, 71 insertions(+), 136 deletions(-) diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index fa569207bee3366..2775a80e77ef8b5 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -1642,38 +1642,33 @@ def test_env_changed(self): success=True), stats=2) - def create_marked_test(self, name, code): - # Create a test: return its name and the line number of each of its - # "# marker" comments - code = textwrap.dedent(code) - name = self.create_test(name, code) - markers = {match[1]: str(lineno) - for lineno, text in enumerate(code.splitlines(), 1) - if (match := re.search(r'# (\w+)$', text))} - return name, markers - def run_tests_github(self, *args, exitcode=0): - # Run tests as in GitHub Actions: return the output and the job - # summary (None if regrtest didn't write it) + # Run tests as in GitHub Actions: return the output, the annotations + # as (file, line, title) tuples and the job summary ('' if not written) filename = os.path.join(self.tmptestdir, 'github_step_summary.md') - self.addCleanup(os_helper.unlink, filename) os_helper.unlink(filename) - env = dict(os.environ) + env = dict(os.environ, GITHUB_STEP_SUMMARY=filename) env.pop('SOURCE_DATE_EPOCH', None) - env['GITHUB_STEP_SUMMARY'] = filename output = self.run_tests(*args, env=env, exitcode=exitcode) + annotations = re.findall(r'^::error file=[^,]*?([^/,]+)' + r'(?:,line=(\d+))?,title=([^:]*)::', + output, re.MULTILINE) + # Every annotation has a file and a title + self.assertEqual(len(annotations), + len(re.findall('^::error', output, re.MULTILINE)), + output) try: with open(filename, encoding='utf-8') as fp: - return output, fp.read() + return output, annotations, fp.read() except FileNotFoundError: - return output, None + return output, annotations, '' def test_github_annotations(self): - # Every test failure is annotated once in the GitHub Actions job log, - # located in the test file, at the failing line of the test file if - # there is a traceback. The job summary lists the failures. - cases, lines = self.create_marked_test('github_cases', """ - import doctest, json, sys, unittest + # Every failure is annotated in the GitHub Actions job log, located + # in its test file, at the failing line ("# fail") for test cases. + # The job summary lists the failures. + code = textwrap.dedent(""" + import doctest, sys, unittest def load_tests(loader, tests, pattern): tests.addTests(doctest.DocTestSuite(sys.modules[__name__])) @@ -1681,158 +1676,98 @@ def load_tests(loader, tests, pattern): def doctest_fail(): ''' - >>> 1 + 1 # doctest + >>> 1 + 1 # fail 3 ''' class Tests(unittest.TestCase): def test_error(self): - json.loads("{") # error + {}['key'] # fail def test_fail(self): self.assertEqual(1, 2) # fail def test_subtest(self): - with self.subTest(x=1.5): - self.fail("subtest") # subtest + with self.subTest(x=1): + self.fail() # fail class SetUpClassTests(unittest.TestCase): + # Fixture errors are only listed in the job summary @classmethod def setUpClass(cls): - raise ValueError("setUpClass") + raise ValueError def test_never(self): pass """) - setup_module, _ = self.create_marked_test( - 'github_setup_module', """ - import unittest - - def setUpModule(): - raise ValueError("setUpModule") - - class Tests(unittest.TestCase): - def test_never(self): - pass - """) - env_changed, _ = self.create_marked_test('github_env_changed', """ + cases = self.create_test('github_cases', code) + fail_lines = {str(lineno) + for lineno, line in enumerate(code.splitlines(), 1) + if line.endswith('# fail')} + env_changed = self.create_test('github_env_changed', textwrap.dedent(""" import os, unittest class Tests(unittest.TestCase): def test_env_changed(self): - os.environ["REGRTEST_GITHUB_ENV_CHANGED"] = "1" - """) - crash, _ = self.create_marked_test('github_crash', """ + os.environ['REGRTEST_GITHUB_ENV_CHANGED'] = '1' + """)) + crash = self.create_test('github_crash', textwrap.dedent(""" import os, unittest class Tests(unittest.TestCase): def test_crash(self): os._exit(1) - """) - - # Annotated test cases: (title, file, line) - test_cases = [ - (f'test_error ({cases}.Tests.test_error)', - f'{cases}.py', lines['error']), - (f'test_fail ({cases}.Tests.test_fail)', - f'{cases}.py', lines['fail']), - (f'test_subtest ({cases}.Tests.test_subtest) (x=1.5)', - f'{cases}.py', lines['subtest']), - # Doctest examples are subtests: "[0]" is the example index - (f'doctest_fail ({cases}) [0]', - f'{cases}.py', lines['doctest']), + """)) + case_titles = [ + f'test_error ({cases}.Tests.test_error)', + f'test_fail ({cases}.Tests.test_fail)', + f'test_subtest ({cases}.Tests.test_subtest) (x=1)', + f'doctest_fail ({cases}) [0]', ] - # Test cases only listed in the job summary: fixture errors - # (setUpClass, setUpModule) are not annotated - failed_titles = [title for title, _, _ in test_cases] + [ - f'setUpClass ({cases}.SetUpClassTests)', - f'setUpModule ({setup_module})', - ] - # Test files: heading in the job summary - test_files = { - cases: f'### {cases} failed (2 errors, 3 failures)', - setup_module: f'### {setup_module} failed (1 error)', - env_changed: f'### {env_changed} failed (env changed)', - crash: f'### {crash} worker non-zero exit code', - } # Representative command lines of the CI jobs - command_lines = { - # "make ci", Windows, macOS, installed Python; the JIT jobs use - # the equivalent "-j0 --verbose2 --verbose3". Only failures of - # the re-run are annotated, not the reported first failures. - 'fast-ci': ['--fast-ci', '-j2'], - # Sanitizers and Hypothesis: env changed is not a failure - 'parallel': ['-j2', '-W'], - # iOS and WASI run tests in the main process: a crash kills it - 'single-process': ['--fast-ci', '--single-process'], - # Profile task of PGO and BOLT builds: failures stop the build - 'pgo': ['--pgo'], - } - for name, args in command_lines.items(): - with self.subTest(name): - tests = list(test_files) - if '--single-process' in args or '--pgo' in args: - tests.remove(crash) - output, summary = self.run_tests_github( + for args in ( + # "make ci": only the failures of the re-run are annotated + ['--fast-ci', '-j2'], + # Sanitizers: env changed is not a failure + ['-j2', '-W'], + # iOS and WASI: a crash would kill the main process + ['--fast-ci', '--single-process'], + # PGO profile task + ['--pgo'], + ): + with self.subTest(args=args): + in_process = '--single-process' in args or '--pgo' in args + tests = [cases, env_changed] + ([] if in_process else [crash]) + output, annotations, summary = self.run_tests_github( *args, *tests, exitcode=EXITCODE_BAD_TEST) - expected = list(test_cases) + expected = [(cases, title) for title in case_titles] if '--fast-ci' in args: - expected.append((env_changed, f'{env_changed}.py', None)) + expected.append((env_changed, env_changed)) if crash in tests: - expected.append((crash, f'{crash}.py', None)) - - # "::error file=...,line=...,title=...::message" - annotations = [] - output_lines = output.splitlines() - for i, line in enumerate(output_lines): - if not line.startswith('::error '): - continue - _, props, message = line.split('::', 2) - props = props.removeprefix('error ').split(',') - props = dict(prop.split('=', 1) for prop in props) - title = props['title'] - annotations.append((title, os.path.basename(props['file']), - props.get('line'))) - before = output_lines[:i] - if title == env_changed: - # Right after the env changed warnings - self.assertStartsWith(before[-1], 'Warning -- ') - elif title != crash: - # Right after the failure report of the test, which - # ends with the exception and an empty line - report = next( - text for text in reversed(before) - if text.startswith(('ERROR: ', 'FAIL: '))) - self.assertEqual(report.split(': ', 1)[1], title) - message = message.replace('%0A', '\n').splitlines() - self.assertEqual(before[-len(message) - 1:], - message + ['']) - self.assertCountEqual(annotations, expected, output) - - # The job summary lists the failed tests in completion order - summary_lines = summary.splitlines() - self.assertEqual(summary_lines[0], - f'## FAILURE: {len(tests)} test files and ' - f'{len(failed_titles)} test cases failed') + expected.append((crash, crash)) self.assertCountEqual( - [line for line in summary_lines - if line.startswith('### ')], - [test_files[name] for name in tests]) - self.assertIn('- os.environ was modified', summary_lines) - self.assertCountEqual( - re.findall(r'(.*)', summary), - failed_titles) - self.assertEqual(summary.count('```pytb\n'), - len(failed_titles)) + [(file, title) for file, _, title in annotations], + [(f'{name}.py', title) for name, title in expected], + output) + for file, line, title in annotations: + # Test files without a failed test case have no line + self.assertIn(line, fail_lines if file == f'{cases}.py' + else {''}, title) + + self.assertIn(f'## FAILURE: {len(tests)} test files and ' + f'5 test cases failed', summary) + for name in tests: + self.assertIn(f'### {name} ', summary) + self.assertIn('- os.environ was modified', summary) + self.assertEqual(summary.count(''), 5) def test_github_summary_success(self): # No annotation and no job summary when all tests pass - testname = self.create_test() - output, summary = self.run_tests_github(testname) - self.assertNotIn('::error', output) - self.assertIsNone(summary) + output, annotations, summary = self.run_tests_github(self.create_test()) + self.assertEqual(annotations, []) + self.assertEqual(summary, '') def test_rerun_fail(self): # FAILURE then FAILURE From 9a94bf16399bdc49f341b8c638e2f6e1bd1bfd40 Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Mon, 12 Oct 2026 15:39:03 -0700 Subject: [PATCH 27/34] Simplify GitHub Actions annotations Locate annotations at the last traceback frame in a test file instead of mapping each test to its file. Pass annotations to workers in RunTests, like use_junit, so tests which run regrtest don't inherit GITHUB_STEP_SUMMARY. Shrink the tests. Co-Authored-By: Claude Opus 5.5 --- Lib/test/libregrtest/main.py | 30 ++-- Lib/test/libregrtest/result.py | 3 +- Lib/test/libregrtest/runtests.py | 1 + Lib/test/libregrtest/setup.py | 2 +- Lib/test/libregrtest/single.py | 4 +- Lib/test/libregrtest/testresult.py | 16 +-- Lib/test/libregrtest/utils.py | 50 +++---- .../test_live_collector_ui.py | 6 +- Lib/test/test_regrtest.py | 131 ++++-------------- 9 files changed, 74 insertions(+), 169 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 20b94eff8aa3fdd..b6bf35e057f01dc 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -285,7 +285,8 @@ def _rerun_failed_tests(self, runtests: RunTests) -> RunTests: forever=False, fail_fast=False, match_tests_dict=match_tests_dict, - output_on_failure=False) + output_on_failure=False, + github_annotations=bool(self.github_summary)) self.logger.set_tests(runtests) msg = f"Re-running {len(tests)} failed tests in verbose mode" @@ -485,18 +486,17 @@ def finalize_tests(self, coverage: trace.CoverageResults | None) -> None: self.write_github_summary() def write_github_summary(self) -> None: - filename = os.environ.get("GITHUB_STEP_SUMMARY") # Tests which failed in the last run (the re-run, if any) failed = self.results.rerun_results # Only write a summary on failure: a CI run has many test jobs exitcode = self.results.get_exitcode(self.fail_env_changed, self.fail_rerun) - if not filename or not failed or not exitcode: + if not self.github_summary or not failed or not exitcode: return cases = [(result.errors or []) + (result.failures or []) for result in failed] ncases = sum(map(len, cases)) - with open(filename, "a", encoding="utf-8") as fp: + with open(self.github_summary, "a", encoding="utf-8") as fp: def write(text: str) -> None: # Separate Markdown blocks with an empty line print(text, end="\n\n", file=fp) @@ -555,6 +555,10 @@ def create_run_tests(self, tests: TestTuple) -> RunTests: hunt_refleak=self.hunt_refleak, test_dir=self.test_dir, use_junit=(self.junit_filename is not None), + # Only annotate failures of the last run: not the first run if + # failed tests will be re-run (with --python, they are not) + github_annotations=(bool(self.github_summary) and + not (self.want_rerun and not self.python_cmd)), coverage=self.coverage, memory_limit=self.memory_limit, gc_threshold=self.gc_threshold, @@ -617,16 +621,10 @@ def _run_tests(self, selected: TestTuple, tests: TestList | None) -> int: if use_load_tracker: self.logger.start_load_tracker() try: - with os_helper.EnvironmentVarGuard() as env: - # In GitHub Actions, only annotate failures of the last run: - # not the first run if failed tests will be re-run (with - # --python, they are not: see rerun_failed_tests()) - if self.want_rerun and not self.python_cmd: - env.unset("GITHUB_STEP_SUMMARY") - if self.num_workers: - self._run_tests_mp(runtests, self.num_workers) - else: - self.run_tests_sequentially(runtests) + if self.num_workers: + self._run_tests_mp(runtests, self.num_workers) + else: + self.run_tests_sequentially(runtests) coverage = self.results.get_coverage_results() self.display_result(runtests) @@ -808,6 +806,10 @@ def _init(self): strip_py_suffix(self.cmdline_args) + # Pass GitHub Actions annotations to workers in RunTests, not to + # regrtest run by tests + self.github_summary = os.environ.pop("GITHUB_STEP_SUMMARY", None) + self._tmp_dir = get_temp_dir(self._tmp_dir) @property diff --git a/Lib/test/libregrtest/result.py b/Lib/test/libregrtest/result.py index e1d05777598f2e9..fafa41e21c49f24 100644 --- a/Lib/test/libregrtest/result.py +++ b/Lib/test/libregrtest/result.py @@ -189,7 +189,8 @@ def print_github_annotation(self, runtests: RunTests) -> None: test file. Test case failures are annotated by the test runner, where they are reported (see RegressionTestResult.printErrorList()). """ - if (self.is_failed(runtests.fail_env_changed) + if (runtests.github_annotations + and self.is_failed(runtests.fail_env_changed) and not self.errors and not self.failures): message = "\n".join([str(self), *(self.env_changed_reasons or ())]) # Test file: "test_x" is test_x.py or test_x/__init__.py, diff --git a/Lib/test/libregrtest/runtests.py b/Lib/test/libregrtest/runtests.py index d86b082fb34a604..541a8146001cabf 100644 --- a/Lib/test/libregrtest/runtests.py +++ b/Lib/test/libregrtest/runtests.py @@ -93,6 +93,7 @@ class RunTests: hunt_refleak: HuntRefleak | None test_dir: StrPath | None use_junit: bool + github_annotations: bool coverage: bool memory_limit: str | None gc_threshold: int | None diff --git a/Lib/test/libregrtest/setup.py b/Lib/test/libregrtest/setup.py index 8909225d0265bf5..d62194acd9c29e5 100644 --- a/Lib/test/libregrtest/setup.py +++ b/Lib/test/libregrtest/setup.py @@ -113,8 +113,8 @@ def setup_tests(runtests: RunTests) -> None: set_match_tests(runtests.match_tests) if runtests.use_junit: - from .testresult import RegressionTestResult support.junit_xml_list = [] + from .testresult import RegressionTestResult RegressionTestResult.USE_XML = True else: support.junit_xml_list = None diff --git a/Lib/test/libregrtest/single.py b/Lib/test/libregrtest/single.py index d7a1c6bea202a65..4f755501301c138 100644 --- a/Lib/test/libregrtest/single.py +++ b/Lib/test/libregrtest/single.py @@ -16,7 +16,7 @@ from .runtests import RunTests from .save_env import saved_test_environment from .setup import setup_tests -from .testresult import get_test_runner +from .testresult import RegressionTestResult, get_test_runner from .parallel_case import ParallelTestCase from .utils import ( TestName, @@ -37,6 +37,8 @@ def run_unittest(test_mod, runtests: RunTests): if loader.errors: raise Exception("errors while loading tests") _filter_suite(tests, match_test) + RegressionTestResult.GITHUB_TEST_FILE = ( + test_mod.__file__ if runtests.github_annotations else None) if runtests.parallel_threads: _parallelize_tests(tests, runtests.parallel_threads) return _run_suite(tests) diff --git a/Lib/test/libregrtest/testresult.py b/Lib/test/libregrtest/testresult.py index bdce67226fff3bf..58ceae44a9dc679 100644 --- a/Lib/test/libregrtest/testresult.py +++ b/Lib/test/libregrtest/testresult.py @@ -11,18 +11,10 @@ from test import support from test.libregrtest.utils import print_github_annotation, sanitize_xml -def test_to_filename(test): - test = getattr(test, 'test_case', test) # subTest() - if dt_test := getattr(test, '_dt_test', None): # doctest.DocTestCase - return dt_test.filename - module = type(test).__module__ - # Fixture errors (setUpClass, setUpModule) are not supported - if module.startswith('unittest.'): - return None - return getattr(sys.modules.get(module), '__file__', None) - class RegressionTestResult(unittest.TextTestResult): USE_XML = False + # Test file of the GitHub Actions annotations: don't annotate if None + GITHUB_TEST_FILE: str | None = None def __init__(self, stream, descriptions, verbosity): super().__init__(stream=stream, descriptions=descriptions, @@ -145,8 +137,8 @@ def printErrorList(self, flavour, errors): super().printErrorList(flavour, [(test, err)]) # Annotate just after the failure report, so that the annotation # links to the report in the job log - if filename := test_to_filename(test): - print_github_annotation(str(test), err, filename, + if self.GITHUB_TEST_FILE: + print_github_annotation(str(test), err, self.GITHUB_TEST_FILE, file=self.stream) def get_xml_element(self): diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index 50de4ae9ab6d674..5a6a6f57a22747b 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -142,56 +142,40 @@ def print_warning(msg: str) -> None: orig_unraisablehook: Callable[..., None] | None = None -# Stdlib directory: Lib/ in a source checkout or an installed lib/python3.X/ -STDLIB_DIR = pathlib.Path(os.__file__).parent -# Stdlib directory relative to the repository -REPOSITORY_LIB_DIR = pathlib.Path("Lib") - - -def repository_path(filename: str) -> str: - """Path of a stdlib file relative to the repository. - - Map STDLIB_DIR to Lib/. Return other paths unchanged. - """ - path = pathlib.Path(filename) - if not path.is_relative_to(STDLIB_DIR): - return filename - return (REPOSITORY_LIB_DIR / path.relative_to(STDLIB_DIR)).as_posix() - - # Escape the data and the properties of GitHub Actions workflow commands GITHUB_ESCAPE_DATA = str.maketrans({"%": "%25", "\r": "%0D", "\n": "%0A"}) GITHUB_ESCAPE_PROPERTY = GITHUB_ESCAPE_DATA | str.maketrans({":": "%3A", ",": "%2C"}) -def print_github_annotation(title: str, message: str, filename: str | None, +def print_github_annotation(title: str, message: str, filename: str, file=None) -> None: - """Print a GitHub Actions error annotation, if run in GitHub Actions. + """Print a GitHub Actions error annotation. message is a traceback or a failure description. Only keep the exception which ends the traceback: the job log has the full traceback. Locate the - annotation in filename, the test file, at the last frame of the traceback - in this file, if any. + annotation at the last traceback frame in a test file, if any, or else + in filename, the test file. """ - if not os.environ.get("GITHUB_STEP_SUMMARY"): - return - if not filename: - raise ValueError(f"missing test file of annotation {title!r}") message = decolor(message) - props = {"file": repository_path(filename)} - # Line of the last traceback frame (' File "filename", line 123') or - # failed doctest example (same, but not indented) in filename - if lines := re.findall(rf'^ *File "{re.escape(filename)}", line (\d+)', - message, re.MULTILINE): - props["line"] = lines[-1] - props["title"] = title + line = None + # Last traceback frame (' File "filename", line 123') or failed doctest + # example (same, but not indented) in a test file + if frames := re.findall(r'^ *File "(.*[/\\]test_[^/\\]*\.py)", line (\d+)', + message, re.MULTILINE): + filename, line = frames[-1] + # In the job, the stdlib can be outside of the repository checkout + path = pathlib.Path(filename) + stdlib_dir = pathlib.Path(os.__file__).parent + if path.is_relative_to(stdlib_dir): + filename = ("Lib" / path.relative_to(stdlib_dir)).as_posix() + props = {"file": filename, "line": line, "title": title} # Only keep the exception: strip the traceback frames, each one a # ' File' line followed by its indented source lines message = re.split(r'^ File .*\n(?: .*\n)*', message, flags=re.MULTILINE)[-1].strip() props_text = ",".join(f"{key}={value.translate(GITHUB_ESCAPE_PROPERTY)}" - for key, value in props.items()) + for key, value in props.items() if value) print(f"::error {props_text}::{message.translate(GITHUB_ESCAPE_DATA)}", file=file, flush=True) diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_live_collector_ui.py b/Lib/test/test_profiling/test_sampling_profiler/test_live_collector_ui.py index 57598715f8b2880..ae4ee969f34d8b3 100644 --- a/Lib/test/test_profiling/test_sampling_profiler/test_live_collector_ui.py +++ b/Lib/test/test_profiling/test_sampling_profiler/test_live_collector_ui.py @@ -11,8 +11,7 @@ import time import unittest from unittest import mock -from test.support import ( - os_helper, requires, requires_remote_subprocess_debugging) +from test.support import requires, requires_remote_subprocess_debugging from test.support.import_helper import import_module # Only run these tests if curses is available @@ -857,9 +856,6 @@ def mock_init_curses_side_effect(self, n_times, mock_self, stdscr): def test_run_failed_module_live(self): """Test that running a existing module that fails exits with clean error.""" - # Don't write to the GitHub Actions job summary of the test suite - env = self.enterContext(os_helper.EnvironmentVarGuard()) - env.unset('GITHUB_STEP_SUMMARY') args = [ "profiling.sampling.cli", "run", "--live", "-m", "test", diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 2775a80e77ef8b5..4fe80533ce9caf6 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -624,9 +624,6 @@ class BaseTestCase(unittest.TestCase): def setUp(self): self.testdir = os.path.realpath(os.path.dirname(__file__)) - # Don't annotate the GitHub Actions job running the test suite - env = self.enterContext(os_helper.EnvironmentVarGuard()) - env.unset('GITHUB_STEP_SUMMARY') self.tmptestdir = tempfile.mkdtemp() self.addCleanup(os_helper.rmtree, self.tmptestdir) @@ -1642,31 +1639,9 @@ def test_env_changed(self): success=True), stats=2) - def run_tests_github(self, *args, exitcode=0): - # Run tests as in GitHub Actions: return the output, the annotations - # as (file, line, title) tuples and the job summary ('' if not written) - filename = os.path.join(self.tmptestdir, 'github_step_summary.md') - os_helper.unlink(filename) - env = dict(os.environ, GITHUB_STEP_SUMMARY=filename) - env.pop('SOURCE_DATE_EPOCH', None) - output = self.run_tests(*args, env=env, exitcode=exitcode) - annotations = re.findall(r'^::error file=[^,]*?([^/,]+)' - r'(?:,line=(\d+))?,title=([^:]*)::', - output, re.MULTILINE) - # Every annotation has a file and a title - self.assertEqual(len(annotations), - len(re.findall('^::error', output, re.MULTILINE)), - output) - try: - with open(filename, encoding='utf-8') as fp: - return output, annotations, fp.read() - except FileNotFoundError: - return output, annotations, '' - - def test_github_annotations(self): - # Every failure is annotated in the GitHub Actions job log, located - # in its test file, at the failing line ("# fail") for test cases. - # The job summary lists the failures. + def test_github_actions(self): + # Failures are annotated in the GitHub Actions job log at the failing + # line ("# fail") of the test file, and listed in the job summary code = textwrap.dedent(""" import doctest, sys, unittest @@ -1681,36 +1656,11 @@ def doctest_fail(): ''' class Tests(unittest.TestCase): - def test_error(self): - {}['key'] # fail - - def test_fail(self): - self.assertEqual(1, 2) # fail - def test_subtest(self): with self.subTest(x=1): self.fail() # fail - - class SetUpClassTests(unittest.TestCase): - # Fixture errors are only listed in the job summary - @classmethod - def setUpClass(cls): - raise ValueError - - def test_never(self): - pass """) cases = self.create_test('github_cases', code) - fail_lines = {str(lineno) - for lineno, line in enumerate(code.splitlines(), 1) - if line.endswith('# fail')} - env_changed = self.create_test('github_env_changed', textwrap.dedent(""" - import os, unittest - - class Tests(unittest.TestCase): - def test_env_changed(self): - os.environ['REGRTEST_GITHUB_ENV_CHANGED'] = '1' - """)) crash = self.create_test('github_crash', textwrap.dedent(""" import os, unittest @@ -1718,56 +1668,33 @@ class Tests(unittest.TestCase): def test_crash(self): os._exit(1) """)) - case_titles = [ - f'test_error ({cases}.Tests.test_error)', - f'test_fail ({cases}.Tests.test_fail)', - f'test_subtest ({cases}.Tests.test_subtest) (x=1)', - f'doctest_fail ({cases}) [0]', - ] - - # Representative command lines of the CI jobs - for args in ( - # "make ci": only the failures of the re-run are annotated - ['--fast-ci', '-j2'], - # Sanitizers: env changed is not a failure - ['-j2', '-W'], - # iOS and WASI: a crash would kill the main process - ['--fast-ci', '--single-process'], - # PGO profile task - ['--pgo'], - ): - with self.subTest(args=args): - in_process = '--single-process' in args or '--pgo' in args - tests = [cases, env_changed] + ([] if in_process else [crash]) - output, annotations, summary = self.run_tests_github( - *args, *tests, exitcode=EXITCODE_BAD_TEST) - - expected = [(cases, title) for title in case_titles] - if '--fast-ci' in args: - expected.append((env_changed, env_changed)) - if crash in tests: - expected.append((crash, crash)) - self.assertCountEqual( - [(file, title) for file, _, title in annotations], - [(f'{name}.py', title) for name, title in expected], - output) - for file, line, title in annotations: - # Test files without a failed test case have no line - self.assertIn(line, fail_lines if file == f'{cases}.py' - else {''}, title) - - self.assertIn(f'## FAILURE: {len(tests)} test files and ' - f'5 test cases failed', summary) - for name in tests: - self.assertIn(f'### {name} ', summary) - self.assertIn('- os.environ was modified', summary) - self.assertEqual(summary.count(''), 5) - - def test_github_summary_success(self): + filename = os.path.join(self.tmptestdir, 'github_step_summary.md') + env = dict(os.environ, GITHUB_STEP_SUMMARY=filename) + env.pop('SOURCE_DATE_EPOCH', None) + output = self.run_tests('--fast-ci', '-j2', cases, crash, + env=env, exitcode=EXITCODE_BAD_TEST) + fail_lines = [str(lineno) + for lineno, line in enumerate(code.splitlines(), 1) + if line.endswith('# fail')] + # Only failures of the re-run are annotated + self.assertCountEqual( + re.findall(r'^::error file=[^,]*?([^/,]+?)(?:,line=(\d+))?,' + r'title=([^:]*)::', output, re.MULTILINE), + [(f'{cases}.py', fail_lines[1], + f'test_subtest ({cases}.Tests.test_subtest) (x=1)'), + (f'{cases}.py', fail_lines[0], f'doctest_fail ({cases}) [0]'), + (f'{crash}.py', '', crash)], + output) + with open(filename, encoding='utf-8') as fp: + summary = fp.read() + self.assertIn('## FAILURE: 2 test files and 2 test cases failed', + summary) + # No annotation and no job summary when all tests pass - output, annotations, summary = self.run_tests_github(self.create_test()) - self.assertEqual(annotations, []) - self.assertEqual(summary, '') + os_helper.unlink(filename) + output = self.run_tests(self.create_test(), env=env) + self.assertNotIn('::error', output) + self.assertFalse(os.path.exists(filename)) def test_rerun_fail(self): # FAILURE then FAILURE From 70cae9942c7c8a47653aed00b390ef8e8acd519f Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Mon, 12 Oct 2026 15:43:02 -0700 Subject: [PATCH 28/34] Set GITHUB_ANNOTATIONS in setup_tests(), like USE_XML Fall back to the first traceback frame (the test) instead of passing the test module file. Co-Authored-By: Claude Opus 5.5 --- Lib/test/libregrtest/setup.py | 3 ++- Lib/test/libregrtest/single.py | 4 +--- Lib/test/libregrtest/testresult.py | 8 +++----- Lib/test/libregrtest/utils.py | 22 +++++++++++++--------- 4 files changed, 19 insertions(+), 18 deletions(-) diff --git a/Lib/test/libregrtest/setup.py b/Lib/test/libregrtest/setup.py index d62194acd9c29e5..74e7132c3d52aa6 100644 --- a/Lib/test/libregrtest/setup.py +++ b/Lib/test/libregrtest/setup.py @@ -112,9 +112,10 @@ def setup_tests(runtests: RunTests) -> None: set_match_tests(runtests.match_tests) + from .testresult import RegressionTestResult + RegressionTestResult.GITHUB_ANNOTATIONS = runtests.github_annotations if runtests.use_junit: support.junit_xml_list = [] - from .testresult import RegressionTestResult RegressionTestResult.USE_XML = True else: support.junit_xml_list = None diff --git a/Lib/test/libregrtest/single.py b/Lib/test/libregrtest/single.py index 4f755501301c138..d7a1c6bea202a65 100644 --- a/Lib/test/libregrtest/single.py +++ b/Lib/test/libregrtest/single.py @@ -16,7 +16,7 @@ from .runtests import RunTests from .save_env import saved_test_environment from .setup import setup_tests -from .testresult import RegressionTestResult, get_test_runner +from .testresult import get_test_runner from .parallel_case import ParallelTestCase from .utils import ( TestName, @@ -37,8 +37,6 @@ def run_unittest(test_mod, runtests: RunTests): if loader.errors: raise Exception("errors while loading tests") _filter_suite(tests, match_test) - RegressionTestResult.GITHUB_TEST_FILE = ( - test_mod.__file__ if runtests.github_annotations else None) if runtests.parallel_threads: _parallelize_tests(tests, runtests.parallel_threads) return _run_suite(tests) diff --git a/Lib/test/libregrtest/testresult.py b/Lib/test/libregrtest/testresult.py index 58ceae44a9dc679..9efa11cd0cf5cd2 100644 --- a/Lib/test/libregrtest/testresult.py +++ b/Lib/test/libregrtest/testresult.py @@ -13,8 +13,7 @@ class RegressionTestResult(unittest.TextTestResult): USE_XML = False - # Test file of the GitHub Actions annotations: don't annotate if None - GITHUB_TEST_FILE: str | None = None + GITHUB_ANNOTATIONS = False def __init__(self, stream, descriptions, verbosity): super().__init__(stream=stream, descriptions=descriptions, @@ -137,9 +136,8 @@ def printErrorList(self, flavour, errors): super().printErrorList(flavour, [(test, err)]) # Annotate just after the failure report, so that the annotation # links to the report in the job log - if self.GITHUB_TEST_FILE: - print_github_annotation(str(test), err, self.GITHUB_TEST_FILE, - file=self.stream) + if self.GITHUB_ANNOTATIONS: + print_github_annotation(str(test), err, file=self.stream) def get_xml_element(self): if not self.USE_XML: diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index 5a6a6f57a22747b..89e89c1e09a20dd 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -148,22 +148,26 @@ def print_warning(msg: str) -> None: ",": "%2C"}) -def print_github_annotation(title: str, message: str, filename: str, - file=None) -> None: +def print_github_annotation(title: str, message: str, + filename: str | None = None, file=None) -> None: """Print a GitHub Actions error annotation. message is a traceback or a failure description. Only keep the exception which ends the traceback: the job log has the full traceback. Locate the - annotation at the last traceback frame in a test file, if any, or else - in filename, the test file. + annotation at the last traceback frame in a test file, or else at the + first frame (the test), or else in filename (the test file). """ message = decolor(message) line = None - # Last traceback frame (' File "filename", line 123') or failed doctest - # example (same, but not indented) in a test file - if frames := re.findall(r'^ *File "(.*[/\\]test_[^/\\]*\.py)", line (\d+)', - message, re.MULTILINE): - filename, line = frames[-1] + # Traceback frames (' File "filename", line 123') and failed doctest + # examples (same, but not indented) + if frames := re.findall(r'^ *File "(.*)", line (\d+)', message, + re.MULTILINE): + test_frames = [frame for frame in frames + if os.path.basename(frame[0]).startswith('test_')] + filename, line = test_frames[-1] if test_frames else frames[0] + if not filename: + raise ValueError(f"missing test file of annotation {title!r}") # In the job, the stdlib can be outside of the repository checkout path = pathlib.Path(filename) stdlib_dir = pathlib.Path(os.__file__).parent From 8f231a6c092619f4930cd16143eb05313c21d4f0 Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Mon, 12 Oct 2026 15:58:11 -0700 Subject: [PATCH 29/34] Simplify findall --- Lib/test/test_regrtest.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 4fe80533ce9caf6..36e81fa5c656c57 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -1673,17 +1673,15 @@ def test_crash(self): env.pop('SOURCE_DATE_EPOCH', None) output = self.run_tests('--fast-ci', '-j2', cases, crash, env=env, exitcode=EXITCODE_BAD_TEST) - fail_lines = [str(lineno) + # One annotation per failure of the re-run, not of the first run: + # one per "# fail" line, and the crash (without a line) + fail_lines = [(f'{cases}.py', str(lineno)) for lineno, line in enumerate(code.splitlines(), 1) if line.endswith('# fail')] - # Only failures of the re-run are annotated self.assertCountEqual( - re.findall(r'^::error file=[^,]*?([^/,]+?)(?:,line=(\d+))?,' - r'title=([^:]*)::', output, re.MULTILINE), - [(f'{cases}.py', fail_lines[1], - f'test_subtest ({cases}.Tests.test_subtest) (x=1)'), - (f'{cases}.py', fail_lines[0], f'doctest_fail ({cases}) [0]'), - (f'{crash}.py', '', crash)], + re.findall(r'^::error file=[^,]*?([^/,]+?)(?:,line=(\d+))?,title=', + output, re.MULTILINE), + fail_lines + [(f'{crash}.py', '')], output) with open(filename, encoding='utf-8') as fp: summary = fp.read() From f2ba9db49524b950ae264681cc00cdd330c91033 Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Mon, 12 Oct 2026 18:00:19 -0700 Subject: [PATCH 30/34] Review fixes. Directly check error annotation list --- Lib/test/libregrtest/setup.py | 2 +- Lib/test/test_regrtest.py | 55 ++++++++++++++++++++++++----------- 2 files changed, 39 insertions(+), 18 deletions(-) diff --git a/Lib/test/libregrtest/setup.py b/Lib/test/libregrtest/setup.py index 74e7132c3d52aa6..8216541e454f82d 100644 --- a/Lib/test/libregrtest/setup.py +++ b/Lib/test/libregrtest/setup.py @@ -12,6 +12,7 @@ from .filter import set_match_tests from .runtests import RunTests +lazy from .testresult import RegressionTestResult from .utils import ( setup_unraisable_hook, setup_threading_excepthook, adjust_rlimit_nofile) @@ -112,7 +113,6 @@ def setup_tests(runtests: RunTests) -> None: set_match_tests(runtests.match_tests) - from .testresult import RegressionTestResult RegressionTestResult.GITHUB_ANNOTATIONS = runtests.github_annotations if runtests.use_junit: support.junit_xml_list = [] diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 36e81fa5c656c57..f445eb2b6a0760c 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -1641,8 +1641,8 @@ def test_env_changed(self): def test_github_actions(self): # Failures are annotated in the GitHub Actions job log at the failing - # line ("# fail") of the test file, and listed in the job summary - code = textwrap.dedent(""" + # line of the test file, and listed in the job summary + cases = self.create_test('github_cases', textwrap.dedent(""" import doctest, sys, unittest def load_tests(loader, tests, pattern): @@ -1651,16 +1651,15 @@ def load_tests(loader, tests, pattern): def doctest_fail(): ''' - >>> 1 + 1 # fail + >>> 1 + 1 3 ''' class Tests(unittest.TestCase): def test_subtest(self): with self.subTest(x=1): - self.fail() # fail - """) - cases = self.create_test('github_cases', code) + self.fail() + """)) crash = self.create_test('github_crash', textwrap.dedent(""" import os, unittest @@ -1671,18 +1670,40 @@ def test_crash(self): filename = os.path.join(self.tmptestdir, 'github_step_summary.md') env = dict(os.environ, GITHUB_STEP_SUMMARY=filename) env.pop('SOURCE_DATE_EPOCH', None) - output = self.run_tests('--fast-ci', '-j2', cases, crash, + # A single worker without --randomize runs the tests in order + output = self.run_tests('-j1', '--rerun', cases, crash, env=env, exitcode=EXITCODE_BAD_TEST) - # One annotation per failure of the re-run, not of the first run: - # one per "# fail" line, and the crash (without a line) - fail_lines = [(f'{cases}.py', str(lineno)) - for lineno, line in enumerate(code.splitlines(), 1) - if line.endswith('# fail')] - self.assertCountEqual( - re.findall(r'^::error file=[^,]*?([^/,]+?)(?:,line=(\d+))?,title=', - output, re.MULTILINE), - fail_lines + [(f'{crash}.py', '')], - output) + + def unescape(text): + for char in '\r\n:,%': + text = text.replace(f'%{ord(char):02X}', char) + return text + + annotations = [] + for props, message in re.findall(r'^::error (.*?)::(.*)$', output, + re.MULTILINE): + props = dict(prop.split('=', 1) for prop in props.split(',')) + annotations.append(({key: unescape(value) + for key, value in props.items()}, + unescape(message))) + # One annotation per failure of the re-run, not of the first run + cases_file = os.path.join(self.tmptestdir, f'{cases}.py') + crash_file = os.path.join(self.tmptestdir, f'{crash}.py') + self.assertEqual(annotations, [ + ({'file': cases_file, 'line': '17', + 'title': f'test_subtest ({cases}.Tests.test_subtest) (x=1)'}, + 'AssertionError: None'), + ({'file': cases_file, 'line': '10', + 'title': f'doctest_fail ({cases}) [0]'}, + 'AssertionError: Failed example:\n' + ' 1 + 1\n' + 'Expected:\n' + ' 3\n' + 'Got:\n' + ' 2'), + ({'file': crash_file, 'title': crash}, + f'{crash} worker non-zero exit code'), + ], output) with open(filename, encoding='utf-8') as fp: summary = fp.read() self.assertIn('## FAILURE: 2 test files and 2 test cases failed', From 42d7918b3ef8a1b18d92e8cb9efb10bafd3915cd Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Mon, 12 Oct 2026 20:29:27 -0700 Subject: [PATCH 31/34] Personal review adjustments, move more into TestRestul --- Lib/test/libregrtest/main.py | 31 ++++------------ Lib/test/libregrtest/result.py | 59 +++++++++++++++++++++--------- Lib/test/libregrtest/setup.py | 3 +- Lib/test/libregrtest/testresult.py | 11 +----- Lib/test/libregrtest/utils.py | 11 +++--- 5 files changed, 56 insertions(+), 59 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index b6bf35e057f01dc..e039a209db6a35a 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -1,4 +1,3 @@ -import html import os import random import re @@ -493,30 +492,14 @@ def write_github_summary(self) -> None: self.fail_rerun) if not self.github_summary or not failed or not exitcode: return - cases = [(result.errors or []) + (result.failures or []) - for result in failed] - ncases = sum(map(len, cases)) + ncases = sum(len(result.test_cases) for result in failed) with open(self.github_summary, "a", encoding="utf-8") as fp: - def write(text: str) -> None: - # Separate Markdown blocks with an empty line - print(text, end="\n\n", file=fp) - - write(f"## {decolor(self.get_state())}: " - f"{count(len(failed), 'test file')} and " - f"{count(ncases, 'test case')} failed") - for result, result_cases in zip(failed, cases): - write(f"### {decolor(str(result))}") - if result.env_changed_reasons: - write("\n".join(f"- {reason}" - for reason in result.env_changed_reasons)) - for name, traceback in result_cases: - # Expand the tracebacks if they fit on a screen: up to 5 - # test cases with a traceback of less than 30 lines - is_open = ncases <= 5 and traceback.count("\n") < 30 - write(f"" - f"{html.escape(name)}") - write(f"```pytb\n{decolor(traceback).rstrip()}\n```") - write("") + fp.write(f"## {decolor(self.get_state())}: " + f"{count(len(failed), 'test file')} and " + f"{count(ncases, 'test case')} failed\n\n") + for result in failed: + # Expand the tracebacks of up to 5 test cases + fp.write(result.format_github_summary(expand=ncases <= 5)) def display_summary(self) -> None: if self.first_runtests is None: diff --git a/Lib/test/libregrtest/result.py b/Lib/test/libregrtest/result.py index fafa41e21c49f24..142740b34a44c62 100644 --- a/Lib/test/libregrtest/result.py +++ b/Lib/test/libregrtest/result.py @@ -1,7 +1,8 @@ import dataclasses +import html import json -import os -from _colorize import get_colors # type: ignore[import-not-found] +import pathlib +from _colorize import decolor, get_colors # type: ignore[import-not-found] from typing import Any from .findtests import findtestdir @@ -182,25 +183,49 @@ def __str__(self) -> str: def has_meaningful_duration(self): return State.has_meaningful_duration(self.state) + @property + def test_cases(self) -> list[tuple[str, str]]: + """Failed test cases: list of (name, traceback).""" + return (self.errors or []) + (self.failures or []) + + def format_github_summary(self, expand: bool) -> str: + """Format the Markdown section of the GitHub Actions job summary.""" + blocks = [f"### {decolor(str(self))}"] + if self.env_changed_reasons: + blocks.append("\n".join(f"- {reason}" + for reason in self.env_changed_reasons)) + for name, traceback in self.test_cases: + # Expand the tracebacks if they fit on a screen: a traceback of + # less than 30 lines + is_open = expand and traceback.count("\n") < 30 + blocks += [f"" + f"{html.escape(name)}", + f"```pytb\n{decolor(traceback).rstrip()}\n```", + ""] + # Separate Markdown blocks with an empty line + return "".join(f"{block}\n\n" for block in blocks) + def print_github_annotation(self, runtests: RunTests) -> None: - """Annotate a failed test without test case failures. + """Annotate each failed test case, or else the failed test file. - For example: crash, timeout, env changed. Locate the annotation in the - test file. Test case failures are annotated by the test runner, where - they are reported (see RegressionTestResult.printErrorList()). + For example, the test file for a crash, timeout or env changed. """ - if (runtests.github_annotations - and self.is_failed(runtests.fail_env_changed) - and not self.errors and not self.failures): + if not (runtests.github_annotations + and self.is_failed(runtests.fail_env_changed)): + return + # Test file: "test_x" is test_x.py or test_x/__init__.py, + # "test.test_x.test_y" is test_x/test_y.py + name = self.test_name.removeprefix("test.") + path = pathlib.Path(findtestdir(runtests.test_dir), *name.split(".")) + filename = path.with_suffix(".py") + if not filename.exists(): + filename = path / "__init__.py" + test_cases = self.test_cases + if not test_cases: message = "\n".join([str(self), *(self.env_changed_reasons or ())]) - # Test file: "test_x" is test_x.py or test_x/__init__.py, - # "test.test_x.test_y" is test_x/test_y.py - name = self.test_name.removeprefix("test.") - path = os.path.join(findtestdir(runtests.test_dir), *name.split(".")) - filename = path + ".py" - if not os.path.exists(filename): - filename = os.path.join(path, "__init__.py") - print_github_annotation(self.test_name, message, filename) + test_cases = [(self.test_name, message)] + for title, message in test_cases: + print_github_annotation(title, message, str(filename)) def set_env_changed(self, *reasons): if self.state is None or self.state == State.PASSED: diff --git a/Lib/test/libregrtest/setup.py b/Lib/test/libregrtest/setup.py index 8216541e454f82d..d62194acd9c29e5 100644 --- a/Lib/test/libregrtest/setup.py +++ b/Lib/test/libregrtest/setup.py @@ -12,7 +12,6 @@ from .filter import set_match_tests from .runtests import RunTests -lazy from .testresult import RegressionTestResult from .utils import ( setup_unraisable_hook, setup_threading_excepthook, adjust_rlimit_nofile) @@ -113,9 +112,9 @@ def setup_tests(runtests: RunTests) -> None: set_match_tests(runtests.match_tests) - RegressionTestResult.GITHUB_ANNOTATIONS = runtests.github_annotations if runtests.use_junit: support.junit_xml_list = [] + from .testresult import RegressionTestResult RegressionTestResult.USE_XML = True else: support.junit_xml_list = None diff --git a/Lib/test/libregrtest/testresult.py b/Lib/test/libregrtest/testresult.py index 9efa11cd0cf5cd2..605f1f4e6a89fb6 100644 --- a/Lib/test/libregrtest/testresult.py +++ b/Lib/test/libregrtest/testresult.py @@ -9,11 +9,10 @@ import traceback import unittest from test import support -from test.libregrtest.utils import print_github_annotation, sanitize_xml +from test.libregrtest.utils import sanitize_xml class RegressionTestResult(unittest.TextTestResult): USE_XML = False - GITHUB_ANNOTATIONS = False def __init__(self, stream, descriptions, verbosity): super().__init__(stream=stream, descriptions=descriptions, @@ -131,14 +130,6 @@ def addUnexpectedSuccess(self, test): self._add_result(test, outcome='UNEXPECTED_SUCCESS') super().addUnexpectedSuccess(test) - def printErrorList(self, flavour, errors): - for test, err in errors: - super().printErrorList(flavour, [(test, err)]) - # Annotate just after the failure report, so that the annotation - # links to the report in the job log - if self.GITHUB_ANNOTATIONS: - print_github_annotation(str(test), err, file=self.stream) - def get_xml_element(self): if not self.USE_XML: raise ValueError("USE_XML is false") diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index 89e89c1e09a20dd..7eeb011ce80b238 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -148,8 +148,7 @@ def print_warning(msg: str) -> None: ",": "%2C"}) -def print_github_annotation(title: str, message: str, - filename: str | None = None, file=None) -> None: +def print_github_annotation(title: str, message: str, filename: str) -> None: """Print a GitHub Actions error annotation. message is a traceback or a failure description. Only keep the exception @@ -157,6 +156,8 @@ def print_github_annotation(title: str, message: str, annotation at the last traceback frame in a test file, or else at the first frame (the test), or else in filename (the test file). """ + if not filename: + raise ValueError(f"missing test file of annotation {title!r}") message = decolor(message) line = None # Traceback frames (' File "filename", line 123') and failed doctest @@ -164,10 +165,8 @@ def print_github_annotation(title: str, message: str, if frames := re.findall(r'^ *File "(.*)", line (\d+)', message, re.MULTILINE): test_frames = [frame for frame in frames - if os.path.basename(frame[0]).startswith('test_')] + if pathlib.Path(frame[0]).name.startswith('test_')] filename, line = test_frames[-1] if test_frames else frames[0] - if not filename: - raise ValueError(f"missing test file of annotation {title!r}") # In the job, the stdlib can be outside of the repository checkout path = pathlib.Path(filename) stdlib_dir = pathlib.Path(os.__file__).parent @@ -181,7 +180,7 @@ def print_github_annotation(title: str, message: str, props_text = ",".join(f"{key}={value.translate(GITHUB_ESCAPE_PROPERTY)}" for key, value in props.items() if value) print(f"::error {props_text}::{message.translate(GITHUB_ESCAPE_DATA)}", - file=file, flush=True) + flush=True) def regrtest_unraisable_hook(unraisable) -> None: From e4d41bfbe5fb30fd079752964b01a55f8c0ef9ff Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Mon, 12 Oct 2026 20:53:52 -0700 Subject: [PATCH 32/34] Self review fixes --- Lib/test/libregrtest/main.py | 8 +++----- Lib/test/libregrtest/result.py | 26 +++++++++++--------------- Lib/test/libregrtest/run_workers.py | 1 - Lib/test/libregrtest/utils.py | 17 ++++++----------- 4 files changed, 20 insertions(+), 32 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index e039a209db6a35a..cb7382604f80bd2 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -492,7 +492,7 @@ def write_github_summary(self) -> None: self.fail_rerun) if not self.github_summary or not failed or not exitcode: return - ncases = sum(len(result.test_cases) for result in failed) + ncases = sum(len(result.failed_test_cases) for result in failed) with open(self.github_summary, "a", encoding="utf-8") as fp: fp.write(f"## {decolor(self.get_state())}: " f"{count(len(failed), 'test file')} and " @@ -538,8 +538,8 @@ def create_run_tests(self, tests: TestTuple) -> RunTests: hunt_refleak=self.hunt_refleak, test_dir=self.test_dir, use_junit=(self.junit_filename is not None), - # Only annotate failures of the last run: not the first run if - # failed tests will be re-run (with --python, they are not) + # Only annotate the failure that results in job failure. + # Ex: When reruning failed tests, only annotate the the final run. github_annotations=(bool(self.github_summary) and not (self.want_rerun and not self.python_cmd)), coverage=self.coverage, @@ -789,8 +789,6 @@ def _init(self): strip_py_suffix(self.cmdline_args) - # Pass GitHub Actions annotations to workers in RunTests, not to - # regrtest run by tests self.github_summary = os.environ.pop("GITHUB_STEP_SUMMARY", None) self._tmp_dir = get_temp_dir(self._tmp_dir) diff --git a/Lib/test/libregrtest/result.py b/Lib/test/libregrtest/result.py index 142740b34a44c62..69071562b14e193 100644 --- a/Lib/test/libregrtest/result.py +++ b/Lib/test/libregrtest/result.py @@ -184,8 +184,7 @@ def has_meaningful_duration(self): return State.has_meaningful_duration(self.state) @property - def test_cases(self) -> list[tuple[str, str]]: - """Failed test cases: list of (name, traceback).""" + def failed_test_cases(self) -> list[tuple[str, str]]: return (self.errors or []) + (self.failures or []) def format_github_summary(self, expand: bool) -> str: @@ -194,37 +193,34 @@ def format_github_summary(self, expand: bool) -> str: if self.env_changed_reasons: blocks.append("\n".join(f"- {reason}" for reason in self.env_changed_reasons)) - for name, traceback in self.test_cases: - # Expand the tracebacks if they fit on a screen: a traceback of - # less than 30 lines + for name, traceback in self.failed_test_cases: + # Only default open short tracebacks. is_open = expand and traceback.count("\n") < 30 blocks += [f"" f"{html.escape(name)}", f"```pytb\n{decolor(traceback).rstrip()}\n```", ""] + # Separate Markdown blocks with an empty line return "".join(f"{block}\n\n" for block in blocks) def print_github_annotation(self, runtests: RunTests) -> None: - """Annotate each failed test case, or else the failed test file. - - For example, the test file for a crash, timeout or env changed. - """ + """Print github error annotation line for each failure.""" if not (runtests.github_annotations and self.is_failed(runtests.fail_env_changed)): return - # Test file: "test_x" is test_x.py or test_x/__init__.py, - # "test.test_x.test_y" is test_x/test_y.py + # Convert python import test name to relative file path. + # Ex: "test.test_x.test_y" is test_x/test_y.py name = self.test_name.removeprefix("test.") path = pathlib.Path(findtestdir(runtests.test_dir), *name.split(".")) filename = path.with_suffix(".py") if not filename.exists(): filename = path / "__init__.py" - test_cases = self.test_cases - if not test_cases: + annotations = self.failed_test_cases + if not annotations: message = "\n".join([str(self), *(self.env_changed_reasons or ())]) - test_cases = [(self.test_name, message)] - for title, message in test_cases: + annotations = [(self.test_name, message)] + for title, message in annotations: print_github_annotation(title, message, str(filename)) def set_env_changed(self, *reasons): diff --git a/Lib/test/libregrtest/run_workers.py b/Lib/test/libregrtest/run_workers.py index 995543d676876cc..3327269ae1c622c 100644 --- a/Lib/test/libregrtest/run_workers.py +++ b/Lib/test/libregrtest/run_workers.py @@ -624,7 +624,6 @@ def _process_result(self, item: QueueOutput) -> TestResult: stdout = mp_result.worker_stdout if stdout: print(stdout, flush=True) - # Annotate after the output: env changed warnings, crash traceback result.print_github_annotation(self.runtests) return result diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index 7eeb011ce80b238..2ac30fb4cba5fc8 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -149,25 +149,20 @@ def print_warning(msg: str) -> None: def print_github_annotation(title: str, message: str, filename: str) -> None: - """Print a GitHub Actions error annotation. - - message is a traceback or a failure description. Only keep the exception - which ends the traceback: the job log has the full traceback. Locate the - annotation at the last traceback frame in a test file, or else at the - first frame (the test), or else in filename (the test file). - """ + """Print a GitHub Actions error annotation for the failure.""" if not filename: - raise ValueError(f"missing test file of annotation {title!r}") + raise ValueError(f"missing filename to annotate {title!r}") message = decolor(message) + + # Try to get a line number from backtrace. line = None - # Traceback frames (' File "filename", line 123') and failed doctest - # examples (same, but not indented) if frames := re.findall(r'^ *File "(.*)", line (\d+)', message, re.MULTILINE): test_frames = [frame for frame in frames if pathlib.Path(frame[0]).name.startswith('test_')] filename, line = test_frames[-1] if test_frames else frames[0] - # In the job, the stdlib can be outside of the repository checkout + + # Normalize stdlib path to be relative inside the repository checkout. path = pathlib.Path(filename) stdlib_dir = pathlib.Path(os.__file__).parent if path.is_relative_to(stdlib_dir): From b0e9c5e42cbe777b0eb76a78b96aefa369c71f14 Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Mon, 12 Oct 2026 21:45:18 -0700 Subject: [PATCH 33/34] Personal review fixes, adjust output locaiton --- Lib/test/libregrtest/result.py | 36 +++++++++++++++++------------- Lib/test/libregrtest/setup.py | 3 ++- Lib/test/libregrtest/single.py | 1 + Lib/test/libregrtest/testresult.py | 16 ++++++++++++- Lib/test/libregrtest/utils.py | 12 +++++----- 5 files changed, 45 insertions(+), 23 deletions(-) diff --git a/Lib/test/libregrtest/result.py b/Lib/test/libregrtest/result.py index 69071562b14e193..448581d5a6108e3 100644 --- a/Lib/test/libregrtest/result.py +++ b/Lib/test/libregrtest/result.py @@ -108,6 +108,9 @@ class TestResult: # short descriptions of how the test altered the execution environment env_changed_reasons: list[str] | None = None + # test file, set once the test module is imported + filename: str | None = None + def is_failed(self, fail_env_changed: bool) -> bool: if self.state == State.ENV_CHANGED: return fail_env_changed @@ -205,23 +208,24 @@ def format_github_summary(self, expand: bool) -> str: return "".join(f"{block}\n\n" for block in blocks) def print_github_annotation(self, runtests: RunTests) -> None: - """Print github error annotation line for each failure.""" - if not (runtests.github_annotations - and self.is_failed(runtests.fail_env_changed)): + """Annotate a failed test file without failed test cases.""" + if (not runtests.github_annotations + or not self.is_failed(runtests.fail_env_changed) + or self.failed_test_cases): return - # Convert python import test name to relative file path. - # Ex: "test.test_x.test_y" is test_x/test_y.py - name = self.test_name.removeprefix("test.") - path = pathlib.Path(findtestdir(runtests.test_dir), *name.split(".")) - filename = path.with_suffix(".py") - if not filename.exists(): - filename = path / "__init__.py" - annotations = self.failed_test_cases - if not annotations: - message = "\n".join([str(self), *(self.env_changed_reasons or ())]) - annotations = [(self.test_name, message)] - for title, message in annotations: - print_github_annotation(title, message, str(filename)) + filename = self.filename + if not filename: + # The worker crashed or timed out before reporting the test file: + # convert python import test name to relative file path. + # Ex: "test.test_x.test_y" is test_x/test_y.py + name = self.test_name.removeprefix("test.") + path = pathlib.Path(findtestdir(runtests.test_dir), + *name.split(".")) + filename = path.with_suffix(".py") + if not filename.exists(): + filename = path / "__init__.py" + message = "\n".join([str(self), *(self.env_changed_reasons or ())]) + print_github_annotation(self.test_name, message, str(filename)) def set_env_changed(self, *reasons): if self.state is None or self.state == State.PASSED: diff --git a/Lib/test/libregrtest/setup.py b/Lib/test/libregrtest/setup.py index d62194acd9c29e5..8216541e454f82d 100644 --- a/Lib/test/libregrtest/setup.py +++ b/Lib/test/libregrtest/setup.py @@ -12,6 +12,7 @@ from .filter import set_match_tests from .runtests import RunTests +lazy from .testresult import RegressionTestResult from .utils import ( setup_unraisable_hook, setup_threading_excepthook, adjust_rlimit_nofile) @@ -112,9 +113,9 @@ def setup_tests(runtests: RunTests) -> None: set_match_tests(runtests.match_tests) + RegressionTestResult.GITHUB_ANNOTATIONS = runtests.github_annotations if runtests.use_junit: support.junit_xml_list = [] - from .testresult import RegressionTestResult RegressionTestResult.USE_XML = True else: support.junit_xml_list = None diff --git a/Lib/test/libregrtest/single.py b/Lib/test/libregrtest/single.py index d7a1c6bea202a65..3aa4ffd00f8f8fa 100644 --- a/Lib/test/libregrtest/single.py +++ b/Lib/test/libregrtest/single.py @@ -153,6 +153,7 @@ def _load_run_test(result: TestResult, runtests: RunTests) -> None: test_name = result.test_name module_name = abs_module_name(test_name, runtests.test_dir) test_mod = importlib.import_module(module_name) + result.filename = test_mod.__file__ if hasattr(test_mod, "test_main"): # https://github.com/python/cpython/issues/89392 diff --git a/Lib/test/libregrtest/testresult.py b/Lib/test/libregrtest/testresult.py index 605f1f4e6a89fb6..79add3d39b373ab 100644 --- a/Lib/test/libregrtest/testresult.py +++ b/Lib/test/libregrtest/testresult.py @@ -9,10 +9,11 @@ import traceback import unittest from test import support -from test.libregrtest.utils import sanitize_xml +from test.libregrtest.utils import print_github_annotation, sanitize_xml class RegressionTestResult(unittest.TextTestResult): USE_XML = False + GITHUB_ANNOTATIONS = False def __init__(self, stream, descriptions, verbosity): super().__init__(stream=stream, descriptions=descriptions, @@ -130,6 +131,14 @@ def addUnexpectedSuccess(self, test): self._add_result(test, outcome='UNEXPECTED_SUCCESS') super().addUnexpectedSuccess(test) + def printErrorList(self, flavour, errors): + for test, err in errors: + super().printErrorList(flavour, [(test, err)]) + # Annotate just after the failure report, so that the annotation + # links to the report in the job log + if self.GITHUB_ANNOTATIONS: + print_github_annotation(str(test), err, stream=self.stream) + def get_xml_element(self): if not self.USE_XML: raise ValueError("USE_XML is false") @@ -146,6 +155,11 @@ def __init__(self, stream, buffer=False): def run(self, test): test(self.result) + # Failures are not reported in quiet mode: annotate them at the end + if self.result.GITHUB_ANNOTATIONS: + for test, err in self.result.errors + self.result.failures: + print_github_annotation(str(test), err, + stream=self.result.stream) return self.result def get_test_runner_class(verbosity, buffer=False): diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index 2ac30fb4cba5fc8..6034fa50b008177 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -148,19 +148,21 @@ def print_warning(msg: str) -> None: ",": "%2C"}) -def print_github_annotation(title: str, message: str, filename: str) -> None: +def print_github_annotation(title: str, message: str, + filename: str | None = None, + stream=None) -> None: """Print a GitHub Actions error annotation for the failure.""" - if not filename: - raise ValueError(f"missing filename to annotate {title!r}") message = decolor(message) - # Try to get a line number from backtrace. + # Try to get a file + line number from backtrace. line = None if frames := re.findall(r'^ *File "(.*)", line (\d+)', message, re.MULTILINE): test_frames = [frame for frame in frames if pathlib.Path(frame[0]).name.startswith('test_')] filename, line = test_frames[-1] if test_frames else frames[0] + if not filename: + raise ValueError(f"missing filename to annotate {title!r}") # Normalize stdlib path to be relative inside the repository checkout. path = pathlib.Path(filename) @@ -175,7 +177,7 @@ def print_github_annotation(title: str, message: str, filename: str) -> None: props_text = ",".join(f"{key}={value.translate(GITHUB_ESCAPE_PROPERTY)}" for key, value in props.items() if value) print(f"::error {props_text}::{message.translate(GITHUB_ESCAPE_DATA)}", - flush=True) + file=stream, flush=True) def regrtest_unraisable_hook(unraisable) -> None: From 01ad16c0af64ca57071bd8333314b4532149697f Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Mon, 12 Oct 2026 21:52:49 -0700 Subject: [PATCH 34/34] Fix mypy --- Lib/test/libregrtest/result.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/Lib/test/libregrtest/result.py b/Lib/test/libregrtest/result.py index 448581d5a6108e3..8f59fcd124900c0 100644 --- a/Lib/test/libregrtest/result.py +++ b/Lib/test/libregrtest/result.py @@ -221,11 +221,12 @@ def print_github_annotation(self, runtests: RunTests) -> None: name = self.test_name.removeprefix("test.") path = pathlib.Path(findtestdir(runtests.test_dir), *name.split(".")) - filename = path.with_suffix(".py") - if not filename.exists(): - filename = path / "__init__.py" + if path.with_suffix(".py").exists(): + filename = str(path.with_suffix(".py")) + else: + filename = str(path / "__init__.py") message = "\n".join([str(self), *(self.env_changed_reasons or ())]) - print_github_annotation(self.test_name, message, str(filename)) + print_github_annotation(self.test_name, message, filename) def set_env_changed(self, *reasons): if self.state is None or self.state == State.PASSED: