diff --git a/README.md b/README.md index 9990919..6ab3d45 100644 --- a/README.md +++ b/README.md @@ -146,6 +146,25 @@ The accepted top-level sections are `test`, `parallel`, `reporter`, `snapshot`, `watch`, `memory`, `ui`, and `coverage`. Options retain the same execution-mode limitations as their CLI equivalents; unsupported combinations fail explicitly. +### Parallel execution semantics + +Programmatic runners can enable bounded execution with `RunnerOptions.parallel` +and set `n_jobs`. A configured value is the exact maximum number of active test +workers; omitting it uses the host's logical CPU count, with a one-worker +fallback when detection is unavailable. + +Suites and nested suites are visited in declaration order. `beforeAll` and +`afterAll` run once on the coordinating thread around that suite and its nested +suites. Direct tests within a suite may overlap. Their inherited `beforeEach` +and `afterEach` hooks run on the same worker as the test, so per-test hook order +is preserved but hooks from different tests may overlap. Hook state and the +allocator passed to tests must therefore be thread-safe. + +Reporter callbacks never run concurrently: test results are emitted in +declaration order after each parallel batch, making output and final totals +deterministic. A parallel batch is already scheduled as a unit, so `bail` cannot +cancel tests that have started. + ### Programmatic Mode For more control, you can manually register tests: diff --git a/src/parallel.zig b/src/parallel.zig index fd937b9..605c8c4 100644 --- a/src/parallel.zig +++ b/src/parallel.zig @@ -3,205 +3,505 @@ const suite = @import("suite.zig"); const reporter = @import("reporter.zig"); const compat = @import("compat.zig"); -/// Options for parallel test execution +/// Options for bounded parallel test execution. pub const ParallelOptions = struct { - /// Number of worker threads (null = auto-detect CPU count) + /// Exact maximum worker count. Null uses the logical CPU count, falling + /// back to one worker when the platform cannot report it. n_jobs: ?usize = null, - /// Whether parallel execution is enabled enabled: bool = false, + filter: ?[]const u8 = null, }; -/// Result from running a test in parallel -pub const TestResult = struct { - test_case: *suite.TestCase, - suite_name: []const u8, - success: bool, - mutex: compat.Mutex = .{}, +pub const ParallelError = error{ + ParallelNotEnabled, + InvalidJobCount, }; -/// Context for parallel test execution -pub const ParallelContext = struct { - allocator: std.mem.Allocator, - reporter: *reporter.Reporter, - results: std.ArrayList(TestResult), - mutex: compat.Mutex = .{}, - - pub fn init(allocator: std.mem.Allocator, rep: *reporter.Reporter) ParallelContext { - return .{ - .allocator = allocator, - .reporter = rep, - .results = .empty, - }; +pub fn resolveWorkerCount(n_jobs: ?usize) !usize { + if (n_jobs) |jobs| { + if (jobs == 0) return ParallelError.InvalidJobCount; + return jobs; } + return @max(std.Thread.getCpuCount() catch 1, 1); +} - pub fn deinit(self: *ParallelContext) void { - self.results.deinit(self.allocator); +const WorkerContext = struct { + allocator: std.mem.Allocator, + test_suite: *suite.TestSuite, + tests: []const *suite.TestCase, + next_index: std.atomic.Value(usize) = std.atomic.Value(usize).init(0), + error_mutex: compat.Mutex = .{}, + + fn worker(self: *WorkerContext) void { + while (true) { + const index = self.next_index.fetchAdd(1, .monotonic); + if (index >= self.tests.len) return; + self.runTest(self.tests[index]); + } } -}; -/// Task for executing a single test -pub const TestTask = struct { - context: *ParallelContext, - test_case: *suite.TestCase, - suite_name: []const u8, - - pub fn run(self: *TestTask) void { - // Execute the test - const success = self.executeTest(); - - // Store result in thread-safe manner - self.context.mutex.lock(); - defer self.context.mutex.unlock(); - - self.context.results.append(self.context.allocator, .{ - .test_case = self.test_case, - .suite_name = self.suite_name, - .success = success, - }) catch |err| { - std.debug.print("Error appending result: {any}\n", .{err}); + fn runTest(self: *WorkerContext, test_case: *suite.TestCase) void { + test_case.status = .running; + const start_time = compat.nanoTimestamp(); + + var before_hooks = self.test_suite.getAllBeforeEachHooks(self.allocator) catch { + self.fail(test_case, "could not collect beforeEach hooks", .{}); + self.finishTiming(test_case, start_time); + return; }; - } + defer before_hooks.deinit(self.allocator); + + var before_failed = false; + for (before_hooks.items) |hook| { + hook(self.allocator) catch |err| { + self.fail(test_case, "beforeEach hook failed: {any}", .{err}); + before_failed = true; + break; + }; + } - fn executeTest(self: *TestTask) bool { - self.test_case.status = .running; + if (!before_failed) { + test_case.test_fn(self.allocator) catch |err| { + self.fail(test_case, "{any}", .{err}); + }; + if (test_case.status == .running) test_case.status = .passed; + } - // Call the test function - self.test_case.test_fn(self.context.allocator) catch |err| { - self.test_case.status = .failed; - const err_msg = std.fmt.allocPrint( - self.context.allocator, - "Test failed: {any}", - .{err}, - ) catch "Out of memory"; - self.test_case.error_message = err_msg; - return false; + var after_hooks = self.test_suite.getAllAfterEachHooks(self.allocator) catch { + self.fail(test_case, "could not collect afterEach hooks", .{}); + self.finishTiming(test_case, start_time); + return; }; + defer after_hooks.deinit(self.allocator); + for (after_hooks.items) |hook| { + hook(self.allocator) catch |err| { + self.fail(test_case, "afterEach hook failed: {any}", .{err}); + break; + }; + } - self.test_case.status = .passed; - return true; + self.finishTiming(test_case, start_time); + } + + fn fail(self: *WorkerContext, test_case: *suite.TestCase, comptime format: []const u8, args: anytype) void { + test_case.status = .failed; + self.error_mutex.lock(); + defer self.error_mutex.unlock(); + test_case.error_message = std.fmt.allocPrint(self.allocator, format, args) catch "Out of memory"; + } + + fn finishTiming(_: *WorkerContext, test_case: *suite.TestCase, start_time: i128) void { + test_case.execution_time_ns = @intCast(compat.nanoTimestamp() - start_time); } }; -/// Run tests in parallel using thread pool -pub fn runTestsParallel( +const RunContext = struct { allocator: std.mem.Allocator, - test_registry: *suite.TestRegistry, + registry: *suite.TestRegistry, rep: *reporter.Reporter, options: ParallelOptions, -) !bool { - if (!options.enabled) { - return error.ParallelNotEnabled; - } + worker_limit: usize, - // Create parallel context - var context = ParallelContext.init(allocator, rep); - defer context.deinit(); + fn runSuite(self: *RunContext, test_suite: *suite.TestSuite) !bool { + try self.rep.onSuiteStart(test_suite.name); - // Count total tests - var total_tests: usize = 0; - for (test_registry.root_suites.items) |test_suite| { - total_tests += test_suite.tests.items.len; - } - - try rep.onRunStart(total_tests); + if (test_suite.shouldSkip()) { + try self.reportSkippedSuite(test_suite); + try self.rep.onSuiteEnd(test_suite.name); + return true; + } - // Collect tasks and spawn threads - var threads = std.ArrayList(std.Thread).empty; - defer threads.deinit(allocator); + test_suite.runBeforeAllHooks(self.allocator) catch |err| { + std.debug.print("beforeAll hook failed: {any}\n", .{err}); + try self.reportSkippedSuite(test_suite); + try self.rep.onSuiteEnd(test_suite.name); + return true; + }; - for (test_registry.root_suites.items) |test_suite| { - try rep.onSuiteStart(test_suite.name); + var scheduled: std.ArrayList(*suite.TestCase) = .empty; + defer scheduled.deinit(self.allocator); for (test_suite.tests.items) |*test_case| { - if (test_case.skip) { + if (!self.shouldRun(test_suite, test_case)) { test_case.status = .skipped; continue; } + try self.rep.onTestStart(test_case.name); + try scheduled.append(self.allocator, test_case); + } - // Create task - const task = try allocator.create(TestTask); - task.* = .{ - .context = &context, - .test_case = test_case, - .suite_name = test_suite.name, - }; + try self.runBatch(test_suite, scheduled.items); + + var all_passed = true; + for (test_suite.tests.items) |*test_case| { + try self.rep.onTestEnd(test_case); + if (test_case.status == .failed) all_passed = false; + } - // Spawn thread for task - const thread = try std.Thread.spawn(.{}, runTestTask, .{task}); - try threads.append(allocator, thread); + for (test_suite.suites.items) |nested_suite| { + if (!try self.runSuite(nested_suite)) all_passed = false; } + + test_suite.runAfterAllHooks(self.allocator) catch |err| { + std.debug.print("afterAll hook failed: {any}\n", .{err}); + all_passed = false; + }; + + try self.rep.onSuiteEnd(test_suite.name); + return all_passed; } - // Wait for all threads to complete - for (threads.items) |thread| { - thread.join(); + fn shouldRun(self: *RunContext, test_suite: *suite.TestSuite, test_case: *suite.TestCase) bool { + if (test_case.skip) return false; + if (self.registry.has_only and !test_case.only and !test_suite.hasOnly()) return false; + if (self.options.filter) |filter| { + if (std.mem.indexOf(u8, test_case.name, filter) == null) return false; + } + return true; } - // Report results (now that all tests are done) - var all_passed = true; - for (test_registry.root_suites.items) |test_suite| { + fn runBatch(self: *RunContext, test_suite: *suite.TestSuite, tests: []const *suite.TestCase) !void { + if (tests.len == 0) return; + + var worker_context = WorkerContext{ + .allocator = self.allocator, + .test_suite = test_suite, + .tests = tests, + }; + const count = @min(self.worker_limit, tests.len); + var threads: std.ArrayList(std.Thread) = .empty; + defer threads.deinit(self.allocator); + + for (0..count) |_| { + const thread = std.Thread.spawn(.{}, WorkerContext.worker, .{&worker_context}) catch |err| { + for (threads.items) |started| started.join(); + return err; + }; + threads.append(self.allocator, thread) catch |err| { + thread.join(); + for (threads.items) |started| started.join(); + return err; + }; + } + for (threads.items) |thread| thread.join(); + } + + fn reportSkippedSuite(self: *RunContext, test_suite: *suite.TestSuite) !void { for (test_suite.tests.items) |*test_case| { - try rep.onTestEnd(test_case); - if (test_case.status == .failed) { - all_passed = false; - } + test_case.status = .skipped; + try self.rep.onTestEnd(test_case); + } + for (test_suite.suites.items) |nested| { + try self.rep.onSuiteStart(nested.name); + try self.reportSkippedSuite(nested); + try self.rep.onSuiteEnd(nested.name); } - try rep.onSuiteEnd(test_suite.name); } +}; - // Create results summary - var results = reporter.TestResults.init(allocator); - defer results.deinit(); +/// Run suites in declaration order while executing each suite's direct tests +/// with a bounded worker set. Reporter callbacks are serialized and emitted in +/// declaration order after each batch completes. +pub fn runTestsParallel( + allocator: std.mem.Allocator, + test_registry: *suite.TestRegistry, + rep: *reporter.Reporter, + options: ParallelOptions, +) !bool { + if (!options.enabled) return ParallelError.ParallelNotEnabled; + const worker_limit = try resolveWorkerCount(options.n_jobs); + + const total_tests = test_registry.countAllTests(); + try rep.onRunStart(total_tests); + var context = RunContext{ + .allocator = allocator, + .registry = test_registry, + .rep = rep, + .options = options, + .worker_limit = worker_limit, + }; + + var all_passed = true; for (test_registry.root_suites.items) |test_suite| { - for (test_suite.tests.items) |test_case| { - results.total += 1; - switch (test_case.status) { - .passed => results.passed += 1, - .failed => results.failed += 1, - .skipped => results.skipped += 1, - else => {}, - } - } + if (!try context.runSuite(test_suite)) all_passed = false; } + var results = reporter.TestResults.init(allocator); + defer results.deinit(); + for (test_registry.root_suites.items) |test_suite| { + try addSuiteResults(&results, test_suite); + } try rep.onRunEnd(&results); - - return all_passed; + return all_passed and results.failed == 0; } -fn runTestTask(task: *TestTask) void { - task.run(); +fn addSuiteResults(results: *reporter.TestResults, test_suite: *suite.TestSuite) !void { + for (test_suite.tests.items) |*test_case| try results.addTest(test_case); + for (test_suite.suites.items) |nested| try addSuiteResults(results, nested); } -// Tests -test "ParallelOptions default values" { +test "ParallelOptions defaults to CPU-based workers" { const options = ParallelOptions{}; + try std.testing.expect(options.n_jobs == null); + try std.testing.expect(!options.enabled); + try std.testing.expectEqual( + @max(std.Thread.getCpuCount() catch 1, 1), + try resolveWorkerCount(null), + ); +} + +test "configured worker count rejects zero" { + try std.testing.expectEqual(@as(usize, 2), try resolveWorkerCount(2)); + try std.testing.expectError(ParallelError.InvalidJobCount, resolveWorkerCount(0)); +} + +var active_tests = std.atomic.Value(usize).init(0); +var peak_tests = std.atomic.Value(usize).init(0); + +fn boundedTest(_: std.mem.Allocator) !void { + const active = active_tests.fetchAdd(1, .monotonic) + 1; + var peak = peak_tests.load(.monotonic); + while (active > peak) { + if (peak_tests.cmpxchgWeak(peak, active, .monotonic, .monotonic)) |observed| { + peak = observed; + } else break; + } + compat.sleep(20 * std.time.ns_per_ms); + _ = active_tests.fetchSub(1, .monotonic); +} + +fn passingTest(_: std.mem.Allocator) !void {} + +fn failingTest(_: std.mem.Allocator) !void { + return error.ExpectedFailure; +} + +var parent_before_each = std.atomic.Value(usize).init(0); +var parent_after_each = std.atomic.Value(usize).init(0); +var child_before_each = std.atomic.Value(usize).init(0); +var child_after_each = std.atomic.Value(usize).init(0); +var parent_before_all = std.atomic.Value(usize).init(0); +var child_before_all = std.atomic.Value(usize).init(0); +var parent_after_all = std.atomic.Value(usize).init(0); +var child_after_all = std.atomic.Value(usize).init(0); + +fn parentBeforeEach(_: std.mem.Allocator) !void { + _ = parent_before_each.fetchAdd(1, .monotonic); +} + +fn parentAfterEach(_: std.mem.Allocator) !void { + _ = parent_after_each.fetchAdd(1, .monotonic); +} + +fn childBeforeEach(_: std.mem.Allocator) !void { + _ = child_before_each.fetchAdd(1, .monotonic); +} + +fn childAfterEach(_: std.mem.Allocator) !void { + _ = child_after_each.fetchAdd(1, .monotonic); +} + +fn parentBeforeAll(_: std.mem.Allocator) !void { + _ = parent_before_all.fetchAdd(1, .monotonic); +} + +fn childBeforeAll(_: std.mem.Allocator) !void { + if (parent_before_all.load(.monotonic) != 1) return error.ParentBeforeAllMissing; + _ = child_before_all.fetchAdd(1, .monotonic); +} - try std.testing.expectEqual(@as(?usize, null), options.n_jobs); - try std.testing.expectEqual(false, options.enabled); +fn childAfterAll(_: std.mem.Allocator) !void { + _ = child_after_all.fetchAdd(1, .monotonic); } -test "ParallelOptions with custom values" { - const options = ParallelOptions{ - .n_jobs = 4, +fn parentAfterAll(_: std.mem.Allocator) !void { + if (child_after_all.load(.monotonic) != 1) return error.ChildAfterAllMissing; + _ = parent_after_all.fetchAdd(1, .monotonic); +} + +const RecordingReporter = struct { + reporter: reporter.Reporter, + run_starts: usize = 0, + run_ends: usize = 0, + suite_starts: usize = 0, + suite_ends: usize = 0, + test_starts: usize = 0, + ended_count: usize = 0, + ended_names: [16][]const u8 = undefined, + total: usize = 0, + passed: usize = 0, + failed: usize = 0, + skipped: usize = 0, + + fn init(allocator: std.mem.Allocator) RecordingReporter { + return .{ + .reporter = .{ + .vtable = &.{ + .onRunStart = onRunStart, + .onRunEnd = onRunEnd, + .onSuiteStart = onSuiteStart, + .onSuiteEnd = onSuiteEnd, + .onTestStart = onTestStart, + .onTestEnd = onTestEnd, + }, + .allocator = allocator, + }, + }; + } + + fn self(rep: *reporter.Reporter) *RecordingReporter { + return @fieldParentPtr("reporter", rep); + } + + fn onRunStart(rep: *reporter.Reporter, _: usize) !void { + self(rep).run_starts += 1; + } + + fn onRunEnd(rep: *reporter.Reporter, results: *reporter.TestResults) !void { + const recording = self(rep); + recording.run_ends += 1; + recording.total = results.total; + recording.passed = results.passed; + recording.failed = results.failed; + recording.skipped = results.skipped; + } + + fn onSuiteStart(rep: *reporter.Reporter, _: []const u8) !void { + self(rep).suite_starts += 1; + } + + fn onSuiteEnd(rep: *reporter.Reporter, _: []const u8) !void { + self(rep).suite_ends += 1; + } + + fn onTestStart(rep: *reporter.Reporter, _: []const u8) !void { + self(rep).test_starts += 1; + } + + fn onTestEnd(rep: *reporter.Reporter, test_case: *const suite.TestCase) !void { + const recording = self(rep); + recording.ended_names[recording.ended_count] = test_case.name; + recording.ended_count += 1; + } +}; + +fn verifyConcurrencyLimit(limit: ?usize) !void { + const allocator = std.heap.smp_allocator; + var registry = suite.TestRegistry.init(allocator); + defer registry.deinit(); + const test_suite = try suite.TestSuite.init(allocator, "bounded"); + for (0..8) |_| try test_suite.addTest(suite.TestCase.init("work", boundedTest)); + try registry.registerSuite(test_suite); + + active_tests.store(0, .monotonic); + peak_tests.store(0, .monotonic); + var recording = RecordingReporter.init(allocator); + try std.testing.expect(try runTestsParallel(allocator, ®istry, &recording.reporter, .{ .enabled = true, - }; + .n_jobs = limit, + })); - try std.testing.expectEqual(@as(?usize, 4), options.n_jobs); - try std.testing.expectEqual(true, options.enabled); + const resolved = try resolveWorkerCount(limit); + try std.testing.expect(peak_tests.load(.monotonic) > 0); + try std.testing.expect(peak_tests.load(.monotonic) <= @min(resolved, 8)); } -test "ParallelContext initialization" { - const allocator = std.testing.allocator; +test "parallel executor bounds one worker" { + try verifyConcurrencyLimit(1); + try std.testing.expectEqual(@as(usize, 1), peak_tests.load(.monotonic)); +} - var rep = reporter.Reporter{ - .vtable = undefined, - .allocator = allocator, - }; +test "parallel executor bounds two workers" { + try verifyConcurrencyLimit(2); + try std.testing.expectEqual(@as(usize, 2), peak_tests.load(.monotonic)); +} + +test "parallel executor bounds default CPU workers" { + try verifyConcurrencyLimit(null); +} - var context = ParallelContext.init(allocator, &rep); - defer context.deinit(); +test "parallel reporting is serialized, ordered, and complete" { + const allocator = std.heap.smp_allocator; + var registry = suite.TestRegistry.init(allocator); + defer registry.deinit(); + const test_suite = try suite.TestSuite.init(allocator, "ordered"); + try test_suite.addTest(suite.TestCase.init("first", passingTest)); + try test_suite.addTest(suite.TestCase.init("second", failingTest)); + var skipped = suite.TestCase.init("third", passingTest); + skipped.skip = true; + try test_suite.addTest(skipped); + try registry.registerSuite(test_suite); + + var recording = RecordingReporter.init(allocator); + try std.testing.expect(!try runTestsParallel(allocator, ®istry, &recording.reporter, .{ + .enabled = true, + .n_jobs = 2, + })); + + try std.testing.expectEqual(@as(usize, 1), recording.run_starts); + try std.testing.expectEqual(@as(usize, 1), recording.run_ends); + try std.testing.expectEqual(@as(usize, 1), recording.suite_starts); + try std.testing.expectEqual(@as(usize, 1), recording.suite_ends); + try std.testing.expectEqual(@as(usize, 2), recording.test_starts); + try std.testing.expectEqual(@as(usize, 3), recording.ended_count); + try std.testing.expectEqualStrings("first", recording.ended_names[0]); + try std.testing.expectEqualStrings("second", recording.ended_names[1]); + try std.testing.expectEqualStrings("third", recording.ended_names[2]); + try std.testing.expectEqual(@as(usize, 3), recording.total); + try std.testing.expectEqual(@as(usize, 1), recording.passed); + try std.testing.expectEqual(@as(usize, 1), recording.failed); + try std.testing.expectEqual(@as(usize, 1), recording.skipped); +} - try std.testing.expectEqual(@as(usize, 0), context.results.items.len); +test "parallel executor preserves nested hook lifecycles" { + const counters = [_]*std.atomic.Value(usize){ + &parent_before_each, + &parent_after_each, + &child_before_each, + &child_after_each, + &parent_before_all, + &child_before_all, + &parent_after_all, + &child_after_all, + }; + for (counters) |counter| counter.store(0, .monotonic); + + const allocator = std.heap.smp_allocator; + var registry = suite.TestRegistry.init(allocator); + defer registry.deinit(); + const parent = try suite.TestSuite.init(allocator, "parent"); + const child = try suite.TestSuite.init(allocator, "child"); + try parent.addBeforeAll(parentBeforeAll); + try parent.addAfterAll(parentAfterAll); + try parent.addBeforeEach(parentBeforeEach); + try parent.addAfterEach(parentAfterEach); + try parent.addTest(suite.TestCase.init("parent test", passingTest)); + try child.addBeforeAll(childBeforeAll); + try child.addAfterAll(childAfterAll); + try child.addBeforeEach(childBeforeEach); + try child.addAfterEach(childAfterEach); + try child.addTest(suite.TestCase.init("child one", passingTest)); + try child.addTest(suite.TestCase.init("child two", passingTest)); + try parent.addSuite(child); + try registry.registerSuite(parent); + + var recording = RecordingReporter.init(allocator); + try std.testing.expect(try runTestsParallel(allocator, ®istry, &recording.reporter, .{ + .enabled = true, + .n_jobs = 2, + })); + + try std.testing.expectEqual(@as(usize, 3), parent_before_each.load(.monotonic)); + try std.testing.expectEqual(@as(usize, 3), parent_after_each.load(.monotonic)); + try std.testing.expectEqual(@as(usize, 2), child_before_each.load(.monotonic)); + try std.testing.expectEqual(@as(usize, 2), child_after_each.load(.monotonic)); + try std.testing.expectEqual(@as(usize, 1), parent_before_all.load(.monotonic)); + try std.testing.expectEqual(@as(usize, 1), child_before_all.load(.monotonic)); + try std.testing.expectEqual(@as(usize, 1), parent_after_all.load(.monotonic)); + try std.testing.expectEqual(@as(usize, 1), child_after_all.load(.monotonic)); + try std.testing.expectEqual(@as(usize, 2), recording.suite_starts); + try std.testing.expectEqual(@as(usize, 2), recording.suite_ends); + try std.testing.expectEqual(@as(usize, 3), recording.total); + try std.testing.expectEqual(@as(usize, 3), recording.passed); } diff --git a/src/test_runner.zig b/src/test_runner.zig index 5599562..e32fdfa 100644 --- a/src/test_runner.zig +++ b/src/test_runner.zig @@ -79,57 +79,35 @@ pub const TestRunner = struct { return RunnerError.NoTestsFound; } - // Notify reporter of run start - try current_reporter.onRunStart(total_tests); - // Run tests in parallel or sequential based on options if (self.options.parallel) { const parallel_opts = parallel.ParallelOptions{ .enabled = true, .n_jobs = self.options.n_jobs, + .filter = self.options.filter, }; - _ = parallel.runTestsParallel( + const all_passed = try parallel.runTestsParallel( self.allocator, self.registry, current_reporter, parallel_opts, - ) catch |err| { - std.debug.print("Parallel execution failed: {any}, falling back to sequential\n", .{err}); - // Fall back to sequential execution - for (self.registry.root_suites.items) |test_suite| { - try self.runSuite(test_suite, current_reporter); - if (self.options.bail and self.results.failed > 0) { - break; - } - } - return false; - }; - - // Update results from parallel execution - self.results.total = 0; - self.results.passed = 0; - self.results.failed = 0; - self.results.skipped = 0; + ); for (self.registry.root_suites.items) |test_suite| { - for (test_suite.tests.items) |test_case| { - self.results.total += 1; - switch (test_case.status) { - .passed => self.results.passed += 1, - .failed => self.results.failed += 1, - .skipped => self.results.skipped += 1, - else => {}, - } - } + try self.addSuiteResults(test_suite); } - } else { - // Sequential execution (original behavior) - for (self.registry.root_suites.items) |test_suite| { - try self.runSuite(test_suite, current_reporter); - if (self.options.bail and self.results.failed > 0) { - break; - } + + try stdout_writer.interface.flush(); + return all_passed and self.results.failed == 0; + } + + // Sequential execution (original behavior) + try current_reporter.onRunStart(total_tests); + for (self.registry.root_suites.items) |test_suite| { + try self.runSuite(test_suite, current_reporter); + if (self.options.bail and self.results.failed > 0) { + break; } } @@ -142,6 +120,11 @@ pub const TestRunner = struct { return self.results.failed == 0; } + fn addSuiteResults(self: *Self, test_suite: *suite.TestSuite) !void { + for (test_suite.tests.items) |*test_case| try self.results.addTest(test_case); + for (test_suite.suites.items) |nested| try self.addSuiteResults(nested); + } + /// Run a single test suite fn runSuite(self: *Self, test_suite: *suite.TestSuite, rep: *reporter_mod.Reporter) !void { // Skip if marked as skip or if has_only and this isn't marked as only