From 6aee67dad25722af65bca4e59ba654b067b486ee Mon Sep 17 00:00:00 2001 From: akushonkamen Date: Fri, 9 Oct 2026 00:13:11 +0800 Subject: [PATCH] Skip failed evaluations in early stopping plateau count Early stopping treated failed and timed-out evaluations as scores. A failure whose metrics fall back to safe_numeric_average (e.g. {"combined_score": 0.0, "error": ...} or the evaluator's timeout shape {"error": 0.0, "timeout": True}) scored 0.0, so the first failure was counted as an infinite improvement over the -inf baseline and poisoned best_score, and every subsequent failure incremented the iterations_without_improvement counter. A run whose evaluator never succeeded could therefore exit through the successful-plateau path (issue #294). For tasks with negative scores, failure results scoring 0.0 kept registering as improvements, permanently pinning best_score. Guard the early stopping block (both the patience-based and the event-based branches) on the absence of "error"/"timeout" markers in the child metrics: those iterations neither update best_score nor count toward patience, so early stopping only triggers on successful evaluations that plateau. Evaluators should include an "error" key in their metrics on failure per the convention suggested in #294. Documented the convention in configs/early_stopping_example.yaml. Fixes #294 --- configs/early_stopping_example.yaml | 5 + openevolve/process_parallel.py | 123 ++++++++++++-------- tests/test_early_stopping_failures.py | 160 ++++++++++++++++++++++++++ 3 files changed, 237 insertions(+), 51 deletions(-) create mode 100644 tests/test_early_stopping_failures.py diff --git a/configs/early_stopping_example.yaml b/configs/early_stopping_example.yaml index c7259ea7f1..0a347babf2 100644 --- a/configs/early_stopping_example.yaml +++ b/configs/early_stopping_example.yaml @@ -10,6 +10,11 @@ log_level: "INFO" early_stopping_patience: 30 # Stop after 30 iterations without improvement convergence_threshold: 0.01 # Minimum improvement of 0.01 required to reset patience early_stopping_metric: "combined_score" # Track the combined_score metric +# +# Note: iterations whose evaluation fails or times out (metrics carrying an +# "error" or "timeout" key, e.g. {"error": 1.0, "combined_score": 0.0}) do not +# count toward the patience counter. Early stopping only triggers on successful +# evaluations that plateau - repeated failures alone never stop the run. # LLM configuration llm: diff --git a/openevolve/process_parallel.py b/openevolve/process_parallel.py index 06eb0f09fe..6241072d4d 100644 --- a/openevolve/process_parallel.py +++ b/openevolve/process_parallel.py @@ -879,59 +879,80 @@ async def run_evolution( # Check early stopping if early_stopping_enabled and child_program.metrics: - # Get the metric to track for early stopping - current_score = None - if self.config.early_stopping_metric in child_program.metrics: - current_score = child_program.metrics[self.config.early_stopping_metric] - elif self.config.early_stopping_metric == "combined_score": - # Default metric not found, use safe average (standard pattern) - current_score = safe_numeric_average(child_program.metrics) - else: - # User specified a custom metric that doesn't exist - logger.warning( - f"Early stopping metric '{self.config.early_stopping_metric}' not found, using safe numeric average" + if child_program.metrics.get( + "error" + ) is not None or child_program.metrics.get("timeout"): + # Failed or timed-out evaluations carry no score. Letting them + # into the counter below would poison the best-score baseline + # with 0.0 on the first failure and then count every subsequent + # failure as "no improvement", so a run that never succeeded + # once would exit through the successful-plateau path + # (issue #294). Skip them: early stopping only counts + # evaluations that actually produced a score. The "error" check + # is presence-based (is not None) because evaluators signal + # failures both with numeric ({"error": 0.0}) and string values. + logger.debug( + "Skipping failed evaluation from early stopping count at iteration %s", + completed_iteration, ) - current_score = safe_numeric_average(child_program.metrics) - - if current_score is not None and isinstance(current_score, (int, float)): - # Check for improvement - if self.config.early_stopping_patience > 0: - improvement = current_score - best_score - if improvement >= self.config.convergence_threshold: - best_score = current_score - iterations_without_improvement = 0 - logger.debug( - f"New best score: {best_score:.4f} (improvement: {improvement:+.4f})" - ) - else: - iterations_without_improvement += 1 - logger.debug( - f"No improvement: {iterations_without_improvement}/{self.config.early_stopping_patience}" - ) - - # Check if we should stop - if ( - iterations_without_improvement - >= self.config.early_stopping_patience - ): - self.early_stopping_triggered = True - logger.info( - f"🛑 Early stopping triggered at iteration {completed_iteration}: " - f"No improvement for {iterations_without_improvement} iterations " - f"(best score: {best_score:.4f})" - ) - break - + else: + # Get the metric to track for early stopping + current_score = None + if self.config.early_stopping_metric in child_program.metrics: + current_score = child_program.metrics[ + self.config.early_stopping_metric + ] + elif self.config.early_stopping_metric == "combined_score": + # Default metric not found, use safe average (standard pattern) + current_score = safe_numeric_average(child_program.metrics) else: - # Event-based early stopping - if current_score == self.config.convergence_threshold: - best_score = current_score - logger.info( - f"🛑 Early stopping (event-based) triggered at iteration {completed_iteration}: " - f"Task successfully solved with score {best_score:.4f}." - ) - self.early_stopping_triggered = True - break + # User specified a custom metric that doesn't exist + logger.warning( + f"Early stopping metric '{self.config.early_stopping_metric}' not found, using safe numeric average" + ) + current_score = safe_numeric_average(child_program.metrics) + + if current_score is not None and isinstance( + current_score, (int, float) + ): + # Check for improvement + if self.config.early_stopping_patience > 0: + improvement = current_score - best_score + if improvement >= self.config.convergence_threshold: + best_score = current_score + iterations_without_improvement = 0 + logger.debug( + f"New best score: {best_score:.4f} (improvement: {improvement:+.4f})" + ) + else: + iterations_without_improvement += 1 + logger.debug( + f"No improvement: {iterations_without_improvement}/{self.config.early_stopping_patience}" + ) + + # Check if we should stop + if ( + iterations_without_improvement + >= self.config.early_stopping_patience + ): + self.early_stopping_triggered = True + logger.info( + f"🛑 Early stopping triggered at iteration {completed_iteration}: " + f"No improvement for {iterations_without_improvement} iterations " + f"(best score: {best_score:.4f})" + ) + break + + else: + # Event-based early stopping + if current_score == self.config.convergence_threshold: + best_score = current_score + logger.info( + f"🛑 Early stopping (event-based) triggered at iteration {completed_iteration}: " + f"Task successfully solved with score {best_score:.4f}." + ) + self.early_stopping_triggered = True + break except _CandidateRejected as rejected: logger.info("Admission strategy rejected program %s", rejected) diff --git a/tests/test_early_stopping_failures.py b/tests/test_early_stopping_failures.py new file mode 100644 index 0000000000..731e0081fc --- /dev/null +++ b/tests/test_early_stopping_failures.py @@ -0,0 +1,160 @@ +""" +Tests for early stopping interaction with failed evaluations (issue #294) + +Failed or timed-out evaluations surface as metrics carrying an "error" or +"timeout" key. Those are not scores: they must not feed the early-stopping +plateau counter, otherwise a run that never succeeds once can exit through +the "successful evaluations plateaued" path. +""" + +import asyncio +import os +import tempfile +import unittest +from unittest.mock import MagicMock, patch + +# Set dummy API key for testing +os.environ["OPENAI_API_KEY"] = "test" + +from openevolve.config import Config +from openevolve.database import Program, ProgramDatabase +from openevolve.process_parallel import ProcessParallelController, SerializableResult + + +class TestEarlyStoppingWithFailedEvaluations(unittest.TestCase): + """Early stopping must only count evaluations that produced a score""" + + def setUp(self): + """Set up test environment""" + self.test_dir = tempfile.mkdtemp() + + # Create test config + self.config = Config() + self.config.max_iterations = 10 + self.config.evaluator.parallel_evaluations = 2 + self.config.evaluator.timeout = 10 + # One island per test program so each owns its MAP-Elites cell and none is + # displaced (MAP-Elites removes programs displaced from their cell). + self.config.database.num_islands = 3 + self.config.database.in_memory = True + self.config.checkpoint_interval = 5 + # Issue #294 scenario: early stopping enabled, reporter's config only set + # patience (convergence_threshold stays at its default of 0.001). + self.config.early_stopping_patience = 3 + + # Create test evaluation file + self.eval_content = """ +def evaluate(program_path): + return {"score": 0.5, "performance": 0.6} +""" + self.eval_file = os.path.join(self.test_dir, "evaluator.py") + with open(self.eval_file, "w") as f: + f.write(self.eval_content) + + # Create test database + self.database = ProgramDatabase(self.config.database) + + # Add some test programs, one per island so each survives as its cell owner + for i in range(3): + program = Program( + id=f"test_{i}", + code=f"def func_{i}(): return {i}", + language="python", + metrics={"score": 0.5 + i * 0.1, "performance": 0.4 + i * 0.1}, + iteration_found=0, + ) + self.database.add(program, target_island=i) + + def tearDown(self): + """Clean up test environment""" + import shutil + + shutil.rmtree(self.test_dir, ignore_errors=True) + + def _run_evolution(self, metrics_for_iteration, max_iterations=5): + """ + Run the controller against mocked per-iteration results. + + metrics_for_iteration is a callable mapping the iteration number to the + metrics dict carried by that iteration's child program. Returns + (controller, submit_mock) after run_evolution completes. + """ + + async def run_test(): + controller = ProcessParallelController(self.config, self.eval_file, self.database) + + def make_result(iteration, island_id): + future = MagicMock() + result = SerializableResult( + child_program_dict={ + "id": f"child_{iteration}", + "code": f"def evolved_{iteration}(): return {iteration}", + "language": "python", + "parent_id": f"test_{island_id}", + "generation": iteration, + "metrics": metrics_for_iteration(iteration), + "iteration_found": iteration, + "metadata": {"changes": "test", "island": island_id}, + }, + parent_id=f"test_{island_id}", + iteration_time=0.1, + iteration=iteration, + ) + future.done.return_value = True + future.result.return_value = result + future.cancel.return_value = True + return future + + with patch.object(controller, "_submit_iteration", side_effect=make_result): + mock_submit = controller._submit_iteration + + controller.start() + await controller.run_evolution( + start_iteration=1, max_iterations=max_iterations, target_score=None + ) + + return controller, mock_submit + + return asyncio.run(run_test()) + + def test_failed_evaluations_do_not_trigger_early_stop(self): + """Repeated evaluator failures (codelion's {"error": 1.0, ...} convention) + must not be counted as a plateau, and every iteration must still run""" + controller, mock_submit = self._run_evolution( + lambda i: {"combined_score": 0.0, "error": "syntax error"} + ) + + self.assertFalse(controller.early_stopping_triggered) + self.assertEqual(mock_submit.call_count, 5) + + def test_timeout_shaped_metrics_do_not_trigger_early_stop(self): + """The evaluator's timeout shape ({"error": 0.0, "timeout": True}) must + also be excluded from the plateau counter""" + controller, mock_submit = self._run_evolution(lambda i: {"error": 0.0, "timeout": True}) + + self.assertFalse(controller.early_stopping_triggered) + self.assertEqual(mock_submit.call_count, 5) + + def test_genuine_plateau_still_stops(self): + """Successful evaluations that plateau must still trigger early stopping""" + controller, _ = self._run_evolution(lambda i: {"combined_score": 0.5}) + + self.assertTrue(controller.early_stopping_triggered) + + def test_negative_best_not_poisoned_by_failures(self): + """After a real (negative) best score, failures scoring 0.0 must not look + like improvements that reset the patience counter""" + controller, mock_submit = self._run_evolution( + lambda i: ( + {"combined_score": -0.5} + if i == 1 + else {"combined_score": 0.0, "error": "runtime failure"} + ) + ) + + self.assertFalse(controller.early_stopping_triggered) + self.assertEqual(mock_submit.call_count, 5) + + +if __name__ == "__main__": + unittest.main()