diff --git a/configs/early_stopping_example.yaml b/configs/early_stopping_example.yaml index c7259ea7f..0a347babf 100644 --- a/configs/early_stopping_example.yaml +++ b/configs/early_stopping_example.yaml @@ -10,6 +10,11 @@ log_level: "INFO" early_stopping_patience: 30 # Stop after 30 iterations without improvement convergence_threshold: 0.01 # Minimum improvement of 0.01 required to reset patience early_stopping_metric: "combined_score" # Track the combined_score metric +# +# Note: iterations whose evaluation fails or times out (metrics carrying an +# "error" or "timeout" key, e.g. {"error": 1.0, "combined_score": 0.0}) do not +# count toward the patience counter. Early stopping only triggers on successful +# evaluations that plateau - repeated failures alone never stop the run. # LLM configuration llm: diff --git a/openevolve/process_parallel.py b/openevolve/process_parallel.py index 06eb0f09f..6241072d4 100644 --- a/openevolve/process_parallel.py +++ b/openevolve/process_parallel.py @@ -879,59 +879,80 @@ async def run_evolution( # Check early stopping if early_stopping_enabled and child_program.metrics: - # Get the metric to track for early stopping - current_score = None - if self.config.early_stopping_metric in child_program.metrics: - current_score = child_program.metrics[self.config.early_stopping_metric] - elif self.config.early_stopping_metric == "combined_score": - # Default metric not found, use safe average (standard pattern) - current_score = safe_numeric_average(child_program.metrics) - else: - # User specified a custom metric that doesn't exist - logger.warning( - f"Early stopping metric '{self.config.early_stopping_metric}' not found, using safe numeric average" + if child_program.metrics.get( + "error" + ) is not None or child_program.metrics.get("timeout"): + # Failed or timed-out evaluations carry no score. Letting them + # into the counter below would poison the best-score baseline + # with 0.0 on the first failure and then count every subsequent + # failure as "no improvement", so a run that never succeeded + # once would exit through the successful-plateau path + # (issue #294). Skip them: early stopping only counts + # evaluations that actually produced a score. The "error" check + # is presence-based (is not None) because evaluators signal + # failures both with numeric ({"error": 0.0}) and string values. + logger.debug( + "Skipping failed evaluation from early stopping count at iteration %s", + completed_iteration, ) - current_score = safe_numeric_average(child_program.metrics) - - if current_score is not None and isinstance(current_score, (int, float)): - # Check for improvement - if self.config.early_stopping_patience > 0: - improvement = current_score - best_score - if improvement >= self.config.convergence_threshold: - best_score = current_score - iterations_without_improvement = 0 - logger.debug( - f"New best score: {best_score:.4f} (improvement: {improvement:+.4f})" - ) - else: - iterations_without_improvement += 1 - logger.debug( - f"No improvement: {iterations_without_improvement}/{self.config.early_stopping_patience}" - ) - - # Check if we should stop - if ( - iterations_without_improvement - >= self.config.early_stopping_patience - ): - self.early_stopping_triggered = True - logger.info( - f"🛑 Early stopping triggered at iteration {completed_iteration}: " - f"No improvement for {iterations_without_improvement} iterations " - f"(best score: {best_score:.4f})" - ) - break - + else: + # Get the metric to track for early stopping + current_score = None + if self.config.early_stopping_metric in child_program.metrics: + current_score = child_program.metrics[ + self.config.early_stopping_metric + ] + elif self.config.early_stopping_metric == "combined_score": + # Default metric not found, use safe average (standard pattern) + current_score = safe_numeric_average(child_program.metrics) else: - # Event-based early stopping - if current_score == self.config.convergence_threshold: - best_score = current_score - logger.info( - f"🛑 Early stopping (event-based) triggered at iteration {completed_iteration}: " - f"Task successfully solved with score {best_score:.4f}." - ) - self.early_stopping_triggered = True - break + # User specified a custom metric that doesn't exist + logger.warning( + f"Early stopping metric '{self.config.early_stopping_metric}' not found, using safe numeric average" + ) + current_score = safe_numeric_average(child_program.metrics) + + if current_score is not None and isinstance( + current_score, (int, float) + ): + # Check for improvement + if self.config.early_stopping_patience > 0: + improvement = current_score - best_score + if improvement >= self.config.convergence_threshold: + best_score = current_score + iterations_without_improvement = 0 + logger.debug( + f"New best score: {best_score:.4f} (improvement: {improvement:+.4f})" + ) + else: + iterations_without_improvement += 1 + logger.debug( + f"No improvement: {iterations_without_improvement}/{self.config.early_stopping_patience}" + ) + + # Check if we should stop + if ( + iterations_without_improvement + >= self.config.early_stopping_patience + ): + self.early_stopping_triggered = True + logger.info( + f"🛑 Early stopping triggered at iteration {completed_iteration}: " + f"No improvement for {iterations_without_improvement} iterations " + f"(best score: {best_score:.4f})" + ) + break + + else: + # Event-based early stopping + if current_score == self.config.convergence_threshold: + best_score = current_score + logger.info( + f"🛑 Early stopping (event-based) triggered at iteration {completed_iteration}: " + f"Task successfully solved with score {best_score:.4f}." + ) + self.early_stopping_triggered = True + break except _CandidateRejected as rejected: logger.info("Admission strategy rejected program %s", rejected) diff --git a/tests/test_early_stopping_failures.py b/tests/test_early_stopping_failures.py new file mode 100644 index 000000000..731e0081f --- /dev/null +++ b/tests/test_early_stopping_failures.py @@ -0,0 +1,160 @@ +""" +Tests for early stopping interaction with failed evaluations (issue #294) + +Failed or timed-out evaluations surface as metrics carrying an "error" or +"timeout" key. Those are not scores: they must not feed the early-stopping +plateau counter, otherwise a run that never succeeds once can exit through +the "successful evaluations plateaued" path. +""" + +import asyncio +import os +import tempfile +import unittest +from unittest.mock import MagicMock, patch + +# Set dummy API key for testing +os.environ["OPENAI_API_KEY"] = "test" + +from openevolve.config import Config +from openevolve.database import Program, ProgramDatabase +from openevolve.process_parallel import ProcessParallelController, SerializableResult + + +class TestEarlyStoppingWithFailedEvaluations(unittest.TestCase): + """Early stopping must only count evaluations that produced a score""" + + def setUp(self): + """Set up test environment""" + self.test_dir = tempfile.mkdtemp() + + # Create test config + self.config = Config() + self.config.max_iterations = 10 + self.config.evaluator.parallel_evaluations = 2 + self.config.evaluator.timeout = 10 + # One island per test program so each owns its MAP-Elites cell and none is + # displaced (MAP-Elites removes programs displaced from their cell). + self.config.database.num_islands = 3 + self.config.database.in_memory = True + self.config.checkpoint_interval = 5 + # Issue #294 scenario: early stopping enabled, reporter's config only set + # patience (convergence_threshold stays at its default of 0.001). + self.config.early_stopping_patience = 3 + + # Create test evaluation file + self.eval_content = """ +def evaluate(program_path): + return {"score": 0.5, "performance": 0.6} +""" + self.eval_file = os.path.join(self.test_dir, "evaluator.py") + with open(self.eval_file, "w") as f: + f.write(self.eval_content) + + # Create test database + self.database = ProgramDatabase(self.config.database) + + # Add some test programs, one per island so each survives as its cell owner + for i in range(3): + program = Program( + id=f"test_{i}", + code=f"def func_{i}(): return {i}", + language="python", + metrics={"score": 0.5 + i * 0.1, "performance": 0.4 + i * 0.1}, + iteration_found=0, + ) + self.database.add(program, target_island=i) + + def tearDown(self): + """Clean up test environment""" + import shutil + + shutil.rmtree(self.test_dir, ignore_errors=True) + + def _run_evolution(self, metrics_for_iteration, max_iterations=5): + """ + Run the controller against mocked per-iteration results. + + metrics_for_iteration is a callable mapping the iteration number to the + metrics dict carried by that iteration's child program. Returns + (controller, submit_mock) after run_evolution completes. + """ + + async def run_test(): + controller = ProcessParallelController(self.config, self.eval_file, self.database) + + def make_result(iteration, island_id): + future = MagicMock() + result = SerializableResult( + child_program_dict={ + "id": f"child_{iteration}", + "code": f"def evolved_{iteration}(): return {iteration}", + "language": "python", + "parent_id": f"test_{island_id}", + "generation": iteration, + "metrics": metrics_for_iteration(iteration), + "iteration_found": iteration, + "metadata": {"changes": "test", "island": island_id}, + }, + parent_id=f"test_{island_id}", + iteration_time=0.1, + iteration=iteration, + ) + future.done.return_value = True + future.result.return_value = result + future.cancel.return_value = True + return future + + with patch.object(controller, "_submit_iteration", side_effect=make_result): + mock_submit = controller._submit_iteration + + controller.start() + await controller.run_evolution( + start_iteration=1, max_iterations=max_iterations, target_score=None + ) + + return controller, mock_submit + + return asyncio.run(run_test()) + + def test_failed_evaluations_do_not_trigger_early_stop(self): + """Repeated evaluator failures (codelion's {"error": 1.0, ...} convention) + must not be counted as a plateau, and every iteration must still run""" + controller, mock_submit = self._run_evolution( + lambda i: {"combined_score": 0.0, "error": "syntax error"} + ) + + self.assertFalse(controller.early_stopping_triggered) + self.assertEqual(mock_submit.call_count, 5) + + def test_timeout_shaped_metrics_do_not_trigger_early_stop(self): + """The evaluator's timeout shape ({"error": 0.0, "timeout": True}) must + also be excluded from the plateau counter""" + controller, mock_submit = self._run_evolution(lambda i: {"error": 0.0, "timeout": True}) + + self.assertFalse(controller.early_stopping_triggered) + self.assertEqual(mock_submit.call_count, 5) + + def test_genuine_plateau_still_stops(self): + """Successful evaluations that plateau must still trigger early stopping""" + controller, _ = self._run_evolution(lambda i: {"combined_score": 0.5}) + + self.assertTrue(controller.early_stopping_triggered) + + def test_negative_best_not_poisoned_by_failures(self): + """After a real (negative) best score, failures scoring 0.0 must not look + like improvements that reset the patience counter""" + controller, mock_submit = self._run_evolution( + lambda i: ( + {"combined_score": -0.5} + if i == 1 + else {"combined_score": 0.0, "error": "runtime failure"} + ) + ) + + self.assertFalse(controller.early_stopping_triggered) + self.assertEqual(mock_submit.call_count, 5) + + +if __name__ == "__main__": + unittest.main()