From 021fefd0be8a73f1827aeb65b67c1453d002c445 Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Mon, 5 Oct 2026 11:24:29 +0200 Subject: [PATCH 1/5] Update tests to reflect new restrictions of OpenML#1290 --- openml/datasets/functions.py | 23 ++++++++++++ tests/test_datasets/test_dataset_functions.py | 36 ++++--------------- 2 files changed, 29 insertions(+), 30 deletions(-) diff --git a/openml/datasets/functions.py b/openml/datasets/functions.py index 7faaf5693..941235eb2 100644 --- a/openml/datasets/functions.py +++ b/openml/datasets/functions.py @@ -905,6 +905,29 @@ def edit_dataset( if not isinstance(data_id, int): raise TypeError(f"`data_id` must be of type `int`, not {type(data_id)}.") + if not any( + field + for field in [ + description, + creator, + contributor, + collection_date, + language, + default_target_attribute, + ignore_attribute, + citation, + row_id_attribute, + original_data_url, + paper_url, + ] + ): + raise ValueError( + "Please provide atleast one field among description, creator, " + "contributor, collection_date, language, citation, " + "original_data_url, default_target_attribute, row_id_attribute, " + "ignore_attribute or paper_url to edit.", + ) + # compose data edit parameters as xml form_data = {"data_id": data_id} # type: openml._api_calls.DATA_TYPE xml = OrderedDict() # type: 'OrderedDict[str, OrderedDict]' diff --git a/tests/test_datasets/test_dataset_functions.py b/tests/test_datasets/test_dataset_functions.py index 80b0b4215..dff4abf00 100644 --- a/tests/test_datasets/test_dataset_functions.py +++ b/tests/test_datasets/test_dataset_functions.py @@ -1414,32 +1414,9 @@ def test_get_dataset_cache_format_feather(self): assert len(categorical) == X.shape[1] assert len(attribute_names) == X.shape[1] - @pytest.mark.test_server() - def test_data_edit_non_critical_field(self): - # Case 1 - # All users can edit non-critical fields of datasets - desc = ( - "This data sets consists of 3 different types of irises' " - "(Setosa, Versicolour, and Virginica) petal and sepal length," - " stored in a 150x4 numpy.ndarray" - ) - did = 128 - result = edit_dataset( - did, - description=desc, - creator="R.A.Fisher", - collection_date="1937", - citation="The use of multiple measurements in taxonomic problems", - language="English", - ) - assert did == result - edited_dataset = openml.datasets.get_dataset(did) - assert edited_dataset.description == desc - @pytest.mark.test_server() def test_data_edit_critical_field(self): - # Case 2 - # only owners (or admin) can edit all critical fields of datasets + # only owners (or admin) can edit all fields of datasets # for this, we need to first clone a dataset to do changes did = fork_dataset(1) self._wait_for_dataset_being_processed(did) @@ -1464,11 +1441,10 @@ def test_data_edit_critical_field(self): os.path.join(openml.config.get_cache_directory(), "datasets", str(did)), ) - @pytest.mark.test_server() def test_data_edit_requires_field(self): # Check server exception when no field to edit is provided self.assertRaisesRegex( - OpenMLServerException, + ValueError, "Please provide atleast one field among description, creator, " "contributor, collection_date, language, citation, " "original_data_url, default_target_attribute, row_id_attribute, " @@ -1516,12 +1492,12 @@ def test_data_edit_cannot_edit_critical_field_if_dataset_has_task(self): ) @pytest.mark.test_server() - def test_edit_data_user_cannot_edit_critical_field_of_other_users_dataset(self): + def test_edit_data_user_cannot_edit_field_of_other_users_dataset(self): # Check server exception when a non-owner or non-admin tries to edit critical fields self.assertRaisesRegex( OpenMLServerException, - "Critical features default_target_attribute, row_id_attribute and ignore_attribute " - "can be edited only by the owner. Fork the dataset if changes are required.", + "Dataset can only be edited by the owner or an administrator. " + "Fork the dataset if changes are required.", edit_dataset, data_id=128, default_target_attribute="y", @@ -2005,4 +1981,4 @@ def test_get_dataset_parquet(requests_mock, test_files_directory, test_server_v1 assert dataset._parquet_url is not None assert dataset.parquet_file is not None assert os.path.isfile(dataset.parquet_file) - assert dataset.data_file is None # is alias for arff path \ No newline at end of file + assert dataset.data_file is None # is alias for arff path From e3c6c67df83462f5438f81b471f485597ff13933 Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Mon, 5 Oct 2026 12:23:19 +0200 Subject: [PATCH 2/5] Update or disable tests affected by new server limit #1757 --- .../test_evaluation_functions.py | 2 ++ .../test_evaluations_example.py | 2 ++ tests/test_runs/test_run.py | 1 + tests/test_runs/test_run_functions.py | 23 ++++++++++++------- tests/test_study/test_study_functions.py | 1 + 5 files changed, 21 insertions(+), 8 deletions(-) diff --git a/tests/test_evaluations/test_evaluation_functions.py b/tests/test_evaluations/test_evaluation_functions.py index e15556d7b..862881175 100644 --- a/tests/test_evaluations/test_evaluation_functions.py +++ b/tests/test_evaluations/test_evaluation_functions.py @@ -155,6 +155,7 @@ def test_evaluation_list_limit(self): ) assert len(evaluations) == 100 + @pytest.mark.skip(reason="Affected by lowered result limit #1757") @pytest.mark.test_server() def test_list_evaluations_empty(self): evaluations = openml.evaluations.list_evaluations("unexisting_measure") @@ -201,6 +202,7 @@ def test_evaluation_list_per_fold(self): assert evaluations[run_id].value is not None assert evaluations[run_id].values is None + @pytest.mark.skip(reason="Affected by lowered result limit #1757") @pytest.mark.production_server() def test_evaluation_list_sort(self): self.use_production_server() diff --git a/tests/test_evaluations/test_evaluations_example.py b/tests/test_evaluations/test_evaluations_example.py index b321f475d..eceea3baa 100644 --- a/tests/test_evaluations/test_evaluations_example.py +++ b/tests/test_evaluations/test_evaluations_example.py @@ -1,11 +1,13 @@ # License: BSD 3-Clause from __future__ import annotations +import pytest import unittest import openml class TestEvaluationsExample(unittest.TestCase): + @pytest.mark.skip(reason="Affected by lowered result limit #1757") def test_example_python_paper(self): # Example script which will appear in the upcoming OpenML-Python paper # This test ensures that the example will keep running! diff --git a/tests/test_runs/test_run.py b/tests/test_runs/test_run.py index 22a8bc936..e3f50b030 100644 --- a/tests/test_runs/test_run.py +++ b/tests/test_runs/test_run.py @@ -25,6 +25,7 @@ class TestRun(TestBase): # Splitting not helpful, these test's don't rely on the server and take # less than 1 seconds + @pytest.mark.skip(reason="Affected by lowered result limit #1757") @pytest.mark.test_server() def test_tagging(self): runs = openml.runs.list_runs(size=1) diff --git a/tests/test_runs/test_run_functions.py b/tests/test_runs/test_run_functions.py index 38128de30..a3b76fcf4 100644 --- a/tests/test_runs/test_run_functions.py +++ b/tests/test_runs/test_run_functions.py @@ -1227,6 +1227,7 @@ def test_initialize_model_from_run(self): assert flowS.components["Imputer"].parameters["strategy"] == '"most_frequent"' assert flowS.components["VarianceThreshold"].parameters["threshold"] == "0.05" + @pytest.mark.skip(reason="Affected by lowered result limit #1757") @pytest.mark.sklearn() @unittest.skipIf( Version(sklearn.__version__) < Version("0.20"), @@ -1505,16 +1506,18 @@ def _check_run(self, run): def test_get_runs_list(self): # TODO: comes from live, no such lists on test self.use_production_server() - runs = openml.runs.list_runs(id=[2], display_errors=True) + runs = openml.runs.list_runs( + id=[2], display_errors=True, size=10) assert len(runs) == 1 for run in runs.to_dict(orient="index").values(): self._check_run(run) @pytest.mark.test_server() def test_list_runs_empty(self): - runs = openml.runs.list_runs(task=[0]) + runs = openml.runs.list_runs(task=[0], size=1) assert runs.empty + @pytest.mark.skip(reason="Affected by lowered result limit #1757") @pytest.mark.production_server() def test_get_runs_list_by_task(self): # TODO: comes from live, no such lists on test @@ -1533,7 +1536,8 @@ def test_get_runs_list_by_task(self): for run in runs.to_dict(orient="index").values(): assert run["task_id"] in task_ids self._check_run(run) - + + @pytest.mark.skip(reason="Affected by lowered result limit #1757") @pytest.mark.production_server() def test_get_runs_list_by_uploader(self): # TODO: comes from live, no such lists on test @@ -1556,6 +1560,7 @@ def test_get_runs_list_by_uploader(self): assert run["uploader"] in uploader_ids self._check_run(run) + @pytest.mark.skip(reason="Affected by lowered result limit #1757") @pytest.mark.production_server() def test_get_runs_list_by_flow(self): # TODO: comes from live, no such lists on test @@ -1606,22 +1611,23 @@ def test_get_runs_list_by_filters(self): # self.assertRaises(openml.exceptions.OpenMLServerError, # openml.runs.list_runs) - runs = openml.runs.list_runs(id=ids) + runs = openml.runs.list_runs(id=ids, size=100) assert len(runs) == 2 - runs = openml.runs.list_runs(task=tasks) + runs = openml.runs.list_runs(task=tasks, size=100) assert len(runs) >= 2 - runs = openml.runs.list_runs(uploader=uploaders_2) + runs = openml.runs.list_runs(uploader=uploaders_2, size=100) assert len(runs) >= 10 - runs = openml.runs.list_runs(flow=flows) + runs = openml.runs.list_runs(flow=flows, size=100) assert len(runs) >= 100 runs = openml.runs.list_runs( id=ids, task=tasks, uploader=uploaders_1, + size=100, ) assert len(runs) == 2 @@ -1831,6 +1837,7 @@ def test_format_prediction_task_regression(self): self.assertListEqual(res, [0] * 5) + @pytest.mark.skip(reason="Affected by lowered result limit #1757") @unittest.skipIf( Version(sklearn.__version__) < Version("0.20"), reason="SimpleImputer doesn't handle mixed type DataFrame as input", @@ -2099,4 +2106,4 @@ def test_joblib_backends(parallel_mock, n_jobs, backend, call_count): # *_time_millis_* not recorded when n_jobs = -1 assert len(res[2]["predictive_accuracy"][0]) == 10 assert len(res[3]["predictive_accuracy"][0]) == 10 - assert parallel_mock.call_count == call_count \ No newline at end of file + assert parallel_mock.call_count == call_count diff --git a/tests/test_study/test_study_functions.py b/tests/test_study/test_study_functions.py index 7dc6b6d2a..d9a51a12a 100644 --- a/tests/test_study/test_study_functions.py +++ b/tests/test_study/test_study_functions.py @@ -151,6 +151,7 @@ def test_publish_empty_study_explicit(self): def test_publish_empty_study_implicit(self): self._test_publish_empty_study_is_allowed(explicit=False) + @pytest.mark.skip(reason="Affected by lowered result limit #1757") @pytest.mark.flaky() @pytest.mark.test_server() def test_publish_study(self): From b6b95cd12245e426d7f3b84d5c959b9b2efce583 Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Mon, 5 Oct 2026 12:23:50 +0200 Subject: [PATCH 3/5] Use python 3.10/3.11 compatible enum lookup --- openml/_config.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/openml/_config.py b/openml/_config.py index d6850b7b0..ff67eaa1d 100644 --- a/openml/_config.py +++ b/openml/_config.py @@ -69,7 +69,7 @@ def _get_servers(mode: ServerMode) -> dict[APIVersion, dict[str, str | None]]: - if mode not in ServerMode: + if mode not in ServerMode._value2member_map_: raise ValueError(f'invalid mode="{mode}" allowed modes: {", ".join(list(ServerMode))}') return deepcopy(_SERVERS_REGISTRY[mode]) @@ -288,13 +288,16 @@ def set_api_version( api_version: APIVersion, fallback_api_version: APIVersion | None = None, ) -> None: - if api_version not in APIVersion: + if api_version not in APIVersion._value2member_map_: raise ValueError( f'invalid api_version="{api_version}" ' f"allowed versions: {', '.join(list(APIVersion))}" ) - if fallback_api_version is not None and fallback_api_version not in APIVersion: + if ( + fallback_api_version is not None + and fallback_api_version not in APIVersion._value2member_map_ + ): raise ValueError( f'invalid fallback_api_version="{fallback_api_version}" ' f"allowed versions: {', '.join(list(APIVersion))}" From 2fb632d7d4a66823a9c1fe1541d2a93c5c961bc7 Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Mon, 5 Oct 2026 12:31:58 +0200 Subject: [PATCH 4/5] Make query more constrained to avoid new server side limits --- tests/test_evaluations/test_evaluation_functions.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/test_evaluations/test_evaluation_functions.py b/tests/test_evaluations/test_evaluation_functions.py index 862881175..8639dd657 100644 --- a/tests/test_evaluations/test_evaluation_functions.py +++ b/tests/test_evaluations/test_evaluation_functions.py @@ -164,6 +164,7 @@ def test_list_evaluations_empty(self): assert isinstance(evaluations, dict) + @pytest.mark.production_server() def test_evaluation_list_per_fold(self): self.use_production_server() @@ -171,6 +172,7 @@ def test_evaluation_list_per_fold(self): task_ids = [6] uploader_ids = [1] flow_ids = [6969] + runs = [6070667, 6070666] evaluations = openml.evaluations.list_evaluations( "predictive_accuracy", @@ -179,10 +181,11 @@ def test_evaluation_list_per_fold(self): tasks=task_ids, flows=flow_ids, uploaders=uploader_ids, + runs=runs, per_fold=True, ) - assert len(evaluations) == size + assert len(evaluations) == len(runs) for run_id in evaluations: assert evaluations[run_id].value is None assert evaluations[run_id].values is not None @@ -196,6 +199,7 @@ def test_evaluation_list_per_fold(self): tasks=task_ids, flows=flow_ids, uploaders=uploader_ids, + runs=runs, per_fold=False, ) for run_id in evaluations: From 434c0f7edf30816dbe54c6bd7e23d2c303a62b30 Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Mon, 5 Oct 2026 14:24:40 +0200 Subject: [PATCH 5/5] Shut down services before cleaning up files --- .github/workflows/test.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index dc0995fc6..574096f2a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -196,7 +196,10 @@ jobs: - name: Cleanup Docker setup if: always() && steps.clone-services.outcome == 'success' + working-directory: ./services run: | + docker compose --profile rest-api --profile minio --profile evaluation-engine down -v + cd .. sudo rm -rf services - name: Check for files left behind by test