MNT: Update tests to reflect server side changes

.github/workflows/test.yml

@@ -196,7 +196,10 @@ jobs:
 
     - name: Cleanup Docker setup
       if: always() && steps.clone-services.outcome == 'success'
+      working-directory: ./services
       run: |
+        docker compose --profile rest-api --profile minio --profile evaluation-engine down -v
+        cd ..
         sudo rm -rf services
 
     - name: Check for files left behind by test

openml/_config.py

@@ -69,7 +69,7 @@
 
 
 def _get_servers(mode: ServerMode) -> dict[APIVersion, dict[str, str | None]]:
-    if mode not in ServerMode:
+    if mode not in ServerMode._value2member_map_:
         raise ValueError(f'invalid mode="{mode}" allowed modes: {", ".join(list(ServerMode))}')
     return deepcopy(_SERVERS_REGISTRY[mode])
 
@@ -288,13 +288,16 @@ def set_api_version(
         api_version: APIVersion,
         fallback_api_version: APIVersion | None = None,
     ) -> None:
-        if api_version not in APIVersion:
+        if api_version not in APIVersion._value2member_map_:
             raise ValueError(
                 f'invalid api_version="{api_version}" '
                 f"allowed versions: {', '.join(list(APIVersion))}"
             )
 
-        if fallback_api_version is not None and fallback_api_version not in APIVersion:
+        if (
+            fallback_api_version is not None
+            and fallback_api_version not in APIVersion._value2member_map_
+        ):
             raise ValueError(
                 f'invalid fallback_api_version="{fallback_api_version}" '
                 f"allowed versions: {', '.join(list(APIVersion))}"

openml/datasets/functions.py

@@ -905,6 +905,29 @@ def edit_dataset(
     if not isinstance(data_id, int):
         raise TypeError(f"`data_id` must be of type `int`, not {type(data_id)}.")
 
+    if not any(
+        field
+        for field in [
+            description,
+            creator,
+            contributor,
+            collection_date,
+            language,
+            default_target_attribute,
+            ignore_attribute,
+            citation,
+            row_id_attribute,
+            original_data_url,
+            paper_url,
+        ]
+    ):
+        raise ValueError(
+            "Please provide atleast one field among description, creator, "
+            "contributor, collection_date, language, citation, "
+            "original_data_url, default_target_attribute, row_id_attribute, "
+            "ignore_attribute or paper_url to edit.",
+        )
+
     # compose data edit parameters as xml
     form_data = {"data_id": data_id}  # type: openml._api_calls.DATA_TYPE
     xml = OrderedDict()  # type: 'OrderedDict[str, OrderedDict]'

tests/test_datasets/test_dataset_functions.py

@@ -1414,32 +1414,9 @@ def test_get_dataset_cache_format_feather(self):
         assert len(categorical) == X.shape[1]
         assert len(attribute_names) == X.shape[1]
 
-    @pytest.mark.test_server()
-    def test_data_edit_non_critical_field(self):
-        # Case 1
-        # All users can edit non-critical fields of datasets
-        desc = (
-            "This data sets consists of 3 different types of irises' "
-            "(Setosa, Versicolour, and Virginica) petal and sepal length,"
-            " stored in a 150x4 numpy.ndarray"
-        )
-        did = 128
-        result = edit_dataset(
-            did,
-            description=desc,
-            creator="R.A.Fisher",
-            collection_date="1937",
-            citation="The use of multiple measurements in taxonomic problems",
-            language="English",
-        )
-        assert did == result
-        edited_dataset = openml.datasets.get_dataset(did)
-        assert edited_dataset.description == desc
-
     @pytest.mark.test_server()
     def test_data_edit_critical_field(self):
-        # Case 2
-        # only owners (or admin) can edit all critical fields of datasets
+        # only owners (or admin) can edit all fields of datasets
         # for this, we need to first clone a dataset to do changes
         did = fork_dataset(1)
         self._wait_for_dataset_being_processed(did)
@@ -1464,11 +1441,10 @@ def test_data_edit_critical_field(self):
                     os.path.join(openml.config.get_cache_directory(), "datasets", str(did)),
                 )
 
-    @pytest.mark.test_server()
     def test_data_edit_requires_field(self):
         # Check server exception when no field to edit is provided
         self.assertRaisesRegex(
-            OpenMLServerException,
+            ValueError,
             "Please provide atleast one field among description, creator, "
             "contributor, collection_date, language, citation, "
             "original_data_url, default_target_attribute, row_id_attribute, "
@@ -1516,12 +1492,12 @@ def test_data_edit_cannot_edit_critical_field_if_dataset_has_task(self):
         )
 
     @pytest.mark.test_server()
-    def test_edit_data_user_cannot_edit_critical_field_of_other_users_dataset(self):
+    def test_edit_data_user_cannot_edit_field_of_other_users_dataset(self):
         # Check server exception when a non-owner or non-admin tries to edit critical fields
         self.assertRaisesRegex(
             OpenMLServerException,
-            "Critical features default_target_attribute, row_id_attribute and ignore_attribute "
-            "can be edited only by the owner. Fork the dataset if changes are required.",
+            "Dataset can only be edited by the owner or an administrator. "
+            "Fork the dataset if changes are required.",
             edit_dataset,
             data_id=128,
             default_target_attribute="y",
@@ -2005,4 +1981,4 @@ def test_get_dataset_parquet(requests_mock, test_files_directory, test_server_v1
     assert dataset._parquet_url is not None
     assert dataset.parquet_file is not None
     assert os.path.isfile(dataset.parquet_file)
-    assert dataset.data_file is None  # is alias for arff path
+    assert dataset.data_file is None  # is alias for arff path

tests/test_evaluations/test_evaluation_functions.py

@@ -155,6 +155,7 @@ def test_evaluation_list_limit(self):
         )
         assert len(evaluations) == 100
 
+    @pytest.mark.skip(reason="Affected by lowered result limit #1757")
     @pytest.mark.test_server()
     def test_list_evaluations_empty(self):
         evaluations = openml.evaluations.list_evaluations("unexisting_measure")
@@ -163,13 +164,15 @@ def test_list_evaluations_empty(self):
 
         assert isinstance(evaluations, dict)
 
+
     @pytest.mark.production_server()
     def test_evaluation_list_per_fold(self):
         self.use_production_server()
         size = 1000
         task_ids = [6]
         uploader_ids = [1]
         flow_ids = [6969]
+        runs = [6070667, 6070666]
 
         evaluations = openml.evaluations.list_evaluations(
             "predictive_accuracy",
@@ -178,10 +181,11 @@ def test_evaluation_list_per_fold(self):
             tasks=task_ids,
             flows=flow_ids,
             uploaders=uploader_ids,
+            runs=runs,
             per_fold=True,
         )
 
-        assert len(evaluations) == size
+        assert len(evaluations) == len(runs)
         for run_id in evaluations:
             assert evaluations[run_id].value is None
             assert evaluations[run_id].values is not None
@@ -195,12 +199,14 @@ def test_evaluation_list_per_fold(self):
             tasks=task_ids,
             flows=flow_ids,
             uploaders=uploader_ids,
+            runs=runs,
             per_fold=False,
         )
         for run_id in evaluations:
             assert evaluations[run_id].value is not None
             assert evaluations[run_id].values is None
 
+    @pytest.mark.skip(reason="Affected by lowered result limit #1757")
     @pytest.mark.production_server()
     def test_evaluation_list_sort(self):
         self.use_production_server()

tests/test_evaluations/test_evaluations_example.py

@@ -1,11 +1,13 @@
 # License: BSD 3-Clause
 from __future__ import annotations
 
+import pytest
 import unittest
 
 import openml
 
 class TestEvaluationsExample(unittest.TestCase):
+    @pytest.mark.skip(reason="Affected by lowered result limit #1757")
     def test_example_python_paper(self):
         # Example script which will appear in the upcoming OpenML-Python paper
         # This test ensures that the example will keep running!

tests/test_runs/test_run.py

@@ -25,6 +25,7 @@ class TestRun(TestBase):
     # Splitting not helpful, these test's don't rely on the server and take
     # less than 1 seconds
 
+    @pytest.mark.skip(reason="Affected by lowered result limit #1757")
     @pytest.mark.test_server()
     def test_tagging(self):
         runs = openml.runs.list_runs(size=1)

tests/test_runs/test_run_functions.py

@@ -1227,6 +1227,7 @@ def test_initialize_model_from_run(self):
         assert flowS.components["Imputer"].parameters["strategy"] == '"most_frequent"'
         assert flowS.components["VarianceThreshold"].parameters["threshold"] == "0.05"
 
+    @pytest.mark.skip(reason="Affected by lowered result limit #1757")
     @pytest.mark.sklearn()
     @unittest.skipIf(
         Version(sklearn.__version__) < Version("0.20"),
@@ -1505,16 +1506,18 @@ def _check_run(self, run):
     def test_get_runs_list(self):
         # TODO: comes from live, no such lists on test
         self.use_production_server()
-        runs = openml.runs.list_runs(id=[2], display_errors=True)
+        runs = openml.runs.list_runs(
+                id=[2], display_errors=True, size=10)
         assert len(runs) == 1
         for run in runs.to_dict(orient="index").values():
             self._check_run(run)
 
     @pytest.mark.test_server()
     def test_list_runs_empty(self):
-        runs = openml.runs.list_runs(task=[0])
+        runs = openml.runs.list_runs(task=[0], size=1)
         assert runs.empty
 
+    @pytest.mark.skip(reason="Affected by lowered result limit #1757")
     @pytest.mark.production_server()
     def test_get_runs_list_by_task(self):
         # TODO: comes from live, no such lists on test
@@ -1533,7 +1536,8 @@ def test_get_runs_list_by_task(self):
         for run in runs.to_dict(orient="index").values():
             assert run["task_id"] in task_ids
             self._check_run(run)
-
+
+    @pytest.mark.skip(reason="Affected by lowered result limit #1757")
     @pytest.mark.production_server()
     def test_get_runs_list_by_uploader(self):
         # TODO: comes from live, no such lists on test
@@ -1556,6 +1560,7 @@ def test_get_runs_list_by_uploader(self):
             assert run["uploader"] in uploader_ids
             self._check_run(run)
 
+    @pytest.mark.skip(reason="Affected by lowered result limit #1757")
     @pytest.mark.production_server()
     def test_get_runs_list_by_flow(self):
         # TODO: comes from live, no such lists on test
@@ -1606,22 +1611,23 @@ def test_get_runs_list_by_filters(self):
         # self.assertRaises(openml.exceptions.OpenMLServerError,
         # openml.runs.list_runs)
 
-        runs = openml.runs.list_runs(id=ids)
+        runs = openml.runs.list_runs(id=ids, size=100)
         assert len(runs) == 2
 
-        runs = openml.runs.list_runs(task=tasks)
+        runs = openml.runs.list_runs(task=tasks, size=100)
         assert len(runs) >= 2
 
-        runs = openml.runs.list_runs(uploader=uploaders_2)
+        runs = openml.runs.list_runs(uploader=uploaders_2, size=100)
         assert len(runs) >= 10
 
-        runs = openml.runs.list_runs(flow=flows)
+        runs = openml.runs.list_runs(flow=flows, size=100)
         assert len(runs) >= 100
 
         runs = openml.runs.list_runs(
             id=ids,
             task=tasks,
             uploader=uploaders_1,
+            size=100,
         )
         assert len(runs) == 2
 
@@ -1831,6 +1837,7 @@ def test_format_prediction_task_regression(self):
         self.assertListEqual(res, [0] * 5)
 
 
+    @pytest.mark.skip(reason="Affected by lowered result limit #1757")
     @unittest.skipIf(
         Version(sklearn.__version__) < Version("0.20"),
         reason="SimpleImputer doesn't handle mixed type DataFrame as input",
@@ -2099,4 +2106,4 @@ def test_joblib_backends(parallel_mock, n_jobs, backend, call_count):
     # *_time_millis_* not recorded when n_jobs = -1
     assert len(res[2]["predictive_accuracy"][0]) == 10
     assert len(res[3]["predictive_accuracy"][0]) == 10
-    assert parallel_mock.call_count == call_count
+    assert parallel_mock.call_count == call_count

tests/test_study/test_study_functions.py

@@ -151,6 +151,7 @@ def test_publish_empty_study_explicit(self):
     def test_publish_empty_study_implicit(self):
         self._test_publish_empty_study_is_allowed(explicit=False)
 
+    @pytest.mark.skip(reason="Affected by lowered result limit #1757")
     @pytest.mark.flaky()
     @pytest.mark.test_server()
     def test_publish_study(self):