From cfc760c7f43447f495e68dfb03bbef957aef3441 Mon Sep 17 00:00:00 2001 From: Kevin Lian Date: Thu, 15 Oct 2026 20:56:59 +0000 Subject: [PATCH 1/3] Add data_quality_monitors bundle resource (direct engine) Adds the data-quality/v1 Monitor API as a direct-engine bundle resource, alongside the legacy quality_monitors resource it supersedes. Co-authored-by: Isaac --- .nextchanges/bundles/data-quality-monitors.md | 1 + .../bind/data_quality_monitor/databricks.yml | 12 + .../bind/data_quality_monitor/out.test.toml | 2 + .../bind/data_quality_monitor/output.txt | 32 ++ .../bind/data_quality_monitor/script | 8 + .../bind/data_quality_monitor/test.toml | 20 + .../bundle/invariant/auto-migrate/test.toml | 2 + .../configs/data_quality_monitor.yml.tmpl | 16 + .../invariant/continue_293/out.test.toml | 1 + .../invariant/delete_idempotent/out.test.toml | 1 + .../destroy_idempotent/out.test.toml | 1 + acceptance/bundle/invariant/migrate/test.toml | 2 + .../bundle/invariant/no_drift/out.test.toml | 1 + acceptance/bundle/invariant/test.toml | 4 + .../databricks.yml | 19 + .../data_quality_monitors-support/mutators.py | 13 + .../out.test.toml | 3 + .../data_quality_monitors-support/output.txt | 35 ++ .../resources.py | 20 + .../data_quality_monitors-support/script | 5 + .../data_quality_monitors-support/test.toml | 4 + acceptance/bundle/refschema/out.fields.txt | 55 ++ .../basic/databricks.yml | 16 + .../data_quality_monitors/basic/out.test.toml | 2 + .../data_quality_monitors/basic/output.txt | 101 ++++ .../data_quality_monitors/basic/script | 27 + .../data_quality_monitors/basic/test.toml | 7 + .../data_profiling/databricks.yml | 15 + .../data_profiling/out.test.toml | 2 + .../data_profiling/output.txt | 88 +++ .../data_profiling/script | 21 + .../data_profiling/test.toml | 7 + .../lifecycle/databricks.yml.tmpl | 14 + .../lifecycle/out.test.toml | 2 + .../lifecycle/output.txt | 35 ++ .../data_quality_monitors/lifecycle/script | 16 + .../data_quality_monitors/lifecycle/test.toml | 8 + .../remote-delete/databricks.yml | 12 + .../remote-delete/out.test.toml | 2 + .../remote-delete/output.txt | 20 + .../remote-delete/script | 9 + .../remote-delete/test.toml | 9 + .../resources/data_quality_monitors/test.toml | 5 + .../apply_bundle_permissions_test.go | 1 + .../mutator/resourcemutator/apply_presets.go | 12 + .../resourcemutator/apply_target_mode_test.go | 16 + .../config/mutator/resourcemutator/run_as.go | 9 + .../mutator/resourcemutator/run_as_test.go | 1 + bundle/config/resources.go | 3 + .../config/resources/data_quality_monitor.go | 85 +++ bundle/config/resources_test.go | 23 +- bundle/direct/dresources/all.go | 1 + bundle/direct/dresources/all_test.go | 11 + .../direct/dresources/apitypes.generated.yml | 2 + .../data_quality_monitors.generated.yml | 21 + .../configs/data_quality_monitors.yml | 21 + .../direct/dresources/data_quality_monitor.go | 74 +++ bundle/internal/schema/annotations.yml | 28 + .../validation/generated/enum_fields.go | 7 + .../validation/generated/required_fields.go | 7 + bundle/schema/jsonschema.json | 523 ++++++++++++++++++ bundle/statemgmt/state_load_test.go | 30 + libs/testserver/data_quality_monitors.go | 85 +++ libs/testserver/fake_workspace.go | 3 + libs/testserver/handlers.go | 18 + python/databricks/bundles/core/__init__.py | 2 + .../bundles/core/_generated/__init__.py | 8 + .../core/_generated/data_quality_monitors.py | 128 +++++ .../bundles/data_quality_monitors/__init__.py | 112 ++++ .../_models/aggregation_granularity.py | 38 ++ .../_models/anomaly_detection_config.py | 46 ++ .../_models/cron_schedule.py | 56 ++ .../_models/data_profiling_config.py | 200 +++++++ .../_models/data_profiling_custom_metric.py | 90 +++ .../data_profiling_custom_metric_type.py | 28 + .../_models/data_quality_monitor.py | 113 ++++ .../_models/inference_log_config.py | 104 ++++ .../_models/inference_problem_type.py | 21 + .../_models/lifecycle.py | 40 ++ .../_models/notification_destination.py | 42 ++ .../_models/notification_settings.py | 46 ++ .../_models/snapshot_config.py | 31 ++ .../_models/time_series_config.py | 56 ++ .../core/_generated/__init__.py | 2 + .../core/_generated/data_quality_monitors.py | 41 ++ python/databricks_tests/core/public_api.txt | 8 + ...tabricks.bundles.data_quality_monitors.rst | 11 + python/docs/index.rst | 1 + 88 files changed, 2878 insertions(+), 2 deletions(-) create mode 100644 .nextchanges/bundles/data-quality-monitors.md create mode 100644 acceptance/bundle/deployment/bind/data_quality_monitor/databricks.yml create mode 100644 acceptance/bundle/deployment/bind/data_quality_monitor/out.test.toml create mode 100644 acceptance/bundle/deployment/bind/data_quality_monitor/output.txt create mode 100644 acceptance/bundle/deployment/bind/data_quality_monitor/script create mode 100644 acceptance/bundle/deployment/bind/data_quality_monitor/test.toml create mode 100644 acceptance/bundle/invariant/configs/data_quality_monitor.yml.tmpl create mode 100644 acceptance/bundle/python/data_quality_monitors-support/databricks.yml create mode 100644 acceptance/bundle/python/data_quality_monitors-support/mutators.py create mode 100644 acceptance/bundle/python/data_quality_monitors-support/out.test.toml create mode 100644 acceptance/bundle/python/data_quality_monitors-support/output.txt create mode 100644 acceptance/bundle/python/data_quality_monitors-support/resources.py create mode 100644 acceptance/bundle/python/data_quality_monitors-support/script create mode 100644 acceptance/bundle/python/data_quality_monitors-support/test.toml create mode 100644 acceptance/bundle/resources/data_quality_monitors/basic/databricks.yml create mode 100644 acceptance/bundle/resources/data_quality_monitors/basic/out.test.toml create mode 100644 acceptance/bundle/resources/data_quality_monitors/basic/output.txt create mode 100644 acceptance/bundle/resources/data_quality_monitors/basic/script create mode 100644 acceptance/bundle/resources/data_quality_monitors/basic/test.toml create mode 100644 acceptance/bundle/resources/data_quality_monitors/data_profiling/databricks.yml create mode 100644 acceptance/bundle/resources/data_quality_monitors/data_profiling/out.test.toml create mode 100644 acceptance/bundle/resources/data_quality_monitors/data_profiling/output.txt create mode 100644 acceptance/bundle/resources/data_quality_monitors/data_profiling/script create mode 100644 acceptance/bundle/resources/data_quality_monitors/data_profiling/test.toml create mode 100644 acceptance/bundle/resources/data_quality_monitors/lifecycle/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/data_quality_monitors/lifecycle/out.test.toml create mode 100644 acceptance/bundle/resources/data_quality_monitors/lifecycle/output.txt create mode 100644 acceptance/bundle/resources/data_quality_monitors/lifecycle/script create mode 100644 acceptance/bundle/resources/data_quality_monitors/lifecycle/test.toml create mode 100644 acceptance/bundle/resources/data_quality_monitors/remote-delete/databricks.yml create mode 100644 acceptance/bundle/resources/data_quality_monitors/remote-delete/out.test.toml create mode 100644 acceptance/bundle/resources/data_quality_monitors/remote-delete/output.txt create mode 100644 acceptance/bundle/resources/data_quality_monitors/remote-delete/script create mode 100644 acceptance/bundle/resources/data_quality_monitors/remote-delete/test.toml create mode 100644 acceptance/bundle/resources/data_quality_monitors/test.toml create mode 100644 bundle/config/resources/data_quality_monitor.go create mode 100644 bundle/direct/dresources/configs/data_quality_monitors.generated.yml create mode 100644 bundle/direct/dresources/configs/data_quality_monitors.yml create mode 100644 bundle/direct/dresources/data_quality_monitor.go create mode 100644 libs/testserver/data_quality_monitors.go create mode 100644 python/databricks/bundles/core/_generated/data_quality_monitors.py create mode 100644 python/databricks/bundles/data_quality_monitors/__init__.py create mode 100644 python/databricks/bundles/data_quality_monitors/_models/aggregation_granularity.py create mode 100644 python/databricks/bundles/data_quality_monitors/_models/anomaly_detection_config.py create mode 100644 python/databricks/bundles/data_quality_monitors/_models/cron_schedule.py create mode 100644 python/databricks/bundles/data_quality_monitors/_models/data_profiling_config.py create mode 100644 python/databricks/bundles/data_quality_monitors/_models/data_profiling_custom_metric.py create mode 100644 python/databricks/bundles/data_quality_monitors/_models/data_profiling_custom_metric_type.py create mode 100644 python/databricks/bundles/data_quality_monitors/_models/data_quality_monitor.py create mode 100644 python/databricks/bundles/data_quality_monitors/_models/inference_log_config.py create mode 100644 python/databricks/bundles/data_quality_monitors/_models/inference_problem_type.py create mode 100644 python/databricks/bundles/data_quality_monitors/_models/lifecycle.py create mode 100644 python/databricks/bundles/data_quality_monitors/_models/notification_destination.py create mode 100644 python/databricks/bundles/data_quality_monitors/_models/notification_settings.py create mode 100644 python/databricks/bundles/data_quality_monitors/_models/snapshot_config.py create mode 100644 python/databricks/bundles/data_quality_monitors/_models/time_series_config.py create mode 100644 python/databricks_tests/core/_generated/data_quality_monitors.py create mode 100644 python/docs/databricks.bundles.data_quality_monitors.rst diff --git a/.nextchanges/bundles/data-quality-monitors.md b/.nextchanges/bundles/data-quality-monitors.md new file mode 100644 index 00000000000..de03ece4904 --- /dev/null +++ b/.nextchanges/bundles/data-quality-monitors.md @@ -0,0 +1 @@ +* Add the `data_quality_monitors` resource for data quality monitors on Unity Catalog schemas and tables (direct engine only). ([#PR_NUMBER](https://github.com/databricks/cli/pull/PR_NUMBER)) diff --git a/acceptance/bundle/deployment/bind/data_quality_monitor/databricks.yml b/acceptance/bundle/deployment/bind/data_quality_monitor/databricks.yml new file mode 100644 index 00000000000..45d1cb4612d --- /dev/null +++ b/acceptance/bundle/deployment/bind/data_quality_monitor/databricks.yml @@ -0,0 +1,12 @@ +bundle: + name: test-bundle + +resources: + data_quality_monitors: + monitor1: + object_type: table + object_id: 11111111-2222-3333-4444-555555555555 + data_profiling_config: + assets_dir: /Workspace/Users/user@example.com/assets + output_schema_id: 66666666-7777-8888-9999-000000000000 + snapshot: {} diff --git a/acceptance/bundle/deployment/bind/data_quality_monitor/out.test.toml b/acceptance/bundle/deployment/bind/data_quality_monitor/out.test.toml new file mode 100644 index 00000000000..19b5b15d11f --- /dev/null +++ b/acceptance/bundle/deployment/bind/data_quality_monitor/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/deployment/bind/data_quality_monitor/output.txt b/acceptance/bundle/deployment/bind/data_quality_monitor/output.txt new file mode 100644 index 00000000000..eacd19e80e3 --- /dev/null +++ b/acceptance/bundle/deployment/bind/data_quality_monitor/output.txt @@ -0,0 +1,32 @@ + +>>> [CLI] bundle deployment bind monitor1 table,[UUID] --auto-approve +Successfully bound data_quality_monitor with an id 'table,[UUID]' +Run 'bundle deploy' to deploy changes to your workspace + +>>> [CLI] bundle summary +Name: test-bundle +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default +Resources: + Data quality monitors: + monitor1: + Name: table,[UUID] + +=== A malformed id is rejected +>>> musterr [CLI] bundle deployment bind monitor1 [UUID] --auto-approve +Error: failed to fetch the resource, err: invalid data quality monitor id "[UUID]" (want ,) + +>>> [CLI] bundle deployment unbind monitor1 + +>>> [CLI] bundle summary +Name: test-bundle +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default +Resources: + Data quality monitors: + monitor1: + Name: table,[UUID] diff --git a/acceptance/bundle/deployment/bind/data_quality_monitor/script b/acceptance/bundle/deployment/bind/data_quality_monitor/script new file mode 100644 index 00000000000..b7b5067446d --- /dev/null +++ b/acceptance/bundle/deployment/bind/data_quality_monitor/script @@ -0,0 +1,8 @@ +trace $CLI bundle deployment bind monitor1 table,11111111-2222-3333-4444-555555555555 --auto-approve +trace $CLI bundle summary + +title "A malformed id is rejected" +trace musterr $CLI bundle deployment bind monitor1 11111111-2222-3333-4444-555555555555 --auto-approve + +trace $CLI bundle deployment unbind monitor1 +trace $CLI bundle summary diff --git a/acceptance/bundle/deployment/bind/data_quality_monitor/test.toml b/acceptance/bundle/deployment/bind/data_quality_monitor/test.toml new file mode 100644 index 00000000000..8bcc29b4015 --- /dev/null +++ b/acceptance/bundle/deployment/bind/data_quality_monitor/test.toml @@ -0,0 +1,20 @@ +Cloud = false + +Ignore = [ + ".databricks", +] + +# The bind flow issues a GET to confirm the remote monitor exists before binding. +[[Server]] +Pattern = "GET /api/data-quality/v1/monitors/{object_type}/{object_id}" +Response.Body = ''' +{ + "object_type": "table", + "object_id": "11111111-2222-3333-4444-555555555555", + "data_profiling_config": { + "assets_dir": "/Workspace/Users/user@example.com/assets", + "output_schema_id": "66666666-7777-8888-9999-000000000000", + "snapshot": {} + } +} +''' diff --git a/acceptance/bundle/invariant/auto-migrate/test.toml b/acceptance/bundle/invariant/auto-migrate/test.toml index 596ae57ee2a..bb1e251f40b 100644 --- a/acceptance/bundle/invariant/auto-migrate/test.toml +++ b/acceptance/bundle/invariant/auto-migrate/test.toml @@ -39,6 +39,8 @@ EnvMatrixExclude.no_cluster_policy = ["INPUT_CONFIG=cluster_policy.yml.tmpl"] EnvMatrixExclude.no_cluster_libraries = ["INPUT_CONFIG=cluster_libraries.yml.tmpl"] # AI Gateway securables are direct-only; the terraform deploy that seeds the migration fails for them. EnvMatrixExclude.no_model_service = ["INPUT_CONFIG=model_service.yml.tmpl"] +# Data quality monitors are direct-only; the terraform deploy that seeds the migration fails for them. +EnvMatrixExclude.no_data_quality_monitor = ["INPUT_CONFIG=data_quality_monitor.yml.tmpl"] # Cross-resource permission references (e.g. ${resources.jobs.job_b.permissions[0].level}) # don't work in terraform mode: the terraform interpolator converts the path to diff --git a/acceptance/bundle/invariant/configs/data_quality_monitor.yml.tmpl b/acceptance/bundle/invariant/configs/data_quality_monitor.yml.tmpl new file mode 100644 index 00000000000..3eb19cc01e5 --- /dev/null +++ b/acceptance/bundle/invariant/configs/data_quality_monitor.yml.tmpl @@ -0,0 +1,16 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + schemas: + foo_schema: + catalog_name: main + name: test-schema-$UNIQUE_NAME + + data_quality_monitors: + foo: + object_type: schema + object_id: ${resources.schemas.foo_schema.schema_id} + anomaly_detection_config: + excluded_table_full_names: + - main.test-schema-$UNIQUE_NAME.excluded diff --git a/acceptance/bundle/invariant/continue_293/out.test.toml b/acceptance/bundle/invariant/continue_293/out.test.toml index 8640ecdc76f..74c3be72804 100644 --- a/acceptance/bundle/invariant/continue_293/out.test.toml +++ b/acceptance/bundle/invariant/continue_293/out.test.toml @@ -11,6 +11,7 @@ EnvMatrix.INPUT_CONFIG = [ "job_apply_policy_default_values_job_cluster.yml.tmpl", "job_apply_policy_default_values_task_cluster.yml.tmpl", "job_apply_policy_default_values_for_each_task.yml.tmpl", + "data_quality_monitor.yml.tmpl", "database_catalog.yml.tmpl", "database_instance.yml.tmpl", "experiment.yml.tmpl", diff --git a/acceptance/bundle/invariant/delete_idempotent/out.test.toml b/acceptance/bundle/invariant/delete_idempotent/out.test.toml index a44900d26f0..737d793d538 100644 --- a/acceptance/bundle/invariant/delete_idempotent/out.test.toml +++ b/acceptance/bundle/invariant/delete_idempotent/out.test.toml @@ -13,6 +13,7 @@ EnvMatrix.INPUT_CONFIG = [ "job_apply_policy_default_values_job_cluster.yml.tmpl", "job_apply_policy_default_values_task_cluster.yml.tmpl", "job_apply_policy_default_values_for_each_task.yml.tmpl", + "data_quality_monitor.yml.tmpl", "database_catalog.yml.tmpl", "database_instance.yml.tmpl", "experiment.yml.tmpl", diff --git a/acceptance/bundle/invariant/destroy_idempotent/out.test.toml b/acceptance/bundle/invariant/destroy_idempotent/out.test.toml index a44900d26f0..737d793d538 100644 --- a/acceptance/bundle/invariant/destroy_idempotent/out.test.toml +++ b/acceptance/bundle/invariant/destroy_idempotent/out.test.toml @@ -13,6 +13,7 @@ EnvMatrix.INPUT_CONFIG = [ "job_apply_policy_default_values_job_cluster.yml.tmpl", "job_apply_policy_default_values_task_cluster.yml.tmpl", "job_apply_policy_default_values_for_each_task.yml.tmpl", + "data_quality_monitor.yml.tmpl", "database_catalog.yml.tmpl", "database_instance.yml.tmpl", "experiment.yml.tmpl", diff --git a/acceptance/bundle/invariant/migrate/test.toml b/acceptance/bundle/invariant/migrate/test.toml index 9e3603b5dd0..b3d3335cc75 100644 --- a/acceptance/bundle/invariant/migrate/test.toml +++ b/acceptance/bundle/invariant/migrate/test.toml @@ -39,6 +39,8 @@ EnvMatrixExclude.no_cluster_policy = ["INPUT_CONFIG=cluster_policy.yml.tmpl"] EnvMatrixExclude.no_cluster_libraries = ["INPUT_CONFIG=cluster_libraries.yml.tmpl"] # AI Gateway securables are direct-only; the terraform deploy that seeds the migration fails for them. EnvMatrixExclude.no_model_service = ["INPUT_CONFIG=model_service.yml.tmpl"] +# Data quality monitors are direct-only; the terraform deploy that seeds the migration fails for them. +EnvMatrixExclude.no_data_quality_monitor = ["INPUT_CONFIG=data_quality_monitor.yml.tmpl"] # Cross-resource permission references (e.g. ${resources.jobs.job_b.permissions[0].level}) # don't work in terraform mode: the terraform interpolator converts the path to diff --git a/acceptance/bundle/invariant/no_drift/out.test.toml b/acceptance/bundle/invariant/no_drift/out.test.toml index 9c32b8dda05..97c73dd7d5e 100644 --- a/acceptance/bundle/invariant/no_drift/out.test.toml +++ b/acceptance/bundle/invariant/no_drift/out.test.toml @@ -13,6 +13,7 @@ EnvMatrix.INPUT_CONFIG = [ "job_apply_policy_default_values_job_cluster.yml.tmpl", "job_apply_policy_default_values_task_cluster.yml.tmpl", "job_apply_policy_default_values_for_each_task.yml.tmpl", + "data_quality_monitor.yml.tmpl", "database_catalog.yml.tmpl", "database_instance.yml.tmpl", "experiment.yml.tmpl", diff --git a/acceptance/bundle/invariant/test.toml b/acceptance/bundle/invariant/test.toml index b4e28af5ca0..80b9c49dcdd 100644 --- a/acceptance/bundle/invariant/test.toml +++ b/acceptance/bundle/invariant/test.toml @@ -27,6 +27,7 @@ EnvMatrix.INPUT_CONFIG = [ "job_apply_policy_default_values_job_cluster.yml.tmpl", "job_apply_policy_default_values_task_cluster.yml.tmpl", "job_apply_policy_default_values_for_each_task.yml.tmpl", + "data_quality_monitor.yml.tmpl", "database_catalog.yml.tmpl", "database_instance.yml.tmpl", "experiment.yml.tmpl", @@ -115,6 +116,9 @@ EnvMatrixExclude.no_model_serving_endpoint_telemetry_on_cloud = ["CONFIG_Cloud=t # UC connection to create on cloud, which the invariant tests don't provision; the # local mock server accepts them, so these run locally only. EnvMatrixExclude.no_model_service_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=model_service.yml.tmpl"] +# Cloud coverage for data quality monitors comes from +# resources/data_quality_monitors/lifecycle; the invariant matrix runs locally. +EnvMatrixExclude.no_data_quality_monitor_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=data_quality_monitor.yml.tmpl"] # Fake SQL endpoint for local tests [[Server]] diff --git a/acceptance/bundle/python/data_quality_monitors-support/databricks.yml b/acceptance/bundle/python/data_quality_monitors-support/databricks.yml new file mode 100644 index 00000000000..3864e71f897 --- /dev/null +++ b/acceptance/bundle/python/data_quality_monitors-support/databricks.yml @@ -0,0 +1,19 @@ +bundle: + name: my_project + +sync: {paths: []} # don't need to copy files + +python: + resources: + - "resources:load_resources" + mutators: + - "mutators:update_data_quality_monitor" + +resources: + data_quality_monitors: + my_data_quality_monitor_1: + object_type: "schema" + object_id: "11111111-2222-3333-4444-555555555555" + anomaly_detection_config: + excluded_table_full_names: + - "main.default.excluded" diff --git a/acceptance/bundle/python/data_quality_monitors-support/mutators.py b/acceptance/bundle/python/data_quality_monitors-support/mutators.py new file mode 100644 index 00000000000..48f1adcac9d --- /dev/null +++ b/acceptance/bundle/python/data_quality_monitors-support/mutators.py @@ -0,0 +1,13 @@ +from dataclasses import replace + +from databricks.bundles.data_quality_monitors import DataQualityMonitor +from databricks.bundles.core import data_quality_monitor_mutator + + +@data_quality_monitor_mutator +def update_data_quality_monitor( + data_quality_monitor: DataQualityMonitor, +) -> DataQualityMonitor: + assert isinstance(data_quality_monitor.object_id, str) + + return replace(data_quality_monitor, object_id=f"{data_quality_monitor.object_id} (updated)") diff --git a/acceptance/bundle/python/data_quality_monitors-support/out.test.toml b/acceptance/bundle/python/data_quality_monitors-support/out.test.toml new file mode 100644 index 00000000000..aeddf3e619e --- /dev/null +++ b/acceptance/bundle/python/data_quality_monitors-support/out.test.toml @@ -0,0 +1,3 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] +EnvMatrix.PYDAB_VERSION = ["current"] diff --git a/acceptance/bundle/python/data_quality_monitors-support/output.txt b/acceptance/bundle/python/data_quality_monitors-support/output.txt new file mode 100644 index 00000000000..ff8be57ef71 --- /dev/null +++ b/acceptance/bundle/python/data_quality_monitors-support/output.txt @@ -0,0 +1,35 @@ + +>>> uv run [UV_ARGS] -q [CLI] bundle validate --output json +{ + "experimental": { + "python": { + "mutators": [ + "mutators:update_data_quality_monitor" + ], + "resources": [ + "resources:load_resources" + ] + } + }, + "resources": { + "data_quality_monitors": { + "my_data_quality_monitor_1": { + "anomaly_detection_config": { + "excluded_table_full_names": [ + "main.default.excluded" + ] + }, + "object_id": "[UUID] (updated)", + "object_type": "schema" + }, + "my_data_quality_monitor_2": { + "data_profiling_config": { + "assets_dir": "/Workspace/Users/user@example.com/assets", + "output_schema_id": "[UUID]" + }, + "object_id": "[UUID] (updated)", + "object_type": "table" + } + } + } +} diff --git a/acceptance/bundle/python/data_quality_monitors-support/resources.py b/acceptance/bundle/python/data_quality_monitors-support/resources.py new file mode 100644 index 00000000000..a55cbffe94a --- /dev/null +++ b/acceptance/bundle/python/data_quality_monitors-support/resources.py @@ -0,0 +1,20 @@ +from databricks.bundles.core import Resources + + +def load_resources() -> Resources: + resources = Resources() + + resources.add_data_quality_monitor( + "my_data_quality_monitor_2", + { + "object_type": "table", + "object_id": "66666666-7777-8888-9999-000000000000", + "data_profiling_config": { + "assets_dir": "/Workspace/Users/user@example.com/assets", + "output_schema_id": "11111111-2222-3333-4444-555555555555", + "snapshot": {}, + }, + }, + ) + + return resources diff --git a/acceptance/bundle/python/data_quality_monitors-support/script b/acceptance/bundle/python/data_quality_monitors-support/script new file mode 100644 index 00000000000..e273fb45a53 --- /dev/null +++ b/acceptance/bundle/python/data_quality_monitors-support/script @@ -0,0 +1,5 @@ + +trace uv run $UV_ARGS -q $CLI bundle validate --output json | \ + jq "pick(.experimental.python, .resources)" + +rm -fr .databricks __pycache__ diff --git a/acceptance/bundle/python/data_quality_monitors-support/test.toml b/acceptance/bundle/python/data_quality_monitors-support/test.toml new file mode 100644 index 00000000000..6015458ef82 --- /dev/null +++ b/acceptance/bundle/python/data_quality_monitors-support/test.toml @@ -0,0 +1,4 @@ +Cloud = false # tests don't interact with APIs + +# data_quality_monitors are only supported in the current version of the wheel +EnvMatrix.PYDAB_VERSION = ["current"] diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index 2c8056be91c..d7de210f999 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -670,6 +670,61 @@ resources.dashboards.*.permissions[*].group_name string ALL resources.dashboards.*.permissions[*].level iam.PermissionLevel ALL resources.dashboards.*.permissions[*].service_principal_name string ALL resources.dashboards.*.permissions[*].user_name string ALL +resources.data_quality_monitors.*.anomaly_detection_config *dataquality.AnomalyDetectionConfig ALL +resources.data_quality_monitors.*.anomaly_detection_config.excluded_table_full_names []string ALL +resources.data_quality_monitors.*.anomaly_detection_config.excluded_table_full_names[*] string ALL +resources.data_quality_monitors.*.data_profiling_config *dataquality.DataProfilingConfig ALL +resources.data_quality_monitors.*.data_profiling_config.assets_dir string ALL +resources.data_quality_monitors.*.data_profiling_config.baseline_table_name string ALL +resources.data_quality_monitors.*.data_profiling_config.custom_metrics []dataquality.DataProfilingCustomMetric ALL +resources.data_quality_monitors.*.data_profiling_config.custom_metrics[*] dataquality.DataProfilingCustomMetric ALL +resources.data_quality_monitors.*.data_profiling_config.custom_metrics[*].definition string ALL +resources.data_quality_monitors.*.data_profiling_config.custom_metrics[*].input_columns []string ALL +resources.data_quality_monitors.*.data_profiling_config.custom_metrics[*].input_columns[*] string ALL +resources.data_quality_monitors.*.data_profiling_config.custom_metrics[*].name string ALL +resources.data_quality_monitors.*.data_profiling_config.custom_metrics[*].output_data_type string ALL +resources.data_quality_monitors.*.data_profiling_config.custom_metrics[*].type dataquality.DataProfilingCustomMetricType ALL +resources.data_quality_monitors.*.data_profiling_config.dashboard_id string ALL +resources.data_quality_monitors.*.data_profiling_config.drift_metrics_table_name string ALL +resources.data_quality_monitors.*.data_profiling_config.effective_warehouse_id string ALL +resources.data_quality_monitors.*.data_profiling_config.inference_log *dataquality.InferenceLogConfig ALL +resources.data_quality_monitors.*.data_profiling_config.inference_log.granularities []dataquality.AggregationGranularity ALL +resources.data_quality_monitors.*.data_profiling_config.inference_log.granularities[*] dataquality.AggregationGranularity ALL +resources.data_quality_monitors.*.data_profiling_config.inference_log.label_column string ALL +resources.data_quality_monitors.*.data_profiling_config.inference_log.model_id_column string ALL +resources.data_quality_monitors.*.data_profiling_config.inference_log.prediction_column string ALL +resources.data_quality_monitors.*.data_profiling_config.inference_log.problem_type dataquality.InferenceProblemType ALL +resources.data_quality_monitors.*.data_profiling_config.inference_log.timestamp_column string ALL +resources.data_quality_monitors.*.data_profiling_config.latest_monitor_failure_message string ALL +resources.data_quality_monitors.*.data_profiling_config.monitor_version int64 ALL +resources.data_quality_monitors.*.data_profiling_config.monitored_table_name string ALL +resources.data_quality_monitors.*.data_profiling_config.notification_settings *dataquality.NotificationSettings ALL +resources.data_quality_monitors.*.data_profiling_config.notification_settings.on_failure *dataquality.NotificationDestination ALL +resources.data_quality_monitors.*.data_profiling_config.notification_settings.on_failure.email_addresses []string ALL +resources.data_quality_monitors.*.data_profiling_config.notification_settings.on_failure.email_addresses[*] string ALL +resources.data_quality_monitors.*.data_profiling_config.output_schema_id string ALL +resources.data_quality_monitors.*.data_profiling_config.profile_metrics_table_name string ALL +resources.data_quality_monitors.*.data_profiling_config.schedule *dataquality.CronSchedule ALL +resources.data_quality_monitors.*.data_profiling_config.schedule.pause_status dataquality.CronSchedulePauseStatus ALL +resources.data_quality_monitors.*.data_profiling_config.schedule.quartz_cron_expression string ALL +resources.data_quality_monitors.*.data_profiling_config.schedule.timezone_id string ALL +resources.data_quality_monitors.*.data_profiling_config.skip_builtin_dashboard bool ALL +resources.data_quality_monitors.*.data_profiling_config.slicing_exprs []string ALL +resources.data_quality_monitors.*.data_profiling_config.slicing_exprs[*] string ALL +resources.data_quality_monitors.*.data_profiling_config.snapshot *dataquality.SnapshotConfig ALL +resources.data_quality_monitors.*.data_profiling_config.status dataquality.DataProfilingStatus ALL +resources.data_quality_monitors.*.data_profiling_config.time_series *dataquality.TimeSeriesConfig ALL +resources.data_quality_monitors.*.data_profiling_config.time_series.granularities []dataquality.AggregationGranularity ALL +resources.data_quality_monitors.*.data_profiling_config.time_series.granularities[*] dataquality.AggregationGranularity ALL +resources.data_quality_monitors.*.data_profiling_config.time_series.timestamp_column string ALL +resources.data_quality_monitors.*.data_profiling_config.warehouse_id string ALL +resources.data_quality_monitors.*.id string INPUT +resources.data_quality_monitors.*.lifecycle resources.Lifecycle INPUT +resources.data_quality_monitors.*.lifecycle.prevent_destroy bool INPUT +resources.data_quality_monitors.*.modified_status string INPUT +resources.data_quality_monitors.*.object_id string ALL +resources.data_quality_monitors.*.object_type string ALL +resources.data_quality_monitors.*.url string INPUT resources.database_catalogs.*.create_database_if_not_exists bool ALL resources.database_catalogs.*.database_instance_name string ALL resources.database_catalogs.*.database_name string ALL diff --git a/acceptance/bundle/resources/data_quality_monitors/basic/databricks.yml b/acceptance/bundle/resources/data_quality_monitors/basic/databricks.yml new file mode 100644 index 00000000000..4c223e5d42d --- /dev/null +++ b/acceptance/bundle/resources/data_quality_monitors/basic/databricks.yml @@ -0,0 +1,16 @@ +bundle: + name: test-bundle + +resources: + schemas: + myschema: + catalog_name: main + name: myschema + + data_quality_monitors: + schema_monitor: + object_type: schema + object_id: ${resources.schemas.myschema.schema_id} + anomaly_detection_config: + excluded_table_full_names: + - main.myschema.table1 diff --git a/acceptance/bundle/resources/data_quality_monitors/basic/out.test.toml b/acceptance/bundle/resources/data_quality_monitors/basic/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/resources/data_quality_monitors/basic/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/data_quality_monitors/basic/output.txt b/acceptance/bundle/resources/data_quality_monitors/basic/output.txt new file mode 100644 index 00000000000..0bf59424cfc --- /dev/null +++ b/acceptance/bundle/resources/data_quality_monitors/basic/output.txt @@ -0,0 +1,101 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Created data_quality_monitors.schema_monitor +Created schemas.myschema +Files: 1 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //data-quality +{ + "method": "POST", + "path": "/api/data-quality/v1/monitors", + "body": { + "anomaly_detection_config": { + "excluded_table_full_names": [ + "main.myschema.table1" + ] + }, + "object_id": "[UUID]", + "object_type": "schema" + } +} + +=== Summary shows the composite id +>>> [CLI] bundle summary -o json +{ + "id": "schema,[UUID]", + "object_type": "schema" +} + +=== Verify deployment +>>> [CLI] data-quality get-monitor schema [UUID] +{ + "excluded_table_full_names": [ + "main.myschema.table1" + ] +} + +=== A second plan has no changes +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +=== Update the config (should update in place, not recreate) +>>> update_file.py databricks.yml main.myschema.table1 main.myschema.table2 + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Updated data_quality_monitors.schema_monitor +Files: 1 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 1 unchanged + +>>> print_requests.py //data-quality +{ + "method": "PATCH", + "path": "/api/data-quality/v1/monitors/schema/[UUID]", + "q": { + "update_mask": "anomaly_detection_config,data_profiling_config" + }, + "body": { + "anomaly_detection_config": { + "excluded_table_full_names": [ + "main.myschema.table2" + ] + }, + "object_id": "[UUID]", + "object_type": "schema" + } +} + +>>> [CLI] data-quality get-monitor schema [UUID] +{ + "excluded_table_full_names": [ + "main.myschema.table2" + ] +} + +=== Change an identity field (should plan a recreate) +>>> update_file.py databricks.yml ${resources.schemas.myschema.schema_id} [UUID] + +>>> [CLI] bundle plan +recreate data_quality_monitors.schema_monitor + +Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.data_quality_monitors.schema_monitor + delete resources.schemas.myschema + +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.myschema + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default + +Destroy: 2 deleted + +>>> print_requests.py //data-quality +{ + "method": "DELETE", + "path": "/api/data-quality/v1/monitors/schema/[UUID]" +} diff --git a/acceptance/bundle/resources/data_quality_monitors/basic/script b/acceptance/bundle/resources/data_quality_monitors/basic/script new file mode 100644 index 00000000000..f7a9bdb9090 --- /dev/null +++ b/acceptance/bundle/resources/data_quality_monitors/basic/script @@ -0,0 +1,27 @@ +trace $CLI bundle deploy +trace print_requests.py //data-quality + +title "Summary shows the composite id" +trace $CLI bundle summary -o json | jq '.resources.data_quality_monitors.schema_monitor | {id, object_type}' + +title "Verify deployment" +SCHEMA_ID=$($CLI schemas get main.myschema | jq -r .schema_id) +trace $CLI data-quality get-monitor schema $SCHEMA_ID | jq .anomaly_detection_config + +title "A second plan has no changes" +trace $CLI bundle plan + +title "Update the config (should update in place, not recreate)" +trace update_file.py databricks.yml main.myschema.table1 main.myschema.table2 +trace $CLI bundle deploy +trace print_requests.py //data-quality +trace $CLI data-quality get-monitor schema $SCHEMA_ID | jq .anomaly_detection_config + +title "Change an identity field (should plan a recreate)" +trace update_file.py databricks.yml '${resources.schemas.myschema.schema_id}' 00000000-0000-0000-0000-000000000000 +trace $CLI bundle plan + +trace $CLI bundle destroy --auto-approve +trace print_requests.py //data-quality + +rm -f out.requests.txt diff --git a/acceptance/bundle/resources/data_quality_monitors/basic/test.toml b/acceptance/bundle/resources/data_quality_monitors/basic/test.toml new file mode 100644 index 00000000000..116f2013783 --- /dev/null +++ b/acceptance/bundle/resources/data_quality_monitors/basic/test.toml @@ -0,0 +1,7 @@ +# Local only: this test inspects the recorded request stream (print_requests), +# which has no equivalent against a real workspace. +Cloud = false + +Ignore = [ + ".databricks", +] diff --git a/acceptance/bundle/resources/data_quality_monitors/data_profiling/databricks.yml b/acceptance/bundle/resources/data_quality_monitors/data_profiling/databricks.yml new file mode 100644 index 00000000000..d09ec414ed9 --- /dev/null +++ b/acceptance/bundle/resources/data_quality_monitors/data_profiling/databricks.yml @@ -0,0 +1,15 @@ +bundle: + name: test-bundle + +resources: + data_quality_monitors: + table_monitor: + object_type: table + object_id: 11111111-2222-3333-4444-555555555555 + data_profiling_config: + assets_dir: /Workspace/Users/user@example.com/assets1 + output_schema_id: 66666666-7777-8888-9999-000000000000 + snapshot: {} + schedule: + quartz_cron_expression: 0 0 12 * * ? + timezone_id: UTC diff --git a/acceptance/bundle/resources/data_quality_monitors/data_profiling/out.test.toml b/acceptance/bundle/resources/data_quality_monitors/data_profiling/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/resources/data_quality_monitors/data_profiling/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/data_quality_monitors/data_profiling/output.txt b/acceptance/bundle/resources/data_quality_monitors/data_profiling/output.txt new file mode 100644 index 00000000000..023c65e80b0 --- /dev/null +++ b/acceptance/bundle/resources/data_quality_monitors/data_profiling/output.txt @@ -0,0 +1,88 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Created data_quality_monitors.table_monitor +Files: 1 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //data-quality +{ + "method": "POST", + "path": "/api/data-quality/v1/monitors", + "body": { + "data_profiling_config": { + "assets_dir": "/Workspace/Users/user@example.com/assets1", + "output_schema_id": "[UUID]", + "schedule": { + "quartz_cron_expression": "0 0 12 * * ?", + "timezone_id": "UTC" + }, + "snapshot": {} + }, + "object_id": "[UUID]", + "object_type": "table" + } +} + +=== Verify deployment, including server-computed fields +>>> [CLI] data-quality get-monitor table [UUID] +{ + "assets_dir": "/Workspace/Users/user@example.com/assets1", + "schedule": { + "quartz_cron_expression": "0 0 12 * * ?", + "timezone_id": "UTC" + }, + "status": "DATA_PROFILING_STATUS_ACTIVE", + "monitor_version": 1 +} + +=== Server-computed fields do not show up as drift +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + +=== Update the schedule (should update in place) +>>> update_file.py databricks.yml 0 0 12 * * ? 0 0 6 * * ? + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Updated data_quality_monitors.table_monitor +Files: 1 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //data-quality +{ + "method": "PATCH", + "path": "/api/data-quality/v1/monitors/table/[UUID]", + "q": { + "update_mask": "anomaly_detection_config,data_profiling_config" + }, + "body": { + "data_profiling_config": { + "assets_dir": "/Workspace/Users/user@example.com/assets1", + "output_schema_id": "[UUID]", + "schedule": { + "quartz_cron_expression": "0 0 6 * * ?", + "timezone_id": "UTC" + }, + "snapshot": {} + }, + "object_id": "[UUID]", + "object_type": "table" + } +} + +=== Change assets_dir (ignored by the update API, should plan a recreate) +>>> update_file.py databricks.yml assets1 assets2 + +>>> [CLI] bundle plan +recreate data_quality_monitors.table_monitor + +Plan: 1 to add, 0 to change, 1 to delete, 0 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.data_quality_monitors.table_monitor + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default + +Destroy: 1 deleted diff --git a/acceptance/bundle/resources/data_quality_monitors/data_profiling/script b/acceptance/bundle/resources/data_quality_monitors/data_profiling/script new file mode 100644 index 00000000000..a3bb766b26d --- /dev/null +++ b/acceptance/bundle/resources/data_quality_monitors/data_profiling/script @@ -0,0 +1,21 @@ +trace $CLI bundle deploy +trace print_requests.py //data-quality + +title "Verify deployment, including server-computed fields" +trace $CLI data-quality get-monitor table 11111111-2222-3333-4444-555555555555 | jq '.data_profiling_config | {assets_dir, schedule, status, monitor_version}' + +title "Server-computed fields do not show up as drift" +trace $CLI bundle plan + +title "Update the schedule (should update in place)" +trace update_file.py databricks.yml '0 0 12 * * ?' '0 0 6 * * ?' +trace $CLI bundle deploy +trace print_requests.py //data-quality + +title "Change assets_dir (ignored by the update API, should plan a recreate)" +trace update_file.py databricks.yml assets1 assets2 +trace $CLI bundle plan + +trace $CLI bundle destroy --auto-approve + +rm -f out.requests.txt diff --git a/acceptance/bundle/resources/data_quality_monitors/data_profiling/test.toml b/acceptance/bundle/resources/data_quality_monitors/data_profiling/test.toml new file mode 100644 index 00000000000..116f2013783 --- /dev/null +++ b/acceptance/bundle/resources/data_quality_monitors/data_profiling/test.toml @@ -0,0 +1,7 @@ +# Local only: this test inspects the recorded request stream (print_requests), +# which has no equivalent against a real workspace. +Cloud = false + +Ignore = [ + ".databricks", +] diff --git a/acceptance/bundle/resources/data_quality_monitors/lifecycle/databricks.yml.tmpl b/acceptance/bundle/resources/data_quality_monitors/lifecycle/databricks.yml.tmpl new file mode 100644 index 00000000000..bd202bb52f8 --- /dev/null +++ b/acceptance/bundle/resources/data_quality_monitors/lifecycle/databricks.yml.tmpl @@ -0,0 +1,14 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + schemas: + myschema: + catalog_name: main + name: dqm_$UNIQUE_NAME + + data_quality_monitors: + schema_monitor: + object_type: schema + object_id: ${resources.schemas.myschema.schema_id} + anomaly_detection_config: {} diff --git a/acceptance/bundle/resources/data_quality_monitors/lifecycle/out.test.toml b/acceptance/bundle/resources/data_quality_monitors/lifecycle/out.test.toml new file mode 100644 index 00000000000..0f167effef1 --- /dev/null +++ b/acceptance/bundle/resources/data_quality_monitors/lifecycle/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/data_quality_monitors/lifecycle/output.txt b/acceptance/bundle/resources/data_quality_monitors/lifecycle/output.txt new file mode 100644 index 00000000000..739b8be4987 --- /dev/null +++ b/acceptance/bundle/resources/data_quality_monitors/lifecycle/output.txt @@ -0,0 +1,35 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Created data_quality_monitors.schema_monitor +Created schemas.myschema +Files: 2 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +=== A second plan has no changes +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +=== Add an exclusion (should update in place) +>>> update_file.py databricks.yml anomaly_detection_config: {} anomaly_detection_config: {excluded_table_full_names: [main.dqm_[UNIQUE_NAME].excluded]} + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Updated data_quality_monitors.schema_monitor +Files: 1 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 1 unchanged + +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.data_quality_monitors.schema_monitor + delete resources.schemas.myschema + +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.myschema + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Destroy: 2 deleted diff --git a/acceptance/bundle/resources/data_quality_monitors/lifecycle/script b/acceptance/bundle/resources/data_quality_monitors/lifecycle/script new file mode 100644 index 00000000000..80bc421c382 --- /dev/null +++ b/acceptance/bundle/resources/data_quality_monitors/lifecycle/script @@ -0,0 +1,16 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +trace $CLI bundle deploy + +title "A second plan has no changes" +trace $CLI bundle plan + +title "Add an exclusion (should update in place)" +trace update_file.py databricks.yml 'anomaly_detection_config: {}' 'anomaly_detection_config: {excluded_table_full_names: [main.dqm_'$UNIQUE_NAME'.excluded]}' +trace $CLI bundle deploy +trace $CLI bundle plan diff --git a/acceptance/bundle/resources/data_quality_monitors/lifecycle/test.toml b/acceptance/bundle/resources/data_quality_monitors/lifecycle/test.toml new file mode 100644 index 00000000000..4b62ffa3f3c --- /dev/null +++ b/acceptance/bundle/resources/data_quality_monitors/lifecycle/test.toml @@ -0,0 +1,8 @@ +# Runs against a real workspace as well: a schema monitor needs only a schema, +# which the bundle creates. +RecordRequests = false + +Ignore = [ + "databricks.yml", + ".databricks", +] diff --git a/acceptance/bundle/resources/data_quality_monitors/remote-delete/databricks.yml b/acceptance/bundle/resources/data_quality_monitors/remote-delete/databricks.yml new file mode 100644 index 00000000000..86e85efd1ab --- /dev/null +++ b/acceptance/bundle/resources/data_quality_monitors/remote-delete/databricks.yml @@ -0,0 +1,12 @@ +bundle: + name: test-bundle + +resources: + data_quality_monitors: + table_monitor: + object_type: table + object_id: 11111111-2222-3333-4444-555555555555 + data_profiling_config: + assets_dir: /Workspace/Users/user@example.com/assets + output_schema_id: 66666666-7777-8888-9999-000000000000 + snapshot: {} diff --git a/acceptance/bundle/resources/data_quality_monitors/remote-delete/out.test.toml b/acceptance/bundle/resources/data_quality_monitors/remote-delete/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/resources/data_quality_monitors/remote-delete/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/data_quality_monitors/remote-delete/output.txt b/acceptance/bundle/resources/data_quality_monitors/remote-delete/output.txt new file mode 100644 index 00000000000..bf119a1c9ba --- /dev/null +++ b/acceptance/bundle/resources/data_quality_monitors/remote-delete/output.txt @@ -0,0 +1,20 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Created data_quality_monitors.table_monitor +Files: 1 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +=== Delete the monitor out of band +>>> [CLI] data-quality delete-monitor table [UUID] + +=== Plan should detect the monitor is gone and re-create it +>>> [CLI] bundle plan +create data_quality_monitors.table_monitor + +Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged + +>>> [CLI] bundle destroy --auto-approve +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default + +Destroy: 0 deleted diff --git a/acceptance/bundle/resources/data_quality_monitors/remote-delete/script b/acceptance/bundle/resources/data_quality_monitors/remote-delete/script new file mode 100644 index 00000000000..fbcd59a7a31 --- /dev/null +++ b/acceptance/bundle/resources/data_quality_monitors/remote-delete/script @@ -0,0 +1,9 @@ +trace $CLI bundle deploy + +title "Delete the monitor out of band" +trace $CLI data-quality delete-monitor table 11111111-2222-3333-4444-555555555555 + +title "Plan should detect the monitor is gone and re-create it" +trace $CLI bundle plan + +trace $CLI bundle destroy --auto-approve diff --git a/acceptance/bundle/resources/data_quality_monitors/remote-delete/test.toml b/acceptance/bundle/resources/data_quality_monitors/remote-delete/test.toml new file mode 100644 index 00000000000..c2bab5b9c0c --- /dev/null +++ b/acceptance/bundle/resources/data_quality_monitors/remote-delete/test.toml @@ -0,0 +1,9 @@ +# Local only: simulates an out-of-band delete with fixed object IDs that don't +# exist in a real workspace. +Cloud = false + +RecordRequests = false + +Ignore = [ + ".databricks", +] diff --git a/acceptance/bundle/resources/data_quality_monitors/test.toml b/acceptance/bundle/resources/data_quality_monitors/test.toml new file mode 100644 index 00000000000..f236bfc8476 --- /dev/null +++ b/acceptance/bundle/resources/data_quality_monitors/test.toml @@ -0,0 +1,5 @@ +# Data quality monitors are deployed through the direct engine only. + +# Lifecycle tests also run against a real workspace. Tests that inspect the +# recorded request stream (e.g. basic) can't run on cloud and override this. +Cloud = true diff --git a/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go b/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go index c71bb2c5834..31a2f5d6d99 100644 --- a/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go +++ b/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go @@ -24,6 +24,7 @@ var unsupportedResources = []string{ "volumes", "schemas", "quality_monitors", + "data_quality_monitors", "registered_models", "model_services", "mcp_services", diff --git a/bundle/config/mutator/resourcemutator/apply_presets.go b/bundle/config/mutator/resourcemutator/apply_presets.go index bb84240d7cc..9f2faab8398 100644 --- a/bundle/config/mutator/resourcemutator/apply_presets.go +++ b/bundle/config/mutator/resourcemutator/apply_presets.go @@ -13,6 +13,7 @@ import ( "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/textutil" "github.com/databricks/databricks-sdk-go/service/catalog" + "github.com/databricks/databricks-sdk-go/service/dataquality" "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/databricks/databricks-sdk-go/service/ml" "github.com/databricks/databricks-sdk-go/service/sql" @@ -197,6 +198,17 @@ func (m *applyPresets) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnos q.Schedule = nil } } + // Same as quality monitors: the schedule's pause_status is read-only, so a + // paused preset can only drop the schedule. + for _, m := range r.DataQualityMonitors { + if m == nil || m.DataProfilingConfig == nil { + continue + } + s := m.DataProfilingConfig.Schedule + if s != nil && s.PauseStatus != dataquality.CronSchedulePauseStatusCronSchedulePauseStatusUnpaused { + m.DataProfilingConfig.Schedule = nil + } + } } // Schemas: Prefix diff --git a/bundle/config/mutator/resourcemutator/apply_target_mode_test.go b/bundle/config/mutator/resourcemutator/apply_target_mode_test.go index b43c1f18e8c..c1dd36c35b8 100644 --- a/bundle/config/mutator/resourcemutator/apply_target_mode_test.go +++ b/bundle/config/mutator/resourcemutator/apply_target_mode_test.go @@ -16,6 +16,7 @@ import ( "github.com/databricks/databricks-sdk-go/service/apps" "github.com/databricks/databricks-sdk-go/service/catalog" "github.com/databricks/databricks-sdk-go/service/compute" + "github.com/databricks/databricks-sdk-go/service/dataquality" "github.com/databricks/databricks-sdk-go/service/iam" "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/databricks/databricks-sdk-go/service/ml" @@ -144,6 +145,20 @@ func mockBundle(mode config.Mode) *bundle.Bundle { }, }, }, + DataQualityMonitors: map[string]*resources.DataQualityMonitor{ + "dataQualityMonitor1": {Monitor: dataquality.Monitor{ + ObjectType: "schema", + ObjectId: "dataQualityMonitor1", + AnomalyDetectionConfig: &dataquality.AnomalyDetectionConfig{}, + }}, + "dataQualityMonitor2": {Monitor: dataquality.Monitor{ + ObjectType: "table", + ObjectId: "dataQualityMonitor2", + DataProfilingConfig: &dataquality.DataProfilingConfig{ + Schedule: &dataquality.CronSchedule{QuartzCronExpression: "0 0 12 * * ?", TimezoneId: "UTC"}, + }, + }}, + }, Catalogs: map[string]*resources.Catalog{ "catalog1": {CreateCatalog: catalog.CreateCatalog{Name: "catalog1"}}, }, @@ -406,6 +421,7 @@ func TestProcessTargetModeDevelopment(t *testing.T) { assert.Equal(t, "qualityMonitor1", b.Config.Resources.QualityMonitors["qualityMonitor1"].TableName) assert.Nil(t, b.Config.Resources.QualityMonitors["qualityMonitor2"].Schedule) assert.Equal(t, catalog.MonitorCronSchedulePauseStatusUnpaused, b.Config.Resources.QualityMonitors["qualityMonitor3"].Schedule.PauseStatus) + assert.Nil(t, b.Config.Resources.DataQualityMonitors["dataQualityMonitor2"].DataProfilingConfig.Schedule) // Schema 1 assert.Equal(t, "dev_lennart_schema1", b.Config.Resources.Schemas["schema1"].Name) diff --git a/bundle/config/mutator/resourcemutator/run_as.go b/bundle/config/mutator/resourcemutator/run_as.go index 65279c7f212..b946db77235 100644 --- a/bundle/config/mutator/resourcemutator/run_as.go +++ b/bundle/config/mutator/resourcemutator/run_as.go @@ -101,6 +101,15 @@ func validateRunAs(b *bundle.Bundle) diag.Diagnostics { )) } + if len(b.Config.Resources.DataQualityMonitors) > 0 { + diags = diags.Extend(reportRunAsNotSupported( + "data_quality_monitors", + b.Config.GetLocation("resources.data_quality_monitors"), + b.Config.Workspace.CurrentUser.UserName, + identity, + )) + } + // Dashboards do not support run_as in the API. if len(b.Config.Resources.Dashboards) > 0 { for key, dashboard := range b.Config.Resources.Dashboards { diff --git a/bundle/config/mutator/resourcemutator/run_as_test.go b/bundle/config/mutator/resourcemutator/run_as_test.go index 5d1aa602d02..0b7bb649c0d 100644 --- a/bundle/config/mutator/resourcemutator/run_as_test.go +++ b/bundle/config/mutator/resourcemutator/run_as_test.go @@ -44,6 +44,7 @@ func allResourceTypes(t *testing.T) []string { "cluster_policies", "clusters", "dashboards", + "data_quality_monitors", "database_catalogs", "database_instances", "experiments", diff --git a/bundle/config/resources.go b/bundle/config/resources.go index f13a44617c4..2d5cc79f153 100644 --- a/bundle/config/resources.go +++ b/bundle/config/resources.go @@ -23,6 +23,7 @@ type Resources struct { ModelProviderServices map[string]*resources.ModelProviderService `json:"model_provider_services,omitempty"` RegisteredModels map[string]*resources.RegisteredModel `json:"registered_models,omitempty"` QualityMonitors map[string]*resources.QualityMonitor `json:"quality_monitors,omitempty"` + DataQualityMonitors map[string]*resources.DataQualityMonitor `json:"data_quality_monitors,omitempty"` Catalogs map[string]*resources.Catalog `json:"catalogs,omitempty"` Schemas map[string]*resources.Schema `json:"schemas,omitempty"` Volumes map[string]*resources.Volume `json:"volumes,omitempty"` @@ -119,6 +120,7 @@ func (r *Resources) AllResources() []ResourceGroup { collectResourceMap(descriptions["model_provider_services"], r.ModelProviderServices), collectResourceMap(descriptions["registered_models"], r.RegisteredModels), collectResourceMap(descriptions["quality_monitors"], r.QualityMonitors), + collectResourceMap(descriptions["data_quality_monitors"], r.DataQualityMonitors), collectResourceMap(descriptions["catalogs"], r.Catalogs), collectResourceMap(descriptions["schemas"], r.Schemas), collectResourceMap(descriptions["external_locations"], r.ExternalLocations), @@ -194,6 +196,7 @@ func SupportedResources() map[string]resources.ResourceDescription { "model_provider_services": (&resources.ModelProviderService{}).ResourceDescription(), "registered_models": (&resources.RegisteredModel{}).ResourceDescription(), "quality_monitors": (&resources.QualityMonitor{}).ResourceDescription(), + "data_quality_monitors": (&resources.DataQualityMonitor{}).ResourceDescription(), "catalogs": (&resources.Catalog{}).ResourceDescription(), "schemas": (&resources.Schema{}).ResourceDescription(), "external_locations": (&resources.ExternalLocation{}).ResourceDescription(), diff --git a/bundle/config/resources/data_quality_monitor.go b/bundle/config/resources/data_quality_monitor.go new file mode 100644 index 00000000000..e0482b6b5f5 --- /dev/null +++ b/bundle/config/resources/data_quality_monitor.go @@ -0,0 +1,85 @@ +package resources + +import ( + "context" + "fmt" + "net/url" + "strings" + + "github.com/databricks/cli/libs/log" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/databricks-sdk-go/marshal" + "github.com/databricks/databricks-sdk-go/service/dataquality" +) + +// DataQualityMonitor is a data quality monitor on a Unity Catalog schema +// (anomaly detection) or table (data profiling), managed through the +// data-quality/v1 API. It supersedes quality_monitors. +type DataQualityMonitor struct { + BaseResource + ID string `json:"id,omitempty" bundle:"readonly"` + dataquality.Monitor +} + +func (m *DataQualityMonitor) UnmarshalJSON(b []byte) error { + return marshal.Unmarshal(b, m) +} + +func (m DataQualityMonitor) MarshalJSON() ([]byte, error) { + return marshal.Marshal(m) +} + +// DataQualityMonitorID composes the bundle ID of a monitor. A monitor has no +// name of its own: it is identified by the object it monitors. The format +// matches the Terraform provider's import ID for databricks_data_quality_monitor. +func DataQualityMonitorID(objectType, objectID string) string { + return objectType + "," + objectID +} + +func ParseDataQualityMonitorID(id string) (objectType, objectID string, err error) { + objectType, objectID, ok := strings.Cut(id, ",") + if !ok || objectType == "" || objectID == "" { + return "", "", fmt.Errorf("invalid data quality monitor id %q (want ,)", id) + } + return objectType, objectID, nil +} + +func (m *DataQualityMonitor) Exists(ctx context.Context, w *databricks.WorkspaceClient, id string) (bool, error) { + objectType, objectID, err := ParseDataQualityMonitorID(id) + if err != nil { + return false, err + } + _, err = w.DataQuality.GetMonitor(ctx, dataquality.GetMonitorRequest{ + ObjectType: objectType, + ObjectId: objectID, + }) + if err != nil { + log.Debugf(ctx, "data quality monitor %s does not exist", id) + if apierr.IsMissing(err) { + return false, nil + } + return false, err + } + return true, nil +} + +func (*DataQualityMonitor) ResourceDescription() ResourceDescription { + return ResourceDescription{ + SingularName: "data_quality_monitor", + PluralName: "data_quality_monitors", + SingularTitle: "Data quality monitor", + PluralTitle: "Data quality monitors", + } +} + +// The object_id is a UUID, which has no Catalog Explorer URL. +func (m *DataQualityMonitor) InitializeURL(baseURL url.URL) {} + +func (m *DataQualityMonitor) GetURL() (string, bool) { + return "", false +} + +func (m *DataQualityMonitor) GetName() string { + return DataQualityMonitorID(m.ObjectType, m.ObjectId) +} diff --git a/bundle/config/resources_test.go b/bundle/config/resources_test.go index c666c8964c0..a96aadfb15d 100644 --- a/bundle/config/resources_test.go +++ b/bundle/config/resources_test.go @@ -22,6 +22,7 @@ import ( "github.com/databricks/databricks-sdk-go/experimental/mocks" "github.com/databricks/databricks-sdk-go/service/apps" "github.com/databricks/databricks-sdk-go/service/catalog" + "github.com/databricks/databricks-sdk-go/service/dataquality" "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/databricks/databricks-sdk-go/service/ml" "github.com/databricks/databricks-sdk-go/service/pipelines" @@ -127,7 +128,10 @@ func TestSupportedResources(t *testing.T) { func TestBundleResourcePluralNamesResolveInWorkspaceURLs(t *testing.T) { // Resources that intentionally have no workspace URL. noURL := map[string]bool{ - "external_locations": true, + // A monitor is addressed by the UUID of the object it monitors, which has + // no Catalog Explorer URL. + "data_quality_monitors": true, + "external_locations": true, // A job run does have a workspace URL, but it's addressed by two IDs // (job + run) so it can't be expressed as a single-ID pattern here; it's // built in JobRun.InitializeURL via workspaceurls.JobRunURL instead. @@ -238,6 +242,11 @@ func TestResourcesBindSupport(t *testing.T) { CreateMonitor: catalog.CreateMonitor{}, }, }, + DataQualityMonitors: map[string]*resources.DataQualityMonitor{ + "my_data_quality_monitor": { + Monitor: dataquality.Monitor{}, + }, + }, ModelServingEndpoints: map[string]*resources.ModelServingEndpoint{ "my_model_serving_endpoint": { CreateServingEndpoint: serving.CreateServingEndpoint{}, @@ -404,6 +413,7 @@ func TestResourcesBindSupport(t *testing.T) { m.GetMockAppsAPI().EXPECT().GetByName(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockAlertsV2API().EXPECT().GetAlertById(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockQualityMonitorsAPI().EXPECT().Get(mock.Anything, mock.Anything).Return(nil, nil) + m.GetMockDataQualityAPI().EXPECT().GetMonitor(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockServingEndpointsAPI().EXPECT().Get(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockAiGatewayAPI().EXPECT().GetModelService(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockAiGatewayAPI().EXPECT().GetMcpService(mock.Anything, mock.Anything).Return(nil, nil) @@ -428,6 +438,11 @@ func TestResourcesBindSupport(t *testing.T) { m.GetMockVectorSearchIndexesAPI().EXPECT().GetIndexByIndexName(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockSecretsUcAPI().EXPECT().GetSecret(mock.Anything, mock.Anything).Return(&catalog.Secret{FullName: "0"}, nil) + // Resources whose ID has a structure that "0" does not satisfy. + bindIDs := map[string]string{ + "data_quality_monitor": "table,0", + } + allResources := supportedResources.AllResources() for _, group := range allResources { if len(group.Resources) == 0 && !unbindableResources[group.Description.SingularName] { @@ -439,7 +454,11 @@ func TestResourcesBindSupport(t *testing.T) { assert.NoError(t, err) // bind operation requires Exists to return true - exists, err := r.Exists(ctx, m.WorkspaceClient, "0") + id := "0" + if v, ok := bindIDs[resource.ResourceDescription().SingularName]; ok { + id = v + } + exists, err := r.Exists(ctx, m.WorkspaceClient, id) assert.NoError(t, err) assert.True(t, exists) } diff --git a/bundle/direct/dresources/all.go b/bundle/direct/dresources/all.go index 27e8496b83e..ff01da10797 100644 --- a/bundle/direct/dresources/all.go +++ b/bundle/direct/dresources/all.go @@ -40,6 +40,7 @@ var SupportedResources = map[string]any{ "mcp_services": (*ResourceMcpService)(nil), "model_provider_services": (*ResourceModelProviderService)(nil), "quality_monitors": (*ResourceQualityMonitor)(nil), + "data_quality_monitors": (*ResourceDataQualityMonitor)(nil), "vector_search_endpoints": (*ResourceVectorSearchEndpoint)(nil), "vector_search_indexes": (*ResourceVectorSearchIndex)(nil), "instance_pools": (*ResourceInstancePool)(nil), diff --git a/bundle/direct/dresources/all_test.go b/bundle/direct/dresources/all_test.go index f00037e48dc..90b7bae77e2 100644 --- a/bundle/direct/dresources/all_test.go +++ b/bundle/direct/dresources/all_test.go @@ -25,6 +25,7 @@ import ( "github.com/databricks/databricks-sdk-go/service/catalog" "github.com/databricks/databricks-sdk-go/service/dashboards" "github.com/databricks/databricks-sdk-go/service/database" + "github.com/databricks/databricks-sdk-go/service/dataquality" "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/databricks/databricks-sdk-go/service/ml" "github.com/databricks/databricks-sdk-go/service/pipelines" @@ -237,6 +238,16 @@ var testConfig map[string]any = map[string]any{ OutputSchemaName: "main.myschema", }, + "data_quality_monitors": &resources.DataQualityMonitor{ + ObjectType: "table", + ObjectId: "11111111-2222-3333-4444-555555555555", + DataProfilingConfig: &dataquality.DataProfilingConfig{ + AssetsDir: "/Workspace/Users/user@example.com/assets", + OutputSchemaId: "66666666-7777-8888-9999-000000000000", + Snapshot: &dataquality.SnapshotConfig{}, + }, + }, + "dashboards": &resources.Dashboard{ DisplayName: "my-dashboard", ParentPath: "/Workspace/Users/user@example.com", diff --git a/bundle/direct/dresources/apitypes.generated.yml b/bundle/direct/dresources/apitypes.generated.yml index af96326fa4c..d85443233d9 100644 --- a/bundle/direct/dresources/apitypes.generated.yml +++ b/bundle/direct/dresources/apitypes.generated.yml @@ -12,6 +12,8 @@ clusters: compute.ClusterSpec dashboards: dashboards.Dashboard +data_quality_monitors: dataquality.Monitor + database_catalogs: database.DatabaseCatalog database_instances: database.DatabaseInstance diff --git a/bundle/direct/dresources/configs/data_quality_monitors.generated.yml b/bundle/direct/dresources/configs/data_quality_monitors.generated.yml new file mode 100644 index 00000000000..5bb28fad822 --- /dev/null +++ b/bundle/direct/dresources/configs/data_quality_monitors.generated.yml @@ -0,0 +1,21 @@ +# Generated, do not edit. + +ignore_remote_changes: + - field: data_profiling_config.dashboard_id + reason: spec:output_only + - field: data_profiling_config.drift_metrics_table_name + reason: spec:output_only + - field: data_profiling_config.effective_warehouse_id + reason: spec:output_only + - field: data_profiling_config.latest_monitor_failure_message + reason: spec:output_only + - field: data_profiling_config.monitor_version + reason: spec:output_only + - field: data_profiling_config.monitored_table_name + reason: spec:output_only + - field: data_profiling_config.profile_metrics_table_name + reason: spec:output_only + - field: data_profiling_config.schedule.pause_status + reason: spec:output_only + - field: data_profiling_config.status + reason: spec:output_only diff --git a/bundle/direct/dresources/configs/data_quality_monitors.yml b/bundle/direct/dresources/configs/data_quality_monitors.yml new file mode 100644 index 00000000000..be871936d76 --- /dev/null +++ b/bundle/direct/dresources/configs/data_quality_monitors.yml @@ -0,0 +1,21 @@ +provided_id_fields: + # object_type + object_id identify the monitored object and compose the ID the + # monitor is fetched by. Both are immutable; a local change recreates. + - field: object_type + reason: id_field + - field: object_id + reason: id_field + +recreate_on_changes: + # Create:REQ Update:IGN in the data-quality API: the update endpoint ignores it. + # Same as quality_monitors.assets_dir. + - field: data_profiling_config.assets_dir + reason: immutable + +ignore_remote_changes: + # Create-time options for the built-in dashboard; quality_monitors treats the + # same fields as not returned on read. + - field: data_profiling_config.skip_builtin_dashboard + reason: input_only + - field: data_profiling_config.warehouse_id + reason: input_only diff --git a/bundle/direct/dresources/data_quality_monitor.go b/bundle/direct/dresources/data_quality_monitor.go new file mode 100644 index 00000000000..50df5cd9bdf --- /dev/null +++ b/bundle/direct/dresources/data_quality_monitor.go @@ -0,0 +1,74 @@ +package dresources + +import ( + "context" + + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/service/dataquality" +) + +// Data quality monitor. +// API: https://docs.databricks.com/api/workspace/dataquality +// Terraform: databricks_data_quality_monitor +type ResourceDataQualityMonitor struct { + client *databricks.WorkspaceClient +} + +// dataQualityMonitorUpdateMask lists every client-settable field; object_type +// and object_id are the identity and recreate on change. This matches the mask +// the Terraform provider sends. +const dataQualityMonitorUpdateMask = "anomaly_detection_config,data_profiling_config" + +func (*ResourceDataQualityMonitor) New(client *databricks.WorkspaceClient) *ResourceDataQualityMonitor { + return &ResourceDataQualityMonitor{client: client} +} + +func (*ResourceDataQualityMonitor) PrepareState(input *resources.DataQualityMonitor) *dataquality.Monitor { + return &input.Monitor +} + +func (r *ResourceDataQualityMonitor) DoRead(ctx context.Context, id string) (*dataquality.Monitor, error) { + objectType, objectID, err := resources.ParseDataQualityMonitorID(id) + if err != nil { + return nil, err + } + return r.client.DataQuality.GetMonitor(ctx, dataquality.GetMonitorRequest{ + ObjectType: objectType, + ObjectId: objectID, + }) +} + +func (r *ResourceDataQualityMonitor) DoCreate(ctx context.Context, config *dataquality.Monitor) (string, *dataquality.Monitor, error) { + resp, err := r.client.DataQuality.CreateMonitor(ctx, dataquality.CreateMonitorRequest{ + Monitor: *config, + }) + if err != nil { + return "", nil, err + } + return resources.DataQualityMonitorID(resp.ObjectType, resp.ObjectId), resp, nil +} + +func (r *ResourceDataQualityMonitor) DoUpdate(ctx context.Context, id string, config *dataquality.Monitor, _ *PlanEntry) (*dataquality.Monitor, error) { + objectType, objectID, err := resources.ParseDataQualityMonitorID(id) + if err != nil { + return nil, err + } + return r.client.DataQuality.UpdateMonitor(ctx, dataquality.UpdateMonitorRequest{ + Monitor: *config, + ObjectType: objectType, + ObjectId: objectID, + UpdateMask: dataQualityMonitorUpdateMask, + }) +} + +func (r *ResourceDataQualityMonitor) DoDelete(ctx context.Context, id string, _ *dataquality.Monitor) error { + objectType, objectID, err := resources.ParseDataQualityMonitorID(id) + if err != nil { + return err + } + return r.client.DataQuality.DeleteMonitor(ctx, dataquality.DeleteMonitorRequest{ + ObjectType: objectType, + ObjectId: objectID, + }) +} diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index 710e4b8373f..500c105af10 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -773,6 +773,34 @@ resources: "warehouse_id": "description": |- The warehouse ID used to run the dashboard. + "data_quality_monitors": + "description": |- + The data quality monitor definitions for the bundle, where each key is the name of the data quality monitor. + "markdown_description": |- + The data quality monitor definitions for the bundle, where each key is the name of the data quality monitor. See [\_](/dev-tools/bundles/resources.md#data_quality_monitors). + "$type": + "markdown_description": |- + The data_quality_monitor resource allows you to define a [data quality monitor](/api/workspace/dataquality/createmonitor) on a Unity Catalog schema (anomaly detection) or table (data profiling). It supersedes the `quality_monitor` resource. For information about data quality monitoring, see [_](/data-governance/unity-catalog/data-quality-monitoring/index.md). + "markdown_examples": |- + The following example defines an anomaly detection monitor on a schema: + + ```yaml + resources: + schemas: + my_schema: + catalog_name: main + name: my_schema + + data_quality_monitors: + my_schema_monitor: + object_type: schema + object_id: ${resources.schemas.my_schema.schema_id} + anomaly_detection_config: {} + ``` + "$fields": + "lifecycle": + "description": |- + Settings that control the deployment lifecycle of the resource, such as preventing it from being destroyed. "database_catalogs": "description": |- PLACEHOLDER diff --git a/bundle/internal/validation/generated/enum_fields.go b/bundle/internal/validation/generated/enum_fields.go index 42401ec6968..2b87d9da2f7 100644 --- a/bundle/internal/validation/generated/enum_fields.go +++ b/bundle/internal/validation/generated/enum_fields.go @@ -58,6 +58,13 @@ var EnumFields = map[string][]string{ "resources.dashboards.*.lifecycle_state": {"ACTIVE", "TRASHED"}, "resources.dashboards.*.permissions[*].level": {"CAN_ATTACH_TO", "CAN_BIND", "CAN_CREATE", "CAN_CREATE_APP", "CAN_EDIT", "CAN_EDIT_METADATA", "CAN_MANAGE", "CAN_MANAGE_PRODUCTION_VERSIONS", "CAN_MANAGE_RUN", "CAN_MANAGE_STAGING_VERSIONS", "CAN_MONITOR", "CAN_MONITOR_ONLY", "CAN_QUERY", "CAN_READ", "CAN_RESTART", "CAN_RUN", "CAN_USE", "CAN_VIEW", "CAN_VIEW_METADATA", "IS_OWNER"}, + "resources.data_quality_monitors.*.data_profiling_config.custom_metrics[*].type": {"DATA_PROFILING_CUSTOM_METRIC_TYPE_AGGREGATE", "DATA_PROFILING_CUSTOM_METRIC_TYPE_DERIVED", "DATA_PROFILING_CUSTOM_METRIC_TYPE_DRIFT"}, + "resources.data_quality_monitors.*.data_profiling_config.inference_log.granularities[*]": {"AGGREGATION_GRANULARITY_1_DAY", "AGGREGATION_GRANULARITY_1_HOUR", "AGGREGATION_GRANULARITY_1_MONTH", "AGGREGATION_GRANULARITY_1_WEEK", "AGGREGATION_GRANULARITY_1_YEAR", "AGGREGATION_GRANULARITY_2_WEEKS", "AGGREGATION_GRANULARITY_30_MINUTES", "AGGREGATION_GRANULARITY_3_WEEKS", "AGGREGATION_GRANULARITY_4_WEEKS", "AGGREGATION_GRANULARITY_5_MINUTES"}, + "resources.data_quality_monitors.*.data_profiling_config.inference_log.problem_type": {"INFERENCE_PROBLEM_TYPE_CLASSIFICATION", "INFERENCE_PROBLEM_TYPE_REGRESSION"}, + "resources.data_quality_monitors.*.data_profiling_config.schedule.pause_status": {"CRON_SCHEDULE_PAUSE_STATUS_PAUSED", "CRON_SCHEDULE_PAUSE_STATUS_UNPAUSED"}, + "resources.data_quality_monitors.*.data_profiling_config.status": {"DATA_PROFILING_STATUS_ACTIVE", "DATA_PROFILING_STATUS_DELETE_PENDING", "DATA_PROFILING_STATUS_ERROR", "DATA_PROFILING_STATUS_FAILED", "DATA_PROFILING_STATUS_PENDING"}, + "resources.data_quality_monitors.*.data_profiling_config.time_series.granularities[*]": {"AGGREGATION_GRANULARITY_1_DAY", "AGGREGATION_GRANULARITY_1_HOUR", "AGGREGATION_GRANULARITY_1_MONTH", "AGGREGATION_GRANULARITY_1_WEEK", "AGGREGATION_GRANULARITY_1_YEAR", "AGGREGATION_GRANULARITY_2_WEEKS", "AGGREGATION_GRANULARITY_30_MINUTES", "AGGREGATION_GRANULARITY_3_WEEKS", "AGGREGATION_GRANULARITY_4_WEEKS", "AGGREGATION_GRANULARITY_5_MINUTES"}, + "resources.database_instances.*.permissions[*].level": {"CAN_ATTACH_TO", "CAN_BIND", "CAN_CREATE", "CAN_CREATE_APP", "CAN_EDIT", "CAN_EDIT_METADATA", "CAN_MANAGE", "CAN_MANAGE_PRODUCTION_VERSIONS", "CAN_MANAGE_RUN", "CAN_MANAGE_STAGING_VERSIONS", "CAN_MONITOR", "CAN_MONITOR_ONLY", "CAN_QUERY", "CAN_READ", "CAN_RESTART", "CAN_RUN", "CAN_USE", "CAN_VIEW", "CAN_VIEW_METADATA", "IS_OWNER"}, "resources.database_instances.*.state": {"AVAILABLE", "DELETING", "FAILING_OVER", "STARTING", "STOPPED", "UPDATING"}, diff --git a/bundle/internal/validation/generated/required_fields.go b/bundle/internal/validation/generated/required_fields.go index 9488299bba0..6e85caee5e4 100644 --- a/bundle/internal/validation/generated/required_fields.go +++ b/bundle/internal/validation/generated/required_fields.go @@ -64,6 +64,13 @@ var RequiredFields = map[string][]string{ "resources.dashboards.*.permissions[*]": {"level"}, + "resources.data_quality_monitors.*": {"object_id", "object_type"}, + "resources.data_quality_monitors.*.data_profiling_config": {"output_schema_id"}, + "resources.data_quality_monitors.*.data_profiling_config.custom_metrics[*]": {"definition", "input_columns", "name", "output_data_type", "type"}, + "resources.data_quality_monitors.*.data_profiling_config.inference_log": {"granularities", "model_id_column", "prediction_column", "problem_type", "timestamp_column"}, + "resources.data_quality_monitors.*.data_profiling_config.schedule": {"quartz_cron_expression", "timezone_id"}, + "resources.data_quality_monitors.*.data_profiling_config.time_series": {"granularities", "timestamp_column"}, + "resources.database_catalogs.*": {"database_instance_name", "database_name", "name"}, "resources.database_instances.*": {"name"}, diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index e8422a92b40..129029d0899 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -834,6 +834,50 @@ } ] }, + "resources.DataQualityMonitor": { + "oneOf": [ + { + "type": "object", + "description": "Monitor for the data quality of unity catalog entities such as schema or table.", + "properties": { + "anomaly_detection_config": { + "description": "[Public Preview] Anomaly Detection Configuration, applicable to `schema` object types.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/dataquality.AnomalyDetectionConfig", + "x-databricks-launch-stage": "PUBLIC_PREVIEW" + }, + "data_profiling_config": { + "description": "[Public Preview] Data Profiling Configuration, applicable to `table` object types. Exactly one `Analysis Configuration`\nmust be present.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/dataquality.DataProfilingConfig", + "x-databricks-launch-stage": "PUBLIC_PREVIEW" + }, + "lifecycle": { + "description": "Settings that control the deployment lifecycle of the resource, such as preventing it from being destroyed.", + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.Lifecycle" + }, + "object_id": { + "description": "[Public Preview] The UUID of the request object. It is `schema_id` for `schema`, and `table_id` for `table`.\n\nFind the `schema_id` from either:\n1. The [schema_id](https://docs.databricks.com/api/workspace/schemas/get#schema_id) of the `Schemas` resource.\n2. In [Catalog Explorer](https://docs.databricks.com/aws/en/catalog-explorer/) \u003e select the `schema` \u003e go to the `Details` tab \u003e the `Schema ID` field.\n\nFind the `table_id` from either:\n1. The [table_id](https://docs.databricks.com/api/workspace/tables/get#table_id) of the `Tables` resource.\n2. In [Catalog Explorer](https://docs.databricks.com/aws/en/catalog-explorer/) \u003e select the `table` \u003e go to the `Details` tab \u003e the `Table ID` field.", + "$ref": "#/$defs/string", + "x-databricks-launch-stage": "PUBLIC_PREVIEW" + }, + "object_type": { + "description": "[Public Preview] The type of the monitored object. Can be one of the following: `schema` or `table`.", + "$ref": "#/$defs/string", + "x-databricks-launch-stage": "PUBLIC_PREVIEW" + } + }, + "additionalProperties": false, + "required": [ + "object_id", + "object_type" + ], + "markdownDescription": "The data_quality_monitor resource allows you to define a [data quality monitor](https://docs.databricks.com/api/workspace/dataquality/createmonitor) on a Unity Catalog schema (anomaly detection) or table (data profiling). It supersedes the `quality_monitor` resource. For information about data quality monitoring, see [link](https://docs.databricks.com/data-governance/unity-catalog/data-quality-monitoring/index.html)." + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\._*\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "resources.DatabaseCatalog": { "oneOf": [ { @@ -4026,6 +4070,11 @@ "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.Dashboard", "markdownDescription": "The dashboard definitions for the bundle, where each key is the name of the dashboard. See [dashboards](https://docs.databricks.com/dev-tools/bundles/resources.html#dashboards)." }, + "data_quality_monitors": { + "description": "The data quality monitor definitions for the bundle, where each key is the name of the data quality monitor.", + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.DataQualityMonitor", + "markdownDescription": "The data quality monitor definitions for the bundle, where each key is the name of the data quality monitor. See [data_quality_monitors](https://docs.databricks.com/dev-tools/bundles/resources.html#data_quality_monitors)." + }, "database_catalogs": { "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.DatabaseCatalog" }, @@ -9064,6 +9113,438 @@ } ] }, + "dataquality.AggregationGranularity": { + "oneOf": [ + { + "type": "string", + "description": "The granularity for aggregating data into time windows based on their timestamp.", + "enum": [ + "AGGREGATION_GRANULARITY_5_MINUTES", + "AGGREGATION_GRANULARITY_30_MINUTES", + "AGGREGATION_GRANULARITY_1_HOUR", + "AGGREGATION_GRANULARITY_1_DAY", + "AGGREGATION_GRANULARITY_1_WEEK", + "AGGREGATION_GRANULARITY_2_WEEKS", + "AGGREGATION_GRANULARITY_3_WEEKS", + "AGGREGATION_GRANULARITY_4_WEEKS", + "AGGREGATION_GRANULARITY_1_MONTH", + "AGGREGATION_GRANULARITY_1_YEAR" + ], + "enumDescriptions": [ + "[Public Preview]", + "[Public Preview]", + "[Public Preview]", + "[Public Preview]", + "[Public Preview]", + "[Public Preview]", + "[Public Preview]", + "[Public Preview]", + "[Public Preview]", + "[Public Preview]" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\._*\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "dataquality.AnomalyDetectionConfig": { + "oneOf": [ + { + "type": "object", + "description": "Anomaly Detection Configurations.", + "properties": { + "excluded_table_full_names": { + "description": "[Beta] List of fully qualified table names to exclude from anomaly detection.", + "$ref": "#/$defs/slice/string", + "x-databricks-launch-stage": "PUBLIC_BETA" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\._*\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "dataquality.CronSchedule": { + "oneOf": [ + { + "type": "object", + "description": "The data quality monitoring workflow cron schedule.", + "properties": { + "quartz_cron_expression": { + "description": "[Public Preview] The expression that determines when to run the monitor. See [examples](https://www.quartz-scheduler.org/documentation/quartz-2.3.0/tutorials/crontrigger.html).", + "$ref": "#/$defs/string", + "x-databricks-launch-stage": "PUBLIC_PREVIEW" + }, + "timezone_id": { + "description": "[Public Preview] A Java timezone id. The schedule for a job will be resolved with respect to this timezone.\nSee `Java TimeZone \u003chttp://docs.oracle.com/javase/7/docs/api/java/util/TimeZone.html\u003e`_ for details.\nThe timezone id (e.g., ``America/Los_Angeles``) in which to evaluate the quartz expression.", + "$ref": "#/$defs/string", + "x-databricks-launch-stage": "PUBLIC_PREVIEW" + } + }, + "additionalProperties": false, + "required": [ + "quartz_cron_expression", + "timezone_id" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\._*\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "dataquality.CronSchedulePauseStatus": { + "oneOf": [ + { + "type": "string", + "description": "The data quality monitoring workflow cron schedule pause status.", + "enum": [ + "CRON_SCHEDULE_PAUSE_STATUS_UNPAUSED", + "CRON_SCHEDULE_PAUSE_STATUS_PAUSED" + ], + "enumDescriptions": [ + "[Public Preview]", + "[Public Preview]" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\._*\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "dataquality.DataProfilingConfig": { + "oneOf": [ + { + "type": "object", + "description": "Data Profiling Configurations.", + "properties": { + "assets_dir": { + "description": "[Public Preview] Field for specifying the absolute path to a custom directory to store data-monitoring\nassets. Normally prepopulated to a default user location via UI and Python APIs.", + "$ref": "#/$defs/string", + "x-databricks-launch-stage": "PUBLIC_PREVIEW" + }, + "baseline_table_name": { + "description": "[Public Preview] Baseline table name.\nBaseline data is used to compute drift from the data in the monitored `table_name`.\nThe baseline table and the monitored table shall have the same schema.", + "$ref": "#/$defs/string", + "x-databricks-launch-stage": "PUBLIC_PREVIEW" + }, + "custom_metrics": { + "description": "[Public Preview] Custom metrics.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/dataquality.DataProfilingCustomMetric", + "x-databricks-launch-stage": "PUBLIC_PREVIEW" + }, + "inference_log": { + "description": "[Public Preview] `Analysis Configuration` for monitoring inference log tables.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/dataquality.InferenceLogConfig", + "x-databricks-launch-stage": "PUBLIC_PREVIEW" + }, + "notification_settings": { + "description": "[Public Preview] Field for specifying notification settings.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/dataquality.NotificationSettings", + "x-databricks-launch-stage": "PUBLIC_PREVIEW" + }, + "output_schema_id": { + "description": "[Public Preview] ID of the schema where output tables are created.", + "$ref": "#/$defs/string", + "x-databricks-launch-stage": "PUBLIC_PREVIEW" + }, + "schedule": { + "description": "[Public Preview] The cron schedule.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/dataquality.CronSchedule", + "x-databricks-launch-stage": "PUBLIC_PREVIEW" + }, + "skip_builtin_dashboard": { + "description": "[Public Preview] Whether to skip creating a default dashboard summarizing data quality metrics.", + "$ref": "#/$defs/bool", + "x-databricks-launch-stage": "PUBLIC_PREVIEW" + }, + "slicing_exprs": { + "description": "[Public Preview] List of column expressions to slice data with for targeted analysis. The data is grouped by\neach expression independently, resulting in a separate slice for each predicate and its\ncomplements. For example `slicing_exprs=[“col_1”, “col_2 \u003e 10”]` will generate the following\nslices: two slices for `col_2 \u003e 10` (True and False), and one slice per unique value in\n`col1`. For high-cardinality columns, only the top 100 unique values by frequency will\ngenerate slices.", + "$ref": "#/$defs/slice/string", + "x-databricks-launch-stage": "PUBLIC_PREVIEW" + }, + "snapshot": { + "description": "[Public Preview] `Analysis Configuration` for monitoring snapshot tables.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/dataquality.SnapshotConfig", + "x-databricks-launch-stage": "PUBLIC_PREVIEW" + }, + "time_series": { + "description": "[Public Preview] `Analysis Configuration` for monitoring time series tables.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/dataquality.TimeSeriesConfig", + "x-databricks-launch-stage": "PUBLIC_PREVIEW" + }, + "warehouse_id": { + "description": "[Public Preview] Optional argument to specify the warehouse for dashboard creation. If not specified, the first running\nwarehouse will be used.", + "$ref": "#/$defs/string", + "x-databricks-launch-stage": "PUBLIC_PREVIEW" + } + }, + "additionalProperties": false, + "required": [ + "output_schema_id" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\._*\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "dataquality.DataProfilingCustomMetric": { + "oneOf": [ + { + "type": "object", + "description": "Custom metric definition.", + "properties": { + "definition": { + "description": "[Public Preview] Jinja template for a SQL expression that specifies how to compute the metric. See [create metric definition](https://docs.databricks.com/en/lakehouse-monitoring/custom-metrics.html#create-definition).", + "$ref": "#/$defs/string", + "x-databricks-launch-stage": "PUBLIC_PREVIEW" + }, + "input_columns": { + "description": "[Public Preview] A list of column names in the input table the metric should be computed for.\nCan use ``\":table\"`` to indicate that the metric needs information from multiple columns.", + "$ref": "#/$defs/slice/string", + "x-databricks-launch-stage": "PUBLIC_PREVIEW" + }, + "name": { + "description": "[Public Preview] Name of the metric in the output tables.", + "$ref": "#/$defs/string", + "x-databricks-launch-stage": "PUBLIC_PREVIEW" + }, + "output_data_type": { + "description": "[Public Preview] The output type of the custom metric.", + "$ref": "#/$defs/string", + "x-databricks-launch-stage": "PUBLIC_PREVIEW" + }, + "type": { + "description": "[Public Preview] The type of the custom metric.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/dataquality.DataProfilingCustomMetricType", + "x-databricks-launch-stage": "PUBLIC_PREVIEW" + } + }, + "additionalProperties": false, + "required": [ + "definition", + "input_columns", + "name", + "output_data_type", + "type" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\._*\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "dataquality.DataProfilingCustomMetricType": { + "oneOf": [ + { + "type": "string", + "description": "The custom metric type.", + "enum": [ + "DATA_PROFILING_CUSTOM_METRIC_TYPE_AGGREGATE", + "DATA_PROFILING_CUSTOM_METRIC_TYPE_DERIVED", + "DATA_PROFILING_CUSTOM_METRIC_TYPE_DRIFT" + ], + "enumDescriptions": [ + "[Public Preview]", + "[Public Preview]", + "[Public Preview]" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\._*\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "dataquality.DataProfilingStatus": { + "oneOf": [ + { + "type": "string", + "description": "The status of the data profiling monitor.", + "enum": [ + "DATA_PROFILING_STATUS_ACTIVE", + "DATA_PROFILING_STATUS_PENDING", + "DATA_PROFILING_STATUS_DELETE_PENDING", + "DATA_PROFILING_STATUS_ERROR", + "DATA_PROFILING_STATUS_FAILED" + ], + "enumDescriptions": [ + "[Public Preview]", + "[Public Preview]", + "[Public Preview]", + "[Public Preview]", + "[Public Preview]" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\._*\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "dataquality.InferenceLogConfig": { + "oneOf": [ + { + "type": "object", + "description": "Inference log configuration.", + "properties": { + "granularities": { + "description": "[Public Preview] List of granularities to use when aggregating data into time windows based on their timestamp.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/dataquality.AggregationGranularity", + "x-databricks-launch-stage": "PUBLIC_PREVIEW" + }, + "label_column": { + "description": "[Public Preview] Column for the label.", + "$ref": "#/$defs/string", + "x-databricks-launch-stage": "PUBLIC_PREVIEW" + }, + "model_id_column": { + "description": "[Public Preview] Column for the model identifier.", + "$ref": "#/$defs/string", + "x-databricks-launch-stage": "PUBLIC_PREVIEW" + }, + "prediction_column": { + "description": "[Public Preview] Column for the prediction.", + "$ref": "#/$defs/string", + "x-databricks-launch-stage": "PUBLIC_PREVIEW" + }, + "problem_type": { + "description": "[Public Preview] Problem type the model aims to solve.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/dataquality.InferenceProblemType", + "x-databricks-launch-stage": "PUBLIC_PREVIEW" + }, + "timestamp_column": { + "description": "[Public Preview] Column for the timestamp.", + "$ref": "#/$defs/string", + "x-databricks-launch-stage": "PUBLIC_PREVIEW" + } + }, + "additionalProperties": false, + "required": [ + "granularities", + "model_id_column", + "prediction_column", + "problem_type", + "timestamp_column" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\._*\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "dataquality.InferenceProblemType": { + "oneOf": [ + { + "type": "string", + "description": "Inference problem type the model aims to solve.", + "enum": [ + "INFERENCE_PROBLEM_TYPE_CLASSIFICATION", + "INFERENCE_PROBLEM_TYPE_REGRESSION" + ], + "enumDescriptions": [ + "[Public Preview]", + "[Public Preview]" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\._*\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "dataquality.NotificationDestination": { + "oneOf": [ + { + "type": "object", + "description": "Destination of the data quality monitoring notification.", + "properties": { + "email_addresses": { + "description": "[Public Preview] The list of email addresses to send the notification to. A maximum of 5 email addresses is supported.", + "$ref": "#/$defs/slice/string", + "x-databricks-launch-stage": "PUBLIC_PREVIEW" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\._*\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "dataquality.NotificationSettings": { + "oneOf": [ + { + "type": "object", + "description": "Settings for sending notifications on the data quality monitoring.", + "properties": { + "on_failure": { + "description": "[Public Preview] Destinations to send notifications on failure/timeout.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/dataquality.NotificationDestination", + "x-databricks-launch-stage": "PUBLIC_PREVIEW" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\._*\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "dataquality.SnapshotConfig": { + "oneOf": [ + { + "type": "object", + "description": "Snapshot analysis configuration.", + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\._*\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "dataquality.TimeSeriesConfig": { + "oneOf": [ + { + "type": "object", + "description": "Time series analysis configuration.", + "properties": { + "granularities": { + "description": "[Public Preview] List of granularities to use when aggregating data into time windows based on their timestamp.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/dataquality.AggregationGranularity", + "x-databricks-launch-stage": "PUBLIC_PREVIEW" + }, + "timestamp_column": { + "description": "[Public Preview] Column for the timestamp.", + "$ref": "#/$defs/string", + "x-databricks-launch-stage": "PUBLIC_PREVIEW" + } + }, + "additionalProperties": false, + "required": [ + "granularities", + "timestamp_column" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\._*\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "iam.PermissionLevel": { "oneOf": [ { @@ -18230,6 +18711,20 @@ } ] }, + "resources.DataQualityMonitor": { + "oneOf": [ + { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.DataQualityMonitor" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\._*\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "resources.DatabaseCatalog": { "oneOf": [ { @@ -19241,6 +19736,34 @@ } ] }, + "dataquality.AggregationGranularity": { + "oneOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/dataquality.AggregationGranularity" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\._*\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "dataquality.DataProfilingCustomMetric": { + "oneOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/dataquality.DataProfilingCustomMetric" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\._*\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "jobs.AlertTaskSubscriber": { "oneOf": [ { diff --git a/bundle/statemgmt/state_load_test.go b/bundle/statemgmt/state_load_test.go index b03ff469284..e6f2788236c 100644 --- a/bundle/statemgmt/state_load_test.go +++ b/bundle/statemgmt/state_load_test.go @@ -10,6 +10,7 @@ import ( "github.com/databricks/databricks-sdk-go/service/catalog" "github.com/databricks/databricks-sdk-go/service/compute" "github.com/databricks/databricks-sdk-go/service/database" + "github.com/databricks/databricks-sdk-go/service/dataquality" "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/databricks/databricks-sdk-go/service/ml" "github.com/databricks/databricks-sdk-go/service/pipelines" @@ -33,6 +34,7 @@ func TestStateToBundleEmptyLocalResources(t *testing.T) { "resources.experiments.test_mlflow_experiment": {ID: "1"}, "resources.model_serving_endpoints.test_model_serving": {ID: "1"}, "resources.model_services.test_model_service": {ID: "main.default.test_model_service"}, + "resources.data_quality_monitors.test_data_quality_monitor": {ID: "table,uuid-1"}, "resources.mcp_services.test_mcp_service": {ID: "main.default.test_mcp_service"}, "resources.model_provider_services.test_model_provider_service": {ID: "main.default.test_model_provider_service"}, "resources.registered_models.test_registered_model": {ID: "1"}, @@ -171,6 +173,9 @@ func TestStateToBundleEmptyLocalResources(t *testing.T) { assert.Equal(t, "main.default.test_model_service", config.Resources.ModelServices["test_model_service"].ID) assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.ModelServices["test_model_service"].ModifiedStatus) + assert.Equal(t, "table,uuid-1", config.Resources.DataQualityMonitors["test_data_quality_monitor"].ID) + assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.DataQualityMonitors["test_data_quality_monitor"].ModifiedStatus) + assert.Equal(t, "main.default.test_mcp_service", config.Resources.McpServices["test_mcp_service"].ID) assert.Equal(t, "main.default.test_model_provider_service", config.Resources.ModelProviderServices["test_model_provider_service"].ID) assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.McpServices["test_mcp_service"].ModifiedStatus) @@ -443,6 +448,11 @@ func TestStateToBundleEmptyRemoteResources(t *testing.T) { }, }, }, + DataQualityMonitors: map[string]*resources.DataQualityMonitor{ + "test_data_quality_monitor": { + Monitor: dataquality.Monitor{ObjectType: "table", ObjectId: "uuid-1"}, + }, + }, McpServices: map[string]*resources.McpService{ "test_mcp_service": { McpServiceConfig: resources.McpServiceConfig{ @@ -573,6 +583,9 @@ func TestStateToBundleEmptyRemoteResources(t *testing.T) { assert.Empty(t, config.Resources.ModelServices["test_model_service"].ID) assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.ModelServices["test_model_service"].ModifiedStatus) + assert.Empty(t, config.Resources.DataQualityMonitors["test_data_quality_monitor"].ID) + assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.DataQualityMonitors["test_data_quality_monitor"].ModifiedStatus) + assert.Empty(t, config.Resources.McpServices["test_mcp_service"].ID) assert.Empty(t, config.Resources.ModelProviderServices["test_model_provider_service"].ID) assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.McpServices["test_mcp_service"].ModifiedStatus) @@ -1018,6 +1031,14 @@ func TestStateToBundleModifiedResources(t *testing.T) { }, }, }, + DataQualityMonitors: map[string]*resources.DataQualityMonitor{ + "test_data_quality_monitor": { + Monitor: dataquality.Monitor{ObjectType: "table", ObjectId: "uuid-1"}, + }, + "test_data_quality_monitor_new": { + Monitor: dataquality.Monitor{ObjectType: "table", ObjectId: "uuid-3"}, + }, + }, McpServices: map[string]*resources.McpService{ "test_mcp_service": { McpServiceConfig: resources.McpServiceConfig{ @@ -1109,6 +1130,8 @@ func TestStateToBundleModifiedResources(t *testing.T) { "resources.cluster_policies.test_cluster_policy_old": {ID: "cp-2"}, "resources.model_services.test_model_service": {ID: "main.default.test_model_service"}, "resources.model_services.test_model_service_old": {ID: "main.default.test_model_service_old"}, + "resources.data_quality_monitors.test_data_quality_monitor": {ID: "table,uuid-1"}, + "resources.data_quality_monitors.test_data_quality_monitor_old": {ID: "table,uuid-2"}, "resources.mcp_services.test_mcp_service": {ID: "main.default.test_mcp_service"}, "resources.model_provider_services.test_model_provider_service": {ID: "main.default.test_model_provider_service"}, "resources.mcp_services.test_mcp_service_old": {ID: "main.default.test_mcp_service_old"}, @@ -1331,6 +1354,13 @@ func TestStateToBundleModifiedResources(t *testing.T) { assert.Empty(t, config.Resources.ModelServices["test_model_service_new"].ID) assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.ModelServices["test_model_service_new"].ModifiedStatus) + assert.Equal(t, "table,uuid-1", config.Resources.DataQualityMonitors["test_data_quality_monitor"].ID) + assert.Empty(t, config.Resources.DataQualityMonitors["test_data_quality_monitor"].ModifiedStatus) + assert.Equal(t, "table,uuid-2", config.Resources.DataQualityMonitors["test_data_quality_monitor_old"].ID) + assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.DataQualityMonitors["test_data_quality_monitor_old"].ModifiedStatus) + assert.Empty(t, config.Resources.DataQualityMonitors["test_data_quality_monitor_new"].ID) + assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.DataQualityMonitors["test_data_quality_monitor_new"].ModifiedStatus) + assert.Equal(t, "main.default.test_mcp_service", config.Resources.McpServices["test_mcp_service"].ID) assert.Equal(t, "main.default.test_model_provider_service", config.Resources.ModelProviderServices["test_model_provider_service"].ID) assert.Empty(t, config.Resources.McpServices["test_mcp_service"].ModifiedStatus) diff --git a/libs/testserver/data_quality_monitors.go b/libs/testserver/data_quality_monitors.go new file mode 100644 index 00000000000..381c1b3f104 --- /dev/null +++ b/libs/testserver/data_quality_monitors.go @@ -0,0 +1,85 @@ +package testserver + +import ( + "encoding/json" + "fmt" + "net/http" + + "github.com/databricks/databricks-sdk-go/service/dataquality" +) + +func dataQualityMonitorKey(objectType, objectID string) string { + return objectType + "/" + objectID +} + +// The SDK sends the Monitor body directly (not wrapped in "monitor"). +func (s *FakeWorkspace) DataQualityMonitorCreate(req Request) Response { + defer s.LockUnlock()() + + var monitor dataquality.Monitor + if err := json.Unmarshal(req.Body, &monitor); err != nil { + return Response{ + Body: fmt.Sprintf("internal error: %s", err), + StatusCode: http.StatusInternalServerError, + } + } + + key := dataQualityMonitorKey(monitor.ObjectType, monitor.ObjectId) + if _, ok := s.DataQualityMonitors[key]; ok { + return Response{ + StatusCode: http.StatusConflict, + Body: map[string]string{"error_code": "RESOURCE_ALREADY_EXISTS", "message": "Monitor already exists for " + key}, + } + } + + setDataProfilingOutputs(monitor.DataProfilingConfig, 1) + s.DataQualityMonitors[key] = monitor + return Response{Body: monitor} +} + +// update_mask is a query parameter; the fake applies both configs, which is +// what the mask the CLI sends covers. +func (s *FakeWorkspace) DataQualityMonitorUpdate(req Request, objectType, objectID string) Response { + defer s.LockUnlock()() + + key := dataQualityMonitorKey(objectType, objectID) + existing, ok := s.DataQualityMonitors[key] + if !ok { + return Response{ + StatusCode: http.StatusNotFound, + Body: map[string]string{"error_code": "RESOURCE_DOES_NOT_EXIST", "message": "Monitor not found for " + key}, + } + } + + var incoming dataquality.Monitor + if err := json.Unmarshal(req.Body, &incoming); err != nil { + return Response{ + Body: fmt.Sprintf("internal error: %s", err), + StatusCode: http.StatusInternalServerError, + } + } + + version := int64(1) + if existing.DataProfilingConfig != nil { + version = existing.DataProfilingConfig.MonitorVersion + 1 + } + existing.AnomalyDetectionConfig = incoming.AnomalyDetectionConfig + existing.DataProfilingConfig = incoming.DataProfilingConfig + setDataProfilingOutputs(existing.DataProfilingConfig, version) + + s.DataQualityMonitors[key] = existing + return Response{Body: existing} +} + +// setDataProfilingOutputs fills server-computed fields so tests exercise drift +// suppression of OUTPUT_ONLY fields. +func setDataProfilingOutputs(config *dataquality.DataProfilingConfig, version int64) { + if config == nil { + return + } + config.Status = dataquality.DataProfilingStatusDataProfilingStatusActive + config.MonitorVersion = version + if config.DashboardId == "" { + config.DashboardId = nextUUID() + } +} diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go index e0073bd656a..b93c8ed8865 100644 --- a/libs/testserver/fake_workspace.go +++ b/libs/testserver/fake_workspace.go @@ -22,6 +22,7 @@ import ( "github.com/databricks/databricks-sdk-go/service/compute" "github.com/databricks/databricks-sdk-go/service/dashboards" "github.com/databricks/databricks-sdk-go/service/database" + "github.com/databricks/databricks-sdk-go/service/dataquality" "github.com/databricks/databricks-sdk-go/service/postgres" "github.com/google/uuid" @@ -206,6 +207,7 @@ type FakeWorkspace struct { Pipelines map[string]pipelines.GetPipelineResponse PipelineUpdates map[string]bool Monitors map[string]catalog.MonitorInfo + DataQualityMonitors map[string]dataquality.Monitor Apps map[string]apps.App Schemas map[string]catalog.SchemaInfo Grants map[string][]catalog.PrivilegeAssignment @@ -503,6 +505,7 @@ func NewFakeWorkspace(url, token string) *FakeWorkspace { Pipelines: map[string]pipelines.GetPipelineResponse{}, PipelineUpdates: map[string]bool{}, Monitors: map[string]catalog.MonitorInfo{}, + DataQualityMonitors: map[string]dataquality.Monitor{}, Apps: map[string]apps.App{}, Catalogs: map[string]catalog.CatalogInfo{}, ExternalLocations: map[string]catalog.ExternalLocationInfo{}, diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index eeaae2bb56f..5c8899b8611 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -664,6 +664,24 @@ func AddDefaultHandlers(server *Server) { return MapDelete(req.Workspace, req.Workspace.Monitors, req.Vars["table_name"]) }) + // Data Quality Monitors: + + server.Handle("POST", "/api/data-quality/v1/monitors", func(req Request) any { + return req.Workspace.DataQualityMonitorCreate(req) + }) + + server.Handle("GET", "/api/data-quality/v1/monitors/{object_type}/{object_id}", func(req Request) any { + return MapGet(req.Workspace, req.Workspace.DataQualityMonitors, dataQualityMonitorKey(req.Vars["object_type"], req.Vars["object_id"])) + }) + + server.Handle("PATCH", "/api/data-quality/v1/monitors/{object_type}/{object_id}", func(req Request) any { + return req.Workspace.DataQualityMonitorUpdate(req, req.Vars["object_type"], req.Vars["object_id"]) + }) + + server.Handle("DELETE", "/api/data-quality/v1/monitors/{object_type}/{object_id}", func(req Request) any { + return MapDelete(req.Workspace, req.Workspace.DataQualityMonitors, dataQualityMonitorKey(req.Vars["object_type"], req.Vars["object_id"])) + }) + // Apps: server.Handle("POST", "/api/2.0/apps/{name}/deployments", func(req Request) any { diff --git a/python/databricks/bundles/core/__init__.py b/python/databricks/bundles/core/__init__.py index 6fb4f7acdd2..ffc7ee85f71 100644 --- a/python/databricks/bundles/core/__init__.py +++ b/python/databricks/bundles/core/__init__.py @@ -20,6 +20,7 @@ "cluster_mutator", "cluster_policy_mutator", "dashboard_mutator", + "data_quality_monitor_mutator", "database_catalog_mutator", "database_instance_mutator", "external_location_mutator", @@ -64,6 +65,7 @@ cluster_mutator, cluster_policy_mutator, dashboard_mutator, + data_quality_monitor_mutator, database_catalog_mutator, database_instance_mutator, external_location_mutator, diff --git a/python/databricks/bundles/core/_generated/__init__.py b/python/databricks/bundles/core/_generated/__init__.py index eb7f3ce6494..fb5ce31b2e4 100644 --- a/python/databricks/bundles/core/_generated/__init__.py +++ b/python/databricks/bundles/core/_generated/__init__.py @@ -20,6 +20,10 @@ _DashboardResources, dashboard_mutator, ) +from databricks.bundles.core._generated.data_quality_monitors import ( + _DataQualityMonitorResources, + data_quality_monitor_mutator, +) from databricks.bundles.core._generated.database_catalogs import ( _DatabaseCatalogResources, database_catalog_mutator, @@ -117,6 +121,7 @@ "cluster_mutator", "cluster_policy_mutator", "dashboard_mutator", + "data_quality_monitor_mutator", "database_catalog_mutator", "database_instance_mutator", "external_location_mutator", @@ -151,6 +156,7 @@ class _GeneratedResources( _ClusterPolicyResources, _ClusterResources, _DashboardResources, + _DataQualityMonitorResources, _DatabaseCatalogResources, _DatabaseInstanceResources, _MlflowExperimentResources, @@ -187,6 +193,7 @@ def _all_resource_types() -> "tuple[_ResourceType, ...]": cluster_policies, clusters, dashboards, + data_quality_monitors, database_catalogs, database_instances, experiments, @@ -220,6 +227,7 @@ def _all_resource_types() -> "tuple[_ResourceType, ...]": cluster_policies._resource_type(), clusters._resource_type(), dashboards._resource_type(), + data_quality_monitors._resource_type(), database_catalogs._resource_type(), database_instances._resource_type(), experiments._resource_type(), diff --git a/python/databricks/bundles/core/_generated/data_quality_monitors.py b/python/databricks/bundles/core/_generated/data_quality_monitors.py new file mode 100644 index 00000000000..63a72d4a988 --- /dev/null +++ b/python/databricks/bundles/core/_generated/data_quality_monitors.py @@ -0,0 +1,128 @@ +# Code generated by pydabs-codegen. DO NOT EDIT. + +from collections.abc import Callable +from typing import TYPE_CHECKING, Optional, overload + +from databricks.bundles.core._bundle import Bundle +from databricks.bundles.core._location import Location +from databricks.bundles.core._resource_mutator import ResourceMutator +from databricks.bundles.core._transform import _transform + +if TYPE_CHECKING: + from databricks.bundles.core._resource_type import _ResourceType + from databricks.bundles.data_quality_monitors._models.data_quality_monitor import ( + DataQualityMonitor, + DataQualityMonitorParam, + ) + + +def _resource_type() -> "_ResourceType": + from databricks.bundles.core._resource_type import _ResourceType + from databricks.bundles.data_quality_monitors._models.data_quality_monitor import ( + DataQualityMonitor, + ) + + return _ResourceType( + resource_type=DataQualityMonitor, + singular_name="data_quality_monitor", + plural_name="data_quality_monitors", + ) + + +class _DataQualityMonitorResources: + """ + Generated data_quality_monitor accessors, mixed into Resources. + """ + + # Provided by the Resources subclass; declared here so the generated methods + # below type-check. + _resources: dict[str, dict] + + if TYPE_CHECKING: + + def add_location(self, path: tuple[str, ...], location: Location) -> None: ... + + def add_diagnostic_error( + self, + msg: str, + *, + detail: Optional[str] = None, + path: Optional[tuple[str, ...]] = None, + location: Optional[Location] = None, + ) -> None: ... + + @property + def data_quality_monitors(self) -> dict[str, "DataQualityMonitor"]: + return self._resources["data_quality_monitors"] + + def add_data_quality_monitor( + self, + resource_name: str, + data_quality_monitor: "DataQualityMonitorParam", + *, + location: Optional[Location] = None, + ) -> None: + """ + Adds the resource data_quality_monitor to the collection of resources. Resource name must be unique across all data_quality_monitors. + + :param resource_name: unique identifier for the data_quality_monitor + :param data_quality_monitor: the data_quality_monitor to add, can be DataQualityMonitor or dict + :param location: optional location of the data_quality_monitor in the source code + """ + from databricks.bundles.data_quality_monitors._models.data_quality_monitor import ( + DataQualityMonitor, + ) + + data_quality_monitor = _transform(DataQualityMonitor, data_quality_monitor) + path = ("resources", "data_quality_monitors", resource_name) + location = location or Location.from_stack_frame(depth=1) + + if self._resources["data_quality_monitors"].get(resource_name): + self.add_diagnostic_error( + msg=f"Duplicate resource name '{resource_name}' for resource 'data_quality_monitor'. Resource names must be unique.", + location=location, + path=path, + ) + else: + if location: + self.add_location(path, location) + + self._resources["data_quality_monitors"][resource_name] = ( + data_quality_monitor + ) + + +@overload +def data_quality_monitor_mutator( + function: Callable[[Bundle, "DataQualityMonitor"], "DataQualityMonitor"], +) -> ResourceMutator["DataQualityMonitor"]: ... + + +@overload +def data_quality_monitor_mutator( + function: Callable[["DataQualityMonitor"], "DataQualityMonitor"], +) -> ResourceMutator["DataQualityMonitor"]: ... + + +def data_quality_monitor_mutator( + function: Callable, +) -> ResourceMutator["DataQualityMonitor"]: + """ + Decorator for defining mutator for data_quality_monitors. Function should return a new instance of the data_quality_monitor + with the desired changes, instead of mutating the input data_quality_monitor. + + Example: + + .. code-block:: python + + @data_quality_monitor_mutator + def my_data_quality_monitor_mutator(bundle: Bundle, data_quality_monitor: DataQualityMonitor) -> DataQualityMonitor: + return replace(data_quality_monitor, ...) + + :param function: Function that mutates data_quality_monitors. + """ + from databricks.bundles.data_quality_monitors._models.data_quality_monitor import ( + DataQualityMonitor, + ) + + return ResourceMutator(resource_type=DataQualityMonitor, function=function) diff --git a/python/databricks/bundles/data_quality_monitors/__init__.py b/python/databricks/bundles/data_quality_monitors/__init__.py new file mode 100644 index 00000000000..05f3ae6c9d9 --- /dev/null +++ b/python/databricks/bundles/data_quality_monitors/__init__.py @@ -0,0 +1,112 @@ +# Code generated by pydabs-codegen. DO NOT EDIT. + +__all__ = [ + "AggregationGranularity", + "AggregationGranularityParam", + "AnomalyDetectionConfig", + "AnomalyDetectionConfigDict", + "AnomalyDetectionConfigParam", + "CronSchedule", + "CronScheduleDict", + "CronScheduleParam", + "DataProfilingConfig", + "DataProfilingConfigDict", + "DataProfilingConfigParam", + "DataProfilingCustomMetric", + "DataProfilingCustomMetricDict", + "DataProfilingCustomMetricParam", + "DataProfilingCustomMetricType", + "DataProfilingCustomMetricTypeParam", + "DataQualityMonitor", + "DataQualityMonitorDict", + "DataQualityMonitorParam", + "InferenceLogConfig", + "InferenceLogConfigDict", + "InferenceLogConfigParam", + "InferenceProblemType", + "InferenceProblemTypeParam", + "Lifecycle", + "LifecycleDict", + "LifecycleParam", + "NotificationDestination", + "NotificationDestinationDict", + "NotificationDestinationParam", + "NotificationSettings", + "NotificationSettingsDict", + "NotificationSettingsParam", + "SnapshotConfig", + "SnapshotConfigDict", + "SnapshotConfigParam", + "TimeSeriesConfig", + "TimeSeriesConfigDict", + "TimeSeriesConfigParam", +] + + +from databricks.bundles.data_quality_monitors._models.aggregation_granularity import ( + AggregationGranularity, + AggregationGranularityParam, +) +from databricks.bundles.data_quality_monitors._models.anomaly_detection_config import ( + AnomalyDetectionConfig, + AnomalyDetectionConfigDict, + AnomalyDetectionConfigParam, +) +from databricks.bundles.data_quality_monitors._models.cron_schedule import ( + CronSchedule, + CronScheduleDict, + CronScheduleParam, +) +from databricks.bundles.data_quality_monitors._models.data_profiling_config import ( + DataProfilingConfig, + DataProfilingConfigDict, + DataProfilingConfigParam, +) +from databricks.bundles.data_quality_monitors._models.data_profiling_custom_metric import ( + DataProfilingCustomMetric, + DataProfilingCustomMetricDict, + DataProfilingCustomMetricParam, +) +from databricks.bundles.data_quality_monitors._models.data_profiling_custom_metric_type import ( + DataProfilingCustomMetricType, + DataProfilingCustomMetricTypeParam, +) +from databricks.bundles.data_quality_monitors._models.data_quality_monitor import ( + DataQualityMonitor, + DataQualityMonitorDict, + DataQualityMonitorParam, +) +from databricks.bundles.data_quality_monitors._models.inference_log_config import ( + InferenceLogConfig, + InferenceLogConfigDict, + InferenceLogConfigParam, +) +from databricks.bundles.data_quality_monitors._models.inference_problem_type import ( + InferenceProblemType, + InferenceProblemTypeParam, +) +from databricks.bundles.data_quality_monitors._models.lifecycle import ( + Lifecycle, + LifecycleDict, + LifecycleParam, +) +from databricks.bundles.data_quality_monitors._models.notification_destination import ( + NotificationDestination, + NotificationDestinationDict, + NotificationDestinationParam, +) +from databricks.bundles.data_quality_monitors._models.notification_settings import ( + NotificationSettings, + NotificationSettingsDict, + NotificationSettingsParam, +) +from databricks.bundles.data_quality_monitors._models.snapshot_config import ( + SnapshotConfig, + SnapshotConfigDict, + SnapshotConfigParam, +) +from databricks.bundles.data_quality_monitors._models.time_series_config import ( + TimeSeriesConfig, + TimeSeriesConfigDict, + TimeSeriesConfigParam, +) diff --git a/python/databricks/bundles/data_quality_monitors/_models/aggregation_granularity.py b/python/databricks/bundles/data_quality_monitors/_models/aggregation_granularity.py new file mode 100644 index 00000000000..45f61a34365 --- /dev/null +++ b/python/databricks/bundles/data_quality_monitors/_models/aggregation_granularity.py @@ -0,0 +1,38 @@ +# Code generated by pydabs-codegen. DO NOT EDIT. + +from enum import Enum +from typing import Literal + + +class AggregationGranularity(Enum): + """ + The granularity for aggregating data into time windows based on their timestamp. + """ + + AGGREGATION_GRANULARITY_5_MINUTES = "AGGREGATION_GRANULARITY_5_MINUTES" + AGGREGATION_GRANULARITY_30_MINUTES = "AGGREGATION_GRANULARITY_30_MINUTES" + AGGREGATION_GRANULARITY_1_HOUR = "AGGREGATION_GRANULARITY_1_HOUR" + AGGREGATION_GRANULARITY_1_DAY = "AGGREGATION_GRANULARITY_1_DAY" + AGGREGATION_GRANULARITY_1_WEEK = "AGGREGATION_GRANULARITY_1_WEEK" + AGGREGATION_GRANULARITY_2_WEEKS = "AGGREGATION_GRANULARITY_2_WEEKS" + AGGREGATION_GRANULARITY_3_WEEKS = "AGGREGATION_GRANULARITY_3_WEEKS" + AGGREGATION_GRANULARITY_4_WEEKS = "AGGREGATION_GRANULARITY_4_WEEKS" + AGGREGATION_GRANULARITY_1_MONTH = "AGGREGATION_GRANULARITY_1_MONTH" + AGGREGATION_GRANULARITY_1_YEAR = "AGGREGATION_GRANULARITY_1_YEAR" + + +AggregationGranularityParam = ( + Literal[ + "AGGREGATION_GRANULARITY_5_MINUTES", + "AGGREGATION_GRANULARITY_30_MINUTES", + "AGGREGATION_GRANULARITY_1_HOUR", + "AGGREGATION_GRANULARITY_1_DAY", + "AGGREGATION_GRANULARITY_1_WEEK", + "AGGREGATION_GRANULARITY_2_WEEKS", + "AGGREGATION_GRANULARITY_3_WEEKS", + "AGGREGATION_GRANULARITY_4_WEEKS", + "AGGREGATION_GRANULARITY_1_MONTH", + "AGGREGATION_GRANULARITY_1_YEAR", + ] + | AggregationGranularity +) diff --git a/python/databricks/bundles/data_quality_monitors/_models/anomaly_detection_config.py b/python/databricks/bundles/data_quality_monitors/_models/anomaly_detection_config.py new file mode 100644 index 00000000000..b948dbb7c8a --- /dev/null +++ b/python/databricks/bundles/data_quality_monitors/_models/anomaly_detection_config.py @@ -0,0 +1,46 @@ +# Code generated by pydabs-codegen. DO NOT EDIT. + +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, TypedDict + +from databricks.bundles.core._transform import _transform +from databricks.bundles.core._transform_to_json import _transform_to_json_value +from databricks.bundles.core._variable import VariableOrList + +if TYPE_CHECKING: + from typing_extensions import Self + + +@dataclass(kw_only=True) +class AnomalyDetectionConfig: + """ + Anomaly Detection Configurations. + """ + + excluded_table_full_names: VariableOrList[str] = field(default_factory=list) + """ + :meta private: [EXPERIMENTAL] + + [Beta] List of fully qualified table names to exclude from anomaly detection. + """ + + @classmethod + def from_dict(cls, value: "AnomalyDetectionConfigDict") -> "Self": + return _transform(cls, value) + + def as_dict(self) -> "AnomalyDetectionConfigDict": + return _transform_to_json_value(self) # type:ignore + + +class AnomalyDetectionConfigDict(TypedDict, total=False): + """""" + + excluded_table_full_names: VariableOrList[str] + """ + :meta private: [EXPERIMENTAL] + + [Beta] List of fully qualified table names to exclude from anomaly detection. + """ + + +AnomalyDetectionConfigParam = AnomalyDetectionConfigDict | AnomalyDetectionConfig diff --git a/python/databricks/bundles/data_quality_monitors/_models/cron_schedule.py b/python/databricks/bundles/data_quality_monitors/_models/cron_schedule.py new file mode 100644 index 00000000000..86d7747ca92 --- /dev/null +++ b/python/databricks/bundles/data_quality_monitors/_models/cron_schedule.py @@ -0,0 +1,56 @@ +# Code generated by pydabs-codegen. DO NOT EDIT. + +from dataclasses import dataclass +from typing import TYPE_CHECKING, TypedDict + +from databricks.bundles.core._transform import _transform +from databricks.bundles.core._transform_to_json import _transform_to_json_value +from databricks.bundles.core._variable import VariableOr + +if TYPE_CHECKING: + from typing_extensions import Self + + +@dataclass(kw_only=True) +class CronSchedule: + """ + The data quality monitoring workflow cron schedule. + """ + + quartz_cron_expression: VariableOr[str] + """ + [Public Preview] The expression that determines when to run the monitor. See [examples](https://www.quartz-scheduler.org/documentation/quartz-2.3.0/tutorials/crontrigger.html). + """ + + timezone_id: VariableOr[str] + """ + [Public Preview] A Java timezone id. The schedule for a job will be resolved with respect to this timezone. + See `Java TimeZone `_ for details. + The timezone id (e.g., ``America/Los_Angeles``) in which to evaluate the quartz expression. + """ + + @classmethod + def from_dict(cls, value: "CronScheduleDict") -> "Self": + return _transform(cls, value) + + def as_dict(self) -> "CronScheduleDict": + return _transform_to_json_value(self) # type:ignore + + +class CronScheduleDict(TypedDict, total=False): + """""" + + quartz_cron_expression: VariableOr[str] + """ + [Public Preview] The expression that determines when to run the monitor. See [examples](https://www.quartz-scheduler.org/documentation/quartz-2.3.0/tutorials/crontrigger.html). + """ + + timezone_id: VariableOr[str] + """ + [Public Preview] A Java timezone id. The schedule for a job will be resolved with respect to this timezone. + See `Java TimeZone `_ for details. + The timezone id (e.g., ``America/Los_Angeles``) in which to evaluate the quartz expression. + """ + + +CronScheduleParam = CronScheduleDict | CronSchedule diff --git a/python/databricks/bundles/data_quality_monitors/_models/data_profiling_config.py b/python/databricks/bundles/data_quality_monitors/_models/data_profiling_config.py new file mode 100644 index 00000000000..760879da4dc --- /dev/null +++ b/python/databricks/bundles/data_quality_monitors/_models/data_profiling_config.py @@ -0,0 +1,200 @@ +# Code generated by pydabs-codegen. DO NOT EDIT. + +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, TypedDict + +from databricks.bundles.core._transform import _transform +from databricks.bundles.core._transform_to_json import _transform_to_json_value +from databricks.bundles.core._variable import ( + VariableOr, + VariableOrList, + VariableOrOptional, +) +from databricks.bundles.data_quality_monitors._models.cron_schedule import ( + CronSchedule, + CronScheduleParam, +) +from databricks.bundles.data_quality_monitors._models.data_profiling_custom_metric import ( + DataProfilingCustomMetric, + DataProfilingCustomMetricParam, +) +from databricks.bundles.data_quality_monitors._models.inference_log_config import ( + InferenceLogConfig, + InferenceLogConfigParam, +) +from databricks.bundles.data_quality_monitors._models.notification_settings import ( + NotificationSettings, + NotificationSettingsParam, +) +from databricks.bundles.data_quality_monitors._models.snapshot_config import ( + SnapshotConfig, + SnapshotConfigParam, +) +from databricks.bundles.data_quality_monitors._models.time_series_config import ( + TimeSeriesConfig, + TimeSeriesConfigParam, +) + +if TYPE_CHECKING: + from typing_extensions import Self + + +@dataclass(kw_only=True) +class DataProfilingConfig: + """ + Data Profiling Configurations. + """ + + output_schema_id: VariableOr[str] + """ + [Public Preview] ID of the schema where output tables are created. + """ + + assets_dir: VariableOrOptional[str] = None + """ + [Public Preview] Field for specifying the absolute path to a custom directory to store data-monitoring + assets. Normally prepopulated to a default user location via UI and Python APIs. + """ + + baseline_table_name: VariableOrOptional[str] = None + """ + [Public Preview] Baseline table name. + Baseline data is used to compute drift from the data in the monitored `table_name`. + The baseline table and the monitored table shall have the same schema. + """ + + custom_metrics: VariableOrList[DataProfilingCustomMetric] = field( + default_factory=list + ) + """ + [Public Preview] Custom metrics. + """ + + inference_log: VariableOrOptional[InferenceLogConfig] = None + """ + [Public Preview] `Analysis Configuration` for monitoring inference log tables. + """ + + notification_settings: VariableOrOptional[NotificationSettings] = None + """ + [Public Preview] Field for specifying notification settings. + """ + + schedule: VariableOrOptional[CronSchedule] = None + """ + [Public Preview] The cron schedule. + """ + + skip_builtin_dashboard: VariableOrOptional[bool] = None + """ + [Public Preview] Whether to skip creating a default dashboard summarizing data quality metrics. + """ + + slicing_exprs: VariableOrList[str] = field(default_factory=list) + """ + [Public Preview] List of column expressions to slice data with for targeted analysis. The data is grouped by + each expression independently, resulting in a separate slice for each predicate and its + complements. For example `slicing_exprs=[“col_1”, “col_2 > 10”]` will generate the following + slices: two slices for `col_2 > 10` (True and False), and one slice per unique value in + `col1`. For high-cardinality columns, only the top 100 unique values by frequency will + generate slices. + """ + + snapshot: VariableOrOptional[SnapshotConfig] = None + """ + [Public Preview] `Analysis Configuration` for monitoring snapshot tables. + """ + + time_series: VariableOrOptional[TimeSeriesConfig] = None + """ + [Public Preview] `Analysis Configuration` for monitoring time series tables. + """ + + warehouse_id: VariableOrOptional[str] = None + """ + [Public Preview] Optional argument to specify the warehouse for dashboard creation. If not specified, the first running + warehouse will be used. + """ + + @classmethod + def from_dict(cls, value: "DataProfilingConfigDict") -> "Self": + return _transform(cls, value) + + def as_dict(self) -> "DataProfilingConfigDict": + return _transform_to_json_value(self) # type:ignore + + +class DataProfilingConfigDict(TypedDict, total=False): + """""" + + output_schema_id: VariableOr[str] + """ + [Public Preview] ID of the schema where output tables are created. + """ + + assets_dir: VariableOrOptional[str] + """ + [Public Preview] Field for specifying the absolute path to a custom directory to store data-monitoring + assets. Normally prepopulated to a default user location via UI and Python APIs. + """ + + baseline_table_name: VariableOrOptional[str] + """ + [Public Preview] Baseline table name. + Baseline data is used to compute drift from the data in the monitored `table_name`. + The baseline table and the monitored table shall have the same schema. + """ + + custom_metrics: VariableOrList[DataProfilingCustomMetricParam] + """ + [Public Preview] Custom metrics. + """ + + inference_log: VariableOrOptional[InferenceLogConfigParam] + """ + [Public Preview] `Analysis Configuration` for monitoring inference log tables. + """ + + notification_settings: VariableOrOptional[NotificationSettingsParam] + """ + [Public Preview] Field for specifying notification settings. + """ + + schedule: VariableOrOptional[CronScheduleParam] + """ + [Public Preview] The cron schedule. + """ + + skip_builtin_dashboard: VariableOrOptional[bool] + """ + [Public Preview] Whether to skip creating a default dashboard summarizing data quality metrics. + """ + + slicing_exprs: VariableOrList[str] + """ + [Public Preview] List of column expressions to slice data with for targeted analysis. The data is grouped by + each expression independently, resulting in a separate slice for each predicate and its + complements. For example `slicing_exprs=[“col_1”, “col_2 > 10”]` will generate the following + slices: two slices for `col_2 > 10` (True and False), and one slice per unique value in + `col1`. For high-cardinality columns, only the top 100 unique values by frequency will + generate slices. + """ + + snapshot: VariableOrOptional[SnapshotConfigParam] + """ + [Public Preview] `Analysis Configuration` for monitoring snapshot tables. + """ + + time_series: VariableOrOptional[TimeSeriesConfigParam] + """ + [Public Preview] `Analysis Configuration` for monitoring time series tables. + """ + + warehouse_id: VariableOrOptional[str] + """ + [Public Preview] Optional argument to specify the warehouse for dashboard creation. If not specified, the first running + warehouse will be used. + """ + + +DataProfilingConfigParam = DataProfilingConfigDict | DataProfilingConfig diff --git a/python/databricks/bundles/data_quality_monitors/_models/data_profiling_custom_metric.py b/python/databricks/bundles/data_quality_monitors/_models/data_profiling_custom_metric.py new file mode 100644 index 00000000000..375e151c3b4 --- /dev/null +++ b/python/databricks/bundles/data_quality_monitors/_models/data_profiling_custom_metric.py @@ -0,0 +1,90 @@ +# Code generated by pydabs-codegen. DO NOT EDIT. + +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, TypedDict + +from databricks.bundles.core._transform import _transform +from databricks.bundles.core._transform_to_json import _transform_to_json_value +from databricks.bundles.core._variable import VariableOr, VariableOrList +from databricks.bundles.data_quality_monitors._models.data_profiling_custom_metric_type import ( + DataProfilingCustomMetricType, + DataProfilingCustomMetricTypeParam, +) + +if TYPE_CHECKING: + from typing_extensions import Self + + +@dataclass(kw_only=True) +class DataProfilingCustomMetric: + """ + Custom metric definition. + """ + + definition: VariableOr[str] + """ + [Public Preview] Jinja template for a SQL expression that specifies how to compute the metric. See [create metric definition](https://docs.databricks.com/en/lakehouse-monitoring/custom-metrics.html#create-definition). + """ + + name: VariableOr[str] + """ + [Public Preview] Name of the metric in the output tables. + """ + + output_data_type: VariableOr[str] + """ + [Public Preview] The output type of the custom metric. + """ + + type: VariableOr[DataProfilingCustomMetricType] + """ + [Public Preview] The type of the custom metric. + """ + + input_columns: VariableOrList[str] = field(default_factory=list) + """ + [Public Preview] A list of column names in the input table the metric should be computed for. + Can use ``":table"`` to indicate that the metric needs information from multiple columns. + """ + + @classmethod + def from_dict(cls, value: "DataProfilingCustomMetricDict") -> "Self": + return _transform(cls, value) + + def as_dict(self) -> "DataProfilingCustomMetricDict": + return _transform_to_json_value(self) # type:ignore + + +class DataProfilingCustomMetricDict(TypedDict, total=False): + """""" + + definition: VariableOr[str] + """ + [Public Preview] Jinja template for a SQL expression that specifies how to compute the metric. See [create metric definition](https://docs.databricks.com/en/lakehouse-monitoring/custom-metrics.html#create-definition). + """ + + name: VariableOr[str] + """ + [Public Preview] Name of the metric in the output tables. + """ + + output_data_type: VariableOr[str] + """ + [Public Preview] The output type of the custom metric. + """ + + type: VariableOr[DataProfilingCustomMetricTypeParam] + """ + [Public Preview] The type of the custom metric. + """ + + input_columns: VariableOrList[str] + """ + [Public Preview] A list of column names in the input table the metric should be computed for. + Can use ``":table"`` to indicate that the metric needs information from multiple columns. + """ + + +DataProfilingCustomMetricParam = ( + DataProfilingCustomMetricDict | DataProfilingCustomMetric +) diff --git a/python/databricks/bundles/data_quality_monitors/_models/data_profiling_custom_metric_type.py b/python/databricks/bundles/data_quality_monitors/_models/data_profiling_custom_metric_type.py new file mode 100644 index 00000000000..076e92239a5 --- /dev/null +++ b/python/databricks/bundles/data_quality_monitors/_models/data_profiling_custom_metric_type.py @@ -0,0 +1,28 @@ +# Code generated by pydabs-codegen. DO NOT EDIT. + +from enum import Enum +from typing import Literal + + +class DataProfilingCustomMetricType(Enum): + """ + The custom metric type. + """ + + DATA_PROFILING_CUSTOM_METRIC_TYPE_AGGREGATE = ( + "DATA_PROFILING_CUSTOM_METRIC_TYPE_AGGREGATE" + ) + DATA_PROFILING_CUSTOM_METRIC_TYPE_DERIVED = ( + "DATA_PROFILING_CUSTOM_METRIC_TYPE_DERIVED" + ) + DATA_PROFILING_CUSTOM_METRIC_TYPE_DRIFT = "DATA_PROFILING_CUSTOM_METRIC_TYPE_DRIFT" + + +DataProfilingCustomMetricTypeParam = ( + Literal[ + "DATA_PROFILING_CUSTOM_METRIC_TYPE_AGGREGATE", + "DATA_PROFILING_CUSTOM_METRIC_TYPE_DERIVED", + "DATA_PROFILING_CUSTOM_METRIC_TYPE_DRIFT", + ] + | DataProfilingCustomMetricType +) diff --git a/python/databricks/bundles/data_quality_monitors/_models/data_quality_monitor.py b/python/databricks/bundles/data_quality_monitors/_models/data_quality_monitor.py new file mode 100644 index 00000000000..cda97b5ad5e --- /dev/null +++ b/python/databricks/bundles/data_quality_monitors/_models/data_quality_monitor.py @@ -0,0 +1,113 @@ +# Code generated by pydabs-codegen. DO NOT EDIT. + +from dataclasses import dataclass +from typing import TYPE_CHECKING, TypedDict + +from databricks.bundles.core._resource import Resource +from databricks.bundles.core._transform import _transform +from databricks.bundles.core._transform_to_json import _transform_to_json_value +from databricks.bundles.core._variable import VariableOr, VariableOrOptional +from databricks.bundles.data_quality_monitors._models.anomaly_detection_config import ( + AnomalyDetectionConfig, + AnomalyDetectionConfigParam, +) +from databricks.bundles.data_quality_monitors._models.data_profiling_config import ( + DataProfilingConfig, + DataProfilingConfigParam, +) +from databricks.bundles.data_quality_monitors._models.lifecycle import ( + Lifecycle, + LifecycleParam, +) + +if TYPE_CHECKING: + from typing_extensions import Self + + +@dataclass(kw_only=True) +class DataQualityMonitor(Resource): + """ + Monitor for the data quality of unity catalog entities such as schema or table. + """ + + object_id: VariableOr[str] + """ + [Public Preview] The UUID of the request object. It is `schema_id` for `schema`, and `table_id` for `table`. + + Find the `schema_id` from either: + 1. The [schema_id](https://docs.databricks.com/api/workspace/schemas/get#schema_id) of the `Schemas` resource. + 2. In [Catalog Explorer](https://docs.databricks.com/aws/en/catalog-explorer/) > select the `schema` > go to the `Details` tab > the `Schema ID` field. + + Find the `table_id` from either: + 1. The [table_id](https://docs.databricks.com/api/workspace/tables/get#table_id) of the `Tables` resource. + 2. In [Catalog Explorer](https://docs.databricks.com/aws/en/catalog-explorer/) > select the `table` > go to the `Details` tab > the `Table ID` field. + """ + + object_type: VariableOr[str] + """ + [Public Preview] The type of the monitored object. Can be one of the following: `schema` or `table`. + """ + + anomaly_detection_config: VariableOrOptional[AnomalyDetectionConfig] = None + """ + [Public Preview] Anomaly Detection Configuration, applicable to `schema` object types. + """ + + data_profiling_config: VariableOrOptional[DataProfilingConfig] = None + """ + [Public Preview] Data Profiling Configuration, applicable to `table` object types. Exactly one `Analysis Configuration` + must be present. + """ + + lifecycle: VariableOrOptional[Lifecycle] = None + """ + Settings that control the deployment lifecycle of the resource, such as preventing it from being destroyed. + """ + + @classmethod + def from_dict(cls, value: "DataQualityMonitorDict") -> "Self": + return _transform(cls, value) + + def as_dict(self) -> "DataQualityMonitorDict": + return _transform_to_json_value(self) # type:ignore + + +class DataQualityMonitorDict(TypedDict, total=False): + """""" + + object_id: VariableOr[str] + """ + [Public Preview] The UUID of the request object. It is `schema_id` for `schema`, and `table_id` for `table`. + + Find the `schema_id` from either: + 1. The [schema_id](https://docs.databricks.com/api/workspace/schemas/get#schema_id) of the `Schemas` resource. + 2. In [Catalog Explorer](https://docs.databricks.com/aws/en/catalog-explorer/) > select the `schema` > go to the `Details` tab > the `Schema ID` field. + + Find the `table_id` from either: + 1. The [table_id](https://docs.databricks.com/api/workspace/tables/get#table_id) of the `Tables` resource. + 2. In [Catalog Explorer](https://docs.databricks.com/aws/en/catalog-explorer/) > select the `table` > go to the `Details` tab > the `Table ID` field. + """ + + object_type: VariableOr[str] + """ + [Public Preview] The type of the monitored object. Can be one of the following: `schema` or `table`. + """ + + anomaly_detection_config: VariableOrOptional[AnomalyDetectionConfigParam] + """ + [Public Preview] Anomaly Detection Configuration, applicable to `schema` object types. + """ + + data_profiling_config: VariableOrOptional[DataProfilingConfigParam] + """ + [Public Preview] Data Profiling Configuration, applicable to `table` object types. Exactly one `Analysis Configuration` + must be present. + """ + + lifecycle: VariableOrOptional[LifecycleParam] + """ + Settings that control the deployment lifecycle of the resource, such as preventing it from being destroyed. + """ + + +DataQualityMonitorParam = DataQualityMonitorDict | DataQualityMonitor diff --git a/python/databricks/bundles/data_quality_monitors/_models/inference_log_config.py b/python/databricks/bundles/data_quality_monitors/_models/inference_log_config.py new file mode 100644 index 00000000000..038d0ba5d5a --- /dev/null +++ b/python/databricks/bundles/data_quality_monitors/_models/inference_log_config.py @@ -0,0 +1,104 @@ +# Code generated by pydabs-codegen. DO NOT EDIT. + +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, TypedDict + +from databricks.bundles.core._transform import _transform +from databricks.bundles.core._transform_to_json import _transform_to_json_value +from databricks.bundles.core._variable import ( + VariableOr, + VariableOrList, + VariableOrOptional, +) +from databricks.bundles.data_quality_monitors._models.aggregation_granularity import ( + AggregationGranularity, + AggregationGranularityParam, +) +from databricks.bundles.data_quality_monitors._models.inference_problem_type import ( + InferenceProblemType, + InferenceProblemTypeParam, +) + +if TYPE_CHECKING: + from typing_extensions import Self + + +@dataclass(kw_only=True) +class InferenceLogConfig: + """ + Inference log configuration. + """ + + model_id_column: VariableOr[str] + """ + [Public Preview] Column for the model identifier. + """ + + prediction_column: VariableOr[str] + """ + [Public Preview] Column for the prediction. + """ + + problem_type: VariableOr[InferenceProblemType] + """ + [Public Preview] Problem type the model aims to solve. + """ + + timestamp_column: VariableOr[str] + """ + [Public Preview] Column for the timestamp. + """ + + granularities: VariableOrList[AggregationGranularity] = field(default_factory=list) + """ + [Public Preview] List of granularities to use when aggregating data into time windows based on their timestamp. + """ + + label_column: VariableOrOptional[str] = None + """ + [Public Preview] Column for the label. + """ + + @classmethod + def from_dict(cls, value: "InferenceLogConfigDict") -> "Self": + return _transform(cls, value) + + def as_dict(self) -> "InferenceLogConfigDict": + return _transform_to_json_value(self) # type:ignore + + +class InferenceLogConfigDict(TypedDict, total=False): + """""" + + model_id_column: VariableOr[str] + """ + [Public Preview] Column for the model identifier. + """ + + prediction_column: VariableOr[str] + """ + [Public Preview] Column for the prediction. + """ + + problem_type: VariableOr[InferenceProblemTypeParam] + """ + [Public Preview] Problem type the model aims to solve. + """ + + timestamp_column: VariableOr[str] + """ + [Public Preview] Column for the timestamp. + """ + + granularities: VariableOrList[AggregationGranularityParam] + """ + [Public Preview] List of granularities to use when aggregating data into time windows based on their timestamp. + """ + + label_column: VariableOrOptional[str] + """ + [Public Preview] Column for the label. + """ + + +InferenceLogConfigParam = InferenceLogConfigDict | InferenceLogConfig diff --git a/python/databricks/bundles/data_quality_monitors/_models/inference_problem_type.py b/python/databricks/bundles/data_quality_monitors/_models/inference_problem_type.py new file mode 100644 index 00000000000..ba7797bbaa9 --- /dev/null +++ b/python/databricks/bundles/data_quality_monitors/_models/inference_problem_type.py @@ -0,0 +1,21 @@ +# Code generated by pydabs-codegen. DO NOT EDIT. + +from enum import Enum +from typing import Literal + + +class InferenceProblemType(Enum): + """ + Inference problem type the model aims to solve. + """ + + INFERENCE_PROBLEM_TYPE_CLASSIFICATION = "INFERENCE_PROBLEM_TYPE_CLASSIFICATION" + INFERENCE_PROBLEM_TYPE_REGRESSION = "INFERENCE_PROBLEM_TYPE_REGRESSION" + + +InferenceProblemTypeParam = ( + Literal[ + "INFERENCE_PROBLEM_TYPE_CLASSIFICATION", "INFERENCE_PROBLEM_TYPE_REGRESSION" + ] + | InferenceProblemType +) diff --git a/python/databricks/bundles/data_quality_monitors/_models/lifecycle.py b/python/databricks/bundles/data_quality_monitors/_models/lifecycle.py new file mode 100644 index 00000000000..697776a198e --- /dev/null +++ b/python/databricks/bundles/data_quality_monitors/_models/lifecycle.py @@ -0,0 +1,40 @@ +# Code generated by pydabs-codegen. DO NOT EDIT. + +from dataclasses import dataclass +from typing import TYPE_CHECKING, TypedDict + +from databricks.bundles.core._transform import _transform +from databricks.bundles.core._transform_to_json import _transform_to_json_value +from databricks.bundles.core._variable import VariableOrOptional + +if TYPE_CHECKING: + from typing_extensions import Self + + +@dataclass(kw_only=True) +class Lifecycle: + """""" + + prevent_destroy: VariableOrOptional[bool] = None + """ + Lifecycle setting to prevent the resource from being destroyed. + """ + + @classmethod + def from_dict(cls, value: "LifecycleDict") -> "Self": + return _transform(cls, value) + + def as_dict(self) -> "LifecycleDict": + return _transform_to_json_value(self) # type:ignore + + +class LifecycleDict(TypedDict, total=False): + """""" + + prevent_destroy: VariableOrOptional[bool] + """ + Lifecycle setting to prevent the resource from being destroyed. + """ + + +LifecycleParam = LifecycleDict | Lifecycle diff --git a/python/databricks/bundles/data_quality_monitors/_models/notification_destination.py b/python/databricks/bundles/data_quality_monitors/_models/notification_destination.py new file mode 100644 index 00000000000..10ce346ec4c --- /dev/null +++ b/python/databricks/bundles/data_quality_monitors/_models/notification_destination.py @@ -0,0 +1,42 @@ +# Code generated by pydabs-codegen. DO NOT EDIT. + +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, TypedDict + +from databricks.bundles.core._transform import _transform +from databricks.bundles.core._transform_to_json import _transform_to_json_value +from databricks.bundles.core._variable import VariableOrList + +if TYPE_CHECKING: + from typing_extensions import Self + + +@dataclass(kw_only=True) +class NotificationDestination: + """ + Destination of the data quality monitoring notification. + """ + + email_addresses: VariableOrList[str] = field(default_factory=list) + """ + [Public Preview] The list of email addresses to send the notification to. A maximum of 5 email addresses is supported. + """ + + @classmethod + def from_dict(cls, value: "NotificationDestinationDict") -> "Self": + return _transform(cls, value) + + def as_dict(self) -> "NotificationDestinationDict": + return _transform_to_json_value(self) # type:ignore + + +class NotificationDestinationDict(TypedDict, total=False): + """""" + + email_addresses: VariableOrList[str] + """ + [Public Preview] The list of email addresses to send the notification to. A maximum of 5 email addresses is supported. + """ + + +NotificationDestinationParam = NotificationDestinationDict | NotificationDestination diff --git a/python/databricks/bundles/data_quality_monitors/_models/notification_settings.py b/python/databricks/bundles/data_quality_monitors/_models/notification_settings.py new file mode 100644 index 00000000000..f957a6ecb7f --- /dev/null +++ b/python/databricks/bundles/data_quality_monitors/_models/notification_settings.py @@ -0,0 +1,46 @@ +# Code generated by pydabs-codegen. DO NOT EDIT. + +from dataclasses import dataclass +from typing import TYPE_CHECKING, TypedDict + +from databricks.bundles.core._transform import _transform +from databricks.bundles.core._transform_to_json import _transform_to_json_value +from databricks.bundles.core._variable import VariableOrOptional +from databricks.bundles.data_quality_monitors._models.notification_destination import ( + NotificationDestination, + NotificationDestinationParam, +) + +if TYPE_CHECKING: + from typing_extensions import Self + + +@dataclass(kw_only=True) +class NotificationSettings: + """ + Settings for sending notifications on the data quality monitoring. + """ + + on_failure: VariableOrOptional[NotificationDestination] = None + """ + [Public Preview] Destinations to send notifications on failure/timeout. + """ + + @classmethod + def from_dict(cls, value: "NotificationSettingsDict") -> "Self": + return _transform(cls, value) + + def as_dict(self) -> "NotificationSettingsDict": + return _transform_to_json_value(self) # type:ignore + + +class NotificationSettingsDict(TypedDict, total=False): + """""" + + on_failure: VariableOrOptional[NotificationDestinationParam] + """ + [Public Preview] Destinations to send notifications on failure/timeout. + """ + + +NotificationSettingsParam = NotificationSettingsDict | NotificationSettings diff --git a/python/databricks/bundles/data_quality_monitors/_models/snapshot_config.py b/python/databricks/bundles/data_quality_monitors/_models/snapshot_config.py new file mode 100644 index 00000000000..c7ae6650923 --- /dev/null +++ b/python/databricks/bundles/data_quality_monitors/_models/snapshot_config.py @@ -0,0 +1,31 @@ +# Code generated by pydabs-codegen. DO NOT EDIT. + +from dataclasses import dataclass +from typing import TYPE_CHECKING, TypedDict + +from databricks.bundles.core._transform import _transform +from databricks.bundles.core._transform_to_json import _transform_to_json_value + +if TYPE_CHECKING: + from typing_extensions import Self + + +@dataclass(kw_only=True) +class SnapshotConfig: + """ + Snapshot analysis configuration. + """ + + @classmethod + def from_dict(cls, value: "SnapshotConfigDict") -> "Self": + return _transform(cls, value) + + def as_dict(self) -> "SnapshotConfigDict": + return _transform_to_json_value(self) # type:ignore + + +class SnapshotConfigDict(TypedDict, total=False): + """""" + + +SnapshotConfigParam = SnapshotConfigDict | SnapshotConfig diff --git a/python/databricks/bundles/data_quality_monitors/_models/time_series_config.py b/python/databricks/bundles/data_quality_monitors/_models/time_series_config.py new file mode 100644 index 00000000000..dd90363888d --- /dev/null +++ b/python/databricks/bundles/data_quality_monitors/_models/time_series_config.py @@ -0,0 +1,56 @@ +# Code generated by pydabs-codegen. DO NOT EDIT. + +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, TypedDict + +from databricks.bundles.core._transform import _transform +from databricks.bundles.core._transform_to_json import _transform_to_json_value +from databricks.bundles.core._variable import VariableOr, VariableOrList +from databricks.bundles.data_quality_monitors._models.aggregation_granularity import ( + AggregationGranularity, + AggregationGranularityParam, +) + +if TYPE_CHECKING: + from typing_extensions import Self + + +@dataclass(kw_only=True) +class TimeSeriesConfig: + """ + Time series analysis configuration. + """ + + timestamp_column: VariableOr[str] + """ + [Public Preview] Column for the timestamp. + """ + + granularities: VariableOrList[AggregationGranularity] = field(default_factory=list) + """ + [Public Preview] List of granularities to use when aggregating data into time windows based on their timestamp. + """ + + @classmethod + def from_dict(cls, value: "TimeSeriesConfigDict") -> "Self": + return _transform(cls, value) + + def as_dict(self) -> "TimeSeriesConfigDict": + return _transform_to_json_value(self) # type:ignore + + +class TimeSeriesConfigDict(TypedDict, total=False): + """""" + + timestamp_column: VariableOr[str] + """ + [Public Preview] Column for the timestamp. + """ + + granularities: VariableOrList[AggregationGranularityParam] + """ + [Public Preview] List of granularities to use when aggregating data into time windows based on their timestamp. + """ + + +TimeSeriesConfigParam = TimeSeriesConfigDict | TimeSeriesConfig diff --git a/python/databricks_tests/core/_generated/__init__.py b/python/databricks_tests/core/_generated/__init__.py index 762fff248ef..aeab4995308 100644 --- a/python/databricks_tests/core/_generated/__init__.py +++ b/python/databricks_tests/core/_generated/__init__.py @@ -7,6 +7,7 @@ cluster_policies, clusters, dashboards, + data_quality_monitors, database_catalogs, database_instances, experiments, @@ -42,6 +43,7 @@ cluster_policies._test_case(), clusters._test_case(), dashboards._test_case(), + data_quality_monitors._test_case(), database_catalogs._test_case(), database_instances._test_case(), experiments._test_case(), diff --git a/python/databricks_tests/core/_generated/data_quality_monitors.py b/python/databricks_tests/core/_generated/data_quality_monitors.py new file mode 100644 index 00000000000..bad5d3fd834 --- /dev/null +++ b/python/databricks_tests/core/_generated/data_quality_monitors.py @@ -0,0 +1,41 @@ +# Code generated by pydabs-codegen. DO NOT EDIT. + +from databricks.bundles.core import Resources, data_quality_monitor_mutator +from databricks.bundles.core._generated.data_quality_monitors import _resource_type +from databricks.bundles.data_quality_monitors._models.anomaly_detection_config import ( + AnomalyDetectionConfig, +) +from databricks.bundles.data_quality_monitors._models.data_profiling_config import ( + DataProfilingConfig, +) +from databricks.bundles.data_quality_monitors._models.data_quality_monitor import ( + DataQualityMonitor, +) +from databricks.bundles.data_quality_monitors._models.lifecycle import Lifecycle +from databricks_tests.core._resource_test_case import ResourceTestCase + + +def _test_case(): + return ( + ResourceTestCase( + add_resource=Resources.add_data_quality_monitor, + dict_example={ + "anomaly_detection_config": {}, + "data_profiling_config": {"output_schema_id": "output_schema_id"}, + "lifecycle": {}, + "object_id": "object_id", + "object_type": "object_type", + }, + dataclass_example=DataQualityMonitor( + anomaly_detection_config=AnomalyDetectionConfig(), + data_profiling_config=DataProfilingConfig( + output_schema_id="output_schema_id" + ), + lifecycle=Lifecycle(), + object_id="object_id", + object_type="object_type", + ), + mutator=data_quality_monitor_mutator, + ), + _resource_type(), + ) diff --git a/python/databricks_tests/core/public_api.txt b/python/databricks_tests/core/public_api.txt index 63e6ad0cf1f..179ccd04edf 100644 --- a/python/databricks_tests/core/public_api.txt +++ b/python/databricks_tests/core/public_api.txt @@ -19,6 +19,7 @@ __all__ = [ cluster_mutator, cluster_policy_mutator, dashboard_mutator, + data_quality_monitor_mutator, database_catalog_mutator, database_instance_mutator, external_location_mutator, @@ -95,6 +96,7 @@ class Resources: def add_cluster(self, resource_name: str, cluster: ClusterParam, *, location: Union[Location, None] = None) -> None def add_cluster_policy(self, resource_name: str, cluster_policy: ClusterPolicyParam, *, location: Union[Location, None] = None) -> None def add_dashboard(self, resource_name: str, dashboard: DashboardParam, *, location: Union[Location, None] = None) -> None + def add_data_quality_monitor(self, resource_name: str, data_quality_monitor: DataQualityMonitorParam, *, location: Union[Location, None] = None) -> None def add_database_catalog(self, resource_name: str, database_catalog: DatabaseCatalogParam, *, location: Union[Location, None] = None) -> None def add_database_instance(self, resource_name: str, database_instance: DatabaseInstanceParam, *, location: Union[Location, None] = None) -> None def add_diagnostic_error(self, msg: str, *, detail: Union[str, None] = None, path: Union[tuple[str, ...], None] = None, location: Union[Location, None] = None) -> None @@ -131,6 +133,7 @@ class Resources: @property cluster_policies -> dict[str, ClusterPolicy] @property clusters -> dict[str, Cluster] @property dashboards -> dict[str, Dashboard] + @property data_quality_monitors -> dict[str, DataQualityMonitor] @property database_catalogs -> dict[str, DatabaseCatalog] @property database_instances -> dict[str, DatabaseInstance] @property diagnostics -> Diagnostics @@ -198,6 +201,10 @@ def cluster_policy_mutator(function: Callable) -> ResourceMutator[ClusterPolicy] @overload def dashboard_mutator(function: Callable[[Dashboard], Dashboard]) -> ResourceMutator[Dashboard] def dashboard_mutator(function: Callable) -> ResourceMutator[Dashboard] +@overload def data_quality_monitor_mutator(function: Callable[[Bundle, DataQualityMonitor], DataQualityMonitor]) -> ResourceMutator[DataQualityMonitor] +@overload def data_quality_monitor_mutator(function: Callable[[DataQualityMonitor], DataQualityMonitor]) -> ResourceMutator[DataQualityMonitor] +def data_quality_monitor_mutator(function: Callable) -> ResourceMutator[DataQualityMonitor] + @overload def database_catalog_mutator(function: Callable[[Bundle, DatabaseCatalog], DatabaseCatalog]) -> ResourceMutator[DatabaseCatalog] @overload def database_catalog_mutator(function: Callable[[DatabaseCatalog], DatabaseCatalog]) -> ResourceMutator[DatabaseCatalog] def database_catalog_mutator(function: Callable) -> ResourceMutator[DatabaseCatalog] @@ -311,6 +318,7 @@ singular_name=catalog plural_name=catalogs resource_type=Catalog singular_name=cluster plural_name=clusters resource_type=Cluster singular_name=cluster_policy plural_name=cluster_policies resource_type=ClusterPolicy singular_name=dashboard plural_name=dashboards resource_type=Dashboard +singular_name=data_quality_monitor plural_name=data_quality_monitors resource_type=DataQualityMonitor singular_name=database_catalog plural_name=database_catalogs resource_type=DatabaseCatalog singular_name=database_instance plural_name=database_instances resource_type=DatabaseInstance singular_name=external_location plural_name=external_locations resource_type=ExternalLocation diff --git a/python/docs/databricks.bundles.data_quality_monitors.rst b/python/docs/databricks.bundles.data_quality_monitors.rst new file mode 100644 index 00000000000..5621bcea762 --- /dev/null +++ b/python/docs/databricks.bundles.data_quality_monitors.rst @@ -0,0 +1,11 @@ +Data Quality Monitors +=============================== + +.. currentmodule:: databricks.bundles.data_quality_monitors + +**Package:** ``databricks.bundles.data_quality_monitors`` + +Classes +--------------- + +.. automodule:: databricks.bundles.data_quality_monitors diff --git a/python/docs/index.rst b/python/docs/index.rst index b758c0febd4..2e8bed3edc8 100644 --- a/python/docs/index.rst +++ b/python/docs/index.rst @@ -16,6 +16,7 @@ See `Bundle configuration in Python Date: Thu, 15 Oct 2026 22:41:27 +0000 Subject: [PATCH 2/3] Fill in changelog PR number --- .nextchanges/bundles/data-quality-monitors.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.nextchanges/bundles/data-quality-monitors.md b/.nextchanges/bundles/data-quality-monitors.md index de03ece4904..2a090acb910 100644 --- a/.nextchanges/bundles/data-quality-monitors.md +++ b/.nextchanges/bundles/data-quality-monitors.md @@ -1 +1 @@ -* Add the `data_quality_monitors` resource for data quality monitors on Unity Catalog schemas and tables (direct engine only). ([#PR_NUMBER](https://github.com/databricks/cli/pull/PR_NUMBER)) +* Add the `data_quality_monitors` resource for data quality monitors on Unity Catalog schemas and tables (direct engine only). ([#7082](https://github.com/databricks/cli/pull/7082)) From fe29bb26f67328839dc24e3ea722ef8eba63b170 Mon Sep 17 00:00:00 2001 From: Kevin Lian Date: Fri, 16 Oct 2026 17:43:22 +0000 Subject: [PATCH 3/3] Update changelog PR number --- .nextchanges/bundles/data-quality-monitors.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.nextchanges/bundles/data-quality-monitors.md b/.nextchanges/bundles/data-quality-monitors.md index 2a090acb910..f5ac21fd3f3 100644 --- a/.nextchanges/bundles/data-quality-monitors.md +++ b/.nextchanges/bundles/data-quality-monitors.md @@ -1 +1 @@ -* Add the `data_quality_monitors` resource for data quality monitors on Unity Catalog schemas and tables (direct engine only). ([#7082](https://github.com/databricks/cli/pull/7082)) +* Add the `data_quality_monitors` resource for data quality monitors on Unity Catalog schemas and tables (direct engine only). ([#7094](https://github.com/databricks/cli/pull/7094))