Migrate from databricks-sdk-go to sdk-go

Taskfile.yml

@@ -778,6 +778,7 @@ tasks:
       # Runs first: regenerates CLI command stubs from the checked-in
       # .codegen/cli.json (no universe checkout needed). Refreshing cli.json
       # itself from the OpenAPI spec is the separate `generate-clijson` task.
+      - task: generate-sdktypes
       - task: generate-cligen
       # Refreshes acceptance/bundle/refschema/out.fields.txt, which feeds
       # generate-direct-apitypes and generate-direct-resources below.
@@ -798,6 +799,7 @@ tasks:
   generate-check:
     desc: Run the reproducible generators (CI drift subset of `generate`)
     cmds:
+      - task: generate-sdktypes
       - task: generate-cligen
       - task: generate-refschema
       - task: generate-schema
@@ -812,13 +814,32 @@ tasks:
   # templates derived from genkit's cli_v0 generator and formats the output
   # in-process (whitespace normalization, unused-import pruning, gofmt), so
   # the result matches the committed files byte-for-byte.
+  # Regenerates the JSON-tagged mirrors of sdk-go types (libs/sdktypes) and the
+  # oneof registry used by libs/sdktypes/sdkconv, for the sdk-go packages listed
+  # in libs/sdktypes/packages.txt ("<module path> <dir>" per line). Inputs are
+  # the sdk-go sources pinned in go.mod.
+  generate-sdktypes:
+    desc: Regenerate libs/sdktypes from the sdk-go packages in go.mod
+    sources:
+      - go.mod
+      - libs/sdktypes/packages.txt
+      - internal/sdktypesgen/*.go
+      - internal/sdkconvgen/*.go
+    cmds:
+      - "sh -c 'sed \"s|^\\([^ ]*\\).*|github.com/databricks/sdk-go/\\1|\" libs/sdktypes/packages.txt | xargs go run ./internal/sdktypesgen libs/sdktypes'"
+      - "sh -c 'sed \"s|^\\([^ ]*\\).*|github.com/databricks/sdk-go/\\1|\" libs/sdktypes/packages.txt | xargs go run ./internal/sdkconvgen libs/sdktypes/sdkconv/oneofs.generated.go'"
+
   generate-cligen:
     desc: Regenerate CLI command stubs from .codegen/cli.json
     sources:
       - .codegen/cli.json
       - .gitattributes.manual
       - internal/cligen/**/*.go
       - internal/cligen/templates/*.tmpl
+      - internal/cligen/http_verbs.json
+      - libs/sdkclients/services.go
+      - libs/sdktypes/**/types.generated.go
+      - go.mod
     generates:
       - cmd/workspace/**/*.go
       - cmd/account/**/*.go
@@ -962,6 +983,7 @@ tasks:
       - "**/*.go"
       - .codegen/cli.json
       - bundle/internal/schema/annotations.yml
+      - bundle/internal/sdkmeta/legacy.json
       - exclude: "**/*_test.go"
       - go.mod
       - go.sum
@@ -976,6 +998,8 @@ tasks:
     desc: Generate enum and required field validation code
     sources:
       - "**/*.go"
+      - .codegen/cli.json
+      - bundle/internal/sdkmeta/legacy.json
       - exclude: "**/*_test.go"
       - go.mod
       - go.sum

acceptance/acceptance_test.go

@@ -26,6 +26,8 @@ import (
 	"time"
 	"unicode/utf8"
 
+	"github.com/databricks/cli/libs/utils"
+
 	"github.com/google/uuid"
 
 	"github.com/databricks/cli/acceptance/internal"
@@ -1013,7 +1015,7 @@ func runTest(t *testing.T,
 	// output) are stable across OSes and don't need per-test slash replacements.
 	cmd.Env = append(cmd.Env, "TESTDIR="+filepath.ToSlash(absDir))
 	cmd.Env = append(cmd.Env, "CLOUD_ENV="+cloudEnv)
-	cmd.Env = append(cmd.Env, "CURRENT_USER_NAME="+user.UserName)
+	cmd.Env = append(cmd.Env, "CURRENT_USER_NAME="+utils.Deref(user.UserName))
 	if !isRunningOnCloud {
 		// Expose a guest token for the as-test-sp helper: the guest prefix plus
 		// the primary identity's uuid suffix make it share the same fake

acceptance/air/run-submit/output.txt

@@ -43,7 +43,9 @@ Stream logs after submission using:
         "permission_level": "CAN_EDIT",
         "service_principal_name": "training-sp"
       }
-    ]
+    ],
+    "request_object_id": "42",
+    "request_object_type": "experiments"
   }
 }
 {
@@ -63,7 +65,9 @@ Stream logs after submission using:
         "permission_level": "CAN_MANAGE_RUN",
         "service_principal_name": "training-sp"
       }
-    ]
+    ],
+    "request_object_id": "123",
+    "request_object_type": "jobs"
   }
 }
 {

acceptance/bundle/artifacts/whl_dynamic/databricks.yml

@@ -21,9 +21,6 @@ resources:
       tasks:
         - task_key: TestTask
           existing_cluster_id: "0717-132531-5opeqon1"
-          python_wheel_task:
-            package_name: "my_test_code"
-            entry_point: "run"
           libraries:
             - whl: ./my_test_code/dist/*.whl
             - whl: prebuilt/other_test_code-0.0.1-py3-none-any.whl

acceptance/bundle/artifacts/whl_dynamic/out.plan_create.json

@@ -63,10 +63,6 @@
                   "whl": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/other_test_code-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl"
                 }
               ],
-              "python_wheel_task": {
-                "entry_point": "run",
-                "package_name": "my_test_code"
-              },
               "task_key": "TestTask"
             }
           ]

acceptance/bundle/artifacts/whl_dynamic/out.plan_update.json

@@ -65,18 +65,12 @@
                   "whl": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/other_test_code-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl"
                 }
               ],
-              "python_wheel_task": {
-                "entry_point": "run",
-                "package_name": "my_test_code"
-              },
               "task_key": "TestTask"
             }
           ]
         }
       },
       "remote_state": {
-        "created_time": [UNIX_TIME_MILLIS],
-        "creator_user_name": "[USERNAME]",
         "deployment": {
           "kind": "BUNDLE",
           "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json"
@@ -95,13 +89,11 @@
           }
         ],
         "format": "MULTI_TASK",
-        "job_id": [TEST_JOB_ID],
         "max_concurrent_runs": 1,
         "name": "[default] My Wheel Job",
         "queue": {
           "enabled": true
         },
-        "run_as_user_name": "[USERNAME]",
         "tasks": [
           {
             "email_notifications": {},
@@ -141,17 +133,17 @@
                 "whl": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/other_test_code-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl"
               }
             ],
-            "python_wheel_task": {
-              "entry_point": "run",
-              "package_name": "my_test_code"
-            },
             "run_if": "ALL_SUCCESS",
             "task_key": "TestTask",
             "timeout_seconds": 0
           }
         ],
         "timeout_seconds": 0,
-        "webhook_notifications": {}
+        "webhook_notifications": {},
+        "created_time": [UNIX_TIME_MILLIS],
+        "creator_user_name": "[USERNAME]",
+        "job_id": [TEST_JOB_ID],
+        "run_as_user_name": "[USERNAME]"
       },
       "changes": {
         "email_notifications": {

acceptance/bundle/deploy/experimental-python/output.txt

@@ -11,6 +11,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged
     "created_time": [UNIX_TIME_MILLIS],
     "creator_user_name": "[USERNAME]",
     "job_id": [JOB1_ID],
+    "run_as_user_name": "[USERNAME]",
     "settings": {
       "deployment": {
         "kind": "BUNDLE",

acceptance/bundle/deploy/python-notebook/output.txt

@@ -11,6 +11,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged
     "created_time": [UNIX_TIME_MILLIS],
     "creator_user_name": "[USERNAME]",
     "job_id": [MY_JOB_ID],
+    "run_as_user_name": "[USERNAME]",
     "settings": {
       "deployment": {
         "kind": "BUNDLE",

acceptance/bundle/deploy/readplan/basic/out.plan_skip.json

@@ -7,24 +7,24 @@
     "resources.jobs.job": {
       "action": "skip",
       "remote_state": {
-        "created_time": [UNIX_TIME_MILLIS],
-        "creator_user_name": "[USERNAME]",
         "deployment": {
           "kind": "BUNDLE",
           "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json"
         },
         "edit_mode": "UI_LOCKED",
         "email_notifications": {},
         "format": "MULTI_TASK",
-        "job_id": [JOB_ID],
         "max_concurrent_runs": 1,
         "name": "test-job",
         "queue": {
           "enabled": true
         },
-        "run_as_user_name": "[USERNAME]",
         "timeout_seconds": 0,
-        "webhook_notifications": {}
+        "webhook_notifications": {},
+        "created_time": [UNIX_TIME_MILLIS],
+        "creator_user_name": "[USERNAME]",
+        "job_id": [JOB_ID],
+        "run_as_user_name": "[USERNAME]"
       },
       "changes": {
         "email_notifications": {

acceptance/bundle/deploy/readplan/grants-remove-principal/output.txt

@@ -30,7 +30,9 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged
           "ALL_PRIVILEGES"
         ]
       }
-    ]
+    ],
+    "securable_full_name": "main.myschema",
+    "securable_type": "schema"
   }
 }
 
@@ -67,7 +69,9 @@ Resources: 0 created, 1 changed, 0 deleted, 1 unchanged
           "ALL_PRIVILEGES"
         ]
       }
-    ]
+    ],
+    "securable_full_name": "main.myschema",
+    "securable_type": "schema"
   }
 }
 The following resources will be deleted:

acceptance/bundle/deploy/readplan/postgres_role/out.plan.role.json

@@ -8,9 +8,9 @@
   "action": "create",
   "new_state": {
     "value": {
-      "parent": "${resources.postgres_branches.branch.name}",
       "postgres_role": "app_role",
-      "role_id": "test-role"
+      "role_id": "test-role",
+      "parent": "${resources.postgres_branches.branch.name}"
     },
     "vars": {
       "parent": "${resources.postgres_branches.branch.name}"

acceptance/bundle/deployment/bind/dashboard/recreation/out.state_after_bind.json

@@ -8,12 +8,12 @@
    "__id__": "[DASHBOARD_ID]",
    "state": {
     "display_name": "test dashboard [UNIQUE_NAME]",
-    "embed_credentials": true,
     "etag": [ETAG],
     "parent_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/resources",
-    "published": true,
+    "warehouse_id": "[TEST_DEFAULT_WAREHOUSE_ID]",
     "serialized_dashboard": "{\"pages\":[{\"displayName\":\"Page One\",\"name\":\"02724bf2\"}]}",
-    "warehouse_id": "[TEST_DEFAULT_WAREHOUSE_ID]"
+    "embed_credentials": true,
+    "published": true
    }
   }
  }

acceptance/bundle/deployment/bind/pipelines/update/out.deploy.requests.json

@@ -30,6 +30,7 @@
         }
       }
     ],
-    "name": "test-pipeline"
+    "name": "test-pipeline",
+    "pipeline_id": "[NEW_PIPELINE_ID]"
   }
 }

acceptance/bundle/dms/no-drift/output.txt

@@ -139,6 +139,7 @@ Resources: 0 created, 1 changed, 0 deleted, 1 unchanged
       }
     ],
     "name": "bar-[UNIQUE_NAME]",
+    "pipeline_id": "[UUID]",
     "schema": "default",
     "serverless": true
   }

acceptance/bundle/generate/dashboard-inplace/output.txt

@@ -20,8 +20,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged
   "parent_path": "/Users/[USERNAME]/.bundle/dashboard update inplace/default/resources",
   "path": "/Users/[USERNAME]/.bundle/dashboard update inplace/default/resources/test dashboard.lvdash.json",
   "serialized_dashboard": "{\"a\":\"b\"}\n",
-  "update_time": "[TIMESTAMP]",
-  "warehouse_id": ""
+  "update_time": "[TIMESTAMP]"
 }
 
 === update the dashboard file using bundle generate

acceptance/bundle/generate/git_job/out.job.yml

@@ -12,6 +12,5 @@ resources:
             source: WORKSPACE
       git_source:
         git_branch: main
-        git_commit: abcdef
         git_provider: github
         git_url: https://git.databricks.com

acceptance/bundle/generate/git_job/test.toml

@@ -8,8 +8,7 @@ Response.Body = '''
         "git_source": {
             "git_url":      "https://git.databricks.com",
             "git_provider": "github",
-            "git_branch":   "main",
-            "git_commit":   "abcdef"
+            "git_branch":   "main"
         },
         "tasks": [
             {

acceptance/bundle/generate/include_warning/test.toml

@@ -8,8 +8,7 @@ Response.Body = '''
         "git_source": {
             "git_url":      "https://git.databricks.com",
             "git_provider": "github",
-            "git_branch":   "main",
-            "git_commit":   "abcdef"
+            "git_branch":   "main"
         },
         "tasks": [
             {

acceptance/bundle/generate/pipeline/test.toml

@@ -2,7 +2,7 @@
 Pattern = "GET /api/2.0/pipelines/1234"
 Response.Body = '''
 {
-    "pipeline_id": 1234,
+    "pipeline_id": "1234",
     "spec": {
         "name": "generate-pipeline",
         "development": true,

acceptance/bundle/generate/pipeline_with_glob/test.toml

@@ -2,7 +2,7 @@
 Pattern = "GET /api/2.0/pipelines/1234"
 Response.Body = '''
 {
-    "pipeline_id": 1234,
+    "pipeline_id": "1234",
     "spec": {
         "name": "generate-pipeline",
         "libraries": [

acceptance/bundle/generate/pipeline_with_sql/test.toml

@@ -2,7 +2,7 @@
 Pattern = "GET /api/2.0/pipelines/1234"
 Response.Body = '''
 {
-    "pipeline_id": 1234,
+    "pipeline_id": "1234",
     "spec": {
         "name": "generate-pipeline",
         "clusters": [

acceptance/bundle/migrate/auto/clean/out.plan_update.json

@@ -22,24 +22,24 @@
         }
       },
       "remote_state": {
-        "created_time": [UNIX_TIME_MILLIS],
-        "creator_user_name": "[USERNAME]",
         "deployment": {
           "kind": "BUNDLE",
           "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json"
         },
         "edit_mode": "UI_LOCKED",
         "email_notifications": {},
         "format": "MULTI_TASK",
-        "job_id": [TEST_JOB_ID],
         "max_concurrent_runs": 1,
         "name": "Test Auto-Migrate Job",
         "queue": {
           "enabled": true
         },
-        "run_as_user_name": "[USERNAME]",
         "timeout_seconds": 0,
-        "webhook_notifications": {}
+        "webhook_notifications": {},
+        "created_time": [UNIX_TIME_MILLIS],
+        "creator_user_name": "[USERNAME]",
+        "job_id": [TEST_JOB_ID],
+        "run_as_user_name": "[USERNAME]"
       },
       "changes": {
         "email_notifications": {