Unified client generation: one client for a module and for a plain Java program

.dagger/modules/e2e/fixtures/clients/standalone/pom.xml

@@ -0,0 +1,16 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <!-- A plain Maven application, not a Dagger module. -->
+    <groupId>io.dagger.e2e</groupId>
+    <artifactId>standalone-client</artifactId>
+    <version>1.0-SNAPSHOT</version>
+
+    <properties>
+        <maven.compiler.release>17</maven.compiler.release>
+        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+    </properties>
+</project>

.dagger/modules/engine-e2e/main.dang

@@ -49,9 +49,16 @@ type EngineE2e {
       .withExec(["test", "-f", modulePath + "/pom.xml"])
       .withExec(["test", "-f", modulePath + "/src/main/java/io/dagger/modules/sdksmoke/SdkSmoke.java"])
       .withExec(["test", "-f", modulePath + "/sdk/src/main/java/io/dagger/client/Dagger.java"])
+      # Core is a client package: generated under the modules root like any
+      # other, and nothing is emitted flat beside the hand-written runtime.
+      .withExec(["test", "-f", modulePath + "/sdk/src/generated/java/io/dagger/client/modules/core/Core.java"])
+      .withExec(["test", "!", "-e", modulePath + "/sdk/src/generated/java/io/dagger/client/Client.java"])
       .withExec(["test", "-f", modulePath + "/src/generated/java/io/dagger/gen/entrypoint/Entrypoint.java"])
       .withExec(["grep", "-q", "github.com/dagger/java-sdk/runtime", modulePath + "/dagger-module.toml"])
 
+    # The scaffolded module's one function is core().container(), so running it
+    # proves core is reachable through its own package at run time, not just
+    # that the sources compile.
     let release = initialized
       .withExec(["dagger", "-m", modulePath, "call", "container", "file", "--path", "/etc/alpine-release", "contents"])
       .stdout
@@ -72,4 +79,81 @@ type EngineE2e {
 
     null
   }
+
+  let clientModulePath: String! = ".dagger/modules/client-smoke"
+  let gitClientRef: String! = "github.com/dagger/sdk-helpers@v1.0.2"
+
+  """
+  A Dagger module that declares a client loads that client at run time.
+
+  Every other check generates a scope or compiles one; none runs the result, so
+  the serve a client package performs on first use — the descriptor it holds,
+  the round trip to the engine, the module it names — is otherwise unproven. The
+  module source calls sdkHelpers() and nothing else: the entry point serves
+  before it builds anything, so constructing the client is already the round
+  trip, and a refusal surfaces as a failed call rather than as a value.
+
+  A git client, because that is the kind a module loads for itself today. A
+  workspace-local client inside a module is still served by the engine and would
+  prove nothing about this path.
+  """
+  pub javaClientRuntimeCheck(ws: Workspace!): Void @check {
+    let withClient = playground(ws)
+      .withExec(["dagger", "-y", "module", "init", "java", "--name", "client-smoke", "--path", clientModulePath])
+      # The client is declared by appending to the scope table `module init`
+      # just wrote, rather than by `dagger module client add java`, which fails
+      # on the pinned engine: calling findClientRoot over the SDK interface
+      # cannot assign its non-null String result into a nullable String field
+      # ("cannot set field of type dagql.Nullable[String] with dagql.String").
+      # That is an engine defect rather than one of this SDK's, and it is worth
+      # a fix upstream — the README documents the command this cannot use.
+      #
+      # If the line ever lands in the wrong table, generation produces no client
+      # package and the module source below stops compiling, so a silent miss is
+      # not among the ways this can go wrong.
+      .withExec(["sh", "-c", "printf 'clients = [\"" + gitClientRef + "\"]\\n' >> dagger.toml"])
+      .withWorkdir(clientModulePath)
+      .withNewFile(
+        "src/main/java/io/dagger/modules/clientsmoke/ClientSmoke.java",
+        clientSmokeSource,
+      )
+      .withExec(["dagger", "-y", "generate"])
+
+    # Back to the workspace root the module path is relative to: the checkout is
+    # mounted at the playground's own cwd, so it has no absolute path to name.
+    let served = withClient
+      .withWorkdir("../../..")
+      .withExec(["dagger", "-m", clientModulePath, "call", "loads-its-client"])
+      .stdout
+    assert(
+      served.contains("served"),
+      "the module did not load its client at run time: " + served,
+    )
+
+    null
+  }
+
+  """
+  Module source that reaches a client and nothing else.
+  """
+  let clientSmokeSource: String! {
+    "package io.dagger.modules.clientsmoke;\n"
+      + "\n"
+      + "import static io.dagger.client.modules.sdkhelpers.SdkHelpers.sdkHelpers;\n"
+      + "\n"
+      + "import io.dagger.client.exception.DaggerQueryException;\n"
+      + "import io.dagger.module.annotation.Function;\n"
+      + "import io.dagger.module.annotation.Object;\n"
+      + "import java.util.concurrent.ExecutionException;\n"
+      + "\n"
+      + "@Object\n"
+      + "public class ClientSmoke {\n"
+      + "  @Function\n"
+      + "  public String loadsItsClient()\n"
+      + "      throws ExecutionException, DaggerQueryException, InterruptedException {\n"
+      + "    sdkHelpers();\n"
+      + "    return \"served\";\n"
+      + "  }\n"
+      + "}\n"
+  }
 }

.dagger/modules/packager/main.dang

@@ -44,20 +44,28 @@ type Packager {
       .withMountedCache("/root/.m2", mavenRepo, sharing: CacheSharingMode.LOCKED)
       .withDirectory("/dagger-io", sdkSource(ws))
       .withWorkdir("/dagger-io")
-      .withExec(["mvn", "--projects", "dagger-codegen-maven-plugin", "--also-make", "install", "-T1C", "-Dmaven.test.skip=true", "-Dfmt.skip=true", "-Dproject.build.outputTimestamp=2024-01-01T00:00:00Z", "--no-transfer-progress"])
-      # Export the two artifacts this module publishes, named one by one rather
-      # than copying io/dagger wholesale. The local repository is shared with
-      # module generation, which installs io/dagger/dagger-java-sdk/<module> and
+      # Install and export in one exec, because the mount is LOCKED per exec and
+      # not across a chain of them. Split in two, the lock is released in
+      # between, and packager:unit-tests — which runs concurrently under one
+      # `dagger check` and installs the same plugin without the fixed output
+      # timestamp — can overwrite the jar in the shared repository before the
+      # export copies it. What then landed under prebuilt/ carried wall-clock
+      # entry timestamps and never matched the committed bytes, so this check
+      # failed on a race rather than on anything in the diff.
+      #
+      # The export names the two artifacts this module publishes rather than
+      # copying io/dagger wholesale. The local repository is shared with module
+      # generation, which installs io/dagger/dagger-java-sdk/<module> and
       # io/dagger/dagger-java-annotation-processor/<module> into it under a
       # per-module version; a wholesale copy sweeps those in too, so what landed
       # under prebuilt/ depended on whether a generation had run first.
       #
-      # Then strip Maven's install-time timestamps so the committed repo is
+      # It then strips Maven's install-time timestamps so the committed repo is
       # byte-reproducible: drop the comment lines from _remote.repositories
       # (keeping the ">=" local-install markers resolution needs) and pin
       # <lastUpdated>. Otherwise every generate re-timestamps these files and
       # the check reports perpetual drift.
-      .withExec(["sh", "-c", "set -e; rm -rf /out; mkdir -p /out/io/dagger; cd /root/.m2/repository/io/dagger; cp -r dagger-codegen-maven-plugin dagger-sdk-parent maven-metadata-local.xml /out/io/dagger/; find /out -name _remote.repositories -exec sed -i '/>=/!d' {} ';'; find /out -name maven-metadata-local.xml -exec sed -i 's|<lastUpdated>[0-9]*</lastUpdated>|<lastUpdated>20240101000000</lastUpdated>|' {} ';'"])
+      .withExec(["sh", "-c", "set -e; mvn --projects dagger-codegen-maven-plugin --also-make install -T1C -Dmaven.test.skip=true -Dfmt.skip=true -Dproject.build.outputTimestamp=2024-01-01T00:00:00Z --no-transfer-progress; rm -rf /out; mkdir -p /out/io/dagger; cd /root/.m2/repository/io/dagger; cp -r dagger-codegen-maven-plugin dagger-sdk-parent maven-metadata-local.xml /out/io/dagger/; find /out -name _remote.repositories -exec sed -i '/>=/!d' {} ';'; find /out -name maven-metadata-local.xml -exec sed -i 's|<lastUpdated>[0-9]*</lastUpdated>|<lastUpdated>20240101000000</lastUpdated>|' {} ';'"])
       .directory("/out")
   }
 

README.md

@@ -17,8 +17,8 @@ time** — the runtime just builds and packages the module.
 > [!IMPORTANT]
 > This SDK implements the module-scope interface from
 > [dagger/dagger#13992](https://github.com/dagger/dagger/pull/13992) and needs an
-> engine that has it. On the released engine (`v1.0.0-beta.11`) the module loads
-> but every call into it fails.
+> engine that has it. That change merged, and `v1.0.0-beta.13` carries it; on an
+> earlier engine the module loads but every call into it fails.
 >
 > It writes module manifests through
 > [`github.com/dagger/sdk-helpers`](https://github.com/dagger/sdk-helpers), the
@@ -63,7 +63,8 @@ generates the SDK bindings in one step:
   src/generated/java/io/dagger/gen/entrypoint/Entrypoint.java   # generated entrypoint
   sdk/src/main/java/...                                         # vendored SDK library
   sdk/src/processor/java/...                                    # vendored annotation processor
-  sdk/src/generated/java/...                                    # client bindings (from the engine schema)
+  sdk/src/generated/java/io/dagger/client/modules/core/...      # the core API (from the engine schema)
+  sdk/src/generated/java/io/dagger/client/modules/<client>/...   # one package per client
 ```
 
 The SDK settings become typed flags on `dagger module init java` and are
@@ -106,6 +107,31 @@ nearest one at or above your current directory, which is how
 project built with anything but Maven has no `pom.xml`, so this SDK reports no
 scope for it.
 
+## Calling the core API
+
+Core is a client package like any other. Its types are generated into
+`io.dagger.client.modules.core`, and the way in is a static method on its root
+type:
+
+```java
+import static io.dagger.client.modules.core.Core.core;
+import io.dagger.client.modules.core.Container;
+
+Container base = core().container().from("alpine:3.24");
+```
+
+`core()` uses the ambient session; `core(dag)` takes one you already hold.
+`io.dagger.client` itself holds only hand-written code — `Dagger`, `Session`,
+`QueryBuilder` and the rest of the runtime — so nothing generated is privileged.
+
+> [!WARNING]
+> `dag().container()` no longer exists. `Dagger.dag()` returns a `Session`, not
+> a generated client, and core is reached as `core()` after a static import of
+> `io.dagger.client.modules.core.Core.core`. Every core type moves with it:
+> `io.dagger.client.Container` becomes
+> `io.dagger.client.modules.core.Container`. `Dagger.connect()` returns an
+> `AutoCloseableSession` in place of `AutoCloseableClient`.
+
 ## Module clients
 
 Module dependencies are replaced by generated module clients:
@@ -114,13 +140,108 @@ Module dependencies are replaced by generated module clients:
 dagger module client add java <module-ref>
 ```
 
+Each client's types are generated into a package of their own,
+`io.dagger.client.modules.<client>`, from that client's own schema and nothing
+else. The way in is a static method on the client's own root type, so one import
+is the whole of the integration:
+
+```java
+import static io.dagger.client.modules.sdkhelpers.SdkHelpers.sdkHelpers;
+
+sdkHelpers().moduleManifest().generate();
+```
+
+Core is not extended with an accessor for it. A client package is
+self-contained: it reaches core types where they live, and nothing in core names
+it. Pass a session explicitly when you have one — `sdkHelpers(dag)` — or let the
+no-argument form use the ambient one. Core is entered the same way, which is the
+whole of the difference between a client and core: none.
+
+A module named `core` is refused, because the generated core API has that
+package. Alias the target to something else.
+
 In a module scope the client set becomes the module's dependency set. Each
 client is recorded in the manifest the module has — `dagger-module.toml`, or the
-`dagger.json` of a pre-1.0 module — and its types are part of the generated
-bindings; a client that is removed is dropped from both.
+`dagger.json` of a pre-1.0 module — and a client that is removed is dropped from
+both the manifest and the bindings.
+
+> [!WARNING]
+> A client's types moved out of `io.dagger.client` in this release, and so did
+> the way in. `dag().sdkHelpers()` becomes `sdkHelpers()` after a static import
+> of `io.dagger.client.modules.sdkhelpers.SdkHelpers.sdkHelpers`, and each type
+> is imported from `io.dagger.client.modules.<client>` rather than from
+> `io.dagger.client`. The `<Client>Arguments` holder moves with the method, onto
+> the client's root type.
+
+## Standalone clients
+
+A Maven project that is no Dagger module can call modules too. Run the same
+command inside it:
+
+```sh
+cd my-java-app          # any directory with a pom.xml
+dagger module client add java github.com/dagger/sdk-helpers@v1.0.2
+dagger generate
+```
+
+`dagger generate` writes the client tree under `dagger/`, all of it SDK-owned
+and regenerated whole:
+
+```
+my-java-app/
+  pom.xml                                                     # gains one profile, see below
+  dagger/src/main/java/io/dagger/client/**                    # the hand-written SDK runtime
+  dagger/src/main/java/io/dagger/client/modules/core/**       # the core API
+  dagger/src/main/java/io/dagger/client/modules/<client>/**   # one package per client
+```
+
+The bindings under `io.dagger.client.modules.<client>` are the same files a
+module gets for the same client. Only what surrounds them differs.
+
+Your own code then reads exactly as a module's does:
+
+```java
+import static io.dagger.client.modules.sdkhelpers.SdkHelpers.sdkHelpers;
+
+public class App {
+  public static void main(String[] args) throws Exception {
+    System.out.println(sdkHelpers().moduleManifest().generate());
+  }
+}
+```
+
+Run it with a `dagger` binary on `PATH` and no wrapper command:
+
+```sh
+mvn package
+java -jar target/my-java-app-1.0-SNAPSHOT.jar
+```
+
+There is no session to join, so the SDK starts one with `dagger session`, and
+each client asks the engine to load its module the first time your code reaches
+for it. Set `_EXPERIMENTAL_DAGGER_CLI_BIN` to point at a specific binary. The
+SDK does not download a CLI; install one first.
+
+The generated code needs Java 17, so the project's `maven.compiler.release`
+(or `maven.compiler.source` and `maven.compiler.target`) has to be 17 or later.
+
+### The one thing written into your pom
+
+Your `pom.xml` is yours, so `dagger generate` adds exactly one element to it: a
+profile with the id `dagger-clients`, carrying a comment that says what wrote
+it. The profile adds `dagger/src/main/java` as a source root, along with the SDK's
+own run-time dependencies. It activates on the presence of the generated tree, so deleting
+`dagger/` makes it inert and deleting the profile removes the integration.
+
+Generation refuses to touch a `dagger-clients` profile that does not carry that
+comment, on the assumption that you wrote it.
+
+### What a client is pinned to
 
-Standalone clients — in a scope that has no Java module — are not generated yet.
-Adding one is refused and the workspace is left unchanged.
+A client added at a git ref records the commit it resolved to, and the generated
+code asks for that commit. A client that is a path in your workspace records the
+path, so a jar built from it only works inside that workspace; a git client is
+the form that travels.
 
 > [!WARNING]
 > The client set is the *whole* dependency set. A module that recorded
@@ -131,9 +252,7 @@ Adding one is refused and the workspace is left unchanged.
 > dagger module client add java <module-ref>
 > ```
 >
-> Then check that each one landed in `dagger.toml` before you generate. On the
-> `sdk-ux-module-max` engine builds this SDK currently needs,
-> `dagger module client add` reports success and writes nothing.
+> Then check that each one landed in `dagger.toml` before you generate.
 
 ## Pre-1.0 modules