fix(dockerclient): version API requests when no API version was configured

.github/workflows/acceptance-testing-bats.yml

@@ -9,9 +9,6 @@ on:
     types: [opened, synchronize, closed]
 jobs:
   test:
-    strategy:
-      matrix:
-        go: ['1.21']
     runs-on: 'ubuntu-latest'
     steps:
       - uses: actions/checkout@v6.0.2
@@ -21,10 +18,37 @@ jobs:
 
       - uses: actions/setup-go@v6.3.0
         with:
-          go-version: ${{ matrix.go }}
+          go-version-file: 'go.mod'
+
+      - name: Install native build dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install --no-install-recommends --yes libbtrfs-dev libgpgme-dev pkg-config
 
       - name: Build Slim (quick dev)
-        run: make build_dev
+        # Mirror src.build.quick.sh; enable native GPGME only for the CI Mint binary.
+        run: |
+          set -e
+          BDIR="${GITHUB_WORKSPACE:?}"
+
+          BUILD_TIME="$(date -u '+%Y-%m-%d_%I:%M:%S%p')"
+          TAG="current"
+          REVISION="current"
+          if hash git 2>/dev/null && [ -e "${BDIR}/.git" ]; then
+            TAG="$(git describe --tags --always)"
+            REVISION="$(git rev-parse HEAD)"
+          fi
+
+          LD_FLAGS="-s -w -X github.com/mintoolkit/mint/pkg/version.appVersionTag=${TAG} -X github.com/mintoolkit/mint/pkg/version.appVersionRev=${REVISION} -X github.com/mintoolkit/mint/pkg/version.appVersionTime=${BUILD_TIME}"
+
+          go generate github.com/mintoolkit/mint/pkg/appbom
+
+          BINDIR="${BDIR}/bin"
+          mkdir -p "$BINDIR"
+          rm -rf "${BINDIR:?}/"*
+
+          CGO_ENABLED=1 go build -ldflags="${LD_FLAGS}" -tags "remote" -mod=vendor -o "${BINDIR}/mint" "${BDIR}/cmd/mint/main.go"
+          CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags="${LD_FLAGS}" -mod=vendor -o "${BINDIR}/mint-sensor" "${BDIR}/cmd/mint-sensor/main.go"
 
       - name: Run the bats test suite
         run: ./test/bats/bin/bats test/debug.bats

.github/workflows/acceptance-testing-e2e.yml

@@ -17,7 +17,7 @@ jobs:
       - name: Set up Go development environment
         uses: actions/setup-go@v6.3.0
         with:
-          go-version: '1.21.4'
+          go-version-file: 'go.mod'
 
       - name: Build Slim (multi-arch)
         run: make build
@@ -65,7 +65,7 @@ jobs:
       - name: Set up Go development environment
         uses: actions/setup-go@v6.3.0
         with:
-          go-version: '1.21.4'
+          go-version-file: 'go.mod'
 
       - name: Download Slim app binaries
         uses: actions/download-artifact@v8.0.1

.github/workflows/unit-testing.yml

@@ -17,7 +17,12 @@ jobs:
       - name: Set up Go development environment
         uses: actions/setup-go@v6.3.0
         with:
-          go-version: '1.21.4'
+          go-version-file: 'go.mod'
+
+      - name: Install native build dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install --no-install-recommends --yes libbtrfs-dev libgpgme-dev pkg-config
 
       - name: Run tests
         run: make test

pkg/app/sensor/app_test.go

@@ -606,14 +606,19 @@ func TestStopSignal_ForceKill(t *testing.T) {
 
 func TestControlCommands_StopTargetApp(t *testing.T) {
 	runID := newTestRun(t)
-	ctx := context.Background()
+	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
+	defer cancel()
 
 	sensor := testsensor.NewSensorOrFail(t, ctx, t.TempDir(), runID, imageSimpleService)
-	defer sensor.Cleanup(t, ctx)
+	defer func() {
+		cleanupCtx, cleanupCancel := context.WithTimeout(context.Background(), 30*time.Second)
+		defer cleanupCancel()
+		sensor.Cleanup(t, cleanupCtx)
+	}()
 
 	sensor.StartStandaloneOrFail(t, ctx, nil)
 
-	go testutil.Delayed(ctx, 5*time.Second, func() {
+	testutil.Delayed(ctx, 5*time.Second, func() {
 		sensor.ExecuteControlCommandOrFail(t, ctx, control.StopTargetAppCommand)
 		sensor.WaitForEventOrFail(t, ctx, event.StopMonitorDone)
 		sensor.WaitForEventOrFail(t, ctx, event.ShutdownSensorDone)

pkg/app/sensor/monitor/composite_test.go

@@ -8,6 +8,7 @@ import (
 
 	log "github.com/sirupsen/logrus"
 
+	"github.com/mintoolkit/mint/pkg/ipc/command"
 	stubmonitor "github.com/mintoolkit/mint/pkg/test/stub/sensor/monitor"
 )
 
@@ -17,6 +18,7 @@ func init() {
 
 func TestCompositeMonitor_Lifecycle(t *testing.T) {
 	mon := &monitor{
+		cmd:    &command.StartMonitor{},
 		fanMon: stubmonitor.NewFanMonitor(context.Background()),
 		ptMon:  stubmonitor.NewPtMonitor(context.Background()),
 	}

pkg/crt/docker/dockerclient/client.go

@@ -174,6 +174,88 @@ func GetUnixSocketAddr() (*SocketInfo, error) {
 	return nil, fmt.Errorf("docker socket not found")
 }
 
+// discoverAPIVersion asks the daemon behind client which API version it
+// speaks. An unreachable or otherwise misbehaving daemon yields "", which
+// leaves the caller with the client it already had, so a connection problem
+// still surfaces as the same error it always did.
+func discoverAPIVersion(client *docker.Client) string {
+	env, err := client.Version()
+	if err != nil {
+		return ""
+	}
+
+	return env.Get("ApiVersion")
+}
+
+// withDiscoveredAPIVersion returns a client whose requests carry the "/vX.Y/"
+// path segment even when the caller configured no API version.
+//
+// go-dockerclient parses an API version into the client's internal
+// requestedAPIVersion only when the string it is given is non-empty, and
+// getURL() consults that field alone when it builds a request path. With no
+// version configured - a plain "mint build"/"slim build" with no
+// DOCKER_API_VERSION set, which is the case in every mintoolkit/mint#95 and
+// slimtoolkit/slim#646 report - every request the client sends omits the
+// version segment. A daemon reached through a proxy, dind (Docker-in-Docker:
+// a CI runner's own container running a Docker daemon, the topology behind
+// those reports), reads an unversioned request as coming from the oldest
+// client it supports and rejects it ("client version ... is too old"). The
+// client's own /version negotiation does not help here: it stores its result
+// in expectedAPIVersion and never copies it into requestedAPIVersion.
+//
+// rebuild constructs the replacement client for the discovered version. Each
+// construction path passes its own constructor, so the rebuilt client keeps
+// that path's transport, credentials and endpoint; this is what lets the TLS
+// and environment paths share this logic with the plain one.
+func withDiscoveredAPIVersion(
+	client *docker.Client,
+	apiVersion string,
+	rebuild func(apiVersion string) (*docker.Client, error),
+) *docker.Client {
+	if client == nil {
+		return client
+	}
+
+	if apiVersion != "" {
+		// An explicitly configured version already populates
+		// requestedAPIVersion, so the lazy self-check buys nothing. This is
+		// what the call sites did by hand before.
+		client.SkipServerVersionCheck = true
+		return client
+	}
+
+	discovered := discoverAPIVersion(client)
+	if discovered == "" {
+		return client
+	}
+
+	versioned, err := rebuild(discovered)
+	if err != nil || versioned == nil {
+		return client
+	}
+
+	// The version was just read from this very daemon, so the lazy self-check
+	// that the first request would otherwise trigger only repeats the probe
+	// that has already happened. Skipping it is exactly what happens today
+	// when a caller sets DOCKER_API_VERSION by hand.
+	versioned.SkipServerVersionCheck = true
+
+	return versioned
+}
+
+// newVersionedClient builds a plain (non-TLS) client for host and gives it the
+// daemon's API version when the caller configured none.
+func newVersionedClient(host, apiVersion string) (*docker.Client, error) {
+	client, err := docker.NewVersionedClient(host, apiVersion)
+	if err != nil {
+		return nil, err
+	}
+
+	return withDiscoveredAPIVersion(client, apiVersion, func(apiVersion string) (*docker.Client, error) {
+		return docker.NewVersionedClient(host, apiVersion)
+	}), nil
+}
+
 // New creates a new Docker client instance
 func New(config *config.DockerClient) (*docker.Client, error) {
 	var client *docker.Client
@@ -205,6 +287,20 @@ func New(config *config.DockerClient) (*docker.Client, error) {
 		return docker.NewVersionedTLSClientFromBytes(host, cert, key, ca, apiVersion)
 	}
 
+	// newVersionedTLSClient is newTLSClient plus the version discovery the
+	// plain path gets: same certificates, same transport, and a "/vX.Y/"
+	// segment on every request when the caller configured no version.
+	newVersionedTLSClient := func(host string, certPath string, verify bool, apiVersion string) (*docker.Client, error) {
+		client, err := newTLSClient(host, certPath, verify, apiVersion)
+		if err != nil {
+			return nil, err
+		}
+
+		return withDiscoveredAPIVersion(client, apiVersion, func(apiVersion string) (*docker.Client, error) {
+			return newTLSClient(host, certPath, verify, apiVersion)
+		}), nil
+	}
+
 	//NOTE:
 	//go-dockerclient doesn't support DOCKER_CONTEXT natively
 	//so we need to lookup the context first to extract its connection info
@@ -270,7 +366,7 @@ func New(config *config.DockerClient) (*docker.Client, error) {
 		config.UseTLS &&
 		config.VerifyTLS &&
 		config.TLSCertPath != "":
-		client, err = newTLSClient(config.Host, config.TLSCertPath, true, config.APIVersion)
+		client, err = newVersionedTLSClient(config.Host, config.TLSCertPath, true, config.APIVersion)
 		if err != nil {
 			return nil, err
 		}
@@ -281,7 +377,7 @@ func New(config *config.DockerClient) (*docker.Client, error) {
 		config.UseTLS &&
 		!config.VerifyTLS &&
 		config.TLSCertPath != "":
-		client, err = newTLSClient(config.Host, config.TLSCertPath, false, config.APIVersion)
+		client, err = newVersionedTLSClient(config.Host, config.TLSCertPath, false, config.APIVersion)
 		if err != nil {
 			return nil, err
 		}
@@ -290,23 +386,19 @@ func New(config *config.DockerClient) (*docker.Client, error) {
 
 	case config.Host != "" &&
 		!config.UseTLS:
-		client, err = docker.NewVersionedClient(config.Host, config.APIVersion)
+		client, err = newVersionedClient(config.Host, config.APIVersion)
 		if err != nil {
 			return nil, err
 		}
 
-		if config.APIVersion != "" {
-			client.SkipServerVersionCheck = true
-		}
-
 		log.Debug("dockerclient.New: new Docker client [3]")
 
 	case config.Host == "" &&
 		!config.VerifyTLS &&
 		config.Env[EnvDockerTLSVerify] == "1" &&
 		config.Env[EnvDockerCertPath] != "" &&
 		config.Env[EnvDockerHost] != "":
-		client, err = newTLSClient(config.Env[EnvDockerHost], config.Env[EnvDockerCertPath], false, config.APIVersion)
+		client, err = newVersionedTLSClient(config.Env[EnvDockerHost], config.Env[EnvDockerCertPath], false, config.APIVersion)
 		if err != nil {
 			return nil, err
 		}
@@ -319,6 +411,12 @@ func New(config *config.DockerClient) (*docker.Client, error) {
 			return nil, err
 		}
 
+		// NewClientFromEnv reads DOCKER_API_VERSION itself, so the same string
+		// decides here whether a version was configured at all.
+		client = withDiscoveredAPIVersion(client, os.Getenv(EnvDockerAPIVer), func(apiVersion string) (*docker.Client, error) {
+			return docker.NewVersionedClientFromEnv(apiVersion)
+		})
+
 		log.Debug("dockerclient.New: new Docker client (env) [5]")
 
 	case config.Host != "" && (strings.HasPrefix(config.Host, "/") || strings.HasPrefix(config.Host, "unix://")):
@@ -342,15 +440,11 @@ func New(config *config.DockerClient) (*docker.Client, error) {
 		}
 
 		config.Host = socketInfo.Address
-		client, err = docker.NewVersionedClient(config.Host, config.APIVersion)
+		client, err = newVersionedClient(config.Host, config.APIVersion)
 		if err != nil {
 			return nil, err
 		}
 
-		if config.APIVersion != "" {
-			client.SkipServerVersionCheck = true
-		}
-
 	case config.Host == "" && config.Env[EnvDockerHost] == "" && contextHost != "":
 		log.Debugf("dockerclient.New: new Docker client - from context ('%s') contextVerifyTLS=%v", contextHost, contextVerifyTLS)
 		if strings.HasPrefix(contextHost, "/") ||
@@ -374,15 +468,11 @@ func New(config *config.DockerClient) (*docker.Client, error) {
 			}
 
 			config.Host = socketInfo.Address
-			client, err = docker.NewVersionedClient(config.Host, config.APIVersion)
+			client, err = newVersionedClient(config.Host, config.APIVersion)
 			if err != nil {
 				return nil, err
 			}
 
-			if config.APIVersion != "" {
-				client.SkipServerVersionCheck = true
-			}
-
 			log.Debugf("dockerclient.New: new Docker client - from context ('%s') - [7]", contextHost)
 		} else {
 			log.Debugf("dockerclient.New: new Docker client - from context - non-unix socket host (%s) [todo]", contextHost)
@@ -403,15 +493,11 @@ func New(config *config.DockerClient) (*docker.Client, error) {
 		}
 
 		config.Host = socketInfo.Address
-		client, err = docker.NewVersionedClient(config.Host, config.APIVersion)
+		client, err = newVersionedClient(config.Host, config.APIVersion)
 		if err != nil {
 			return nil, err
 		}
 
-		if config.APIVersion != "" {
-			client.SkipServerVersionCheck = true
-		}
-
 		log.Debug("dockerclient.New: new Docker client (default) [6]")
 
 	default: