Skip to content

CI: hermetic spawns in cli/covgap tests (#823) #1441

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Mikola Lysenko (mikolalysenko) merged 3 commits into from
Oct 12, 2026
Merged

CI: hermetic spawns in cli/covgap tests (#823) #1441

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
8e02bca
Start fix for #823 (cli/covgap spawns)
claude Oct 12, 2026
e974944
Make cli and covgap tests ignore SOCKET_*
claude Oct 12, 2026
6d44a29
Close the in-process env race in the builder
claude Oct 12, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 7 additions & 31 deletions crates/socket-patch-cli/tests/cli/covgap_api_client.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,7 @@
//! stderr *text* — the "you configured the sha512- storage hash, not the
//! token" hint — which only a spawned process can observe.

use crate::common::binary;

use std::process::Command;
use crate::common::{binary, hermetic_command};

use wiremock::matchers::{method, path};
use wiremock::{Mock, MockServer, ResponseTemplate};
Expand Down Expand Up @@ -73,14 +71,10 @@ async fn run_get_with_hash_shaped_token(extra: &[&str]) -> std::process::Output
"sha512-deadbeefdeadbeef",
]);
// `mock` is dropped (and its `.expect(1)`s verified) on return.
Command::new(binary())
// Hermetic env: ambient state must not short-circuit auto-resolution
// (no env slug, no offline gate, no socket-cli config).
hermetic_command(&binary())
.args(&args)
// Ambient state must not short-circuit auto-resolution: no env
// slug, no offline gate, no socket-cli config (`socket login`).
.env_remove("SOCKET_ORG_SLUG")
.env_remove("SOCKET_OFFLINE")
.env_remove("SOCKET_API_TOKEN")
.env("SOCKET_NO_CONFIG", "1")
.current_dir(tmp.path())
.output()
.expect("run socket-patch get")
Expand Down Expand Up @@ -244,12 +238,7 @@ async fn unresolved_org_reaches_get_uuid_dry_run_json_warnings() {
let cwd = tmp.path();
let uri = mock.uri();
let token = format!("sktsec_{}_api", "x".repeat(44));
let mut cmd = Command::new(binary());
for (key, _) in std::env::vars() {
if key.starts_with("SOCKET_") {
cmd.env_remove(key);
}
}
let mut cmd = hermetic_command(&binary());
let out = cmd
.args([
"get",
Expand All @@ -268,7 +257,6 @@ async fn unresolved_org_reaches_get_uuid_dry_run_json_warnings() {
"--api-token",
&token,
])
.env("SOCKET_NO_CONFIG", "1")
.current_dir(cwd)
.output()
.expect("run socket-patch get");
Expand Down Expand Up @@ -377,12 +365,7 @@ async fn unresolved_org_routes_the_whole_scan_vex_run_to_the_proxy_once() {
let vex_path = cwd.join("out.vex.json");
let uri = mock.uri();
let token = format!("sktsec_{}_api", "x".repeat(44));
let mut cmd = Command::new(binary());
for (key, _) in std::env::vars() {
if key.starts_with("SOCKET_") {
cmd.env_remove(key);
}
}
let mut cmd = hermetic_command(&binary());
let out = cmd
.args([
"scan",
Expand All @@ -400,7 +383,6 @@ async fn unresolved_org_routes_the_whole_scan_vex_run_to_the_proxy_once() {
"--api-token",
&token,
])
.env("SOCKET_NO_CONFIG", "1")
.current_dir(cwd)
.output()
.expect("run socket-patch scan");
Expand Down Expand Up @@ -485,12 +467,7 @@ async fn unresolved_org_reaches_apply_and_vendor_json_warnings() {
std::fs::write(cwd.join(".socket/manifest.json"), r#"{"patches":{}}"#).unwrap();
let uri = mock.uri();
let token = format!("sktsec_{}_api", "x".repeat(44));
let mut cmd = Command::new(binary());
for (key, _) in std::env::vars() {
if key.starts_with("SOCKET_") {
cmd.env_remove(key);
}
}
let mut cmd = hermetic_command(&binary());
let out = cmd
.args([
command,
Expand All @@ -504,7 +481,6 @@ async fn unresolved_org_reaches_apply_and_vendor_json_warnings() {
"--api-token",
&token,
])
.env("SOCKET_NO_CONFIG", "1")
.current_dir(cwd)
.output()
.unwrap_or_else(|e| panic!("run socket-patch {command}: {e}"));
Expand Down
22 changes: 6 additions & 16 deletions crates/socket-patch-cli/tests/cli/covgap_commands_list.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,6 @@ use crate::vlt_vendored;
// `--debug` provenance echoes (GlobalArgs::telemetry_credentials, config layer)
// ---------------------------------------------------------------------------

const BINARY: &str = env!("CARGO_BIN_EXE_socket-patch");

/// The platform env var that positions the socket-cli data dir.
const DATA_DIR_VAR: &str = if cfg!(windows) {
"LOCALAPPDATA"
Expand Down Expand Up @@ -60,28 +58,20 @@ fn write_empty_manifest(root: &Path) {
std::fs::write(dir.join("manifest.json"), r#"{ "patches": {} }"#).unwrap();
}

/// `socket-patch list` against `project`, hermetic like
/// cli_config_fallback.rs's `list_cmd`: every ambient `SOCKET_*` scrubbed,
/// the data dir pointed at the fixture, the config layer re-enabled,
/// telemetry off (the provenance test re-enables it), the update notifier
/// off. `debug` adds the `--debug` flag under test.
/// `socket-patch list` against `project` in the shared hermetic env (every
/// ambient `SOCKET_*` scrubbed, telemetry and the update notifier off; the
/// provenance test re-enables telemetry), with the data dir pointed at the
/// fixture and the config layer re-enabled. `debug` adds the `--debug`
/// flag under test.
fn list_cmd(project: &Path, data_dir: &Path, debug: bool) -> Command {
let mut cmd = Command::new(BINARY);
let mut cmd = common::hermetic_command(&common::binary());
cmd.arg("list");
if debug {
cmd.arg("--debug");
}
cmd.arg("--cwd").arg(project);
for (key, _) in std::env::vars_os() {
let name = key.to_string_lossy();
if name.starts_with("SOCKET_") {
cmd.env_remove(&key);
}
}
cmd.env(DATA_DIR_VAR, data_dir);
cmd.env("SOCKET_NO_CONFIG", "0");
cmd.env("SOCKET_TELEMETRY_DISABLED", "1");
cmd.env("SOCKET_NO_UPDATE_CHECK", "1");
cmd
}

Expand Down
45 changes: 10 additions & 35 deletions crates/socket-patch-cli/tests/cli/telemetry_e2e.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
//! count). Each test runs the released binary in a tempdir against
//! the mock URI.

use crate::common::binary;
use crate::common::{binary, hermetic_command};

use std::path::Path;
use std::process::Command;
Expand Down Expand Up @@ -97,45 +97,20 @@ fn build_cmd_with_token(
ORG_SLUG,
];
args.extend_from_slice(extra_args);
let mut cmd = Command::new(binary());
cmd.args(&args).current_dir(cwd);
// The binary binds a wide `SOCKET_*` env surface; an ambient value
// silently changes what these tests exercise — SOCKET_ECOSYSTEMS=cargo
// makes scan skip the npm fixture (scannedPackages 0), SOCKET_CWD aims
// the crawl outside the tempdir, SOCKET_GLOBAL flips to global mode.
// The highest-risk vars are seeded with hostile values and then
// scrubbed — `env_remove` clears the seed too, so the child never sees
// it, but if a scrub line is ever dropped the seed (rather than a
// developer's ambient shell, which this suite can't rely on) turns the
// tests red immediately. Same pattern as `common::run_with_env`.
cmd.env("SOCKET_CWD", "/nonexistent")
.env("SOCKET_ECOSYSTEMS", "cargo")
.env("SOCKET_GLOBAL", "true")
.env("SOCKET_GLOBAL_PREFIX", "/nonexistent")
.env("SOCKET_DRY_RUN", "true")
.env("SOCKET_MANIFEST_PATH", "/nonexistent/manifest.json")
.env_remove("SOCKET_CWD")
.env_remove("SOCKET_ECOSYSTEMS")
.env_remove("SOCKET_GLOBAL")
.env_remove("SOCKET_GLOBAL_PREFIX")
.env_remove("SOCKET_DRY_RUN")
.env_remove("SOCKET_MANIFEST_PATH");
// Prefix-scrub whatever else the ambient shell carries (SOCKET_YES,
// SOCKET_STRICT, SOCKET_VENDOR_SOURCE, ...). Unlike common/mod.rs this
// scrub covers the telemetry opt-outs too: this suite's entire point is
// asserting telemetry POSTs against a local wiremock, so an opted-out
// dev shell must not vacuously green the count-0 asserts.
for (key, _) in std::env::vars_os() {
let name = key.to_string_lossy();
if name.starts_with("SOCKET_") && name != "SOCKET_NO_CONFIG" {
cmd.env_remove(&key);
}
}
// Default: disable the test-environment short-circuit
// (`is_telemetry_disabled()` flips on `VITEST=true`).
// the crawl outside the tempdir, SOCKET_GLOBAL flips to global mode. The
// hermetic builder scrubs all of them.
let mut cmd = hermetic_command(&binary());
cmd.args(&args).current_dir(cwd);
// This suite's entire point is asserting telemetry POSTs against a
// local wiremock, so undo the builder's forced opt-out (an opted-out
// dev shell must not vacuously green the count-0 asserts), and disable
// the test-environment short-circuit (`is_telemetry_disabled()` flips
// on `VITEST=true`).
cmd.env_remove("VITEST");
cmd.env_remove("SOCKET_TELEMETRY_DISABLED");
cmd.env_remove("SOCKET_OFFLINE");
// An ambient VIRTUAL_ENV hijacks the python crawler (its site-packages
// get crawled as project packages), breaking the exact
// `scannedPackages` oracles below.
Expand Down
11 changes: 2 additions & 9 deletions crates/socket-patch-cli/tests/cli_apply_silent.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,23 +25,16 @@

#[path = "common/mod.rs"]
mod common;
use common::binary;
use common::{binary, hermetic_command};

use std::path::Path;
use std::process::Command;

use socket_patch_cli::args::GLOBAL_ARG_ENV_VARS;

/// Run `socket-patch apply` in `cwd` with a scrubbed SOCKET_* environment
/// so ambient developer/CI configuration (tokens, silent toggles) can't
/// change the branch under test.
fn run_apply(cwd: &Path, args: &[&str]) -> (i32, String, String) {
let mut cmd = Command::new(binary());
let mut cmd = hermetic_command(&binary());
cmd.arg("apply").args(args).current_dir(cwd);
for var in GLOBAL_ARG_ENV_VARS {
cmd.env_remove(var);
}
cmd.env("SOCKET_TELEMETRY_DISABLED", "1");
let out = cmd.output().expect("run socket-patch apply");
(
out.status.code().unwrap_or(-1),
Expand Down
15 changes: 6 additions & 9 deletions crates/socket-patch-cli/tests/cli_argv_non_utf8.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,9 @@

use std::ffi::OsStr;
use std::os::unix::ffi::OsStrExt;
use std::process::Command;

const BINARY: &str = env!("CARGO_BIN_EXE_socket-patch");
#[path = "common/hermetic.rs"]
mod hermetic;

/// An argument that is valid on the OS level but not valid UTF-8.
fn non_utf8_arg() -> &'static OsStr {
Expand All @@ -28,16 +28,13 @@ fn non_utf8_arg() -> &'static OsStr {

/// Run the binary with the given args in a hermetic env and capture output.
fn run(args: &[&OsStr]) -> (Option<i32>, String, String) {
let mut cmd = Command::new(BINARY);
// Hermetic env: ambient SOCKET_* vars can never perturb where the
// invocation fails (the assertion is about the argv path, not env
// handling).
let mut cmd = hermetic::binary_command();
for a in args {
cmd.arg(a);
}
// Scrub the global env-var surface so ambient SOCKET_* vars can never
// perturb where the invocation fails (the assertion is about the argv
// path, not env handling).
for var in socket_patch_cli::args::GLOBAL_ARG_ENV_VARS {
cmd.env_remove(var);
}
let out = cmd.output().expect("spawn socket-patch");
(
out.status.code(),
Expand Down
50 changes: 17 additions & 33 deletions crates/socket-patch-cli/tests/cli_config_fallback.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,8 @@ use base64::Engine as _;
use wiremock::matchers::{method, path};
use wiremock::{Mock, MockServer, ResponseTemplate};

const BINARY: &str = env!("CARGO_BIN_EXE_socket-patch");
#[path = "common/hermetic.rs"]
mod hermetic;

/// The platform env var that positions the socket-cli data dir.
const DATA_DIR_VAR: &str = if cfg!(windows) {
Expand All @@ -50,34 +51,26 @@ fn write_config(data_dir: &Path, json: &serde_json::Value) {
std::fs::write(dir.join("config.json"), encoded).unwrap();
}

/// Build a hermetic human-mode `socket-patch scan -e npm --cwd <project>` command: every
/// ambient `SOCKET_*` var is scrubbed (including the inherited
/// `SOCKET_NO_CONFIG=1` guard — tests re-add exactly what they need), the
/// data dir points at `data_dir`, and the project dir is an empty npm
/// project so the crawl finds nothing and no batch request fires.
/// Build a hermetic human-mode `socket-patch scan -e npm --cwd <project>`
/// command: the shared hermetic env (every ambient `SOCKET_*` scrubbed,
/// telemetry off) with the config layer re-enabled, the data dir pointed
/// at `data_dir`, and the project dir an empty npm project so the crawl
/// finds nothing and no batch request fires.
fn scan_cmd(project: &Path, data_dir: &Path) -> Command {
let mut cmd = Command::new(BINARY);
let mut cmd = hermetic::binary_command();
// Human mode: core's proxy advisory (the oracle below) is muted under
// `--json`/`--silent`.
cmd.args(["scan", "-e", "npm", "--cwd"]).arg(project);
for (key, _) in std::env::vars_os() {
let name = key.to_string_lossy();
if name.starts_with("SOCKET_") {
cmd.env_remove(&key);
}
}
// Ambient VIRTUAL_ENV would be harmless under `-e npm`, but scrub it
// anyway to mirror the other scan harnesses.
cmd.env_remove("VIRTUAL_ENV");
cmd.env(DATA_DIR_VAR, data_dir);
// Re-enable the config layer (the workspace-level SOCKET_NO_CONFIG=1
// guard was scrubbed above; set an explicit falsy value so the intent
// is visible). Gate tests override this with "1".
// Re-enable the config layer (the hermetic builder forces
// SOCKET_NO_CONFIG=1; caller env lands last). Gate tests override this
// with "1". Telemetry stays off (it is fire-and-forget to the *real*
// proxy when a run is unauthenticated); the one test that exercises
// telemetry-follows-config re-enables it explicitly.
cmd.env("SOCKET_NO_CONFIG", "0");
// Telemetry is fire-and-forget to the *real* proxy when a run is
// unauthenticated — keep these tests off the network. The one test
// that exercises telemetry-follows-config re-enables it explicitly.
cmd.env("SOCKET_TELEMETRY_DISABLED", "1");
cmd
}

Expand Down Expand Up @@ -441,23 +434,14 @@ fn write_empty_manifest(root: &Path) {

/// `socket-patch list` against `project`, with the same hermetic env as
/// [`scan_cmd`]: every ambient `SOCKET_*` scrubbed, the data dir pointed at
/// the fixture, the config layer re-enabled, telemetry off by default.
/// the fixture, the config layer re-enabled, telemetry off by default, and
/// the passive update check off so the only request a run can make is the
/// telemetry POST under test.
fn list_cmd(project: &Path, data_dir: &Path) -> Command {
let mut cmd = Command::new(BINARY);
let mut cmd = hermetic::binary_command();
cmd.arg("list").arg("--cwd").arg(project);
for (key, _) in std::env::vars_os() {
let name = key.to_string_lossy();
if name.starts_with("SOCKET_") {
cmd.env_remove(&key);
}
}
cmd.env(DATA_DIR_VAR, data_dir);
cmd.env("SOCKET_NO_CONFIG", "0");
cmd.env("SOCKET_TELEMETRY_DISABLED", "1");
// The scrub above also dropped the workspace-level notifier guard; keep
// the passive update check off so the only request a run can make is the
// telemetry POST under test.
cmd.env("SOCKET_NO_UPDATE_CHECK", "1");
cmd
}

Expand Down
14 changes: 2 additions & 12 deletions crates/socket-patch-cli/tests/cli_get_silent.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,26 +9,16 @@

#[path = "common/mod.rs"]
mod common;
use common::binary;
use common::{binary, hermetic_command};

use std::path::Path;
use std::process::Command;

use socket_patch_cli::args::GLOBAL_ARG_ENV_VARS;

/// Run `socket-patch get` in `cwd` with a scrubbed SOCKET_* environment
/// so ambient developer/CI configuration (tokens, org slugs, silent
/// toggles) can't change the branch under test.
fn run_get(cwd: &Path, args: &[&str]) -> (i32, String) {
let mut cmd = Command::new(binary());
let mut cmd = hermetic_command(&binary());
cmd.arg("get").args(args).current_dir(cwd);
for var in GLOBAL_ARG_ENV_VARS {
cmd.env_remove(var);
}
for var in ["SOCKET_SAVE_ONLY", "SOCKET_ALL_RELEASES"] {
cmd.env_remove(var);
}
cmd.env("SOCKET_TELEMETRY_DISABLED", "1");
let out = cmd.output().expect("run socket-patch get");
(
out.status.code().unwrap_or(-1),
Expand Down
Loading
Loading