feat(websocket): flush the log buffer when a server is unreachable

CHANGELOG.md

@@ -12,6 +12,11 @@
 - Warn when the connected agent's startup scripts fail or time out, with
   **Show Logs** and **Open in Dashboard** actions. These failures used to appear
   only in the Coder output channel.
+- Flush the buffered connection logs into the Coder output channel after several
+  consecutive failed reconnect attempts, so the detail leading up to a "hangs on
+  connecting" problem is captured even when the server is simply unreachable and
+  the socket never reaches a terminal failure. The extension keeps retrying until
+  the server is reachable again.
 
 ## [v1.16.4](https://github.com/coder/vscode-coder/releases/tag/v1.16.4) 2026-09-23
 

src/api/coderApi.ts

@@ -45,6 +45,7 @@ import {
 import {
 	ConnectionState,
 	ReconnectingWebSocket,
+	type ConnectionFailureReason,
 	type ReconnectingWebSocketOptions,
 	type SocketFactory,
 } from "../websocket/reconnectingWebSocket";
@@ -72,7 +73,6 @@ import type {
 } from "coder/site/src/api/typesGenerated";
 import type { ClientOptions } from "ws";
 
-import type { ConnectionStateReason } from "../instrumentation/websocket";
 import type { Logger } from "../logging/logger";
 import type {
 	CloseEvent,
@@ -130,7 +130,7 @@ export class CoderApi extends Api implements vscode.Disposable {
 		private readonly httpRequestsTelemetry: HttpRequestsTelemetry,
 		private readonly authConfigTracker: AuthConfigTracker,
 		private readonly onConnectionFailure?: (
-			reason: ConnectionStateReason,
+			reason: ConnectionFailureReason,
 			route: string,
 		) => void,
 	) {
@@ -154,7 +154,7 @@ export class CoderApi extends Api implements vscode.Disposable {
 		output: Logger,
 		telemetry: TelemetryReporter = NOOP_TELEMETRY_REPORTER,
 		onConnectionFailure?: (
-			reason: ConnectionStateReason,
+			reason: ConnectionFailureReason,
 			route: string,
 		) => void,
 	): CoderApi {

src/instrumentation/EVENTS.md

@@ -16,7 +16,7 @@ new ones live in `CONVENTIONS.md`.
 | [Remote setup](#remote-setup)                   | `remote.setup`                                                                                                                                                                                                                                                          |
 | [SSH](#ssh)                                     | `ssh.process.discovered`, `ssh.process.lost`, `ssh.process.recovered`, `ssh.process.replaced`, `ssh.process.disposed`, `ssh.network.sampled`                                                                                                                            |
 | [HTTP](#http)                                   | `http.requests`                                                                                                                                                                                                                                                         |
-| [WebSocket connections](#websocket-connections) | `connection.state_transitioned`, `connection.opened`, `connection.dropped`, `connection.reconnect_resolved`                                                                                                                                                             |
+| [WebSocket connections](#websocket-connections) | `connection.state_transitioned`, `connection.opened`, `connection.dropped`, `connection.reconnect_resolved`, `connection.unreachable`                                                                                                                                   |
 | [Workspace](#workspace)                         | `workspace.start.triggered`, `workspace.update.triggered`, `workspace.start.prompted`, `workspace.update.prompted`, `workspace.open`, `workspace.picker.prompted`, `workspace.dev_container.open`, `workspace.state_transitioned`, `workspace.agent.state_transitioned` |
 
 Signal kinds, which each category groups its events by:
@@ -471,6 +471,17 @@ success or termination).
 | `max_backoff_ms` (measurement)    | largest backoff scheduled                              |
 | `total_duration_ms` (measurement) | cycle wall time                                        |
 
+#### `connection.unreachable`
+
+Emitted once when the reconnect loop has failed enough consecutive times to
+treat the server as unreachable. The socket keeps retrying, and the counter
+resets on a successful open, so a later outage emits again.
+
+| Attribute                | Values                      |
+| ------------------------ | --------------------------- |
+| `route`                  | normalized route            |
+| `attempts` (measurement) | consecutive failed attempts |
+
 ## Workspace
 
 Emitted by `WorkspaceOperationTelemetry` (start and update),

src/instrumentation/websocket.ts

@@ -127,6 +127,18 @@ export class WebSocketTelemetry {
 		}
 	}
 
+	/**
+	 * The reconnect loop has failed enough consecutive times to treat the server
+	 * as unreachable. Surfaced so Support can query sustained unreachability.
+	 */
+	public unreachable(route: string, attempts: number): void {
+		this.#telemetry.log(
+			"connection.unreachable",
+			{ route: normalizeRoute(route) },
+			{ attempts },
+		);
+	}
+
 	public reset(): void {
 		this.#connectStartedAtMs = undefined;
 		this.#connectionOpenedAtMs = undefined;

src/websocket/reconnectingWebSocket.ts

@@ -111,9 +111,17 @@ function reduceState(
 
 export type SocketFactory<TData> = () => Promise<UnidirectionalStream<TData>>;
 
+export type ConnectionFailureReason = ConnectionStateReason | "unreachable";
+
 /** Default failure callback for callers that do not observe connection failures. */
 const NOOP_CONNECTION_FAILURE = (): void => undefined;
 
+/**
+ * Consecutive failed reconnect attempts before the buffer is flushed once and
+ * the server is treated as unreachable.
+ */
+const MAX_RECONNECT_FAILURES_BEFORE_FLUSH = 6;
+
 export interface ReconnectingWebSocketOptions {
 	initialBackoffMs?: number;
 	maxBackoffMs?: number;
@@ -123,8 +131,14 @@ export interface ReconnectingWebSocketOptions {
 	route: string;
 	/** Callback invoked when a refreshable certificate error is detected. Returns true if refresh succeeded. */
 	onCertificateRefreshNeeded: () => Promise<boolean>;
-	/** Callback invoked when the connection fails terminally (not a transient drop). */
-	onConnectionFailure?: (reason: ConnectionStateReason, route: string) => void;
+	/**
+	 * Callback invoked on a terminal failure, or once per outage when the server
+	 * stays unreachable. Retrying continues in the unreachable case.
+	 */
+	onConnectionFailure?: (
+		reason: ConnectionFailureReason,
+		route: string,
+	) => void;
 }
 
 export class ReconnectingWebSocket<
@@ -152,6 +166,7 @@ export class ReconnectingWebSocket<
 	#lastRoute: string;
 	#backoffMs: number;
 	#reconnectTimeoutId: NodeJS.Timeout | null = null;
+	#consecutiveConnectFailures = 0;
 	#state: ConnectionState = ConnectionState.IDLE;
 	#certRefreshAttempted = false; // Tracks if cert refresh was already attempted this connection cycle
 	readonly #onDispose?: () => void;
@@ -275,6 +290,7 @@ export class ReconnectingWebSocket<
 		if (this.#state === ConnectionState.DISCONNECTED) {
 			this.#backoffMs = this.#options.initialBackoffMs;
 			this.#certRefreshAttempted = false; // User-initiated reconnect, allow retry
+			this.#consecutiveConnectFailures = 0;
 		}
 
 		if (this.#reconnectTimeoutId !== null) {
@@ -373,6 +389,7 @@ export class ReconnectingWebSocket<
 				// Reset backoff on successful connection
 				this.#backoffMs = this.#options.initialBackoffMs;
 				this.#certRefreshAttempted = false;
+				this.#consecutiveConnectFailures = 0;
 				this.executeHandlers("open", event);
 			});
 
@@ -448,6 +465,20 @@ export class ReconnectingWebSocket<
 		if (!this.#dispatch({ type: "SCHEDULE_RETRY" }, reason)) {
 			return;
 		}
+
+		this.#consecutiveConnectFailures += 1;
+		if (
+			this.#consecutiveConnectFailures === MAX_RECONNECT_FAILURES_BEFORE_FLUSH
+		) {
+			this.#logger.warn(
+				`Server unreachable after ${this.#consecutiveConnectFailures} reconnect attempts for ${this.#route}, still retrying`,
+			);
+			this.#telemetry.unreachable(
+				this.#route,
+				this.#consecutiveConnectFailures,
+			);
+			this.#options.onConnectionFailure("unreachable", this.#route);
+		}
 		const jitter =
 			this.#backoffMs * this.#options.jitterFactor * (Math.random() * 2 - 1);
 		const delayMs = Math.max(0, this.#backoffMs + jitter);

test/unit/instrumentation/websocket.test.ts

@@ -31,6 +31,23 @@ describe("WebSocketTelemetry", () => {
 		});
 	});
 
+	describe("unreachable", () => {
+		it("emits connection.unreachable with the normalized route and attempts", () => {
+			const { ws, sink } = setup();
+
+			ws.unreachable(
+				"wss://coder.example.com/api/v2/workspaces/123e4567-e89b-12d3-a456-426614174000/watch-ws?token=secret",
+				6,
+			);
+
+			const [event] = sink.eventsNamed("connection.unreachable");
+			expect(event).toMatchObject({
+				properties: { route: "/api/v2/workspaces/{id}/watch-ws" },
+				measurements: { attempts: 6 },
+			});
+		});
+	});
+
 	describe("opened", () => {
 		it("emits connection.opened with route and connect duration", () => {
 			const { ws, sink } = setup();

test/unit/websocket/reconnectingWebSocket.test.ts

@@ -12,6 +12,7 @@ import {
 import {
 	ConnectionState,
 	ReconnectingWebSocket,
+	type ConnectionFailureReason,
 	type SocketFactory,
 } from "@/websocket/reconnectingWebSocket";
 import { HandshakeError } from "@/websocket/utils";
@@ -25,7 +26,6 @@ import { createMockLogger } from "../../mocks/testHelpers";
 
 import type { CloseEvent, Event as WsEvent } from "ws";
 
-import type { ConnectionStateReason } from "@/instrumentation/websocket";
 import type { UnidirectionalStream } from "@/websocket/eventStreamConnection";
 
 describe("ReconnectingWebSocket", () => {
@@ -862,6 +862,143 @@ describe("ReconnectingWebSocket", () => {
 			ws.close();
 		});
 	});
+
+	describe("Unreachable server", () => {
+		// Mirrors MAX_RECONNECT_FAILURES_BEFORE_FLUSH in the implementation.
+		const FAILURES_BEFORE_FLUSH = 6;
+		// Pathname of the mock socket's URL, which becomes the logged route.
+		const ROUTE = "/api/test";
+		// Constant backoff, so advancing by it is exactly one failed attempt.
+		const BACKOFF_MS = 100;
+
+		async function setupUnreachable(
+			options: { telemetry?: TelemetryReporter } = {},
+		) {
+			const onConnectionFailure =
+				vi.fn<(reason: ConnectionFailureReason, route: string) => void>();
+			const { ws, sockets, setFactoryError } =
+				await createReconnectingWebSocketWithErrorControl({
+					...options,
+					onConnectionFailure,
+					initialBackoffMs: BACKOFF_MS,
+					maxBackoffMs: BACKOFF_MS,
+					jitterFactor: 0,
+				});
+			sockets[0].fireOpen();
+
+			// Drop the healthy socket and make every reconnect fail. The close is
+			// the first failed attempt; each advance is the next.
+			const startOutage = (): void => {
+				setFactoryError(new Error("connect ECONNREFUSED"));
+				sockets.at(-1)?.fireClose({
+					code: WebSocketCloseCode.ABNORMAL,
+					reason: "Connection lost",
+				});
+			};
+			const failNextAttempt = async (): Promise<void> => {
+				await vi.advanceTimersByTimeAsync(BACKOFF_MS);
+			};
+			// Fail exactly enough attempts to reach the flush.
+			const failUntilFlush = async (): Promise<void> => {
+				startOutage();
+				for (let i = 0; i < FAILURES_BEFORE_FLUSH - 1; i++) {
+					await failNextAttempt();
+				}
+			};
+			const recover = async (): Promise<void> => {
+				setFactoryError(null);
+				await vi.advanceTimersByTimeAsync(BACKOFF_MS);
+				sockets.at(-1)?.fireOpen();
+			};
+			return {
+				ws,
+				sockets,
+				onConnectionFailure,
+				startOutage,
+				failNextAttempt,
+				failUntilFlush,
+				recover,
+			};
+		}
+
+		it("flushes once with the unreachable reason after N failed attempts", async () => {
+			const { ws, onConnectionFailure, startOutage, failNextAttempt } =
+				await setupUnreachable();
+
+			startOutage(); // attempt 1
+			for (let i = 0; i < FAILURES_BEFORE_FLUSH - 2; i++) {
+				await failNextAttempt(); // through attempt N-1
+			}
+			expect(onConnectionFailure).not.toHaveBeenCalled();
+
+			await failNextAttempt(); // attempt N
+			expect(onConnectionFailure).toHaveBeenCalledTimes(1);
+			expect(onConnectionFailure).toHaveBeenCalledWith("unreachable", ROUTE);
+
+			ws.close();
+		});
+
+		it("keeps retrying through a long outage without flushing again, then recovers", async () => {
+			const { ws, onConnectionFailure, failUntilFlush, recover } =
+				await setupUnreachable();
+
+			await failUntilFlush();
+			expect(onConnectionFailure).toHaveBeenCalledTimes(1);
+
+			// A long outage (sleep, network loss) never gives up or re-flushes.
+			await vi.advanceTimersByTimeAsync(5 * 60_000);
+			expect(ws.state).toBe(ConnectionState.AWAITING_RETRY);
+			expect(onConnectionFailure).toHaveBeenCalledTimes(1);
+
+			await recover();
+			expect(ws.state).toBe(ConnectionState.CONNECTED);
+			expect(onConnectionFailure).toHaveBeenCalledTimes(1);
+
+			ws.close();
+		});
+
+		it("flushes again after a successful open resets the counter", async () => {
+			const { ws, onConnectionFailure, failUntilFlush, recover } =
+				await setupUnreachable();
+
+			await failUntilFlush();
+			expect(onConnectionFailure).toHaveBeenCalledTimes(1);
+
+			await recover();
+
+			await failUntilFlush();
+			expect(onConnectionFailure).toHaveBeenCalledTimes(2);
+
+			ws.close();
+		});
+
+		it("does not flush a transient outage that recovers before N", async () => {
+			const { ws, onConnectionFailure, startOutage, failNextAttempt, recover } =
+				await setupUnreachable();
+
+			startOutage();
+			await failNextAttempt();
+			await failNextAttempt();
+			await recover();
+
+			expect(onConnectionFailure).not.toHaveBeenCalled();
+			ws.close();
+		});
+
+		it("emits connection.unreachable at the flush", async () => {
+			enableLocalTelemetry();
+			const sink = new TestSink();
+			const telemetry = createTestTelemetryService(sink);
+			const { ws, failUntilFlush } = await setupUnreachable({
+				telemetry,
+			});
+
+			await failUntilFlush();
+
+			expect(sink.eventsNamed("connection.unreachable")).toHaveLength(1);
+			ws.close();
+		});
+	});
 });
 
 type MockSocket = UnidirectionalStream<unknown> & {
@@ -933,15 +1070,21 @@ function createMockSocket(): MockSocket {
 }
 
 type ConnectionFailureSpy = ReturnType<
-	typeof vi.fn<(reason: ConnectionStateReason, route: string) => void>
+	typeof vi.fn<(reason: ConnectionFailureReason, route: string) => void>
 >;
 
 interface FactoryOptions {
 	onDispose?: () => void;
 	onCertificateRefreshNeeded?: () => Promise<boolean>;
-	onConnectionFailure?: (reason: ConnectionStateReason, route: string) => void;
+	onConnectionFailure?: (
+		reason: ConnectionFailureReason,
+		route: string,
+	) => void;
 	route?: string;
 	telemetry?: TelemetryReporter;
+	initialBackoffMs?: number;
+	maxBackoffMs?: number;
+	jitterFactor?: number;
 }
 
 async function createReconnectingWebSocket(
@@ -953,7 +1096,7 @@ async function createReconnectingWebSocket(
 }> {
 	const sockets: MockSocket[] = [];
 	const onConnectionFailure =
-		vi.fn<(reason: ConnectionStateReason, route: string) => void>();
+		vi.fn<(reason: ConnectionFailureReason, route: string) => void>();
 	const factory = vi.fn(() => {
 		const socket = createMockSocket();
 		sockets.push(socket);
@@ -1008,6 +1151,9 @@ async function fromFactory<T>(
 			onCertificateRefreshNeeded:
 				options.onCertificateRefreshNeeded ?? (() => Promise.resolve(false)),
 			onConnectionFailure: options.onConnectionFailure ?? vi.fn(),
+			initialBackoffMs: options.initialBackoffMs,
+			maxBackoffMs: options.maxBackoffMs,
+			jitterFactor: options.jitterFactor,
 		},
 		options.onDispose,
 	);