From 68da19edfecc3e31f2a612bb58d150dceaa4c84c Mon Sep 17 00:00:00 2001 From: Camiel van Schoonhoven Date: Tue, 8 Sep 2026 11:13:49 -0700 Subject: [PATCH 01/36] feat(pipeline-storage): type-only host contract and fail-open detection Declares the interface a host page may expose on window.__TANGLE_PIPELINE_STORAGE_HOST__ and the detection that reads it. Nothing consumes either yet. Detection is deliberately fail-open: a missing global, a wrong shape, a throwing getter or a contract version we do not understand all resolve to undefined, which leaves local storage as the only storage. Co-Authored-By: Claude Opus 5 (1M context) --- src/services/pipelineStorage/host/contract.ts | 33 ++++++ .../pipelineStorage/host/detectHost.test.ts | 101 ++++++++++++++++++ .../pipelineStorage/host/detectHost.ts | 40 +++++++ 3 files changed, 174 insertions(+) create mode 100644 src/services/pipelineStorage/host/contract.ts create mode 100644 src/services/pipelineStorage/host/detectHost.test.ts create mode 100644 src/services/pipelineStorage/host/detectHost.ts diff --git a/src/services/pipelineStorage/host/contract.ts b/src/services/pipelineStorage/host/contract.ts new file mode 100644 index 0000000000..acdec1be77 --- /dev/null +++ b/src/services/pipelineStorage/host/contract.ts @@ -0,0 +1,33 @@ +export const PIPELINE_STORAGE_HOST_VERSION = 1; + +export interface HostPipelineSummary { + key: string; + externalId: string; + displayName: string | null; + contentVersion: string; + createdAt?: string; + modifiedAt?: string; +} + +export interface HostPipeline extends HostPipelineSummary { + spec: unknown; +} + +export type HostErrorCode = + "unauthenticated" | "not_found" | "rate_limited" | "conflict" | "unavailable"; + +export interface PipelineStorageHost { + readonly version: number; + readonly label: string; + list(): Promise; + read(key: string): Promise; + write(key: string, spec: unknown): Promise; + delete(key: string): Promise; + has(key: string): Promise; +} + +declare global { + interface Window { + __TANGLE_PIPELINE_STORAGE_HOST__?: PipelineStorageHost; + } +} diff --git a/src/services/pipelineStorage/host/detectHost.test.ts b/src/services/pipelineStorage/host/detectHost.test.ts new file mode 100644 index 0000000000..d9c32929b0 --- /dev/null +++ b/src/services/pipelineStorage/host/detectHost.test.ts @@ -0,0 +1,101 @@ +import { afterEach, describe, expect, it } from "vitest"; + +import { + PIPELINE_STORAGE_HOST_VERSION, + type PipelineStorageHost, +} from "./contract"; +import { getPipelineStorageHost } from "./detectHost"; + +function installHost(host: unknown) { + Object.defineProperty(window, "__TANGLE_PIPELINE_STORAGE_HOST__", { + value: host, + configurable: true, + writable: true, + }); +} + +function createFakeHost( + overrides: Partial> = {}, +) { + return { + version: PIPELINE_STORAGE_HOST_VERSION, + label: "Shared storage", + list: async () => [], + read: async () => ({}), + write: async () => ({}), + delete: async () => undefined, + has: async () => false, + ...overrides, + }; +} + +afterEach(() => { + delete window.__TANGLE_PIPELINE_STORAGE_HOST__; +}); + +describe("getPipelineStorageHost", () => { + it("returns the host when the global satisfies the contract", () => { + const host = createFakeHost(); + installHost(host); + + expect(getPipelineStorageHost()).toBe(host); + }); + + it("returns undefined when the global is missing", () => { + expect(getPipelineStorageHost()).toBeUndefined(); + }); + + it.each(["list", "read", "write", "delete", "has"] as const)( + "returns undefined when %s is not a function", + (method) => { + installHost(createFakeHost({ [method]: "not-callable" })); + + expect(getPipelineStorageHost()).toBeUndefined(); + }, + ); + + it("returns undefined when the version is above the supported version", () => { + installHost(createFakeHost({ version: PIPELINE_STORAGE_HOST_VERSION + 1 })); + + expect(getPipelineStorageHost()).toBeUndefined(); + }); + + it("accepts a host declaring an older version", () => { + const host = createFakeHost({ version: PIPELINE_STORAGE_HOST_VERSION - 1 }); + installHost(host); + + expect(getPipelineStorageHost()).toBe(host); + }); + + it("returns undefined when the version is not a number", () => { + installHost(createFakeHost({ version: "1" })); + + expect(getPipelineStorageHost()).toBeUndefined(); + }); + + it.each([undefined, "", " "])( + "returns undefined when the label is %p", + (label) => { + installHost(createFakeHost({ label })); + + expect(getPipelineStorageHost()).toBeUndefined(); + }, + ); + + it("returns undefined when the global is not an object", () => { + installHost("a host, honest"); + + expect(getPipelineStorageHost()).toBeUndefined(); + }); + + it("returns undefined when reading the global throws", () => { + Object.defineProperty(window, "__TANGLE_PIPELINE_STORAGE_HOST__", { + get() { + throw new Error("cross-origin"); + }, + configurable: true, + }); + + expect(getPipelineStorageHost()).toBeUndefined(); + }); +}); diff --git a/src/services/pipelineStorage/host/detectHost.ts b/src/services/pipelineStorage/host/detectHost.ts new file mode 100644 index 0000000000..a7479c0552 --- /dev/null +++ b/src/services/pipelineStorage/host/detectHost.ts @@ -0,0 +1,40 @@ +import { + PIPELINE_STORAGE_HOST_VERSION, + type PipelineStorageHost, +} from "./contract"; + +const HOST_METHODS = [ + "list", + "read", + "write", + "delete", + "has", +] as const satisfies readonly (keyof PipelineStorageHost)[]; + +/** + * The host page and this app deploy independently, so a contract version we do + * not understand has to read as "no host at all" rather than as a host we can + * half-drive. Every failure mode — missing global, wrong shape, throwing + * getter, newer version — resolves to `undefined` and leaves local storage as + * the only storage. + */ +export function getPipelineStorageHost(): PipelineStorageHost | undefined { + try { + if (typeof window === "undefined") return undefined; + + const host = window.__TANGLE_PIPELINE_STORAGE_HOST__; + if (!host || typeof host !== "object") return undefined; + if (typeof host.version !== "number") return undefined; + if (host.version > PIPELINE_STORAGE_HOST_VERSION) return undefined; + if (typeof host.label !== "string" || host.label.trim() === "") { + return undefined; + } + if (HOST_METHODS.some((method) => typeof host[method] !== "function")) { + return undefined; + } + + return host; + } catch { + return undefined; + } +} From edf6e0d1d4dc4e808b18b4ef082a8d03b8209283 Mon Sep 17 00:00:00 2001 From: Camiel van Schoonhoven Date: Tue, 8 Sep 2026 11:14:57 -0700 Subject: [PATCH 02/36] refactor(pipeline-storage): let a driver report the identity of what it wrote A driver's write() now returns a PipelineFileDescriptor instead of void, so a store that mints its own ids, names and content versions can say so rather than having the registry invent them. addFile adopts the reported externalId when there is one and mints a UUID otherwise, and listing prefers the same field so repeated listings resolve to stable ids. addFile also writes before it registers: the previous order left a registry row pointing at a file that was never created whenever the write was rejected. Co-Authored-By: Claude Opus 5 (1M context) --- .../SessionStoragePipelineDriver.ts | 6 +- .../googleDrive/GoogleDriveStorageDriver.ts | 22 +- src/services/pipelineStorage/PipelineFile.ts | 9 +- .../pipelineStorage/PipelineFolder.test.ts | 247 ++++++++++++++++++ .../pipelineStorage/PipelineFolder.ts | 55 ++-- .../drivers/LocalFileSystemDriver.ts | 6 +- .../drivers/RootFolderDbStorageDriver.ts | 6 +- src/services/pipelineStorage/types.ts | 6 +- 8 files changed, 320 insertions(+), 37 deletions(-) create mode 100644 src/services/pipelineStorage/PipelineFolder.test.ts diff --git a/src/providers/TourProvider/tourPipelineStorage/SessionStoragePipelineDriver.ts b/src/providers/TourProvider/tourPipelineStorage/SessionStoragePipelineDriver.ts index 4e17937f4c..9ac3887f9e 100644 --- a/src/providers/TourProvider/tourPipelineStorage/SessionStoragePipelineDriver.ts +++ b/src/providers/TourProvider/tourPipelineStorage/SessionStoragePipelineDriver.ts @@ -37,8 +37,12 @@ export class SessionStoragePipelineDriver implements PipelineStorageDriver { return content; } - async write(storageKey: string, content: string): Promise { + async write( + storageKey: string, + content: string, + ): Promise { sessionStorage.setItem(this.key(storageKey), content); + return { storageKey }; } async delete(storageKey: string): Promise { diff --git a/src/services/googleDrive/GoogleDriveStorageDriver.ts b/src/services/googleDrive/GoogleDriveStorageDriver.ts index e6d02ca829..72b54af929 100644 --- a/src/services/googleDrive/GoogleDriveStorageDriver.ts +++ b/src/services/googleDrive/GoogleDriveStorageDriver.ts @@ -74,6 +74,7 @@ export class GoogleDriveStorageDriver implements PipelineStorageDriver { .filter((f) => PIPELINE_YAML_PATTERN.test(f.name)) .map((f) => ({ storageKey: f.name, + externalId: f.id, modifiedAt: f.modifiedTime ? new Date(f.modifiedTime) : undefined, createdAt: f.createdTime ? new Date(f.createdTime) : undefined, })); @@ -89,15 +90,20 @@ export class GoogleDriveStorageDriver implements PipelineStorageDriver { return response.text(); } - async write(storageKey: string, content: string): Promise { + async write( + storageKey: string, + content: string, + ): Promise { const fileName = toFileName(storageKey); const existingId = await this.resolveFileId(storageKey); if (existingId) { await this.updateFileContent(existingId, content); - } else { - await this.createFile(fileName, content); + return { storageKey, externalId: existingId }; } + + const createdId = await this.createFile(fileName, content); + return { storageKey, ...(createdId ? { externalId: createdId } : {}) }; } async rename(oldStorageKey: string, newStorageKey: string): Promise { @@ -149,7 +155,10 @@ export class GoogleDriveStorageDriver implements PipelineStorageDriver { return data.files?.[0]?.id ?? null; } - private async createFile(fileName: string, content: string): Promise { + private async createFile( + fileName: string, + content: string, + ): Promise { const metadata = { name: fileName, parents: [this.folderId], @@ -185,6 +194,11 @@ export class GoogleDriveStorageDriver implements PipelineStorageDriver { if (!response.ok) { throw new Error(`Google Drive upload failed: ${response.status}`); } + + const created = (await response.json().catch(() => null)) as { + id?: string; + } | null; + return created?.id; } private async updateFileContent( diff --git a/src/services/pipelineStorage/PipelineFile.ts b/src/services/pipelineStorage/PipelineFile.ts index ca3c6adb38..36234a2303 100644 --- a/src/services/pipelineStorage/PipelineFile.ts +++ b/src/services/pipelineStorage/PipelineFile.ts @@ -37,7 +37,14 @@ export class PipelineFile { } async write(content: string): Promise { - await this.folder.driver.write(this.storageKey, content); + const descriptor = await this.folder.driver.write(this.storageKey, content); + + if (descriptor.contentVersion !== undefined) { + await updateEntry(this.id, { + contentVersion: descriptor.contentVersion, + }); + } + emitPipelineFileChanged({ storageKey: this.storageKey, source: "v2" }); emitUserPipelineWritten(); } diff --git a/src/services/pipelineStorage/PipelineFolder.test.ts b/src/services/pipelineStorage/PipelineFolder.test.ts new file mode 100644 index 0000000000..1214032f1b --- /dev/null +++ b/src/services/pipelineStorage/PipelineFolder.test.ts @@ -0,0 +1,247 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { + type PipelineFileChange, + subscribePipelineFileChanged, +} from "./pipelineFileEvents"; +import { PipelineFolder } from "./PipelineFolder"; +import { + type PipelineFileDescriptor, + type PipelineRegistryEntry, + type PipelineStorageDriver, + ROOT_FOLDER_ID, +} from "./types"; + +const registry = vi.hoisted(() => new Map()); + +vi.mock("./db", () => ({ pipelineStorageDb: { folders: {} } })); + +vi.mock("./createDriver", () => ({ + createDriver: () => { + throw new Error("createDriver should not be reached in these tests"); + }, +})); + +vi.mock("./pipelineRegistry", () => ({ + addEntry: vi.fn(async (entry: PipelineRegistryEntry) => { + registry.set(entry.id, entry); + }), + updateEntry: vi.fn( + async (id: string, changes: Partial) => { + const existing = registry.get(id); + if (existing) registry.set(id, { ...existing, ...changes }); + }, + ), + deleteEntry: vi.fn(async (id: string) => { + registry.delete(id); + }), + findById: vi.fn(async (id: string) => registry.get(id)), + findByStorageKey: vi.fn(async (storageKey: string) => + [...registry.values()].find((entry) => entry.storageKey === storageKey), + ), + findByRemoteStorageKey: vi.fn(async () => []), + getAllByFolderId: vi.fn(async (folderId: string) => + [...registry.values()].filter((entry) => entry.folderId === folderId), + ), + assertStorageKeyUnique: vi.fn(async (storageKey: string) => { + const clash = [...registry.values()].some( + (entry) => entry.storageKey === storageKey, + ); + if (clash) throw new Error(`Storage key already in use: ${storageKey}`); + }), + deleteFoldersAndDetachEntries: vi.fn(async () => undefined), +})); + +interface FakeDriverOptions { + write?: ( + storageKey: string, + content: string, + ) => Promise; +} + +interface FakeDriver extends PipelineStorageDriver { + descriptors: PipelineFileDescriptor[]; + contents: Map; +} + +function createFakeDriver(options: FakeDriverOptions = {}): FakeDriver { + const contents = new Map(); + const descriptors: PipelineFileDescriptor[] = []; + + return { + type: "fake", + allowsMoveIn: true, + allowsMoveOut: true, + descriptors, + contents, + async list() { + return descriptors; + }, + async read(storageKey: string) { + return contents.get(storageKey) ?? ""; + }, + write: + options.write ?? + (async (storageKey: string, content: string) => { + contents.set(storageKey, content); + return { storageKey }; + }), + async rename() {}, + async delete(storageKey: string) { + contents.delete(storageKey); + }, + async hasKey(storageKey: string) { + return contents.has(storageKey); + }, + }; +} + +function createFolder(driver: PipelineStorageDriver): PipelineFolder { + return new PipelineFolder({ + id: ROOT_FOLDER_ID, + name: "Pipelines", + parentId: null, + driver, + }); +} + +const unsubscribes: (() => void)[] = []; + +function collectChanges(): PipelineFileChange[] { + const changes: PipelineFileChange[] = []; + unsubscribes.push( + subscribePipelineFileChanged((change) => changes.push(change)), + ); + return changes; +} + +beforeEach(() => { + registry.clear(); +}); + +afterEach(() => { + unsubscribes.splice(0).forEach((unsubscribe) => unsubscribe()); +}); + +describe("PipelineFolder.addFile", () => { + it("adopts the identity the driver reports for the new file", async () => { + const folder = createFolder( + createFakeDriver({ + write: async (storageKey) => ({ + storageKey, + externalId: "external-1", + displayName: "Churn model", + contentVersion: "v1", + }), + }), + ); + + const file = await folder.addFile("opaque-key", "name: Churn model"); + + expect(file.id).toBe("external-1"); + expect(registry.get("external-1")).toEqual({ + id: "external-1", + storageKey: "opaque-key", + folderId: ROOT_FOLDER_ID, + contentVersion: "v1", + }); + }); + + it("mints an id when the driver reports no identity of its own", async () => { + const folder = createFolder(createFakeDriver()); + + const file = await folder.addFile("my-pipeline", "name: My pipeline"); + + expect(file.id).not.toBe("my-pipeline"); + expect(registry.get(file.id)).toMatchObject({ + storageKey: "my-pipeline", + contentVersion: undefined, + }); + }); + + it("registers the key the driver actually wrote, not the one requested", async () => { + const folder = createFolder( + createFakeDriver({ + write: async () => ({ storageKey: "assigned-by-store" }), + }), + ); + + const file = await folder.addFile("requested", "name: Requested"); + + expect(file.storageKey).toBe("assigned-by-store"); + expect(registry.get(file.id)?.storageKey).toBe("assigned-by-store"); + }); + + it("leaves no registry entry behind when the write is rejected", async () => { + const folder = createFolder( + createFakeDriver({ + write: async () => { + throw new Error("quota exceeded"); + }, + }), + ); + + await expect(folder.addFile("rejected", "name: Rejected")).rejects.toThrow( + "quota exceeded", + ); + + expect(registry.size).toBe(0); + }); +}); + +describe("PipelineFolder.listPipelines", () => { + it("keeps ids stable across repeated listings", async () => { + const driver = createFakeDriver(); + driver.descriptors.push({ storageKey: "key-1", externalId: "external-1" }); + const folder = createFolder(driver); + + const [first] = await folder.listPipelines(); + const [second] = await folder.listPipelines(); + + expect(first.id).toBe("external-1"); + expect(second.id).toBe("external-1"); + expect(registry.size).toBe(1); + }); + + it("keeps a minted id stable across repeated listings", async () => { + const driver = createFakeDriver(); + driver.descriptors.push({ storageKey: "key-1" }); + const folder = createFolder(driver); + + const [first] = await folder.listPipelines(); + const [second] = await folder.listPipelines(); + + expect(second.id).toBe(first.id); + }); +}); + +describe("PipelineFolder.findFile", () => { + it("returns nothing for a key the driver does not hold", async () => { + const folder = createFolder(createFakeDriver()); + + expect(await folder.findFile("missing")).toBeUndefined(); + expect(registry.size).toBe(0); + }); + + it("reuses the registry row of a key the driver holds", async () => { + const folder = createFolder(createFakeDriver()); + const added = await folder.addFile("key-1", "name: One"); + + const found = await folder.findFile("key-1"); + + expect(found?.id).toBe(added.id); + expect(registry.size).toBe(1); + }); +}); + +describe("emitted events", () => { + it("does not emit a remote change for a local write", async () => { + const folder = createFolder(createFakeDriver()); + const file = await folder.addFile("key-1", "name: One"); + + const changes = collectChanges(); + await file.write("name: Two"); + + expect(changes).toEqual([{ storageKey: "key-1", source: "v2" }]); + }); +}); diff --git a/src/services/pipelineStorage/PipelineFolder.ts b/src/services/pipelineStorage/PipelineFolder.ts index a34ce31c65..516e552399 100644 --- a/src/services/pipelineStorage/PipelineFolder.ts +++ b/src/services/pipelineStorage/PipelineFolder.ts @@ -12,6 +12,7 @@ import { import { type DriverConfig, type FolderEntry, + type PipelineFileDescriptor, type PipelineStorageDriver, ROOT_FOLDER_ID, } from "./types"; @@ -91,11 +92,8 @@ export class PipelineFolder { const descriptors = await this.driver.list(); return Promise.all( - descriptors.map((d) => - resolveOrCreateRegistryEntry(d.storageKey, this, { - createdAt: d.createdAt, - modifiedAt: d.modifiedAt, - }), + descriptors.map((descriptor) => + resolveOrCreateRegistryEntry(descriptor, this), ), ); } @@ -104,21 +102,28 @@ export class PipelineFolder { const hasKey = await this.driver.hasKey(storageKey); if (!hasKey) return undefined; - return resolveOrCreateRegistryEntry(storageKey, this); + return resolveOrCreateRegistryEntry({ storageKey }, this); } async assignFile(storageKey: string): Promise { - return resolveOrCreateRegistryEntry(storageKey, this); + return resolveOrCreateRegistryEntry({ storageKey }, this); } async addFile(storageKey: string, content: string): Promise { await assertStorageKeyUnique(storageKey); - const id = crypto.randomUUID(); - await addEntry({ id, storageKey, folderId: this.id }); - await this.driver.write(storageKey, content); + // Writing before registering means a rejected write leaves no registry row + // pointing at a file that was never created. + const descriptor = await this.driver.write(storageKey, content); + const id = descriptor.externalId ?? crypto.randomUUID(); + await addEntry({ + id, + storageKey: descriptor.storageKey, + folderId: this.id, + contentVersion: descriptor.contentVersion, + }); - return new PipelineFile({ id, storageKey, folder: this }); + return new PipelineFile({ id, folder: this, ...descriptor }); } async listSubfolders(): Promise { @@ -233,28 +238,22 @@ async function collectDescendantIds(parentId: string): Promise { return ids; } -interface FileMetadata { - createdAt?: Date; - modifiedAt?: Date; -} - async function resolveOrCreateRegistryEntry( - storageKey: string, + descriptor: PipelineFileDescriptor, folder: PipelineFolder, - metadata?: FileMetadata, ): Promise { - const existing = await findByStorageKey(storageKey); + const existing = await findByStorageKey(descriptor.storageKey); if (existing) { - return new PipelineFile({ - id: existing.id, - storageKey: existing.storageKey, - folder, - ...metadata, - }); + return new PipelineFile({ id: existing.id, folder, ...descriptor }); } - const id = crypto.randomUUID(); - await addEntry({ id, storageKey, folderId: folder.id }); - return new PipelineFile({ id, storageKey, folder, ...metadata }); + const id = descriptor.externalId ?? crypto.randomUUID(); + await addEntry({ + id, + storageKey: descriptor.storageKey, + folderId: folder.id, + contentVersion: descriptor.contentVersion, + }); + return new PipelineFile({ id, folder, ...descriptor }); } diff --git a/src/services/pipelineStorage/drivers/LocalFileSystemDriver.ts b/src/services/pipelineStorage/drivers/LocalFileSystemDriver.ts index a3e6c306c5..0cfa115c41 100644 --- a/src/services/pipelineStorage/drivers/LocalFileSystemDriver.ts +++ b/src/services/pipelineStorage/drivers/LocalFileSystemDriver.ts @@ -71,7 +71,10 @@ export class LocalFileSystemDriver implements PipelineStorageDriver { return file.text(); } - async write(storageKey: string, content: string): Promise { + async write( + storageKey: string, + content: string, + ): Promise { const fileName = toFileName(storageKey); const fileHandle = await this.dirHandle.getFileHandle(fileName, { create: true, @@ -79,6 +82,7 @@ export class LocalFileSystemDriver implements PipelineStorageDriver { const writable = await fileHandle.createWritable(); await writable.write(content); await writable.close(); + return { storageKey }; } async rename(oldStorageKey: string, newStorageKey: string): Promise { diff --git a/src/services/pipelineStorage/drivers/RootFolderDbStorageDriver.ts b/src/services/pipelineStorage/drivers/RootFolderDbStorageDriver.ts index 5ca0be8186..3f958392bd 100644 --- a/src/services/pipelineStorage/drivers/RootFolderDbStorageDriver.ts +++ b/src/services/pipelineStorage/drivers/RootFolderDbStorageDriver.ts @@ -43,8 +43,12 @@ export class RootFolderDbStorageDriver implements PipelineStorageDriver { return entry.componentRef.text; } - async write(storageKey: string, content: string): Promise { + async write( + storageKey: string, + content: string, + ): Promise { await writeComponentToFileListFromText(LIST_NAME, storageKey, content); + return { storageKey }; } async rename(oldStorageKey: string, newStorageKey: string): Promise { diff --git a/src/services/pipelineStorage/types.ts b/src/services/pipelineStorage/types.ts index 57d52a7441..d5773677b6 100644 --- a/src/services/pipelineStorage/types.ts +++ b/src/services/pipelineStorage/types.ts @@ -7,6 +7,9 @@ export const ROOT_FOLDER_ID = "__root__"; export interface PipelineFileDescriptor { storageKey: string; + externalId?: string; + displayName?: string; + contentVersion?: string; createdAt?: Date; modifiedAt?: Date; } @@ -25,7 +28,7 @@ export interface PipelineStorageDriver { readonly allowsMoveOut: boolean; list(): Promise; read(storageKey: string): Promise; - write(storageKey: string, content: string): Promise; + write(storageKey: string, content: string): Promise; rename(oldStorageKey: string, newStorageKey: string): Promise; delete(storageKey: string): Promise; hasKey(storageKey: string): Promise; @@ -41,6 +44,7 @@ export interface PipelineRegistryEntry { id: string; storageKey: string; folderId: string; + contentVersion?: string; } export interface FolderEntry { From cea3ce88ae1cb540abcfa778979ec94ad2fae89f Mon Sep 17 00:00:00 2001 From: Camiel van Schoonhoven Date: Tue, 8 Sep 2026 11:15:51 -0700 Subject: [PATCH 03/36] refactor(pipeline-storage): separate a pipeline's displayed name from its key MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PipelineFile gains a displayName that falls back to the storage key, so every store whose key is the name reads exactly as before while a store with opaque keys can supply something a person recognises. The three read sites — the folder table's rows, its filter, and the delete confirmation — switch to it; the folder table still navigates by key. useLoadSpec stops assuming the route param equals the storage key. It matched change events against ref.name, which silently stopped reloading the editor once the two could differ, so it now compares against the key of the file the query actually loaded. Co-Authored-By: Claude Opus 5 (1M context) --- .../EditorMenuBar/components/FileMenu.tsx | 2 +- .../v2/pages/Editor/hooks/useLoadSpec.ts | 43 ++++++++++--------- .../FolderPipelineTable.tsx | 2 +- .../components/PipelineRows.tsx | 6 +-- src/services/pipelineStorage/PipelineFile.ts | 8 ++++ .../pipelineStorage/PipelineFolder.test.ts | 23 ++++++++++ 6 files changed, 59 insertions(+), 25 deletions(-) diff --git a/src/routes/v2/pages/Editor/components/EditorMenuBar/components/FileMenu.tsx b/src/routes/v2/pages/Editor/components/EditorMenuBar/components/FileMenu.tsx index be5ca5e362..4bb2edb593 100644 --- a/src/routes/v2/pages/Editor/components/EditorMenuBar/components/FileMenu.tsx +++ b/src/routes/v2/pages/Editor/components/EditorMenuBar/components/FileMenu.tsx @@ -211,7 +211,7 @@ export function FileMenu() { { void handleDeletePipeline(); setDeleteDialogOpen(false); diff --git a/src/routes/v2/pages/Editor/hooks/useLoadSpec.ts b/src/routes/v2/pages/Editor/hooks/useLoadSpec.ts index 8fcd4f4c8e..e8a44858e9 100644 --- a/src/routes/v2/pages/Editor/hooks/useLoadSpec.ts +++ b/src/routes/v2/pages/Editor/hooks/useLoadSpec.ts @@ -4,7 +4,7 @@ import { registerRootStore, type UndoStore as MobxUndoStore, } from "mobx-keystone"; -import { useEffect } from "react"; +import { useEffect, useRef } from "react"; import { type ComponentSpec, @@ -54,10 +54,10 @@ async function backfillFromLegacyStore( return storage.rootFolder.assignFile(name); } -async function resolveSpecData( +async function resolvePipelineFile( ref: PipelineRef, storage: PipelineStorageService, -): Promise { +): Promise { let pipelineFile = ref.fileId ? await storage.findPipelineById(ref.fileId) : await storage.resolvePipelineByName(ref.name); @@ -69,8 +69,8 @@ async function resolveSpecData( if (!pipelineFile) { throw new Error(`Pipeline "${ref.name}" not found`); } - const yamlContent = await pipelineFile.read(); - return yaml.load(yamlContent, PIPELINE_YAML_LOAD_OPTIONS); + + return pipelineFile; } export const EDITOR_SPEC_QUERY_KEY = "editor-v2-spec"; @@ -79,37 +79,35 @@ export function useLoadSpec(ref: PipelineRef) { const storage = usePipelineStorage(); const queryClient = useQueryClient(); const queryKey = [EDITOR_SPEC_QUERY_KEY, ref.fileId ?? ref.name]; + const loadedStorageKey = useRef(undefined); - // When the v1 editor writes to the same IndexedDB file, drop our cached - // deserialization so the next read of this query goes back to disk. We + // When something outside this editor writes the same file, drop our cached + // deserialization so the next read of this query goes back to storage. We // ignore "v2" emissions because those are our own autosaves — reloading the // spec under our feet would discard MobX editor state (selection, undo, …) // and the in-memory model is already authoritative for v2. useEffect(() => { const matchKey = [EDITOR_SPEC_QUERY_KEY, ref.fileId ?? ref.name]; + // Writes are announced by storage key. Matching on ref.name only holds + // while the route param and the storage key are the same string, which a + // store with opaque keys breaks — and the symptom is silent: the editor + // just stops reloading. Compare against the key of the file this query + // actually loaded, falling back to ref.name until it resolves. + const matchedKey = () => loadedStorageKey.current ?? ref.name; + const state = queryClient.getQueryState(matchKey); const lastFetched = state?.dataUpdatedAt; if (lastFetched !== undefined) { - const candidates = [ref.name, ref.fileId].filter( - (k): k is string => typeof k === "string", - ); - const lastForeign = candidates - .map((key) => getLastForeignWriteTime(key, "v2") ?? 0) - .reduce((a, b) => Math.max(a, b), 0); + const lastForeign = getLastForeignWriteTime(matchedKey(), "v2") ?? 0; if (lastForeign > lastFetched) { queryClient.invalidateQueries({ queryKey: matchKey }); } } - // Contract: v1 emits with storageKey === the pipeline name, which must - // match this query's ref.name (or ref.fileId). If v1 ever writes under a - // different key (e.g. after a rename), invalidation silently stops and v2 - // looks stale until a refresh. This coupling is acceptable only because the - // v1 editor is slated for removal once v2 becomes the default. return subscribePipelineFileChanged(({ storageKey, source }) => { if (source === "v2") return; - if (storageKey !== ref.name && storageKey !== ref.fileId) return; + if (storageKey !== matchedKey()) return; queryClient.invalidateQueries({ queryKey: matchKey }); }); }, [queryClient, ref.fileId, ref.name]); @@ -117,8 +115,13 @@ export function useLoadSpec(ref: PipelineRef) { return useSuspenseQuery({ queryKey, queryFn: async (): Promise => { + const filePromise = resolvePipelineFile(ref, storage); + const [specData, undoHistory] = await Promise.all([ - resolveSpecData(ref, storage), + filePromise.then(async (file) => { + loadedStorageKey.current = file.storageKey; + return yaml.load(await file.read(), PIPELINE_YAML_LOAD_OPTIONS); + }), loadUndoHistory(ref.name).catch(() => null), ]); diff --git a/src/routes/v2/pages/PipelineFolders/components/FolderPipelineTable/FolderPipelineTable.tsx b/src/routes/v2/pages/PipelineFolders/components/FolderPipelineTable/FolderPipelineTable.tsx index 9686af1afe..54f7bd54a3 100644 --- a/src/routes/v2/pages/PipelineFolders/components/FolderPipelineTable/FolderPipelineTable.tsx +++ b/src/routes/v2/pages/PipelineFolders/components/FolderPipelineTable/FolderPipelineTable.tsx @@ -101,7 +101,7 @@ export const FolderPipelineTable = withSuspenseWrapper( const filteredPipelines = pipelines .filter((p) => - p.storageKey.toLowerCase().includes(searchQuery.toLowerCase()), + p.displayName.toLowerCase().includes(searchQuery.toLowerCase()), ) .sort( (a, b) => diff --git a/src/routes/v2/pages/PipelineFolders/components/FolderPipelineTable/components/PipelineRows.tsx b/src/routes/v2/pages/PipelineFolders/components/FolderPipelineTable/components/PipelineRows.tsx index 3bf41c679a..aab24f66f0 100644 --- a/src/routes/v2/pages/PipelineFolders/components/FolderPipelineTable/components/PipelineRows.tsx +++ b/src/routes/v2/pages/PipelineFolders/components/FolderPipelineTable/components/PipelineRows.tsx @@ -42,7 +42,7 @@ export function PipelineRows({ return ( <> {pipelines.map((file) => { - const name = file.storageKey; + const name = file.displayName; const pipelineItem: DragItem = { type: "pipeline", id: file.id }; const items = getDragItems(pipelineItem); @@ -55,9 +55,9 @@ export function PipelineRows({ onDelete={onDelete} isSelected={selectedPipelines.has(file.id)} onSelect={(checked) => onSelectPipeline(file.id, checked)} - onPipelineClick={(clickedName: string) => + onPipelineClick={() => folderNav?.onPipelineClick!({ - name: clickedName, + name: file.storageKey, fileId: file.id, }) } diff --git a/src/services/pipelineStorage/PipelineFile.ts b/src/services/pipelineStorage/PipelineFile.ts index 36234a2303..2cdc45ca37 100644 --- a/src/services/pipelineStorage/PipelineFile.ts +++ b/src/services/pipelineStorage/PipelineFile.ts @@ -10,6 +10,7 @@ interface PipelineFileInit { id: string; storageKey: string; folder: PipelineFolder; + displayName?: string; createdAt?: Date; modifiedAt?: Date; } @@ -22,10 +23,17 @@ export class PipelineFile { @observable accessor storageKey: string; @observable accessor folder: PipelineFolder; + private readonly assignedDisplayName?: string; + + get displayName(): string { + return this.assignedDisplayName ?? this.storageKey; + } + constructor(options: PipelineFileInit) { this.id = options.id; this.storageKey = options.storageKey; this.folder = options.folder; + this.assignedDisplayName = options.displayName; this.createdAt = options.createdAt; this.modifiedAt = options.modifiedAt; diff --git a/src/services/pipelineStorage/PipelineFolder.test.ts b/src/services/pipelineStorage/PipelineFolder.test.ts index 1214032f1b..044408a2cb 100644 --- a/src/services/pipelineStorage/PipelineFolder.test.ts +++ b/src/services/pipelineStorage/PipelineFolder.test.ts @@ -139,6 +139,7 @@ describe("PipelineFolder.addFile", () => { const file = await folder.addFile("opaque-key", "name: Churn model"); expect(file.id).toBe("external-1"); + expect(file.displayName).toBe("Churn model"); expect(registry.get("external-1")).toEqual({ id: "external-1", storageKey: "opaque-key", @@ -213,6 +214,28 @@ describe("PipelineFolder.listPipelines", () => { expect(second.id).toBe(first.id); }); + + it("carries the display name the driver reports", async () => { + const driver = createFakeDriver(); + driver.descriptors.push({ + storageKey: "opaque-key", + displayName: "Churn model", + }); + + const [file] = await createFolder(driver).listPipelines(); + + expect(file.displayName).toBe("Churn model"); + expect(file.storageKey).toBe("opaque-key"); + }); + + it("falls back to the storage key when the driver reports no name", async () => { + const driver = createFakeDriver(); + driver.descriptors.push({ storageKey: "my-pipeline" }); + + const [file] = await createFolder(driver).listPipelines(); + + expect(file.displayName).toBe("my-pipeline"); + }); }); describe("PipelineFolder.findFile", () => { From 1222ea45674e85b64c936cc6678ba02551166a4e Mon Sep 17 00:00:00 2001 From: Camiel van Schoonhoven Date: Tue, 8 Sep 2026 11:16:17 -0700 Subject: [PATCH 04/36] feat(pipeline-storage): project a pipeline to a portable document before saving A saved pipeline currently carries whatever the in-memory model has accumulated, including per-viewer component-library state (favorited, owned, published_by, superseded_by) and stale input values. Those travel with the file and become someone else's view of their own library. toPortablePipelineSpec reduces a spec to an allowlist of pipeline- describing fields, recursively through subgraphs, and is wired into the two pipeline save paths. dynamicData arguments pass through verbatim, since their shape is not modelled here and filtering inside them would only fail at run time. Allowed keys are copied in source order, so a document that loses nothing serialises byte-for-byte as it did before. Co-Authored-By: Claude Opus 5 (1M context) --- src/models/componentSpec/index.ts | 2 + .../componentSpec/serialization/index.ts | 2 + .../portablePipelineSpec.test.ts | 228 ++++++++++++++++++ .../serialization/portablePipelineSpec.ts | 182 ++++++++++++++ .../componentSpec/serialization/serialize.ts | 8 + .../components/fileMenu.actions.ts | 5 +- .../v2/pages/Editor/store/autoSaveStore.ts | 4 +- 7 files changed, 427 insertions(+), 4 deletions(-) create mode 100644 src/models/componentSpec/serialization/portablePipelineSpec.test.ts create mode 100644 src/models/componentSpec/serialization/portablePipelineSpec.ts diff --git a/src/models/componentSpec/index.ts b/src/models/componentSpec/index.ts index 6a1ba1e510..c4a4a167fc 100644 --- a/src/models/componentSpec/index.ts +++ b/src/models/componentSpec/index.ts @@ -15,6 +15,8 @@ export { serializeComponentSpec, serializeComponentSpecToText, serializeComponentSpecToYaml, + serializePipelineDocumentToText, + toPortablePipelineSpec, YamlDeserializer, } from "./serialization"; diff --git a/src/models/componentSpec/serialization/index.ts b/src/models/componentSpec/serialization/index.ts index b6cbdaa977..3d2cf1684e 100644 --- a/src/models/componentSpec/serialization/index.ts +++ b/src/models/componentSpec/serialization/index.ts @@ -1,7 +1,9 @@ export { collectIdStack } from "./collectIdStack"; +export { toPortablePipelineSpec } from "./portablePipelineSpec"; export { serializeComponentSpec, serializeComponentSpecToText, serializeComponentSpecToYaml, + serializePipelineDocumentToText, } from "./serialize"; export { YamlDeserializer } from "./yamlDeserializer"; diff --git a/src/models/componentSpec/serialization/portablePipelineSpec.test.ts b/src/models/componentSpec/serialization/portablePipelineSpec.test.ts new file mode 100644 index 0000000000..c25f7841fb --- /dev/null +++ b/src/models/componentSpec/serialization/portablePipelineSpec.test.ts @@ -0,0 +1,228 @@ +import { describe, expect, it } from "vitest"; + +import type { + ArgumentType, + ComponentSpec, + TaskSpec, +} from "@/utils/componentSpec"; + +import { toPortablePipelineSpec } from "./portablePipelineSpec"; + +const LIBRARY_FIELDS = { + favorited: true, + owned: true, + deprecated: true, + published_by: "someone", + superseded_by: "some-digest", +}; + +function graphPipeline(tasks: Record): ComponentSpec { + return { + name: "Pipeline", + implementation: { graph: { tasks } }, + }; +} + +function graphTasks(spec: ComponentSpec): Record { + if (!("graph" in spec.implementation)) { + throw new Error("expected a graph implementation"); + } + return spec.implementation.graph.tasks; +} + +function nestedSpec(task: TaskSpec): ComponentSpec { + const spec = task.componentRef.spec; + if (!spec) throw new Error("expected a nested spec"); + return spec; +} + +// A document reaching the projection may carry keys the wire types do not +// describe — that is the whole reason the projection exists — so malformed +// fixtures are built loosely and narrowed here rather than at each use. +function malformed(value: unknown): T { + return value as T; +} + +describe("toPortablePipelineSpec", () => { + it("strips component-library fields at the root", () => { + const spec = malformed({ + ...graphPipeline({}), + ...LIBRARY_FIELDS, + }); + + const portable = toPortablePipelineSpec(spec); + + expect(Object.keys(portable)).toEqual(["name", "implementation"]); + }); + + it("strips component-library fields from a nested task componentRef", () => { + const spec = graphPipeline({ + train: { + componentRef: { + name: "Train", + digest: "sha256:abc", + ...LIBRARY_FIELDS, + }, + }, + }); + + const portable = toPortablePipelineSpec(spec); + + expect(Object.keys(graphTasks(portable).train.componentRef)).toEqual([ + "name", + "digest", + ]); + }); + + it("strips a stale value from an input while keeping its default", () => { + const spec: ComponentSpec = { + ...graphPipeline({}), + inputs: [{ name: "epochs", type: "Integer", default: "10", value: "42" }], + }; + + const portable = toPortablePipelineSpec(spec); + + expect(portable.inputs?.[0]).toEqual({ + name: "epochs", + type: "Integer", + default: "10", + }); + }); + + it("preserves annotations on the pipeline, its tasks and its ports", () => { + const position = JSON.stringify({ x: 120, y: 340 }); + const spec: ComponentSpec = { + ...graphPipeline({ + train: { + componentRef: { name: "Train" }, + annotations: { "editor.position": position }, + }, + }), + metadata: { annotations: { "pipeline.owner": "team" } }, + inputs: [ + { name: "epochs", annotations: { "editor.position": position } }, + ], + outputs: [ + { name: "model", annotations: { "editor.position": position } }, + ], + }; + + const portable = toPortablePipelineSpec(spec); + + expect(portable.metadata).toEqual({ + annotations: { "pipeline.owner": "team" }, + }); + expect(graphTasks(portable).train.annotations).toEqual({ + "editor.position": position, + }); + expect(portable.inputs?.[0].annotations).toEqual({ + "editor.position": position, + }); + expect(portable.outputs?.[0].annotations).toEqual({ + "editor.position": position, + }); + }); + + it("preserves a container implementation two subgraph levels down", () => { + const container = { + image: "python:3.11", + command: ["python", "-m", "train"], + args: ["--fast"], + env: { LOG_LEVEL: "debug" }, + }; + const middle = graphPipeline({ + train: { + componentRef: { + name: "Train", + spec: { name: "Train", implementation: { container } }, + ...LIBRARY_FIELDS, + }, + }, + }); + const spec = graphPipeline({ + outer: { + componentRef: { name: "Subgraph", spec: middle, ...LIBRARY_FIELDS }, + }, + }); + + const portable = toPortablePipelineSpec(spec); + + const innerTasks = graphTasks(nestedSpec(graphTasks(portable).outer)); + const leaf = nestedSpec(innerTasks.train); + + expect(leaf.implementation).toEqual({ container }); + expect(innerTasks.train.componentRef).not.toHaveProperty("favorited"); + }); + + it("preserves placeholder objects inside a container command", () => { + const command = [ + "python", + { inputValue: "epochs" }, + { outputPath: "model" }, + { if: { cond: { isPresent: "epochs" }, then: ["--epochs"] } }, + ]; + const spec: ComponentSpec = { + name: "Train", + implementation: { container: { image: "python:3.11", command } }, + }; + + const portable = toPortablePipelineSpec(spec); + + expect(portable.implementation).toEqual({ + container: { image: "python:3.11", command }, + }); + }); + + it("passes a nested dynamicData payload through unchanged", () => { + const dynamicData = { + source: "vault", + lookup: { key: "api-token", scope: { env: "staging", tags: ["a", "b"] } }, + }; + const spec = graphPipeline({ + train: { + componentRef: { name: "Train" }, + arguments: { token: malformed({ dynamicData }) }, + }, + }); + + const portable = toPortablePipelineSpec(spec); + + const token = graphTasks(portable).train.arguments?.token; + expect(token).toEqual({ dynamicData }); + expect(JSON.stringify(token)).toBe(JSON.stringify({ dynamicData })); + }); + + it("drops an argument that carries dynamicData alongside a sibling key", () => { + const spec = graphPipeline({ + train: { + componentRef: { name: "Train" }, + arguments: { + safe: "plain", + mixed: malformed({ + dynamicData: { source: "vault" }, + taskOutput: { taskId: "prep", outputName: "data" }, + }), + }, + }, + }); + + const portable = toPortablePipelineSpec(spec); + + expect(graphTasks(portable).train.arguments).toEqual({ safe: "plain" }); + }); + + it("keeps the source key order so an unchanged document round-trips", () => { + const spec = { + name: "Pipeline", + description: "A pipeline", + metadata: { annotations: { a: "1" } }, + inputs: [{ name: "epochs", type: "Integer" }], + outputs: [{ name: "model", type: "Model" }], + implementation: { graph: { tasks: {} } }, + } satisfies ComponentSpec; + + const portable = toPortablePipelineSpec(spec); + + expect(JSON.stringify(portable)).toBe(JSON.stringify(spec)); + }); +}); diff --git a/src/models/componentSpec/serialization/portablePipelineSpec.ts b/src/models/componentSpec/serialization/portablePipelineSpec.ts new file mode 100644 index 0000000000..20957e91d6 --- /dev/null +++ b/src/models/componentSpec/serialization/portablePipelineSpec.ts @@ -0,0 +1,182 @@ +import type { + ArgumentType, + ComponentReference, + ComponentSpec, + ContainerImplementation, + GraphSpec, + ImplementationType, + InputSpec, + MetadataSpec, + OutputSpec, + TaskSpec, +} from "@/utils/componentSpec"; +import { + isDynamicDataArgument, + isGraphImplementation, +} from "@/utils/componentSpec"; + +const COMPONENT_SPEC_KEYS = new Set([ + "name", + "description", + "metadata", + "inputs", + "outputs", + "implementation", +]); +const COMPONENT_REFERENCE_KEYS = new Set([ + "name", + "digest", + "tag", + "url", + "spec", + "text", +]); +const METADATA_KEYS = new Set(["annotations", "labels"]); +const INPUT_KEYS = new Set([ + "name", + "type", + "description", + "default", + "optional", + "annotations", +]); +const OUTPUT_KEYS = new Set(["name", "type", "description", "annotations"]); +const TASK_KEYS = new Set([ + "componentRef", + "arguments", + "isEnabled", + "executionOptions", + "annotations", +]); +const GRAPH_KEYS = new Set(["tasks", "outputValues"]); +const CONTAINER_KEYS = new Set(["image", "command", "args", "env"]); + +/** + * Reduces a pipeline to the fields that describe the pipeline itself, dropping + * the per-viewer component-library state (`favorited`, `owned`, `published_by`, + * `superseded_by`) that the in-memory model accumulates, so a saved document + * does not carry one person's library into someone else's copy. + */ +export function toPortablePipelineSpec(spec: ComponentSpec): ComponentSpec { + const picked = pickKeys(spec, COMPONENT_SPEC_KEYS); + const result: ComponentSpec = { + ...picked, + implementation: pickImplementation(spec.implementation), + }; + + if (result.metadata) result.metadata = pickMetadata(result.metadata); + if (result.inputs) result.inputs = result.inputs.map(pickInput); + if (result.outputs) result.outputs = result.outputs.map(pickOutput); + + return result; +} + +function pickImplementation( + implementation: ImplementationType, +): ImplementationType { + if (isGraphImplementation(implementation)) { + return { graph: pickGraph(implementation.graph) }; + } + return pickContainer(implementation); +} + +function pickContainer( + implementation: ContainerImplementation, +): ContainerImplementation { + return { + container: { + ...pickKeys(implementation.container, CONTAINER_KEYS), + image: implementation.container.image, + }, + }; +} + +function pickGraph(graph: GraphSpec): GraphSpec { + return { + ...pickKeys(graph, GRAPH_KEYS), + tasks: mapValues(graph.tasks, pickTask), + }; +} + +function pickTask(task: TaskSpec): TaskSpec { + const result: TaskSpec = { + ...pickKeys(task, TASK_KEYS), + componentRef: pickComponentReference(task.componentRef), + }; + + if (result.arguments) result.arguments = pickArguments(result.arguments); + + return result; +} + +function pickComponentReference( + componentRef: ComponentReference, +): ComponentReference { + const result = pickKeys(componentRef, COMPONENT_REFERENCE_KEYS); + if (result.spec) result.spec = toPortablePipelineSpec(result.spec); + return result; +} + +function pickArguments( + args: Record, +): Record { + const result: Record = {}; + + for (const [name, argument] of Object.entries(args)) { + if (isDynamicDataArgument(argument)) { + // A dynamicData payload addresses secrets and execution-time context + // whose shape this app does not model, so it is passed through verbatim: + // filtering inside it would strip what a run needs and only fail at run + // time. That is safe only while nothing else rides on the same object, so + // an argument carrying a sibling key is dropped rather than half-copied. + if (Object.keys(argument).length === 1) result[name] = argument; + continue; + } + result[name] = argument; + } + + return result; +} + +function pickMetadata(metadata: MetadataSpec): MetadataSpec { + return pickKeys(metadata, METADATA_KEYS); +} + +function pickInput(input: InputSpec): InputSpec { + return { ...pickKeys(input, INPUT_KEYS), name: input.name }; +} + +function pickOutput(output: OutputSpec): OutputSpec { + return { ...pickKeys(output, OUTPUT_KEYS), name: output.name }; +} + +function mapValues( + source: Record, + transform: (value: T) => T, +): Record { + const result: Record = {}; + for (const [key, value] of Object.entries(source)) { + result[key] = transform(value); + } + return result; +} + +/** + * Copies the allowed keys in the order the source declares them, so a document + * that loses nothing also serialises byte-for-byte as it did before. + */ +function pickKeys( + source: T, + allowed: ReadonlySet, +): Partial { + const result: Partial = {}; + + for (const key of Object.keys(source)) { + if (!allowed.has(key)) continue; + const typedKey = key as keyof T; + if (source[typedKey] === undefined) continue; + result[typedKey] = source[typedKey]; + } + + return result; +} diff --git a/src/models/componentSpec/serialization/serialize.ts b/src/models/componentSpec/serialization/serialize.ts index d58748b2a2..74b15ac8f3 100644 --- a/src/models/componentSpec/serialization/serialize.ts +++ b/src/models/componentSpec/serialization/serialize.ts @@ -3,6 +3,7 @@ import { componentSpecToText, componentSpecToYaml } from "@/utils/yaml"; import type { ComponentSpec } from "../entities/componentSpec"; import type { ComponentSpecJson } from "../entities/types"; import { JsonSerializer } from "./jsonSerializer"; +import { toPortablePipelineSpec } from "./portablePipelineSpec"; const serializer = new JsonSerializer(); @@ -20,3 +21,10 @@ export function serializeComponentSpecToYaml(spec: ComponentSpec): string { export function serializeComponentSpecToText(spec: ComponentSpec): string { return componentSpecToText(serializeComponentSpec(spec)); } + +/** Serialize a ComponentSpec model to the document persisted for a pipeline. */ +export function serializePipelineDocumentToText(spec: ComponentSpec): string { + return componentSpecToText( + toPortablePipelineSpec(serializeComponentSpec(spec)), + ); +} diff --git a/src/routes/v2/pages/Editor/components/EditorMenuBar/components/fileMenu.actions.ts b/src/routes/v2/pages/Editor/components/EditorMenuBar/components/fileMenu.actions.ts index 7ff7b46cb1..19c48c67ae 100644 --- a/src/routes/v2/pages/Editor/components/EditorMenuBar/components/fileMenu.actions.ts +++ b/src/routes/v2/pages/Editor/components/EditorMenuBar/components/fileMenu.actions.ts @@ -4,6 +4,7 @@ import { exportPipeline } from "@/components/shared/ReactFlow/FlowSidebar/sectio import { serializeComponentSpec, serializeComponentSpecToYaml, + toPortablePipelineSpec, } from "@/models/componentSpec"; import type { NavigationStore } from "@/routes/v2/shared/store/navigationStore"; import type { PipelineFile } from "@/services/pipelineStorage/PipelineFile"; @@ -28,10 +29,10 @@ export async function savePipelineAs( const componentSpec = navigation.rootSpec; if (!componentSpec) return undefined; - const serialized = { + const serialized = toPortablePipelineSpec({ ...serializeComponentSpec(componentSpec), name: newName, - }; + }); const componentText = componentSpecToYaml(serialized); return storage.rootFolder.addFile(newName, componentText); diff --git a/src/routes/v2/pages/Editor/store/autoSaveStore.ts b/src/routes/v2/pages/Editor/store/autoSaveStore.ts index fdaae8cd45..7cba5191ad 100644 --- a/src/routes/v2/pages/Editor/store/autoSaveStore.ts +++ b/src/routes/v2/pages/Editor/store/autoSaveStore.ts @@ -3,7 +3,7 @@ import { action, makeObservable, observable, reaction } from "mobx"; import type { ComponentSpec } from "@/models/componentSpec"; import { collectIdStack, - serializeComponentSpecToText, + serializePipelineDocumentToText, } from "@/models/componentSpec"; import { saveUndoHistory } from "@/routes/v2/pages/Editor/utils/undoHistoryStorage"; import { AUTOSAVE_DEBOUNCE_TIME_MS } from "@/utils/constants"; @@ -86,7 +86,7 @@ export class AutoSaveStore { private serializeSpec(): string | null { if (!this.spec) return null; try { - return serializeComponentSpecToText(this.spec); + return serializePipelineDocumentToText(this.spec); } catch { return null; } From 6001d2bc31ada1f43e2ce0097ecfa3c3d5d86e61 Mon Sep 17 00:00:00 2001 From: Camiel van Schoonhoven Date: Tue, 8 Sep 2026 11:16:47 -0700 Subject: [PATCH 05/36] feat(pipeline-storage): drive a host-provided store as a folder HostStorageDriver adapts the detected host to PipelineStorageDriver. It owns the YAML boundary in both directions: the host exchanges pipeline documents as structured data, so the driver parses and projects on write and re-serialises on read, and refuses a document it cannot recognise. Errors are duck-typed off the rejection value, because instanceof does not survive a window boundary, and every unrecognised code degrades to "unavailable". Each code maps to its own message built from the label the host supplies, so nothing about the store is named here. A save that fails now leaves the editor's autosave indicator showing the reason instead of a cloud tick. When a host is present, a folder pointing at it is seeded once, and its name follows the host's label. When no host is present none of this is reachable: createDriver refuses to build the driver and no folder appears. Co-Authored-By: Claude Opus 5 (1M context) --- .../components/AutoSaveIndicator.tsx | 17 +- .../v2/pages/Editor/store/autoSaveStore.ts | 19 +- src/services/pipelineStorage/createDriver.ts | 11 + src/services/pipelineStorage/db.ts | 33 +- .../drivers/HostStorageDriver.test.ts | 399 ++++++++++++++++++ .../drivers/HostStorageDriver.ts | 151 +++++++ src/services/pipelineStorage/types.ts | 4 + 7 files changed, 624 insertions(+), 10 deletions(-) create mode 100644 src/services/pipelineStorage/drivers/HostStorageDriver.test.ts create mode 100644 src/services/pipelineStorage/drivers/HostStorageDriver.ts diff --git a/src/routes/v2/pages/Editor/components/EditorMenuBar/components/AutoSaveIndicator.tsx b/src/routes/v2/pages/Editor/components/EditorMenuBar/components/AutoSaveIndicator.tsx index 5860b6551e..57fb5e680d 100644 --- a/src/routes/v2/pages/Editor/components/EditorMenuBar/components/AutoSaveIndicator.tsx +++ b/src/routes/v2/pages/Editor/components/EditorMenuBar/components/AutoSaveIndicator.tsx @@ -41,8 +41,13 @@ function IdleLayer({ children }: { children: ReactNode }) { ); } -function getTooltipText(isSaving: boolean, lastSavedAt: Date | null): string { +function getTooltipText( + isSaving: boolean, + lastSavedAt: Date | null, + saveError: string | null, +): string { if (isSaving) return "Saving..."; + if (saveError) return saveError; if (lastSavedAt) { return `Last saved at ${lastSavedAt.toLocaleTimeString()}`; } @@ -51,8 +56,8 @@ function getTooltipText(isSaving: boolean, lastSavedAt: Date | null): string { export const AutoSaveIndicator = observer(function AutoSaveIndicator() { const { autoSave } = useEditorSession(); - const { isSaving, lastSavedAt } = autoSave; - const tooltipText = getTooltipText(isSaving, lastSavedAt); + const { isSaving, lastSavedAt, saveError } = autoSave; + const tooltipText = getTooltipText(isSaving, lastSavedAt, saveError); const handleClick = () => { void autoSave.save(); @@ -75,7 +80,11 @@ export const AutoSaveIndicator = observer(function AutoSaveIndicator() { - + {saveError ? ( + + ) : ( + + )} diff --git a/src/routes/v2/pages/Editor/store/autoSaveStore.ts b/src/routes/v2/pages/Editor/store/autoSaveStore.ts index 7cba5191ad..23c8d93926 100644 --- a/src/routes/v2/pages/Editor/store/autoSaveStore.ts +++ b/src/routes/v2/pages/Editor/store/autoSaveStore.ts @@ -8,6 +8,7 @@ import { import { saveUndoHistory } from "@/routes/v2/pages/Editor/utils/undoHistoryStorage"; import { AUTOSAVE_DEBOUNCE_TIME_MS } from "@/utils/constants"; import { debounce } from "@/utils/debounce"; +import { getErrorMessage } from "@/utils/string"; import type { PipelineFileStore } from "./pipelineFileStore"; import type { UndoStore } from "./undoStore"; @@ -17,6 +18,7 @@ const AUTOSAVE_MIN_SAVING_INDICATOR_MS = 600; export class AutoSaveStore { @observable accessor isSaving = false; @observable accessor lastSavedAt: Date | null = null; + @observable accessor saveError: string | null = null; private spec: ComponentSpec | null = null; private pipelineName: string | null = null; @@ -41,6 +43,7 @@ export class AutoSaveStore { this.pipelineName = pipelineName; this.isSaving = false; this.lastSavedAt = null; + this.saveError = null; // The freshly-loaded spec matches what's on disk, so seed the baseline to // avoid flushing an unchanged pipeline on dispose. this.lastSavedYaml = this.serializeSpec(); @@ -81,6 +84,12 @@ export class AutoSaveStore { @action setSaved(date: Date) { this.lastSavedAt = date; this.isSaving = false; + this.saveError = null; + } + + @action private setSaveError(message: string) { + this.saveError = message; + this.isSaving = false; } private serializeSpec(): string | null { @@ -114,7 +123,7 @@ export class AutoSaveStore { return new Date(); } catch (error) { console.error("Auto-save failed:", error); - return null; + return getErrorMessage(error); } })(); @@ -122,12 +131,12 @@ export class AutoSaveStore { setTimeout(resolve, AUTOSAVE_MIN_SAVING_INDICATOR_MS), ); - const [savedAt] = await Promise.all([savePromise, minDisplayPromise]); + const [outcome] = await Promise.all([savePromise, minDisplayPromise]); - if (savedAt) { - this.setSaved(savedAt); + if (outcome instanceof Date) { + this.setSaved(outcome); } else { - this.setSaving(false); + this.setSaveError(outcome); } } diff --git a/src/services/pipelineStorage/createDriver.ts b/src/services/pipelineStorage/createDriver.ts index e2c0f9402d..3fddebf1a0 100644 --- a/src/services/pipelineStorage/createDriver.ts +++ b/src/services/pipelineStorage/createDriver.ts @@ -1,8 +1,10 @@ import { getGoogleDriveAuth } from "../googleDrive/GoogleDriveAuthService"; // google-drive import { GoogleDriveStorageDriver } from "../googleDrive/GoogleDriveStorageDriver"; // google-drive import { FolderIndexDbStorageDriver } from "./drivers/FolderIndexDbStorageDriver"; +import { HostStorageDriver } from "./drivers/HostStorageDriver"; import { LocalFileSystemDriver } from "./drivers/LocalFileSystemDriver"; import { RootFolderDbStorageDriver } from "./drivers/RootFolderDbStorageDriver"; +import { getPipelineStorageHost } from "./host/detectHost"; import type { DriverConfig, PipelineStorageDriver } from "./types"; export function createDriver(config: DriverConfig): PipelineStorageDriver { @@ -13,6 +15,15 @@ export function createDriver(config: DriverConfig): PipelineStorageDriver { return new FolderIndexDbStorageDriver(config.folderId); case "local-fs": return new LocalFileSystemDriver(config.handle); + case "host": { + const host = getPipelineStorageHost(); + if (!host) { + throw new Error( + "Host-provided pipeline storage is not available on this page", + ); + } + return new HostStorageDriver(host); + } case "google-drive": // google-drive return new GoogleDriveStorageDriver( config.folderId, diff --git a/src/services/pipelineStorage/db.ts b/src/services/pipelineStorage/db.ts index 0a288d60ba..2ca064b5b9 100644 --- a/src/services/pipelineStorage/db.ts +++ b/src/services/pipelineStorage/db.ts @@ -2,8 +2,10 @@ import { Dexie, type EntityTable } from "dexie"; import { USER_PIPELINES_LIST_NAME } from "@/utils/constants"; +import { getPipelineStorageHost } from "./host/detectHost"; import { type FolderEntry, + HOST_FOLDER_ID, type PipelineRegistryEntry, ROOT_FOLDER_ID, } from "./types"; @@ -19,6 +21,11 @@ pipelineStorageDb.version(1).stores({ }); pipelineStorageDb.on("ready", async () => { + await seedRegistryFromLegacyList(); + await seedHostFolder(); +}); + +async function seedRegistryFromLegacyList() { const count = await pipelineStorageDb.pipeline_registry.count(); if (count > 0) return; @@ -49,4 +56,28 @@ pipelineStorageDb.on("ready", async () => { console.error(e); throw e; } -}); +} + +async function seedHostFolder() { + const host = getPipelineStorageHost(); + if (!host) return; + + const existing = await pipelineStorageDb.folders.get(HOST_FOLDER_ID); + + if (!existing) { + await pipelineStorageDb.folders.add({ + id: HOST_FOLDER_ID, + name: host.label, + parentId: ROOT_FOLDER_ID, + driverConfig: { driverType: "host" }, + createdAt: Date.now(), + }); + return; + } + + if (existing.name !== host.label) { + await pipelineStorageDb.folders.update(HOST_FOLDER_ID, { + name: host.label, + }); + } +} diff --git a/src/services/pipelineStorage/drivers/HostStorageDriver.test.ts b/src/services/pipelineStorage/drivers/HostStorageDriver.test.ts new file mode 100644 index 0000000000..b35ef5aa42 --- /dev/null +++ b/src/services/pipelineStorage/drivers/HostStorageDriver.test.ts @@ -0,0 +1,399 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; + +import type { ComponentSpec } from "@/utils/componentSpec"; +import { componentSpecFromYaml, componentSpecToYaml } from "@/utils/yaml"; + +import { createDriver } from "../createDriver"; +import type { + HostErrorCode, + HostPipeline, + HostPipelineSummary, + PipelineStorageHost, +} from "../host/contract"; +import type { PipelineStorageDriver } from "../types"; +import { HostStorageDriver, HostStorageError } from "./HostStorageDriver"; + +const LABEL = "Shared storage"; + +interface FakeHost extends PipelineStorageHost { + readonly writtenSpecs: unknown[]; + readonly readCalls: string[]; +} + +function createFakeHost(seed: Record = {}): FakeHost { + const store = new Map(Object.entries(seed)); + const writtenSpecs: unknown[] = []; + const readCalls: string[] = []; + let revision = 0; + + return { + version: 1, + label: LABEL, + writtenSpecs, + readCalls, + async list(): Promise { + return [...store.values()].map(({ spec: _spec, ...summary }) => summary); + }, + async read(key: string): Promise { + readCalls.push(key); + const pipeline = store.get(key); + if (!pipeline) throw { code: "not_found" }; + return pipeline; + }, + async write(key: string, spec: unknown): Promise { + writtenSpecs.push(spec); + revision += 1; + const existing = store.get(key); + const pipeline: HostPipeline = { + key, + externalId: existing?.externalId ?? `external-${store.size + 1}`, + displayName: readSpecName(spec), + contentVersion: `v${revision}`, + spec, + }; + store.set(key, pipeline); + const { spec: _spec, ...summary } = pipeline; + return summary; + }, + async delete(key: string): Promise { + store.delete(key); + }, + async has(key: string): Promise { + return store.has(key); + }, + }; +} + +function readSpecName(spec: unknown): string | null { + if (typeof spec !== "object" || spec === null) return null; + const name: unknown = Reflect.get(spec, "name"); + return typeof name === "string" ? name : null; +} + +function hostPipeline( + key: string, + overrides: Partial = {}, +): HostPipeline { + return { + key, + externalId: `external-${key}`, + displayName: "Churn model", + contentVersion: "v1", + spec: { name: "Churn model", implementation: { graph: { tasks: {} } } }, + ...overrides, + }; +} + +function rejectingHost(rejection: unknown): PipelineStorageHost { + const reject = () => Promise.reject(rejection); + return { + version: 1, + label: LABEL, + list: reject, + read: reject, + write: reject, + delete: reject, + has: reject, + }; +} + +describe("HostStorageDriver.list", () => { + it("maps summaries to descriptors without reading any pipeline", async () => { + const host = createFakeHost({ + "key-1": hostPipeline("key-1", { + createdAt: "2026-01-01T00:00:00.000Z", + modifiedAt: "2026-02-01T00:00:00.000Z", + }), + }); + + const descriptors = await new HostStorageDriver(host).list(); + + expect(descriptors).toEqual([ + { + storageKey: "key-1", + externalId: "external-key-1", + displayName: "Churn model", + contentVersion: "v1", + createdAt: new Date("2026-01-01T00:00:00.000Z"), + modifiedAt: new Date("2026-02-01T00:00:00.000Z"), + }, + ]); + expect(host.readCalls).toEqual([]); + }); + + it("falls back to a placeholder display name when the host has none", async () => { + const host = createFakeHost({ + "key-1": hostPipeline("key-1", { displayName: null }), + }); + + const [descriptor] = await new HostStorageDriver(host).list(); + + expect(descriptor.displayName).toBe("Untitled pipeline"); + }); + + it("omits timestamps the host reports unparseably", async () => { + const host = createFakeHost({ + "key-1": hostPipeline("key-1", { createdAt: "not-a-date" }), + }); + + const [descriptor] = await new HostStorageDriver(host).list(); + + expect(descriptor.createdAt).toBeUndefined(); + }); +}); + +describe("HostStorageDriver.write", () => { + it("projects the document before handing it to the host", async () => { + const host = createFakeHost(); + const yamlText = componentSpecToYaml({ + name: "Churn model", + inputs: [{ name: "epochs", default: "10", value: "42" }], + implementation: { + graph: { + tasks: { + train: { + componentRef: { + name: "Train", + favorited: true, + owned: true, + spec: { + name: "Train", + implementation: { container: { image: "python:3.11" } }, + }, + }, + }, + }, + }, + }, + }); + + await new HostStorageDriver(host).write("key-1", yamlText); + + const written = JSON.stringify(host.writtenSpecs[0]); + expect(written).not.toContain("favorited"); + expect(written).not.toContain("owned"); + expect(written).not.toContain('"value"'); + expect(written).toContain("epochs"); + }); + + it("returns the descriptor the host reports for the write", async () => { + const host = createFakeHost(); + const yamlText = componentSpecToYaml({ + name: "Churn model", + implementation: { graph: { tasks: {} } }, + }); + + const descriptor = await new HostStorageDriver(host).write( + "key-1", + yamlText, + ); + + expect(descriptor).toMatchObject({ + storageKey: "key-1", + displayName: "Churn model", + contentVersion: "v1", + }); + expect(descriptor.externalId).toBeDefined(); + }); + + it("reports a fresh contentVersion on every write", async () => { + const driver = new HostStorageDriver(createFakeHost()); + const yamlText = componentSpecToYaml({ + name: "Churn model", + implementation: { graph: { tasks: {} } }, + }); + + const first = await driver.write("key-1", yamlText); + const second = await driver.write("key-1", yamlText); + + expect(second.contentVersion).not.toBe(first.contentVersion); + }); +}); + +describe("HostStorageDriver round trip", () => { + it("returns YAML for a pipeline it previously saved", async () => { + const spec: ComponentSpec = { + name: "Churn model", + description: "Predicts churn", + inputs: [{ name: "epochs", type: "Integer", default: "10" }], + implementation: { + graph: { + tasks: { + train: { + componentRef: { + name: "Train", + spec: { + name: "Train", + implementation: { container: { image: "python:3.11" } }, + }, + }, + annotations: { "editor.position": '{"x":10,"y":20}' }, + }, + }, + }, + }, + }; + const driver = new HostStorageDriver(createFakeHost()); + + await driver.write("key-1", componentSpecToYaml(spec)); + const yamlText = await driver.read("key-1"); + + expect(componentSpecFromYaml(yamlText)).toEqual(spec); + }); + + it("rejects a pipeline the host returns in an unreadable shape", async () => { + const host = createFakeHost({ + "key-1": hostPipeline("key-1", { spec: { nope: true } }), + }); + + await expect(new HostStorageDriver(host).read("key-1")).rejects.toThrow( + HostStorageError, + ); + }); +}); + +describe("HostStorageDriver.rename", () => { + it("refuses, because a key carries no name for the host", async () => { + const driver: PipelineStorageDriver = new HostStorageDriver( + createFakeHost(), + ); + + await expect(driver.rename("key-1", "key-2")).rejects.toThrow(LABEL); + }); +}); + +describe("HostStorageDriver.delete and hasKey", () => { + it("removes the pipeline from the host", async () => { + const host = createFakeHost({ "key-1": hostPipeline("key-1") }); + const driver = new HostStorageDriver(host); + + expect(await driver.hasKey("key-1")).toBe(true); + await driver.delete("key-1"); + + expect(await driver.hasKey("key-1")).toBe(false); + }); +}); + +describe("HostStorageDriver error mapping", () => { + const cases: [HostErrorCode, RegExp][] = [ + ["unauthenticated", /session has expired/], + ["not_found", /no longer exists/], + ["rate_limited", /is busy/], + ["conflict", /changed in/], + ["unavailable", /could not be reached/], + ]; + + it.each(cases)( + "maps the %s code to its own message", + async (code, matcher) => { + const driver = new HostStorageDriver(rejectingHost({ code })); + + await expect(driver.list()).rejects.toMatchObject({ + code, + message: expect.stringMatching(matcher), + }); + }, + ); + + it.each([ + ["an unrecognised code", { code: "teapot" }], + ["no code at all", new Error("boom")], + ["a non-object rejection", "boom"], + ])("degrades %s to unavailable", async (_case, rejection) => { + const driver = new HostStorageDriver(rejectingHost(rejection)); + + await expect(driver.read("key-1")).rejects.toMatchObject({ + code: "unavailable", + }); + }); + + it("names the host label from the contract rather than a hardcoded name", async () => { + const host = { + ...rejectingHost({ code: "unavailable" }), + label: "Team drive", + }; + + await expect(new HostStorageDriver(host).list()).rejects.toThrow( + /Team drive/, + ); + }); + + it("keeps the original rejection as the error cause", async () => { + const rejection = { code: "conflict", detail: "revision mismatch" }; + const driver = new HostStorageDriver(rejectingHost(rejection)); + + const error = await driver.list().catch((thrown: unknown) => thrown); + + expect(error).toBeInstanceOf(HostStorageError); + expect((error as HostStorageError).cause).toBe(rejection); + }); + + it("maps every failing operation, not just reads", async () => { + const driver = new HostStorageDriver(rejectingHost({ code: "conflict" })); + const yamlText = componentSpecToYaml({ + name: "Churn model", + implementation: { graph: { tasks: {} } }, + }); + + await expect(driver.write("key-1", yamlText)).rejects.toMatchObject({ + code: "conflict", + }); + await expect(driver.delete("key-1")).rejects.toMatchObject({ + code: "conflict", + }); + await expect(driver.hasKey("key-1")).rejects.toMatchObject({ + code: "conflict", + }); + }); +}); + +describe("createDriver without a detected host", () => { + afterEach(() => { + delete window.__TANGLE_PIPELINE_STORAGE_HOST__; + }); + + it.each([ + ["no host global", undefined], + ["a host with a non-function member", { ...createFakeHost(), write: null }], + [ + "a host declaring an unsupported version", + { ...createFakeHost(), version: 99 }, + ], + ])("refuses to build a host driver with %s", (_case, host) => { + if (host) { + Object.defineProperty(window, "__TANGLE_PIPELINE_STORAGE_HOST__", { + value: host, + configurable: true, + writable: true, + }); + } + + expect(() => createDriver({ driverType: "host" })).toThrow(); + }); + + it("leaves the local drivers untouched", () => { + expect(() => createDriver({ driverType: "root-indexdb" })).not.toThrow(); + expect(() => + createDriver({ driverType: "folder-indexdb", folderId: "folder-1" }), + ).not.toThrow(); + }); +}); + +describe("HostStorageDriver capabilities", () => { + it("owns its listing and accepts no moves", () => { + const driver = new HostStorageDriver(createFakeHost()); + + expect(driver.listingIsAuthoritative).toBe(true); + expect(driver.allowsMoveIn).toBe(false); + expect(driver.allowsMoveOut).toBe(false); + }); + + it("does not touch the host until an operation is called", () => { + const host = createFakeHost(); + const list = vi.spyOn(host, "list"); + + new HostStorageDriver(host); + + expect(list).not.toHaveBeenCalled(); + }); +}); diff --git a/src/services/pipelineStorage/drivers/HostStorageDriver.ts b/src/services/pipelineStorage/drivers/HostStorageDriver.ts new file mode 100644 index 0000000000..482399f1d7 --- /dev/null +++ b/src/services/pipelineStorage/drivers/HostStorageDriver.ts @@ -0,0 +1,151 @@ +import { toPortablePipelineSpec } from "@/models/componentSpec/serialization/portablePipelineSpec"; +import { isValidComponentSpec } from "@/utils/componentSpec"; +import { componentSpecFromYaml, componentSpecToYaml } from "@/utils/yaml"; + +import type { + HostErrorCode, + HostPipelineSummary, + PipelineStorageHost, +} from "../host/contract"; +import { + HOST_DRIVER_TYPE, + type PipelineFileDescriptor, + type PipelineStorageDriver, +} from "../types"; + +export interface HostDriverConfig { + driverType: "host"; +} + +const UNTITLED_PIPELINE_NAME = "Untitled pipeline"; + +export class HostStorageError extends Error { + readonly name = "HostStorageError"; + + constructor( + readonly code: HostErrorCode, + message: string, + options?: ErrorOptions, + ) { + super(message, options); + } +} + +export class HostStorageDriver implements PipelineStorageDriver { + readonly type = HOST_DRIVER_TYPE; + readonly allowsMoveIn = false; + readonly allowsMoveOut = false; + readonly listingIsAuthoritative = true; + + constructor(private readonly host: PipelineStorageHost) {} + + async list(): Promise { + const summaries = await this.call(() => this.host.list()); + return summaries.map((summary) => toDescriptor(summary)); + } + + async read(storageKey: string): Promise { + const pipeline = await this.call(() => this.host.read(storageKey)); + + if (!isValidComponentSpec(pipeline.spec)) { + throw new HostStorageError( + "unavailable", + `Pipeline "${storageKey}" came back in a format this editor cannot read.`, + ); + } + + return componentSpecToYaml(pipeline.spec); + } + + async write( + storageKey: string, + content: string, + ): Promise { + const spec = toPortablePipelineSpec(componentSpecFromYaml(content)); + const summary = await this.call(() => this.host.write(storageKey, spec)); + return toDescriptor(summary); + } + + async rename(): Promise { + throw new Error( + `Pipelines in ${this.host.label} cannot be renamed by key. Save the pipeline under a different name instead.`, + ); + } + + async delete(storageKey: string): Promise { + await this.call(() => this.host.delete(storageKey)); + } + + async hasKey(storageKey: string): Promise { + return this.call(() => this.host.has(storageKey)); + } + + private async call(operation: () => Promise): Promise { + try { + return await operation(); + } catch (error) { + throw this.toStorageError(error); + } + } + + private toStorageError(error: unknown): HostStorageError { + if (error instanceof HostStorageError) return error; + + const code = readErrorCode(error); + return new HostStorageError(code, describe(code, this.host.label), { + cause: error, + }); + } +} + +function toDescriptor(summary: HostPipelineSummary): PipelineFileDescriptor { + return { + storageKey: summary.key, + externalId: summary.externalId, + displayName: summary.displayName ?? UNTITLED_PIPELINE_NAME, + contentVersion: summary.contentVersion, + createdAt: toDate(summary.createdAt), + modifiedAt: toDate(summary.modifiedAt), + }; +} + +function toDate(value: string | undefined): Date | undefined { + if (!value) return undefined; + const date = new Date(value); + return Number.isNaN(date.getTime()) ? undefined : date; +} + +/** + * Errors cross a window boundary, where `instanceof` does not survive, so the + * code is duck-typed off the rejection value and anything unrecognised — an + * older or newer host vocabulary included — degrades to "unavailable". + */ +function readErrorCode(error: unknown): HostErrorCode { + if (typeof error !== "object" || error === null) return "unavailable"; + + const code: unknown = Reflect.get(error, "code"); + switch (code) { + case "unauthenticated": + case "not_found": + case "rate_limited": + case "conflict": + return code; + default: + return "unavailable"; + } +} + +function describe(code: HostErrorCode, label: string): string { + switch (code) { + case "unauthenticated": + return `Your ${label} session has expired. Reload the page to sign in again.`; + case "not_found": + return `This pipeline no longer exists in ${label}.`; + case "rate_limited": + return `${label} is busy. Wait a moment before trying again.`; + case "conflict": + return `This pipeline changed in ${label} since it was opened. Reload it before saving again.`; + case "unavailable": + return `${label} could not be reached. Try again in a moment.`; + } +} diff --git a/src/services/pipelineStorage/types.ts b/src/services/pipelineStorage/types.ts index d5773677b6..a1a2e605d4 100644 --- a/src/services/pipelineStorage/types.ts +++ b/src/services/pipelineStorage/types.ts @@ -1,9 +1,12 @@ import type { GoogleDriveDriverConfig } from "../googleDrive/types"; // google-drive import type { FolderIndexDbDriverConfig } from "./drivers/FolderIndexDbStorageDriver"; +import type { HostDriverConfig } from "./drivers/HostStorageDriver"; import type { LocalFileSystemDriverConfig } from "./drivers/LocalFileSystemDriver"; import type { RootFolderDbDriverConfig } from "./drivers/RootFolderDbStorageDriver"; export const ROOT_FOLDER_ID = "__root__"; +export const HOST_FOLDER_ID = "__host__"; +export const HOST_DRIVER_TYPE = "host"; export interface PipelineFileDescriptor { storageKey: string; @@ -38,6 +41,7 @@ export type DriverConfig = | RootFolderDbDriverConfig | FolderIndexDbDriverConfig | LocalFileSystemDriverConfig + | HostDriverConfig | GoogleDriveDriverConfig; // google-drive export interface PipelineRegistryEntry { From 33ca22be9e5b5c48c2b262b1b0bcbe44c4bef359 Mon Sep 17 00:00:00 2001 From: Camiel van Schoonhoven Date: Tue, 8 Sep 2026 11:17:05 -0700 Subject: [PATCH 06/36] feat(pipeline-storage): reconcile with an authoritative listing and migrate lazily A store that owns its own listing means a file missing from a listing is gone, not merely unlisted, so its registry row is dropped and a moved content version is announced as a remote change so an open editor reloads it. Our own writes record the version they produced, which is what stops a save from reading back as someone else's change. Existing local pipelines migrate one at a time: the first save of a pipeline while a host is present also writes it to the host and remembers the key. The local copy stays authoritative and is never removed. Deleting a pipeline from the host forgets that key everywhere, because writing to a deleted key can revive the dead record and the pipeline would come back wearing its identity. The registry needs an index to find rows by their host key, hence version(2). Co-Authored-By: Claude Opus 5 (1M context) --- src/services/pipelineStorage/PipelineFile.ts | 8 ++ .../pipelineStorage/PipelineFolder.test.ts | 92 +++++++++++++++++++ .../pipelineStorage/PipelineFolder.ts | 43 +++++++++ src/services/pipelineStorage/db.ts | 6 ++ .../pipelineStorage/host/hostMirror.ts | 47 ++++++++++ .../pipelineStorage/pipelineFileEvents.ts | 2 +- .../pipelineStorage/pipelineRegistry.ts | 9 ++ src/services/pipelineStorage/types.ts | 2 + 8 files changed, 208 insertions(+), 1 deletion(-) create mode 100644 src/services/pipelineStorage/host/hostMirror.ts diff --git a/src/services/pipelineStorage/PipelineFile.ts b/src/services/pipelineStorage/PipelineFile.ts index 2cdc45ca37..88f0b45f52 100644 --- a/src/services/pipelineStorage/PipelineFile.ts +++ b/src/services/pipelineStorage/PipelineFile.ts @@ -2,9 +2,11 @@ import { action, makeObservable, observable, runInAction } from "mobx"; import { emitUserPipelineWritten } from "@/utils/userPipelineWriteEvents"; +import { clearMirrorsOfHostKey, mirrorWriteToHost } from "./host/hostMirror"; import { emitPipelineFileChanged } from "./pipelineFileEvents"; import type { PipelineFolder } from "./PipelineFolder"; import { deleteEntry, updateEntry } from "./pipelineRegistry"; +import { HOST_DRIVER_TYPE } from "./types"; interface PipelineFileInit { id: string; @@ -53,6 +55,8 @@ export class PipelineFile { }); } + await mirrorWriteToHost(this, content); + emitPipelineFileChanged({ storageKey: this.storageKey, source: "v2" }); emitUserPipelineWritten(); } @@ -86,5 +90,9 @@ export class PipelineFile { async deleteFile(): Promise { await this.folder.driver.delete(this.storageKey); await deleteEntry(this.id); + + if (this.folder.driver.type === HOST_DRIVER_TYPE) { + await clearMirrorsOfHostKey(this.storageKey); + } } } diff --git a/src/services/pipelineStorage/PipelineFolder.test.ts b/src/services/pipelineStorage/PipelineFolder.test.ts index 044408a2cb..f3d3bad011 100644 --- a/src/services/pipelineStorage/PipelineFolder.test.ts +++ b/src/services/pipelineStorage/PipelineFolder.test.ts @@ -53,6 +53,7 @@ vi.mock("./pipelineRegistry", () => ({ })); interface FakeDriverOptions { + listingIsAuthoritative?: boolean; write?: ( storageKey: string, content: string, @@ -72,6 +73,7 @@ function createFakeDriver(options: FakeDriverOptions = {}): FakeDriver { type: "fake", allowsMoveIn: true, allowsMoveOut: true, + listingIsAuthoritative: options.listingIsAuthoritative, descriptors, contents, async list() { @@ -238,6 +240,96 @@ describe("PipelineFolder.listPipelines", () => { }); }); +describe("PipelineFolder reconciliation", () => { + it("drops rows an authoritative listing no longer reports", async () => { + const driver = createFakeDriver({ listingIsAuthoritative: true }); + driver.descriptors.push({ storageKey: "key-1", externalId: "external-1" }); + const folder = createFolder(driver); + await folder.listPipelines(); + + driver.descriptors.length = 0; + const files = await folder.listPipelines(); + + expect(files).toEqual([]); + expect(registry.size).toBe(0); + }); + + it("keeps rows a non-authoritative listing omits", async () => { + const driver = createFakeDriver(); + driver.descriptors.push({ storageKey: "key-1", externalId: "external-1" }); + const folder = createFolder(driver); + await folder.listPipelines(); + + driver.descriptors.length = 0; + await folder.listPipelines(); + + expect(registry.has("external-1")).toBe(true); + }); + + it("announces a content version that moved elsewhere", async () => { + const driver = createFakeDriver({ listingIsAuthoritative: true }); + driver.descriptors.push({ + storageKey: "key-1", + externalId: "external-1", + contentVersion: "v1", + }); + const folder = createFolder(driver); + await folder.listPipelines(); + + const changes = collectChanges(); + driver.descriptors[0].contentVersion = "v2"; + await folder.listPipelines(); + + expect(changes).toEqual([{ storageKey: "key-1", source: "remote" }]); + expect(registry.get("external-1")?.contentVersion).toBe("v2"); + }); + + it("stays quiet when the content version is unchanged", async () => { + const driver = createFakeDriver({ listingIsAuthoritative: true }); + driver.descriptors.push({ + storageKey: "key-1", + externalId: "external-1", + contentVersion: "v1", + }); + const folder = createFolder(driver); + await folder.listPipelines(); + + const changes = collectChanges(); + await folder.listPipelines(); + + expect(changes).toEqual([]); + }); + + it("does not mistake our own write for a change made elsewhere", async () => { + let revision = 0; + const driver = createFakeDriver({ + listingIsAuthoritative: true, + write: async (storageKey) => { + revision += 1; + return { + storageKey, + externalId: "external-1", + contentVersion: `v${revision}`, + }; + }, + }); + const folder = createFolder(driver); + const file = await folder.addFile("key-1", "name: One"); + driver.descriptors.push({ + storageKey: "key-1", + externalId: "external-1", + contentVersion: "v1", + }); + + const changes = collectChanges(); + await file.write("name: Two"); + driver.descriptors[0].contentVersion = "v2"; + await folder.listPipelines(); + + expect(changes.map((change) => change.source)).toEqual(["v2"]); + }); +}); + describe("PipelineFolder.findFile", () => { it("returns nothing for a key the driver does not hold", async () => { const folder = createFolder(createFakeDriver()); diff --git a/src/services/pipelineStorage/PipelineFolder.ts b/src/services/pipelineStorage/PipelineFolder.ts index 516e552399..a9873253d1 100644 --- a/src/services/pipelineStorage/PipelineFolder.ts +++ b/src/services/pipelineStorage/PipelineFolder.ts @@ -3,11 +3,15 @@ import { action, makeObservable, observable } from "mobx"; import { createDriver } from "./createDriver"; import { pipelineStorageDb } from "./db"; import { PipelineFile } from "./PipelineFile"; +import { emitPipelineFileChanged } from "./pipelineFileEvents"; import { addEntry, assertStorageKeyUnique, + deleteEntry, deleteFoldersAndDetachEntries, findByStorageKey, + getAllByFolderId, + updateEntry, } from "./pipelineRegistry"; import { type DriverConfig, @@ -91,6 +95,10 @@ export class PipelineFolder { async listPipelines(): Promise { const descriptors = await this.driver.list(); + if (this.driver.listingIsAuthoritative) { + await reconcileRegistryToListing(this.id, descriptors); + } + return Promise.all( descriptors.map((descriptor) => resolveOrCreateRegistryEntry(descriptor, this), @@ -257,3 +265,38 @@ async function resolveOrCreateRegistryEntry( }); return new PipelineFile({ id, folder, ...descriptor }); } + +/** + * Aligns the registry with a store that owns its own listing: rows the store no + * longer reports are dropped, and a moved `contentVersion` is announced so an + * editor holding the file reloads it. + */ +async function reconcileRegistryToListing( + folderId: string, + descriptors: PipelineFileDescriptor[], +): Promise { + const known = await getAllByFolderId(folderId); + const listed = new Map(descriptors.map((d) => [d.storageKey, d])); + + for (const entry of known) { + const descriptor = listed.get(entry.storageKey); + + if (!descriptor) { + await deleteEntry(entry.id); + continue; + } + + if ( + descriptor.contentVersion === undefined || + descriptor.contentVersion === entry.contentVersion + ) { + continue; + } + + await updateEntry(entry.id, { contentVersion: descriptor.contentVersion }); + emitPipelineFileChanged({ + storageKey: entry.storageKey, + source: "remote", + }); + } +} diff --git a/src/services/pipelineStorage/db.ts b/src/services/pipelineStorage/db.ts index 2ca064b5b9..cb444c0249 100644 --- a/src/services/pipelineStorage/db.ts +++ b/src/services/pipelineStorage/db.ts @@ -20,6 +20,12 @@ pipelineStorageDb.version(1).stores({ folders: "id, parentId", }); +pipelineStorageDb.version(2).stores({ + pipeline_registry: + "id, &storageKey, folderId, [folderId+storageKey], remoteStorageKey", + folders: "id, parentId", +}); + pipelineStorageDb.on("ready", async () => { await seedRegistryFromLegacyList(); await seedHostFolder(); diff --git a/src/services/pipelineStorage/host/hostMirror.ts b/src/services/pipelineStorage/host/hostMirror.ts new file mode 100644 index 0000000000..36b2a7c6da --- /dev/null +++ b/src/services/pipelineStorage/host/hostMirror.ts @@ -0,0 +1,47 @@ +import type { PipelineFile } from "../PipelineFile"; +import { + findById, + findByRemoteStorageKey, + updateEntry, +} from "../pipelineRegistry"; +import { HOST_DRIVER_TYPE } from "../types"; +import { getPipelineStorageHost } from "./detectHost"; + +/** + * Copies a locally-stored pipeline to the host the first time it is saved with + * a host present, so a person's existing work becomes reachable there without a + * bulk migration. The local copy is authoritative and is never removed. + */ +export async function mirrorWriteToHost( + file: PipelineFile, + content: string, +): Promise { + if (file.folder.driver.type === HOST_DRIVER_TYPE) return; + + const host = getPipelineStorageHost(); + if (!host) return; + + const entry = await findById(file.id); + if (!entry || entry.remoteStorageKey) return; + + const { HostStorageDriver } = await import("../drivers/HostStorageDriver"); + const remoteStorageKey = crypto.randomUUID(); + await new HostStorageDriver(host).write(remoteStorageKey, content); + await updateEntry(file.id, { remoteStorageKey }); +} + +/** + * A key the host has deleted must never be written to again: the host may + * revive the deleted record rather than mint a new one, and the pipeline would + * come back wearing the dead record's identity. Forgetting the key is what + * makes the next save create a fresh record instead. + */ +export async function clearMirrorsOfHostKey( + remoteStorageKey: string, +): Promise { + const mirrored = await findByRemoteStorageKey(remoteStorageKey); + + for (const entry of mirrored) { + await updateEntry(entry.id, { remoteStorageKey: undefined }); + } +} diff --git a/src/services/pipelineStorage/pipelineFileEvents.ts b/src/services/pipelineStorage/pipelineFileEvents.ts index 84029e1f9d..e0187bcc49 100644 --- a/src/services/pipelineStorage/pipelineFileEvents.ts +++ b/src/services/pipelineStorage/pipelineFileEvents.ts @@ -1,4 +1,4 @@ -export type PipelineFileSource = "v1" | "v2"; +export type PipelineFileSource = "v1" | "v2" | "remote"; export interface PipelineFileChange { storageKey: string; diff --git a/src/services/pipelineStorage/pipelineRegistry.ts b/src/services/pipelineStorage/pipelineRegistry.ts index 6f06b06c62..8257ac5f2e 100644 --- a/src/services/pipelineStorage/pipelineRegistry.ts +++ b/src/services/pipelineStorage/pipelineRegistry.ts @@ -31,6 +31,15 @@ export async function findByStorageKey( .first(); } +export async function findByRemoteStorageKey( + remoteStorageKey: string, +): Promise { + return pipelineStorageDb.pipeline_registry + .where("remoteStorageKey") + .equals(remoteStorageKey) + .toArray(); +} + export async function getAllByFolderId( folderId: string, ): Promise { diff --git a/src/services/pipelineStorage/types.ts b/src/services/pipelineStorage/types.ts index a1a2e605d4..9f10c295f1 100644 --- a/src/services/pipelineStorage/types.ts +++ b/src/services/pipelineStorage/types.ts @@ -29,6 +29,7 @@ export interface PipelineStorageDriver { readonly permissions?: DriverPermissions; readonly allowsMoveIn: boolean; readonly allowsMoveOut: boolean; + readonly listingIsAuthoritative?: boolean; list(): Promise; read(storageKey: string): Promise; write(storageKey: string, content: string): Promise; @@ -49,6 +50,7 @@ export interface PipelineRegistryEntry { storageKey: string; folderId: string; contentVersion?: string; + remoteStorageKey?: string; } export interface FolderEntry { From 7a308a39dc7a0273a3f7cbf156354fb028040fd0 Mon Sep 17 00:00:00 2001 From: Camiel van Schoonhoven Date: Tue, 8 Sep 2026 11:37:41 -0700 Subject: [PATCH 07/36] fix(pipeline-storage): remove the host folder when the page loads without a host MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Seeding the folder was only half the job. The folder outlives the page that installed it, so any later load without a host left a row whose driver cannot be built — and one such row takes the whole folder listing down with it, not just itself, because listSubfolders maps every entry through createDriver. The pipelines page collapsed to the generic retry icon after a few silent retries. Syncing now runs in both directions, and the registry rows that only ever described the host's files go with the folder. A local pipeline's record of what it already copied to the host survives, so nothing is copied twice when the host comes back. Co-Authored-By: Claude Opus 5 (1M context) --- src/services/pipelineStorage/db.ts | 35 +--- .../pipelineStorage/host/hostFolder.test.ts | 187 ++++++++++++++++++ .../pipelineStorage/host/hostFolder.ts | 56 ++++++ 3 files changed, 250 insertions(+), 28 deletions(-) create mode 100644 src/services/pipelineStorage/host/hostFolder.test.ts create mode 100644 src/services/pipelineStorage/host/hostFolder.ts diff --git a/src/services/pipelineStorage/db.ts b/src/services/pipelineStorage/db.ts index cb444c0249..d9aaa5e5a2 100644 --- a/src/services/pipelineStorage/db.ts +++ b/src/services/pipelineStorage/db.ts @@ -2,19 +2,22 @@ import { Dexie, type EntityTable } from "dexie"; import { USER_PIPELINES_LIST_NAME } from "@/utils/constants"; -import { getPipelineStorageHost } from "./host/detectHost"; +import { syncHostFolder } from "./host/hostFolder"; import { type FolderEntry, - HOST_FOLDER_ID, type PipelineRegistryEntry, ROOT_FOLDER_ID, } from "./types"; -export const pipelineStorageDb = new Dexie("tangle_pipelines") as Dexie & { +export type PipelineStorageDb = Dexie & { pipeline_registry: EntityTable; folders: EntityTable; }; +export const pipelineStorageDb = new Dexie( + "tangle_pipelines", +) as PipelineStorageDb; + pipelineStorageDb.version(1).stores({ pipeline_registry: "id, &storageKey, folderId, [folderId+storageKey]", folders: "id, parentId", @@ -28,7 +31,7 @@ pipelineStorageDb.version(2).stores({ pipelineStorageDb.on("ready", async () => { await seedRegistryFromLegacyList(); - await seedHostFolder(); + await syncHostFolder(pipelineStorageDb); }); async function seedRegistryFromLegacyList() { @@ -63,27 +66,3 @@ async function seedRegistryFromLegacyList() { throw e; } } - -async function seedHostFolder() { - const host = getPipelineStorageHost(); - if (!host) return; - - const existing = await pipelineStorageDb.folders.get(HOST_FOLDER_ID); - - if (!existing) { - await pipelineStorageDb.folders.add({ - id: HOST_FOLDER_ID, - name: host.label, - parentId: ROOT_FOLDER_ID, - driverConfig: { driverType: "host" }, - createdAt: Date.now(), - }); - return; - } - - if (existing.name !== host.label) { - await pipelineStorageDb.folders.update(HOST_FOLDER_ID, { - name: host.label, - }); - } -} diff --git a/src/services/pipelineStorage/host/hostFolder.test.ts b/src/services/pipelineStorage/host/hostFolder.test.ts new file mode 100644 index 0000000000..942ca85c46 --- /dev/null +++ b/src/services/pipelineStorage/host/hostFolder.test.ts @@ -0,0 +1,187 @@ +import { afterEach, describe, expect, it } from "vitest"; + +import type { PipelineStorageDb } from "../db"; +import type { FolderEntry, PipelineRegistryEntry } from "../types"; +import { HOST_FOLDER_ID, ROOT_FOLDER_ID } from "../types"; +import { PIPELINE_STORAGE_HOST_VERSION } from "./contract"; +import { syncHostFolder } from "./hostFolder"; + +function createFakeDb(seed: { + folders?: FolderEntry[]; + registry?: PipelineRegistryEntry[]; +}) { + const folders = new Map((seed.folders ?? []).map((f) => [f.id, f])); + const registry = new Map((seed.registry ?? []).map((e) => [e.id, e])); + + const db = { + folders: { + get: async (id: string) => folders.get(id), + add: async (entry: FolderEntry) => { + folders.set(entry.id, entry); + }, + update: async (id: string, changes: Partial) => { + const existing = folders.get(id); + if (existing) folders.set(id, { ...existing, ...changes }); + }, + delete: async (id: string) => { + folders.delete(id); + }, + }, + pipeline_registry: { + where: (field: keyof PipelineRegistryEntry) => ({ + equals: (value: string) => ({ + delete: async () => { + for (const [id, entry] of registry) { + if (entry[field] === value) registry.delete(id); + } + }, + }), + }), + }, + transaction: async ( + _mode: string, + _a: unknown, + _b: unknown, + body: () => Promise, + ) => body(), + }; + + return { db: db as unknown as PipelineStorageDb, folders, registry }; +} + +function installHost(label: string) { + Object.defineProperty(window, "__TANGLE_PIPELINE_STORAGE_HOST__", { + value: { + version: PIPELINE_STORAGE_HOST_VERSION, + label, + list: async () => [], + read: async () => ({}), + write: async () => ({}), + delete: async () => undefined, + has: async () => false, + }, + configurable: true, + writable: true, + }); +} + +function hostFolderEntry(name = "Shared storage"): FolderEntry { + return { + id: HOST_FOLDER_ID, + name, + parentId: ROOT_FOLDER_ID, + driverConfig: { driverType: "host" }, + createdAt: 0, + }; +} + +function localFolderEntry(): FolderEntry { + return { + id: "local-folder", + name: "My folder", + parentId: ROOT_FOLDER_ID, + driverConfig: { driverType: "folder-indexdb", folderId: "local-folder" }, + createdAt: 0, + }; +} + +afterEach(() => { + delete window.__TANGLE_PIPELINE_STORAGE_HOST__; +}); + +describe("syncHostFolder with a host present", () => { + it("adds the folder under the host's label", async () => { + installHost("Shared storage"); + const { db, folders } = createFakeDb({}); + + await syncHostFolder(db); + + expect(folders.get(HOST_FOLDER_ID)).toMatchObject({ + name: "Shared storage", + parentId: ROOT_FOLDER_ID, + driverConfig: { driverType: "host" }, + }); + }); + + it("adds the folder only once", async () => { + installHost("Shared storage"); + const { db, folders } = createFakeDb({}); + + await syncHostFolder(db); + const created = folders.get(HOST_FOLDER_ID); + await syncHostFolder(db); + + expect(folders.get(HOST_FOLDER_ID)).toBe(created); + }); + + it("follows a label the host has changed", async () => { + installHost("Team storage"); + const { db, folders } = createFakeDb({ + folders: [hostFolderEntry("Shared storage")], + }); + + await syncHostFolder(db); + + expect(folders.get(HOST_FOLDER_ID)?.name).toBe("Team storage"); + }); +}); + +describe("syncHostFolder with no host present", () => { + it("removes a folder left behind by an earlier page load", async () => { + const { db, folders } = createFakeDb({ folders: [hostFolderEntry()] }); + + await syncHostFolder(db); + + expect(folders.has(HOST_FOLDER_ID)).toBe(false); + }); + + it("removes the registry rows that described the host's files", async () => { + const { db, registry } = createFakeDb({ + folders: [hostFolderEntry()], + registry: [ + { id: "remote-1", storageKey: "opaque-1", folderId: HOST_FOLDER_ID }, + { id: "local-1", storageKey: "My pipeline", folderId: ROOT_FOLDER_ID }, + ], + }); + + await syncHostFolder(db); + + expect([...registry.keys()]).toEqual(["local-1"]); + }); + + it("keeps a local row's record of what it already copied to the host", async () => { + const { db, registry } = createFakeDb({ + folders: [hostFolderEntry()], + registry: [ + { + id: "local-1", + storageKey: "My pipeline", + folderId: ROOT_FOLDER_ID, + remoteStorageKey: "opaque-1", + }, + ], + }); + + await syncHostFolder(db); + + expect(registry.get("local-1")?.remoteStorageKey).toBe("opaque-1"); + }); + + it("leaves every other folder alone", async () => { + const { db, folders } = createFakeDb({ + folders: [hostFolderEntry(), localFolderEntry()], + }); + + await syncHostFolder(db); + + expect([...folders.keys()]).toEqual(["local-folder"]); + }); + + it("does nothing when there is no host folder to remove", async () => { + const { db, folders } = createFakeDb({ folders: [localFolderEntry()] }); + + await syncHostFolder(db); + + expect([...folders.keys()]).toEqual(["local-folder"]); + }); +}); diff --git a/src/services/pipelineStorage/host/hostFolder.ts b/src/services/pipelineStorage/host/hostFolder.ts new file mode 100644 index 0000000000..c202279a0a --- /dev/null +++ b/src/services/pipelineStorage/host/hostFolder.ts @@ -0,0 +1,56 @@ +import type { PipelineStorageDb } from "../db"; +import { HOST_FOLDER_ID, ROOT_FOLDER_ID } from "../types"; +import { getPipelineStorageHost } from "./detectHost"; + +/** + * Brings the host folder into line with whether a host is actually on the page, + * in both directions. Adding it is only half the job: the folder outlives the + * page that installed it, and a row whose driver cannot be built takes the + * whole folder listing down with it, so a page loading without a host has to + * remove the folder rather than leave it behind. + */ +export async function syncHostFolder(db: PipelineStorageDb): Promise { + const host = getPipelineStorageHost(); + + if (!host) { + await removeHostFolder(db); + return; + } + + const existing = await db.folders.get(HOST_FOLDER_ID); + + if (!existing) { + await db.folders.add({ + id: HOST_FOLDER_ID, + name: host.label, + parentId: ROOT_FOLDER_ID, + driverConfig: { driverType: "host" }, + createdAt: Date.now(), + }); + return; + } + + if (existing.name !== host.label) { + await db.folders.update(HOST_FOLDER_ID, { name: host.label }); + } +} + +/** + * The registry rows go with it. They only ever recorded which local id stood + * for which remote file, so without the folder they describe nothing, and the + * pipelines themselves are untouched on the host. Rows elsewhere keep their + * `remoteStorageKey`, so a pipeline already copied to the host is not copied + * again when the host returns. + */ +async function removeHostFolder(db: PipelineStorageDb): Promise { + const existing = await db.folders.get(HOST_FOLDER_ID); + if (!existing) return; + + await db.transaction("rw", db.folders, db.pipeline_registry, async () => { + await db.pipeline_registry + .where("folderId") + .equals(HOST_FOLDER_ID) + .delete(); + await db.folders.delete(HOST_FOLDER_ID); + }); +} From b41ab61abc6d07b39f142e71d0ccd8fe9ae568bc Mon Sep 17 00:00:00 2001 From: Camiel van Schoonhoven Date: Tue, 8 Sep 2026 14:56:25 -0700 Subject: [PATCH 08/36] refactor(pipeline-storage): make a host-provided store the only store MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Sitting a host-provided store beside browser storage as an extra folder does not hold up. The copy taken on first save is frozen at that moment while the local original keeps being edited, and nothing tells anyone the two have drifted. Editing was the point, so the copy is the wrong artefact. Storage mode is now one decision taken as the app boots: with a host on the page the root folder *is* the host, and browser storage is not a sibling, a fallback, or a cache. The decision holds on to the host it found rather than re-reading the global, because detection is fail-open and a host that later disappears would otherwise read as "no host" and send the next write to the browser — the one outcome this exists to stop. The contract has no folders, so the host root is flat: it lists no subfolders, refuses to create one, and refuses to open a folder id left over from browser storage rather than quietly resolving it locally. Seeding the registry from the legacy list is skipped for the same reason — those names are keys the host has never heard of. The registry's host-key column, the host folder row and the copy-on-save path all go, and version(3) clears out what earlier page loads left behind: a folder row whose driver cannot be built takes the whole listing down. `utils/URL.ts` reaches for `RUNS_BASE_PATH` through the router rather than the route constants it actually lives in, which drags the entire route tree into a leaf util and makes anything importing it uncycleable in tests. Co-Authored-By: Claude Opus 5 (1M context) --- src/services/pipelineStorage/PipelineFile.ts | 8 - .../pipelineStorage/PipelineFolder.test.ts | 1 - .../pipelineStorage/PipelineFolder.ts | 9 + .../PipelineStorageService.test.ts | 133 +++++++++++++ .../pipelineStorage/PipelineStorageService.ts | 40 ++-- src/services/pipelineStorage/createDriver.ts | 4 +- src/services/pipelineStorage/db.ts | 45 ++++- .../drivers/HostStorageDriver.test.ts | 8 +- .../pipelineStorage/host/hostFolder.test.ts | 187 ------------------ .../pipelineStorage/host/hostFolder.ts | 56 ------ .../pipelineStorage/host/hostMirror.ts | 47 ----- .../pipelineStorage/pipelineRegistry.ts | 9 - src/services/pipelineStorage/storageMode.ts | 39 ++++ src/services/pipelineStorage/types.ts | 2 - src/utils/URL.test.ts | 4 - src/utils/URL.ts | 2 +- tests/e2e/fixtures/pipelineStorageHost.ts | 178 +++++++++++++++++ tests/e2e/pipeline-storage-host.spec.ts | 66 +++++++ 18 files changed, 507 insertions(+), 331 deletions(-) create mode 100644 src/services/pipelineStorage/PipelineStorageService.test.ts delete mode 100644 src/services/pipelineStorage/host/hostFolder.test.ts delete mode 100644 src/services/pipelineStorage/host/hostFolder.ts delete mode 100644 src/services/pipelineStorage/host/hostMirror.ts create mode 100644 src/services/pipelineStorage/storageMode.ts create mode 100644 tests/e2e/fixtures/pipelineStorageHost.ts create mode 100644 tests/e2e/pipeline-storage-host.spec.ts diff --git a/src/services/pipelineStorage/PipelineFile.ts b/src/services/pipelineStorage/PipelineFile.ts index 88f0b45f52..2cdc45ca37 100644 --- a/src/services/pipelineStorage/PipelineFile.ts +++ b/src/services/pipelineStorage/PipelineFile.ts @@ -2,11 +2,9 @@ import { action, makeObservable, observable, runInAction } from "mobx"; import { emitUserPipelineWritten } from "@/utils/userPipelineWriteEvents"; -import { clearMirrorsOfHostKey, mirrorWriteToHost } from "./host/hostMirror"; import { emitPipelineFileChanged } from "./pipelineFileEvents"; import type { PipelineFolder } from "./PipelineFolder"; import { deleteEntry, updateEntry } from "./pipelineRegistry"; -import { HOST_DRIVER_TYPE } from "./types"; interface PipelineFileInit { id: string; @@ -55,8 +53,6 @@ export class PipelineFile { }); } - await mirrorWriteToHost(this, content); - emitPipelineFileChanged({ storageKey: this.storageKey, source: "v2" }); emitUserPipelineWritten(); } @@ -90,9 +86,5 @@ export class PipelineFile { async deleteFile(): Promise { await this.folder.driver.delete(this.storageKey); await deleteEntry(this.id); - - if (this.folder.driver.type === HOST_DRIVER_TYPE) { - await clearMirrorsOfHostKey(this.storageKey); - } } } diff --git a/src/services/pipelineStorage/PipelineFolder.test.ts b/src/services/pipelineStorage/PipelineFolder.test.ts index f3d3bad011..78d6a05310 100644 --- a/src/services/pipelineStorage/PipelineFolder.test.ts +++ b/src/services/pipelineStorage/PipelineFolder.test.ts @@ -39,7 +39,6 @@ vi.mock("./pipelineRegistry", () => ({ findByStorageKey: vi.fn(async (storageKey: string) => [...registry.values()].find((entry) => entry.storageKey === storageKey), ), - findByRemoteStorageKey: vi.fn(async () => []), getAllByFolderId: vi.fn(async (folderId: string) => [...registry.values()].filter((entry) => entry.folderId === folderId), ), diff --git a/src/services/pipelineStorage/PipelineFolder.ts b/src/services/pipelineStorage/PipelineFolder.ts index a9873253d1..88cd1229d2 100644 --- a/src/services/pipelineStorage/PipelineFolder.ts +++ b/src/services/pipelineStorage/PipelineFolder.ts @@ -28,6 +28,7 @@ interface PipelineFolderInit { driver: PipelineStorageDriver; favorite?: boolean; createdAt?: number; + isFlat?: boolean; } class FolderNotFoundError extends Error { @@ -60,6 +61,7 @@ export class PipelineFolder { readonly id: string; readonly isRoot: boolean; + readonly isFlat: boolean; readonly parentId: string | null; readonly driver: PipelineStorageDriver; readonly createdAt: number; @@ -81,6 +83,7 @@ export class PipelineFolder { constructor(options: PipelineFolderInit) { this.isRoot = options.id === ROOT_FOLDER_ID; + this.isFlat = options.isFlat ?? false; this.id = options.id; this.name = options.name; @@ -135,6 +138,8 @@ export class PipelineFolder { } async listSubfolders(): Promise { + if (this.isFlat) return []; + const entries = await queryChildFolders(this.id); return sortByName(entries).map((entry) => PipelineFolder.fromEntry(entry)); @@ -144,6 +149,10 @@ export class PipelineFolder { name: string; driverConfig?: DriverConfig; }): Promise { + if (this.isFlat) { + throw new Error(`"${this.name}" does not support folders`); + } + const id = crypto.randomUUID(); const driverConfig: DriverConfig = options.driverConfig ?? { driverType: "folder-indexdb", diff --git a/src/services/pipelineStorage/PipelineStorageService.test.ts b/src/services/pipelineStorage/PipelineStorageService.test.ts new file mode 100644 index 0000000000..1f59e69a10 --- /dev/null +++ b/src/services/pipelineStorage/PipelineStorageService.test.ts @@ -0,0 +1,133 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import type { PipelineStorageHost } from "./host/contract"; +import { PipelineStorageService } from "./PipelineStorageService"; +import { resetStorageModeForTests } from "./storageMode"; +import { HOST_DRIVER_TYPE, ROOT_FOLDER_ID } from "./types"; + +vi.mock("./db", () => ({ + pipelineStorageDb: { + folders: { + toArray: async () => [ + { + id: "folder-1", + name: "My folder", + parentId: ROOT_FOLDER_ID, + driverConfig: { driverType: "folder-indexdb", folderId: "folder-1" }, + createdAt: 0, + favorite: true, + }, + ], + filter: (predicate: (entry: { favorite?: boolean }) => boolean) => ({ + toArray: async () => + [ + { + id: "folder-1", + name: "My folder", + parentId: ROOT_FOLDER_ID, + driverConfig: { + driverType: "folder-indexdb", + folderId: "folder-1", + }, + createdAt: 0, + favorite: true, + }, + ].filter(predicate), + }), + }, + }, +})); + +const LABEL = "Shared storage"; + +function installHost(): void { + const host: PipelineStorageHost = { + version: 1, + label: LABEL, + list: async () => [], + read: async () => { + throw new Error("not seeded"); + }, + write: async () => { + throw new Error("not seeded"); + }, + delete: async () => undefined, + has: async () => false, + }; + + Object.defineProperty(window, "__TANGLE_PIPELINE_STORAGE_HOST__", { + value: host, + configurable: true, + writable: true, + }); +} + +beforeEach(() => { + resetStorageModeForTests(); +}); + +afterEach(() => { + delete window.__TANGLE_PIPELINE_STORAGE_HOST__; + resetStorageModeForTests(); +}); + +describe("with no host on the page", () => { + it("keeps browser storage as the root", () => { + const service = new PipelineStorageService(); + + expect(service.mode).toEqual({ kind: "local" }); + expect(service.rootFolder.driver.type).not.toBe(HOST_DRIVER_TYPE); + expect(service.rootFolder.isFlat).toBe(false); + }); + + it("still resolves folders", async () => { + const service = new PipelineStorageService(); + + expect(await service.getAllFolders()).toHaveLength(1); + expect(await service.getFavoriteFolders()).toHaveLength(1); + }); +}); + +describe("with a host on the page", () => { + beforeEach(() => { + installHost(); + }); + + it("makes the host the only store, under the host's own label", () => { + const service = new PipelineStorageService(); + + expect(service.mode).toEqual({ kind: "host", label: LABEL }); + expect(service.rootFolder.id).toBe(ROOT_FOLDER_ID); + expect(service.rootFolder.driver.type).toBe(HOST_DRIVER_TYPE); + expect(service.rootFolder.name).toBe(LABEL); + }); + + it("reports no folders rather than browser-stored ones", async () => { + const service = new PipelineStorageService(); + + expect(await service.getAllFolders()).toEqual([]); + expect(await service.getFavoriteFolders()).toEqual([]); + expect(await service.rootFolder.listSubfolders()).toEqual([]); + }); + + it("refuses to open a folder left over from browser storage", async () => { + const service = new PipelineStorageService(); + + await expect(service.findFolderById("folder-1")).rejects.toThrow(); + }); + + it("refuses to create a folder", async () => { + const service = new PipelineStorageService(); + + await expect( + service.rootFolder.createSubfolder({ name: "New folder" }), + ).rejects.toThrow(); + }); + + it("stays on the host even if the host global disappears mid-session", () => { + const service = new PipelineStorageService(); + delete window.__TANGLE_PIPELINE_STORAGE_HOST__; + + expect(new PipelineStorageService().mode).toEqual(service.mode); + }); +}); diff --git a/src/services/pipelineStorage/PipelineStorageService.ts b/src/services/pipelineStorage/PipelineStorageService.ts index 0ec593abb3..2726faf4d9 100644 --- a/src/services/pipelineStorage/PipelineStorageService.ts +++ b/src/services/pipelineStorage/PipelineStorageService.ts @@ -5,20 +5,17 @@ import { pipelineStorageDb } from "./db"; import { PipelineFile } from "./PipelineFile"; import { PipelineFolder } from "./PipelineFolder"; import { findById, findByStorageKey } from "./pipelineRegistry"; -import { type PipelineStorageDriver, ROOT_FOLDER_ID } from "./types"; - -const ROOT_DRIVER_CONFIG = { - driverType: "folder-indexdb", - folderId: ROOT_FOLDER_ID, -} as const; +import { resolveStorageMode, type StorageMode } from "./storageMode"; +import { ROOT_FOLDER_ID } from "./types"; export class PipelineStorageService { @observable accessor rootFolder: PipelineFolder; + readonly mode: StorageMode; + constructor() { - this.rootFolder = createRoot({ - driver: createDriver(ROOT_DRIVER_CONFIG), - }); + this.mode = resolveStorageMode(); + this.rootFolder = createRoot(this.mode); makeObservable(this); } @@ -50,15 +47,23 @@ export class PipelineStorageService { return this.rootFolder; } + if (this.rootFolder.isFlat) { + throw new Error(`Folder not available in ${this.rootFolder.name}: ${id}`); + } + return PipelineFolder.resolveById(id); } async getAllFolders(): Promise { + if (this.rootFolder.isFlat) return []; + const entries = await pipelineStorageDb.folders.toArray(); return entries.map((entry) => PipelineFolder.fromEntry(entry)); } async getFavoriteFolders(): Promise { + if (this.rootFolder.isFlat) return []; + const entries = await pipelineStorageDb.folders .filter((f) => f.favorite === true) .toArray(); @@ -70,11 +75,24 @@ export class PipelineStorageService { } } -function createRoot(options?: { driver: PipelineStorageDriver }) { +function createRoot(mode: StorageMode): PipelineFolder { + if (mode.kind === "host") { + return new PipelineFolder({ + id: ROOT_FOLDER_ID, + name: mode.label, + parentId: null, + driver: createDriver({ driverType: "host" }), + isFlat: true, + }); + } + return new PipelineFolder({ id: ROOT_FOLDER_ID, name: "Root", parentId: null, - driver: options?.driver ?? createDriver({ driverType: "root-indexdb" }), + driver: createDriver({ + driverType: "folder-indexdb", + folderId: ROOT_FOLDER_ID, + }), }); } diff --git a/src/services/pipelineStorage/createDriver.ts b/src/services/pipelineStorage/createDriver.ts index 3fddebf1a0..058e42d540 100644 --- a/src/services/pipelineStorage/createDriver.ts +++ b/src/services/pipelineStorage/createDriver.ts @@ -4,7 +4,7 @@ import { FolderIndexDbStorageDriver } from "./drivers/FolderIndexDbStorageDriver import { HostStorageDriver } from "./drivers/HostStorageDriver"; import { LocalFileSystemDriver } from "./drivers/LocalFileSystemDriver"; import { RootFolderDbStorageDriver } from "./drivers/RootFolderDbStorageDriver"; -import { getPipelineStorageHost } from "./host/detectHost"; +import { getStorageHost } from "./storageMode"; import type { DriverConfig, PipelineStorageDriver } from "./types"; export function createDriver(config: DriverConfig): PipelineStorageDriver { @@ -16,7 +16,7 @@ export function createDriver(config: DriverConfig): PipelineStorageDriver { case "local-fs": return new LocalFileSystemDriver(config.handle); case "host": { - const host = getPipelineStorageHost(); + const host = getStorageHost(); if (!host) { throw new Error( "Host-provided pipeline storage is not available on this page", diff --git a/src/services/pipelineStorage/db.ts b/src/services/pipelineStorage/db.ts index d9aaa5e5a2..6cb1b81a73 100644 --- a/src/services/pipelineStorage/db.ts +++ b/src/services/pipelineStorage/db.ts @@ -2,7 +2,7 @@ import { Dexie, type EntityTable } from "dexie"; import { USER_PIPELINES_LIST_NAME } from "@/utils/constants"; -import { syncHostFolder } from "./host/hostFolder"; +import { isHostStorage } from "./storageMode"; import { type FolderEntry, type PipelineRegistryEntry, @@ -29,12 +29,53 @@ pipelineStorageDb.version(2).stores({ folders: "id, parentId", }); +pipelineStorageDb + .version(3) + .stores({ + pipeline_registry: "id, &storageKey, folderId, [folderId+storageKey]", + folders: "id, parentId", + }) + .upgrade(async (tx) => { + await tx + .table("pipeline_registry") + .toCollection() + .modify((entry: Record) => { + delete entry.remoteStorageKey; + }); + + /** + * An earlier shape of this feature kept the host alongside local storage as + * a child folder. A row whose driver cannot be built takes the whole folder + * listing down with it, and the host is now the root rather than a child, + * so those rows have nothing left to describe. + */ + const hostFolders = await tx + .table("folders") + .filter((folder) => folder.driverConfig.driverType === "host") + .toArray(); + + for (const folder of hostFolders) { + await tx + .table("pipeline_registry") + .where("folderId") + .equals(folder.id) + .delete(); + await tx.table("folders").delete(folder.id); + } + }); + pipelineStorageDb.on("ready", async () => { await seedRegistryFromLegacyList(); - await syncHostFolder(pipelineStorageDb); }); +/** + * The registry indexes storage keys within the one store the app is using. In + * host mode those keys are the host's, so seeding it with local pipeline names + * would claim files the host has never heard of. + */ async function seedRegistryFromLegacyList() { + if (isHostStorage()) return; + const count = await pipelineStorageDb.pipeline_registry.count(); if (count > 0) return; diff --git a/src/services/pipelineStorage/drivers/HostStorageDriver.test.ts b/src/services/pipelineStorage/drivers/HostStorageDriver.test.ts index b35ef5aa42..cf3ad7c654 100644 --- a/src/services/pipelineStorage/drivers/HostStorageDriver.test.ts +++ b/src/services/pipelineStorage/drivers/HostStorageDriver.test.ts @@ -1,4 +1,4 @@ -import { afterEach, describe, expect, it, vi } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import type { ComponentSpec } from "@/utils/componentSpec"; import { componentSpecFromYaml, componentSpecToYaml } from "@/utils/yaml"; @@ -10,6 +10,7 @@ import type { HostPipelineSummary, PipelineStorageHost, } from "../host/contract"; +import { resetStorageModeForTests } from "../storageMode"; import type { PipelineStorageDriver } from "../types"; import { HostStorageDriver, HostStorageError } from "./HostStorageDriver"; @@ -348,8 +349,13 @@ describe("HostStorageDriver error mapping", () => { }); describe("createDriver without a detected host", () => { + beforeEach(() => { + resetStorageModeForTests(); + }); + afterEach(() => { delete window.__TANGLE_PIPELINE_STORAGE_HOST__; + resetStorageModeForTests(); }); it.each([ diff --git a/src/services/pipelineStorage/host/hostFolder.test.ts b/src/services/pipelineStorage/host/hostFolder.test.ts deleted file mode 100644 index 942ca85c46..0000000000 --- a/src/services/pipelineStorage/host/hostFolder.test.ts +++ /dev/null @@ -1,187 +0,0 @@ -import { afterEach, describe, expect, it } from "vitest"; - -import type { PipelineStorageDb } from "../db"; -import type { FolderEntry, PipelineRegistryEntry } from "../types"; -import { HOST_FOLDER_ID, ROOT_FOLDER_ID } from "../types"; -import { PIPELINE_STORAGE_HOST_VERSION } from "./contract"; -import { syncHostFolder } from "./hostFolder"; - -function createFakeDb(seed: { - folders?: FolderEntry[]; - registry?: PipelineRegistryEntry[]; -}) { - const folders = new Map((seed.folders ?? []).map((f) => [f.id, f])); - const registry = new Map((seed.registry ?? []).map((e) => [e.id, e])); - - const db = { - folders: { - get: async (id: string) => folders.get(id), - add: async (entry: FolderEntry) => { - folders.set(entry.id, entry); - }, - update: async (id: string, changes: Partial) => { - const existing = folders.get(id); - if (existing) folders.set(id, { ...existing, ...changes }); - }, - delete: async (id: string) => { - folders.delete(id); - }, - }, - pipeline_registry: { - where: (field: keyof PipelineRegistryEntry) => ({ - equals: (value: string) => ({ - delete: async () => { - for (const [id, entry] of registry) { - if (entry[field] === value) registry.delete(id); - } - }, - }), - }), - }, - transaction: async ( - _mode: string, - _a: unknown, - _b: unknown, - body: () => Promise, - ) => body(), - }; - - return { db: db as unknown as PipelineStorageDb, folders, registry }; -} - -function installHost(label: string) { - Object.defineProperty(window, "__TANGLE_PIPELINE_STORAGE_HOST__", { - value: { - version: PIPELINE_STORAGE_HOST_VERSION, - label, - list: async () => [], - read: async () => ({}), - write: async () => ({}), - delete: async () => undefined, - has: async () => false, - }, - configurable: true, - writable: true, - }); -} - -function hostFolderEntry(name = "Shared storage"): FolderEntry { - return { - id: HOST_FOLDER_ID, - name, - parentId: ROOT_FOLDER_ID, - driverConfig: { driverType: "host" }, - createdAt: 0, - }; -} - -function localFolderEntry(): FolderEntry { - return { - id: "local-folder", - name: "My folder", - parentId: ROOT_FOLDER_ID, - driverConfig: { driverType: "folder-indexdb", folderId: "local-folder" }, - createdAt: 0, - }; -} - -afterEach(() => { - delete window.__TANGLE_PIPELINE_STORAGE_HOST__; -}); - -describe("syncHostFolder with a host present", () => { - it("adds the folder under the host's label", async () => { - installHost("Shared storage"); - const { db, folders } = createFakeDb({}); - - await syncHostFolder(db); - - expect(folders.get(HOST_FOLDER_ID)).toMatchObject({ - name: "Shared storage", - parentId: ROOT_FOLDER_ID, - driverConfig: { driverType: "host" }, - }); - }); - - it("adds the folder only once", async () => { - installHost("Shared storage"); - const { db, folders } = createFakeDb({}); - - await syncHostFolder(db); - const created = folders.get(HOST_FOLDER_ID); - await syncHostFolder(db); - - expect(folders.get(HOST_FOLDER_ID)).toBe(created); - }); - - it("follows a label the host has changed", async () => { - installHost("Team storage"); - const { db, folders } = createFakeDb({ - folders: [hostFolderEntry("Shared storage")], - }); - - await syncHostFolder(db); - - expect(folders.get(HOST_FOLDER_ID)?.name).toBe("Team storage"); - }); -}); - -describe("syncHostFolder with no host present", () => { - it("removes a folder left behind by an earlier page load", async () => { - const { db, folders } = createFakeDb({ folders: [hostFolderEntry()] }); - - await syncHostFolder(db); - - expect(folders.has(HOST_FOLDER_ID)).toBe(false); - }); - - it("removes the registry rows that described the host's files", async () => { - const { db, registry } = createFakeDb({ - folders: [hostFolderEntry()], - registry: [ - { id: "remote-1", storageKey: "opaque-1", folderId: HOST_FOLDER_ID }, - { id: "local-1", storageKey: "My pipeline", folderId: ROOT_FOLDER_ID }, - ], - }); - - await syncHostFolder(db); - - expect([...registry.keys()]).toEqual(["local-1"]); - }); - - it("keeps a local row's record of what it already copied to the host", async () => { - const { db, registry } = createFakeDb({ - folders: [hostFolderEntry()], - registry: [ - { - id: "local-1", - storageKey: "My pipeline", - folderId: ROOT_FOLDER_ID, - remoteStorageKey: "opaque-1", - }, - ], - }); - - await syncHostFolder(db); - - expect(registry.get("local-1")?.remoteStorageKey).toBe("opaque-1"); - }); - - it("leaves every other folder alone", async () => { - const { db, folders } = createFakeDb({ - folders: [hostFolderEntry(), localFolderEntry()], - }); - - await syncHostFolder(db); - - expect([...folders.keys()]).toEqual(["local-folder"]); - }); - - it("does nothing when there is no host folder to remove", async () => { - const { db, folders } = createFakeDb({ folders: [localFolderEntry()] }); - - await syncHostFolder(db); - - expect([...folders.keys()]).toEqual(["local-folder"]); - }); -}); diff --git a/src/services/pipelineStorage/host/hostFolder.ts b/src/services/pipelineStorage/host/hostFolder.ts deleted file mode 100644 index c202279a0a..0000000000 --- a/src/services/pipelineStorage/host/hostFolder.ts +++ /dev/null @@ -1,56 +0,0 @@ -import type { PipelineStorageDb } from "../db"; -import { HOST_FOLDER_ID, ROOT_FOLDER_ID } from "../types"; -import { getPipelineStorageHost } from "./detectHost"; - -/** - * Brings the host folder into line with whether a host is actually on the page, - * in both directions. Adding it is only half the job: the folder outlives the - * page that installed it, and a row whose driver cannot be built takes the - * whole folder listing down with it, so a page loading without a host has to - * remove the folder rather than leave it behind. - */ -export async function syncHostFolder(db: PipelineStorageDb): Promise { - const host = getPipelineStorageHost(); - - if (!host) { - await removeHostFolder(db); - return; - } - - const existing = await db.folders.get(HOST_FOLDER_ID); - - if (!existing) { - await db.folders.add({ - id: HOST_FOLDER_ID, - name: host.label, - parentId: ROOT_FOLDER_ID, - driverConfig: { driverType: "host" }, - createdAt: Date.now(), - }); - return; - } - - if (existing.name !== host.label) { - await db.folders.update(HOST_FOLDER_ID, { name: host.label }); - } -} - -/** - * The registry rows go with it. They only ever recorded which local id stood - * for which remote file, so without the folder they describe nothing, and the - * pipelines themselves are untouched on the host. Rows elsewhere keep their - * `remoteStorageKey`, so a pipeline already copied to the host is not copied - * again when the host returns. - */ -async function removeHostFolder(db: PipelineStorageDb): Promise { - const existing = await db.folders.get(HOST_FOLDER_ID); - if (!existing) return; - - await db.transaction("rw", db.folders, db.pipeline_registry, async () => { - await db.pipeline_registry - .where("folderId") - .equals(HOST_FOLDER_ID) - .delete(); - await db.folders.delete(HOST_FOLDER_ID); - }); -} diff --git a/src/services/pipelineStorage/host/hostMirror.ts b/src/services/pipelineStorage/host/hostMirror.ts deleted file mode 100644 index 36b2a7c6da..0000000000 --- a/src/services/pipelineStorage/host/hostMirror.ts +++ /dev/null @@ -1,47 +0,0 @@ -import type { PipelineFile } from "../PipelineFile"; -import { - findById, - findByRemoteStorageKey, - updateEntry, -} from "../pipelineRegistry"; -import { HOST_DRIVER_TYPE } from "../types"; -import { getPipelineStorageHost } from "./detectHost"; - -/** - * Copies a locally-stored pipeline to the host the first time it is saved with - * a host present, so a person's existing work becomes reachable there without a - * bulk migration. The local copy is authoritative and is never removed. - */ -export async function mirrorWriteToHost( - file: PipelineFile, - content: string, -): Promise { - if (file.folder.driver.type === HOST_DRIVER_TYPE) return; - - const host = getPipelineStorageHost(); - if (!host) return; - - const entry = await findById(file.id); - if (!entry || entry.remoteStorageKey) return; - - const { HostStorageDriver } = await import("../drivers/HostStorageDriver"); - const remoteStorageKey = crypto.randomUUID(); - await new HostStorageDriver(host).write(remoteStorageKey, content); - await updateEntry(file.id, { remoteStorageKey }); -} - -/** - * A key the host has deleted must never be written to again: the host may - * revive the deleted record rather than mint a new one, and the pipeline would - * come back wearing the dead record's identity. Forgetting the key is what - * makes the next save create a fresh record instead. - */ -export async function clearMirrorsOfHostKey( - remoteStorageKey: string, -): Promise { - const mirrored = await findByRemoteStorageKey(remoteStorageKey); - - for (const entry of mirrored) { - await updateEntry(entry.id, { remoteStorageKey: undefined }); - } -} diff --git a/src/services/pipelineStorage/pipelineRegistry.ts b/src/services/pipelineStorage/pipelineRegistry.ts index 8257ac5f2e..6f06b06c62 100644 --- a/src/services/pipelineStorage/pipelineRegistry.ts +++ b/src/services/pipelineStorage/pipelineRegistry.ts @@ -31,15 +31,6 @@ export async function findByStorageKey( .first(); } -export async function findByRemoteStorageKey( - remoteStorageKey: string, -): Promise { - return pipelineStorageDb.pipeline_registry - .where("remoteStorageKey") - .equals(remoteStorageKey) - .toArray(); -} - export async function getAllByFolderId( folderId: string, ): Promise { diff --git a/src/services/pipelineStorage/storageMode.ts b/src/services/pipelineStorage/storageMode.ts new file mode 100644 index 0000000000..6d1e61136c --- /dev/null +++ b/src/services/pipelineStorage/storageMode.ts @@ -0,0 +1,39 @@ +import type { PipelineStorageHost } from "./host/contract"; +import { getPipelineStorageHost } from "./host/detectHost"; + +export type StorageMode = { kind: "local" } | { kind: "host"; label: string }; + +let resolved: StorageMode | undefined; +let resolvedHost: PipelineStorageHost | undefined; + +/** + * Decided once and then frozen for the life of the page, holding on to the host + * itself rather than re-reading the global. Detection is fail-open, so a host + * whose global is removed or whose getter starts throwing would otherwise read + * as "no host" and quietly send the next write to browser storage — the one + * outcome host mode exists to prevent. + */ +export function resolveStorageMode(): StorageMode { + if (!resolved) { + resolvedHost = getPipelineStorageHost(); + resolved = resolvedHost + ? { kind: "host", label: resolvedHost.label } + : { kind: "local" }; + } + + return resolved; +} + +export function getStorageHost(): PipelineStorageHost | undefined { + resolveStorageMode(); + return resolvedHost; +} + +export function isHostStorage(): boolean { + return resolveStorageMode().kind === "host"; +} + +export function resetStorageModeForTests(): void { + resolved = undefined; + resolvedHost = undefined; +} diff --git a/src/services/pipelineStorage/types.ts b/src/services/pipelineStorage/types.ts index 9f10c295f1..029f3290b0 100644 --- a/src/services/pipelineStorage/types.ts +++ b/src/services/pipelineStorage/types.ts @@ -5,7 +5,6 @@ import type { LocalFileSystemDriverConfig } from "./drivers/LocalFileSystemDrive import type { RootFolderDbDriverConfig } from "./drivers/RootFolderDbStorageDriver"; export const ROOT_FOLDER_ID = "__root__"; -export const HOST_FOLDER_ID = "__host__"; export const HOST_DRIVER_TYPE = "host"; export interface PipelineFileDescriptor { @@ -50,7 +49,6 @@ export interface PipelineRegistryEntry { storageKey: string; folderId: string; contentVersion?: string; - remoteStorageKey?: string; } export interface FolderEntry { diff --git a/src/utils/URL.test.ts b/src/utils/URL.test.ts index faf1aa67d1..2ff94f3847 100644 --- a/src/utils/URL.test.ts +++ b/src/utils/URL.test.ts @@ -12,10 +12,6 @@ import { toAbsoluteHttpUrl, } from "./URL"; -vi.mock("@/routes/router", () => ({ - RUNS_BASE_PATH: "/runs", -})); - // Kept ahead of the download tests, which delete `global.URL` in their teardown. describe("toAbsoluteHttpUrl", () => { it("accepts absolute http and https urls", () => { diff --git a/src/utils/URL.ts b/src/utils/URL.ts index 8602ab1be4..24e13a2d97 100644 --- a/src/utils/URL.ts +++ b/src/utils/URL.ts @@ -1,4 +1,4 @@ -import { RUNS_BASE_PATH } from "@/routes/router"; +import { RUNS_BASE_PATH } from "@/routes/appRoutes"; import { BASE_URL, IS_GITHUB_PAGES } from "@/utils/constants"; const convertGcsUrlToBrowserUrl = ( diff --git a/tests/e2e/fixtures/pipelineStorageHost.ts b/tests/e2e/fixtures/pipelineStorageHost.ts new file mode 100644 index 0000000000..54405820d8 --- /dev/null +++ b/tests/e2e/fixtures/pipelineStorageHost.ts @@ -0,0 +1,178 @@ +import type { Page } from "@playwright/test"; + +type HostFailMode = "none" | "unavailable" | "unauthenticated" | "rate_limited"; + +interface HostSeedPipeline { + key: string; + displayName: string; + spec: unknown; +} + +export interface HostStorageOptions { + label?: string; + seed?: HostSeedPipeline[]; + failMode?: HostFailMode; + latencyMs?: number; +} + +interface HostRecord { + key: string; + externalId: string; + displayName: string | null; + contentVersion: string; + spec: unknown; +} + +interface HostTestState { + records(): HostRecord[]; +} + +declare global { + interface Window { + __TANGLE_TEST_HOST__?: HostTestState; + } +} + +const DEFAULT_LABEL = "Shared storage"; + +/** + * Stands in for the page that embeds this app. It has to be installed with + * `addInitScript` rather than `evaluate`, because storage mode is decided while + * the app boots and never revisited. + */ +export async function installPipelineStorageHost( + page: Page, + options: HostStorageOptions = {}, +): Promise { + await page.addInitScript( + (config: Required) => { + const store = new Map(); + let revision = 0; + + for (const seeded of config.seed) { + revision += 1; + store.set(seeded.key, { + key: seeded.key, + externalId: `external-${revision}`, + displayName: seeded.displayName, + contentVersion: `v${revision}`, + spec: seeded.spec, + }); + } + + async function gate(): Promise { + if (config.latencyMs > 0) { + await new Promise((resolve) => setTimeout(resolve, config.latencyMs)); + } + + if (config.failMode !== "none") { + throw Object.assign(new Error(`host is ${config.failMode}`), { + code: config.failMode, + }); + } + } + + function summaryOf(record: HostRecord) { + const { spec: _spec, ...summary } = record; + return summary; + } + + function nameOf(spec: unknown): string | null { + if (typeof spec !== "object" || spec === null) return null; + const name: unknown = Reflect.get(spec, "name"); + return typeof name === "string" ? name : null; + } + + window.__TANGLE_PIPELINE_STORAGE_HOST__ = { + version: 1, + label: config.label, + async list() { + await gate(); + return [...store.values()].map(summaryOf); + }, + async read(key: string) { + await gate(); + const found = store.get(key); + if (!found) { + throw Object.assign(new Error(`no pipeline for ${key}`), { + code: "not_found", + }); + } + return found; + }, + async write(key: string, spec: unknown) { + await gate(); + const existing = store.get(key); + revision += 1; + const record: HostRecord = { + key, + externalId: existing?.externalId ?? `external-${revision}`, + displayName: nameOf(spec), + contentVersion: `v${revision}`, + spec, + }; + store.set(key, record); + return summaryOf(record); + }, + async delete(key: string) { + await gate(); + store.delete(key); + }, + async has(key: string) { + await gate(); + return store.has(key); + }, + }; + + window.__TANGLE_TEST_HOST__ = { + records: () => [...store.values()], + }; + }, + { + label: options.label ?? DEFAULT_LABEL, + seed: options.seed ?? [], + failMode: options.failMode ?? "none", + latencyMs: options.latencyMs ?? 0, + } satisfies Required, + ); +} + +export async function readHostRecords(page: Page): Promise { + return page.evaluate(() => window.__TANGLE_TEST_HOST__?.records() ?? []); +} + +/** + * The negative assertion host mode exists for: with a host present, nothing may + * reach the browser's own pipeline store, whatever the host does. + */ +export async function readLocallyStoredPipelineKeys( + page: Page, +): Promise { + return page.evaluate(async () => { + const database = await new Promise((resolve) => { + const request = indexedDB.open("components"); + request.onsuccess = () => resolve(request.result); + request.onerror = () => resolve(undefined); + }); + + if (!database) return []; + + const storeName = "file_store_user_pipelines"; + if (!database.objectStoreNames.contains(storeName)) { + database.close(); + return []; + } + + const keys = await new Promise((resolve) => { + const request = database + .transaction(storeName, "readonly") + .objectStore(storeName) + .getAllKeys(); + request.onsuccess = () => resolve(request.result.map(String)); + request.onerror = () => resolve([]); + }); + + database.close(); + return keys; + }); +} diff --git a/tests/e2e/pipeline-storage-host.spec.ts b/tests/e2e/pipeline-storage-host.spec.ts new file mode 100644 index 0000000000..c0c34084fd --- /dev/null +++ b/tests/e2e/pipeline-storage-host.spec.ts @@ -0,0 +1,66 @@ +import { expect, test } from "@playwright/test"; + +import { + type HostStorageOptions, + installPipelineStorageHost, + readHostRecords, + readLocallyStoredPipelineKeys, +} from "./fixtures/pipelineStorageHost"; + +const LABEL = "Shared storage"; + +const SEED = [ + { + key: "0f8c1a2b-0000-4000-8000-000000000001", + displayName: "Churn model", + spec: { name: "Churn model", implementation: { graph: { tasks: {} } } }, + }, + { + key: "0f8c1a2b-0000-4000-8000-000000000002", + displayName: "Nightly refresh", + spec: { name: "Nightly refresh", implementation: { graph: { tasks: {} } } }, + }, +]; + +async function installSeededHost( + page: Parameters[0], + options: HostStorageOptions = {}, +) { + await installPipelineStorageHost(page, { + label: LABEL, + seed: SEED, + ...options, + }); +} + +test.describe("host-provided pipeline storage", () => { + test("lists what the host holds", async ({ page }) => { + await installSeededHost(page); + + await page.goto("/pipeline-folders"); + + await expect(page.getByText("Churn model")).toBeVisible(); + await expect(page.getByText("Nightly refresh")).toBeVisible(); + }); + + test("keeps the browser's own pipeline store empty", async ({ page }) => { + await installSeededHost(page); + + await page.goto("/pipeline-folders"); + await expect(page.getByText("Churn model")).toBeVisible(); + + expect(await readLocallyStoredPipelineKeys(page)).toEqual([]); + expect(await readHostRecords(page)).toHaveLength(SEED.length); + }); + + test("writes nothing locally when the host cannot be reached", async ({ + page, + }) => { + await installSeededHost(page, { failMode: "unavailable" }); + + await page.goto("/pipeline-folders"); + + await expect(page.getByText("Churn model")).toBeHidden(); + expect(await readLocallyStoredPipelineKeys(page)).toEqual([]); + }); +}); From 58c4df5c97a0bb9ca6efe86f266ac8f8143d16e2 Mon Sep 17 00:00:00 2001 From: Camiel van Schoonhoven Date: Tue, 8 Sep 2026 15:06:42 -0700 Subject: [PATCH 09/36] refactor(pipeline-storage): resolve a route reference in one place Three call sites each turned a `PipelineRef` into a file their own way, and none of them agreed: one fell back from a missing id to the name and opened whatever happened to share it, one resolved the file a second time inside a floating promise, and one gave up silently. With opaque storage keys the name is a cosmetic slug, so guessing from it opens the wrong pipeline and then writes to it. `PipelineStorageService.resolve(ref)` is now the only way in. An id that cannot be found is an error rather than an invitation to fall back; a name resolves against the authoritative listing, and a name shared by two pipelines is refused instead of guessed at. A miss no longer reaches for browser storage. `useLoadSpec` returns the resolved file, so `useSpecLifecycle` initialises synchronously and a resolve failure surfaces as a suspense error instead of a null file. Autosave with no open file now throws rather than reporting success and advancing its saved marker. Co-Authored-By: Claude Opus 5 (1M context) --- .../TourPipelineStorageService.ts | 1 + src/routes/v2/pages/Editor/EditorV2.tsx | 4 +- .../components/DriverPermissionGate.tsx | 9 +- .../v2/pages/Editor/hooks/useLoadSpec.ts | 47 ++------ .../v2/pages/Editor/hooks/useSpecLifecycle.ts | 31 ++--- .../pages/Editor/store/autoSaveStore.test.ts | 68 +++++++++++ .../v2/pages/Editor/store/autoSaveStore.ts | 7 +- .../PipelineStorageService.test.ts | 106 +++++++++++++++++- .../pipelineStorage/PipelineStorageService.ts | 101 +++++++++++++++-- 9 files changed, 291 insertions(+), 83 deletions(-) create mode 100644 src/routes/v2/pages/Editor/store/autoSaveStore.test.ts diff --git a/src/providers/TourProvider/tourPipelineStorage/TourPipelineStorageService.ts b/src/providers/TourProvider/tourPipelineStorage/TourPipelineStorageService.ts index deb29e4ebd..2c9acf6dc9 100644 --- a/src/providers/TourProvider/tourPipelineStorage/TourPipelineStorageService.ts +++ b/src/providers/TourProvider/tourPipelineStorage/TourPipelineStorageService.ts @@ -14,6 +14,7 @@ export class TourPipelineStorageService extends PipelineStorageService { name: "Tour", parentId: null, driver: new SessionStoragePipelineDriver(), + isFlat: true, }); } diff --git a/src/routes/v2/pages/Editor/EditorV2.tsx b/src/routes/v2/pages/Editor/EditorV2.tsx index af7ee78d24..9f402a55d0 100644 --- a/src/routes/v2/pages/Editor/EditorV2.tsx +++ b/src/routes/v2/pages/Editor/EditorV2.tsx @@ -74,14 +74,14 @@ const PipelineEditorSkeleton = () => ( const PipelineEditor = withSuspenseWrapper( observer(({ pipelineRef }: PipelineEditorProps) => { const { - data: { spec: rootSpec, restoredUndoStore }, + data: { spec: rootSpec, file: pipelineFile, restoredUndoStore }, } = useLoadSpec(pipelineRef); const { navigation } = useSharedStores(); const tourMode = useTourMode(); useWindowPersistence(tourMode ? TOUR_WINDOW_LAYOUT_ID : "editor"); useDockAreaAccordion(); - useSpecLifecycle(rootSpec, pipelineRef, restoredUndoStore); + useSpecLifecycle(rootSpec, pipelineRef, pipelineFile, restoredUndoStore); useSelectionWindowSync(); usePropertiesWindowPositioning(); useLinkedWindowCleanup(); diff --git a/src/routes/v2/pages/Editor/components/DriverPermissionGate.tsx b/src/routes/v2/pages/Editor/components/DriverPermissionGate.tsx index f738eca3bb..a23d32657d 100644 --- a/src/routes/v2/pages/Editor/components/DriverPermissionGate.tsx +++ b/src/routes/v2/pages/Editor/components/DriverPermissionGate.tsx @@ -19,15 +19,16 @@ interface DriverPermissionGateProps { children: ReactNode; } +/** + * A folder that cannot be resolved is not a permission problem, so the gate + * opens and lets whatever loads the pipeline report the real failure. + */ async function resolveFolder( ref: PipelineRef, storage: PipelineStorageService, ): Promise { try { - const file = ref.fileId - ? await storage.findPipelineById(ref.fileId) - : undefined; - return file?.folder ?? null; + return (await storage.resolve(ref)).folder; } catch { return null; } diff --git a/src/routes/v2/pages/Editor/hooks/useLoadSpec.ts b/src/routes/v2/pages/Editor/hooks/useLoadSpec.ts index e8a44858e9..3fb03661d3 100644 --- a/src/routes/v2/pages/Editor/hooks/useLoadSpec.ts +++ b/src/routes/v2/pages/Editor/hooks/useLoadSpec.ts @@ -18,19 +18,18 @@ import { createUndoStoreWithEvents, loadUndoHistory, } from "@/routes/v2/pages/Editor/utils/undoHistoryStorage"; -import { RootFolderDbStorageDriver } from "@/services/pipelineStorage/drivers/RootFolderDbStorageDriver"; import type { PipelineFile } from "@/services/pipelineStorage/PipelineFile"; import { getLastForeignWriteTime, subscribePipelineFileChanged, } from "@/services/pipelineStorage/pipelineFileEvents"; import { usePipelineStorage } from "@/services/pipelineStorage/PipelineStorageProvider"; -import type { PipelineStorageService } from "@/services/pipelineStorage/PipelineStorageService"; import type { PipelineRef } from "@/services/pipelineStorage/types"; import { PIPELINE_YAML_LOAD_OPTIONS } from "@/utils/yaml"; interface LoadedSpec { spec: ComponentSpec; + file: PipelineFile; restoredUndoStore?: MobxUndoStore; } @@ -42,37 +41,6 @@ function deserializeSpec(data: unknown, idGen?: IdGenerator): ComponentSpec { return spec; } -async function backfillFromLegacyStore( - name: string, - storage: PipelineStorageService, -): Promise { - const legacyDriver = new RootFolderDbStorageDriver(); - - const existsInLegacy = await legacyDriver.hasKey(name); - if (!existsInLegacy) return undefined; - - return storage.rootFolder.assignFile(name); -} - -async function resolvePipelineFile( - ref: PipelineRef, - storage: PipelineStorageService, -): Promise { - let pipelineFile = ref.fileId - ? await storage.findPipelineById(ref.fileId) - : await storage.resolvePipelineByName(ref.name); - - if (!pipelineFile) { - pipelineFile = await backfillFromLegacyStore(ref.name, storage); - } - - if (!pipelineFile) { - throw new Error(`Pipeline "${ref.name}" not found`); - } - - return pipelineFile; -} - export const EDITOR_SPEC_QUERY_KEY = "editor-v2-spec"; export function useLoadSpec(ref: PipelineRef) { @@ -115,12 +83,15 @@ export function useLoadSpec(ref: PipelineRef) { return useSuspenseQuery({ queryKey, queryFn: async (): Promise => { - const filePromise = resolvePipelineFile(ref, storage); + const filePromise = storage.resolve(ref); - const [specData, undoHistory] = await Promise.all([ + const [{ file, specData }, undoHistory] = await Promise.all([ filePromise.then(async (file) => { loadedStorageKey.current = file.storageKey; - return yaml.load(await file.read(), PIPELINE_YAML_LOAD_OPTIONS); + return { + file, + specData: yaml.load(await file.read(), PIPELINE_YAML_LOAD_OPTIONS), + }; }), loadUndoHistory(ref.name).catch(() => null), ]); @@ -128,7 +99,7 @@ export function useLoadSpec(ref: PipelineRef) { const loadedSpec = deserializeSpecData(specData, undoHistory); await hydrateLoadedSpecRefs(loadedSpec.spec); - return loadedSpec; + return { ...loadedSpec, file }; }, staleTime: Infinity, retry: false, @@ -138,7 +109,7 @@ export function useLoadSpec(ref: PipelineRef) { function deserializeSpecData( specData: unknown, undoHistory: Awaited> | null, -): LoadedSpec { +): Omit { if (undoHistory) { try { const replayIdGen = new ReplayIdGenerator(undoHistory.idStack); diff --git a/src/routes/v2/pages/Editor/hooks/useSpecLifecycle.ts b/src/routes/v2/pages/Editor/hooks/useSpecLifecycle.ts index e9ddb7b007..6ba0fcb2d2 100644 --- a/src/routes/v2/pages/Editor/hooks/useSpecLifecycle.ts +++ b/src/routes/v2/pages/Editor/hooks/useSpecLifecycle.ts @@ -6,26 +6,13 @@ import { useEffect, useRef } from "react"; import type { ComponentSpec } from "@/models/componentSpec"; import { useEditorSession } from "@/routes/v2/pages/Editor/store/EditorSessionContext"; import { useSharedStores } from "@/routes/v2/shared/store/SharedStoreContext"; -import { usePipelineStorage } from "@/services/pipelineStorage/PipelineStorageProvider"; -import type { PipelineStorageService } from "@/services/pipelineStorage/PipelineStorageService"; +import type { PipelineFile } from "@/services/pipelineStorage/PipelineFile"; import type { PipelineRef } from "@/services/pipelineStorage/types"; -/** - * todo: make public and export to re-use - */ -async function resolvePipelineFile( - ref: PipelineRef, - storage: PipelineStorageService, -) { - if (ref.fileId) { - return storage.findPipelineById(ref.fileId); - } - return storage.resolvePipelineByName(ref.name); -} - export function useSpecLifecycle( rootSpec: ComponentSpec, pipelineRef: PipelineRef, + pipelineFile: PipelineFile, restoredUndoStore?: MobxUndoStore, ) { const { editor, navigation, windows: windowStore } = useSharedStores(); @@ -34,7 +21,6 @@ export function useSpecLifecycle( autoSave, pipelineFile: pipelineFileStore, } = useEditorSession(); - const storage = usePipelineStorage(); const prevTaskEntityIdsRef = useRef>(new Set()); useEffect(() => { @@ -46,13 +32,10 @@ export function useSpecLifecycle( const saveName = pipelineRef.name ?? rootSpec.name; - void (async () => { - if (saveName) { - const file = await resolvePipelineFile(pipelineRef, storage); - pipelineFileStore.init(file ?? null); - autoSave.init(rootSpec, saveName); - } - })(); + if (saveName) { + pipelineFileStore.init(pipelineFile); + autoSave.init(rootSpec, saveName); + } prevTaskEntityIdsRef.current = new Set(rootSpec.tasks.map((t) => t.$id)); @@ -95,6 +78,7 @@ export function useSpecLifecycle( }, [ rootSpec, pipelineRef, + pipelineFile, restoredUndoStore, editor, navigation, @@ -102,6 +86,5 @@ export function useSpecLifecycle( undo, autoSave, pipelineFileStore, - storage, ]); } diff --git a/src/routes/v2/pages/Editor/store/autoSaveStore.test.ts b/src/routes/v2/pages/Editor/store/autoSaveStore.test.ts new file mode 100644 index 0000000000..14ecf8d86c --- /dev/null +++ b/src/routes/v2/pages/Editor/store/autoSaveStore.test.ts @@ -0,0 +1,68 @@ +import { describe, expect, it, vi } from "vitest"; + +import type { ComponentSpec } from "@/models/componentSpec"; +import type { PipelineFile } from "@/services/pipelineStorage/PipelineFile"; + +import { AutoSaveStore } from "./autoSaveStore"; +import { PipelineFileStore } from "./pipelineFileStore"; +import type { UndoStore } from "./undoStore"; + +vi.mock("@/models/componentSpec", () => ({ + collectIdStack: () => [], + serializePipelineDocumentToText: (spec: { name: string }) => + `name: ${spec.name}`, +})); + +vi.mock("@/routes/v2/pages/Editor/utils/undoHistoryStorage", () => ({ + saveUndoHistory: vi.fn(async () => undefined), +})); + +function createStore(file: PipelineFile | null) { + const fileStore = new PipelineFileStore(); + fileStore.init(file); + + const undoStore = { undoManager: null } as unknown as UndoStore; + + return new AutoSaveStore(undoStore, fileStore); +} + +function createSpec(name: string): ComponentSpec { + return { name } as unknown as ComponentSpec; +} + +describe("AutoSaveStore.save", () => { + it("writes the serialized pipeline to the open file", async () => { + const write = vi.fn(async () => undefined); + const store = createStore({ write } as unknown as PipelineFile); + + store.init(createSpec("Churn model"), "Churn model"); + await store.save(); + + expect(write).toHaveBeenCalledWith("name: Churn model"); + expect(store.saveError).toBeNull(); + expect(store.lastSavedAt).toBeInstanceOf(Date); + }); + + it("reports a failure when there is no file to write to", async () => { + const store = createStore(null); + + store.init(createSpec("Churn model"), "Churn model"); + await store.save(); + + expect(store.saveError).not.toBeNull(); + expect(store.lastSavedAt).toBeNull(); + }); + + it("surfaces the reason the write was rejected", async () => { + const store = createStore({ + write: async () => { + throw new Error("Shared storage could not be reached."); + }, + } as unknown as PipelineFile); + + store.init(createSpec("Churn model"), "Churn model"); + await store.save(); + + expect(store.saveError).toContain("could not be reached"); + }); +}); diff --git a/src/routes/v2/pages/Editor/store/autoSaveStore.ts b/src/routes/v2/pages/Editor/store/autoSaveStore.ts index 23c8d93926..703cadbdcc 100644 --- a/src/routes/v2/pages/Editor/store/autoSaveStore.ts +++ b/src/routes/v2/pages/Editor/store/autoSaveStore.ts @@ -117,7 +117,12 @@ export class AutoSaveStore { const savePromise = (async () => { try { - await this.pipelineFileStore.activePipelineFile?.write(yamlText); + const file = this.pipelineFileStore.activePipelineFile; + if (!file) { + throw new Error(`No open file to save "${pipelineName}" to.`); + } + + await file.write(yamlText); await this.persistUndoHistory(); this.lastSavedYaml = yamlText; return new Date(); diff --git a/src/services/pipelineStorage/PipelineStorageService.test.ts b/src/services/pipelineStorage/PipelineStorageService.test.ts index 1f59e69a10..615e3279a3 100644 --- a/src/services/pipelineStorage/PipelineStorageService.test.ts +++ b/src/services/pipelineStorage/PipelineStorageService.test.ts @@ -1,9 +1,38 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import type { PipelineStorageHost } from "./host/contract"; -import { PipelineStorageService } from "./PipelineStorageService"; +import type { HostPipelineSummary, PipelineStorageHost } from "./host/contract"; +import { + AmbiguousPipelineNameError, + PipelineNotFoundError, + PipelineStorageService, +} from "./PipelineStorageService"; import { resetStorageModeForTests } from "./storageMode"; -import { HOST_DRIVER_TYPE, ROOT_FOLDER_ID } from "./types"; +import { + HOST_DRIVER_TYPE, + type PipelineRegistryEntry, + ROOT_FOLDER_ID, +} from "./types"; + +const registry = new Map(); + +vi.mock("./pipelineRegistry", () => ({ + addEntry: async (entry: PipelineRegistryEntry) => { + registry.set(entry.id, entry); + }, + updateEntry: async (id: string, updates: Partial) => { + const entry = registry.get(id); + if (entry) registry.set(id, { ...entry, ...updates }); + }, + deleteEntry: async (id: string) => { + registry.delete(id); + }, + findById: async (id: string) => registry.get(id), + findByStorageKey: async (storageKey: string) => + [...registry.values()].find((entry) => entry.storageKey === storageKey), + getAllByFolderId: async (folderId: string) => + [...registry.values()].filter((entry) => entry.folderId === folderId), + assertStorageKeyUnique: async () => undefined, +})); vi.mock("./db", () => ({ pipelineStorageDb: { @@ -40,11 +69,20 @@ vi.mock("./db", () => ({ const LABEL = "Shared storage"; -function installHost(): void { +function summary(key: string, displayName: string): HostPipelineSummary { + return { + key, + externalId: `id-${key}`, + displayName, + contentVersion: "1", + }; +} + +function installHost(listing: HostPipelineSummary[] = []): void { const host: PipelineStorageHost = { version: 1, label: LABEL, - list: async () => [], + list: async () => listing, read: async () => { throw new Error("not seeded"); }, @@ -52,7 +90,7 @@ function installHost(): void { throw new Error("not seeded"); }, delete: async () => undefined, - has: async () => false, + has: async (key) => listing.some((entry) => entry.key === key), }; Object.defineProperty(window, "__TANGLE_PIPELINE_STORAGE_HOST__", { @@ -63,6 +101,7 @@ function installHost(): void { } beforeEach(() => { + registry.clear(); resetStorageModeForTests(); }); @@ -131,3 +170,58 @@ describe("with a host on the page", () => { expect(new PipelineStorageService().mode).toEqual(service.mode); }); }); + +describe("resolving a route reference against a host", () => { + it("opens the pipeline whose key the route carries", async () => { + installHost([summary("opaque-key-1", "Churn model")]); + + const file = await new PipelineStorageService().resolve({ + name: "opaque-key-1", + }); + + expect(file.storageKey).toBe("opaque-key-1"); + }); + + it("opens a pipeline by its displayed name when only one has it", async () => { + installHost([ + summary("opaque-key-1", "Churn model"), + summary("opaque-key-2", "Ranking model"), + ]); + + const file = await new PipelineStorageService().resolve({ + name: "Ranking model", + }); + + expect(file.storageKey).toBe("opaque-key-2"); + }); + + it("refuses to guess between pipelines sharing a name", async () => { + installHost([ + summary("opaque-key-1", "Churn model"), + summary("opaque-key-2", "Churn model"), + ]); + + await expect( + new PipelineStorageService().resolve({ name: "Churn model" }), + ).rejects.toThrow(AmbiguousPipelineNameError); + }); + + it("reports a missing pipeline rather than reaching for browser storage", async () => { + installHost([summary("opaque-key-1", "Churn model")]); + + await expect( + new PipelineStorageService().resolve({ name: "Churn model v2" }), + ).rejects.toThrow(PipelineNotFoundError); + }); + + it("finds a pipeline the registry has never seen by its id", async () => { + installHost([summary("opaque-key-1", "Churn model")]); + + const file = await new PipelineStorageService().resolve({ + name: "whatever-the-link-said", + fileId: "id-opaque-key-1", + }); + + expect(file.storageKey).toBe("opaque-key-1"); + }); +}); diff --git a/src/services/pipelineStorage/PipelineStorageService.ts b/src/services/pipelineStorage/PipelineStorageService.ts index 2726faf4d9..3953568801 100644 --- a/src/services/pipelineStorage/PipelineStorageService.ts +++ b/src/services/pipelineStorage/PipelineStorageService.ts @@ -2,11 +2,20 @@ import { makeObservable, observable } from "mobx"; import { createDriver } from "./createDriver"; import { pipelineStorageDb } from "./db"; +import { RootFolderDbStorageDriver } from "./drivers/RootFolderDbStorageDriver"; import { PipelineFile } from "./PipelineFile"; import { PipelineFolder } from "./PipelineFolder"; import { findById, findByStorageKey } from "./pipelineRegistry"; import { resolveStorageMode, type StorageMode } from "./storageMode"; -import { ROOT_FOLDER_ID } from "./types"; +import { type PipelineRef, ROOT_FOLDER_ID } from "./types"; + +export class PipelineNotFoundError extends Error { + readonly name = "PipelineNotFoundError"; +} + +export class AmbiguousPipelineNameError extends Error { + readonly name = "AmbiguousPipelineNameError"; +} export class PipelineStorageService { @observable accessor rootFolder: PipelineFolder; @@ -19,20 +28,55 @@ export class PipelineStorageService { makeObservable(this); } + /** + * The one way to turn a route's reference into a file. `fileId` is the real + * identity, so a reference carrying one that cannot be found is an error + * rather than an invitation to fall back to the name and open whatever + * happens to share it. + */ + async resolve(ref: PipelineRef): Promise { + if (ref.fileId) return this.findPipelineById(ref.fileId); + + const byName = await this.resolvePipelineByName(ref.name); + if (byName) return byName; + + const adopted = await this.adoptFromLegacyStore(ref.name); + if (adopted) return adopted; + + throw new PipelineNotFoundError(`Pipeline "${ref.name}" not found`); + } + async findPipelineById(id: string): Promise { const entry = await findById(id); - if (!entry) { - throw new Error(`Pipeline not found: ${id}`); + + if (entry) { + return new PipelineFile({ + id: entry.id, + storageKey: entry.storageKey, + folder: await this.findFolderById(entry.folderId), + }); } - return new PipelineFile({ - id: entry.id, - storageKey: entry.storageKey, - folder: await this.findFolderById(entry.folderId), - }); + /** + * A registry row caches what the store itself reported, so a miss means + * "not seen on this device yet" rather than "does not exist" — a shared + * link opened in a fresh browser lands here. + */ + const listed = await this.rootFolder.listPipelines(); + const found = listed.find((file) => file.id === id); + + if (!found) { + throw new PipelineNotFoundError(`Pipeline not found: ${id}`); + } + + return found; } async resolvePipelineByName(name: string): Promise { + if (this.rootFolder.isFlat) { + return resolveInFlatStore(this.rootFolder, name); + } + const existing = await findByStorageKey(name); if (!existing) return this.rootFolder.findFile(name); @@ -42,6 +86,22 @@ export class PipelineStorageService { return folder.findFile(name); } + /** + * Pipelines predating the registry live in the legacy list under their name + * and have no row pointing at them, so opening one by name has to claim it. + * A store with opaque keys has no such history and no such names. + */ + private async adoptFromLegacyStore( + name: string, + ): Promise { + if (this.rootFolder.isFlat) return undefined; + + const exists = await new RootFolderDbStorageDriver().hasKey(name); + if (!exists) return undefined; + + return this.rootFolder.assignFile(name); + } + async findFolderById(id: string): Promise { if (id === ROOT_FOLDER_ID) { return this.rootFolder; @@ -75,6 +135,31 @@ export class PipelineStorageService { } } +/** + * Opaque keys mean the route's slug may be either the key or the displayed + * name, and one listing answers both. Two pipelines may legitimately share a + * name, so an ambiguous slug is refused rather than guessed at. + */ +async function resolveInFlatStore( + folder: PipelineFolder, + name: string, +): Promise { + const files = await folder.listPipelines(); + + const byKey = files.find((file) => file.storageKey === name); + if (byKey) return byKey; + + const byName = files.filter((file) => file.displayName === name); + + if (byName.length > 1) { + throw new AmbiguousPipelineNameError( + `More than one pipeline is called "${name}". Open it from the pipeline list instead.`, + ); + } + + return byName[0]; +} + function createRoot(mode: StorageMode): PipelineFolder { if (mode.kind === "host") { return new PipelineFolder({ From e8ffacae38547beee4fb71bd3d22e6806c20cd72 Mon Sep 17 00:00:00 2001 From: Camiel van Schoonhoven Date: Tue, 8 Sep 2026 15:18:36 -0700 Subject: [PATCH 10/36] refactor(routing): carry a pipeline's identity into the editor route MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `getDefaultEditorPath(name)` could only produce a path, so every caller handed the editor a display name and nothing else. With opaque storage keys a name identifies nothing: two pipelines may share one, and the editor has to guess. `getDefaultEditorTarget(ref)` returns `{to, params, search}`, putting the file id in the search params where it survives navigation; the `…Href` variant covers the two `window.open` sites and the places that need a plain string. Callers pass a ref, so a call site that learns the id later only has to start supplying it. Two places dropped the id even when it was in the URL: switching editor versions rebuilt the path from scratch, and the v2-to-v1 redirect discarded search entirely. Both now carry it through. Undo history was keyed on the pipeline name in a Dexie primary key, where two pipelines sharing a name replay each other's edits. It is keyed on the file id now; the old table is dropped rather than migrated, because the events only mean anything against the spec ids they were recorded from. Co-Authored-By: Claude Opus 5 (1M context) --- .../Home/PipelineSection/PipelineRow.tsx | 9 +++-- src/components/Learn/useImportPipeline.ts | 6 ++-- src/components/PipelineRun/RunToolbar.tsx | 4 +-- .../components/InspectPipelineButton.test.tsx | 6 +++- .../components/InspectPipelineButton.tsx | 11 +++--- src/components/shared/EditorVersionToggle.tsx | 1 + src/components/shared/ImportPipeline.tsx | 6 ++-- src/components/shared/NewPipelineButton.tsx | 11 +++--- src/components/shared/VersionToggle.tsx | 8 ++++- src/routes/Dashboard/TypePill.tsx | 6 ++-- src/routes/Import/Import.test.tsx | 4 ++- src/routes/Import/index.tsx | 6 ++-- src/routes/editorRoutes.ts | 34 +++++++++++++++---- src/routes/router.ts | 5 +-- .../v2/pages/Editor/hooks/useLoadSpec.ts | 20 +++++------ .../v2/pages/Editor/store/autoSaveStore.ts | 5 +-- .../pages/Editor/utils/undoHistoryStorage.ts | 25 ++++++++++---- .../pages/RunView/hooks/useRunViewActions.ts | 4 +-- src/services/pipelineRunService.ts | 4 +-- 19 files changed, 107 insertions(+), 68 deletions(-) diff --git a/src/components/Home/PipelineSection/PipelineRow.tsx b/src/components/Home/PipelineSection/PipelineRow.tsx index 00ea4c9f18..fb11e0c92c 100644 --- a/src/components/Home/PipelineSection/PipelineRow.tsx +++ b/src/components/Home/PipelineSection/PipelineRow.tsx @@ -30,7 +30,10 @@ import { import { Paragraph, Text } from "@/components/ui/typography"; import { cn } from "@/lib/utils"; import { useAnalytics } from "@/providers/AnalyticsProvider"; -import { getDefaultEditorPath } from "@/routes/editorRoutes"; +import { + getDefaultEditorHref, + getDefaultEditorTarget, +} from "@/routes/editorRoutes"; import { deletePipeline } from "@/services/pipelineService"; import { getPipelineTagsFromSpec } from "@/utils/annotations"; import type { ComponentReferenceWithSpec } from "@/utils/componentStore"; @@ -111,11 +114,11 @@ const PipelineRow = withSuspenseWrapper( if (e.ctrlKey || e.metaKey) { rowTrack("pipeline_opened", { open_mode: "editor_new_tab" }); - window.open(getDefaultEditorPath(name), "_blank"); + window.open(getDefaultEditorHref({ name }), "_blank"); return; } rowTrack("pipeline_opened", { open_mode: "editor_same_tab" }); - navigate({ to: getDefaultEditorPath(name) }); + navigate(getDefaultEditorTarget({ name })); }; const handleCheckboxChange = (checked: boolean | "indeterminate") => { diff --git a/src/components/Learn/useImportPipeline.ts b/src/components/Learn/useImportPipeline.ts index a436a591ad..5b11aab3a8 100644 --- a/src/components/Learn/useImportPipeline.ts +++ b/src/components/Learn/useImportPipeline.ts @@ -2,7 +2,7 @@ import { useMutation } from "@tanstack/react-query"; import { useNavigate } from "@tanstack/react-router"; import useToastNotification from "@/hooks/useToastNotification"; -import { getDefaultEditorPath } from "@/routes/editorRoutes"; +import { getDefaultEditorTarget } from "@/routes/editorRoutes"; import { importPipelineFromUrl } from "./importPipelineFromUrl"; @@ -14,9 +14,7 @@ export function useImportPipeline() { mutationFn: async (url: string) => await importPipelineFromUrl(url), onSuccess: (result) => { notify(`Pipeline "${result.name}" created successfully`, "success"); - navigate({ - to: getDefaultEditorPath(result.name), - }); + navigate(getDefaultEditorTarget({ name: result.name })); }, }); } diff --git a/src/components/PipelineRun/RunToolbar.tsx b/src/components/PipelineRun/RunToolbar.tsx index ef22656b07..10c77091e0 100644 --- a/src/components/PipelineRun/RunToolbar.tsx +++ b/src/components/PipelineRun/RunToolbar.tsx @@ -5,7 +5,7 @@ import { useUserDetails } from "@/hooks/useUserDetails"; import { cn } from "@/lib/utils"; import { useComponentSpec } from "@/providers/ComponentSpecProvider"; import { useExecutionData } from "@/providers/ExecutionDataProvider"; -import { getDefaultEditorPath } from "@/routes/editorRoutes"; +import { getDefaultEditorHref } from "@/routes/editorRoutes"; import { extractCanonicalName } from "@/utils/canonicalPipelineName"; import { countInProgressFromStats, @@ -31,7 +31,7 @@ export const RunToolbar = () => { const { data: currentUserDetails } = useUserDetails(); const editorRoute = componentSpec.name - ? getDefaultEditorPath(componentSpec.name) + ? getDefaultEditorHref({ name: componentSpec.name }) : ""; const canAccessEditorSpec = useCheckComponentSpecFromPath( diff --git a/src/components/PipelineRun/components/InspectPipelineButton.test.tsx b/src/components/PipelineRun/components/InspectPipelineButton.test.tsx index 35a7717f9c..9e6ca046f0 100644 --- a/src/components/PipelineRun/components/InspectPipelineButton.test.tsx +++ b/src/components/PipelineRun/components/InspectPipelineButton.test.tsx @@ -16,6 +16,10 @@ describe("", () => { render(); const inspectButton = screen.getByTestId("inspect-pipeline-button"); act(() => fireEvent.click(inspectButton)); - expect(mockNavigate).toHaveBeenCalledWith({ to: "/editor-v2/foo" }); + expect(mockNavigate).toHaveBeenCalledWith({ + to: "/editor-v2/$pipelineName", + params: { pipelineName: "foo" }, + search: {}, + }); }); }); diff --git a/src/components/PipelineRun/components/InspectPipelineButton.tsx b/src/components/PipelineRun/components/InspectPipelineButton.tsx index e37b1d20d1..302c164b52 100644 --- a/src/components/PipelineRun/components/InspectPipelineButton.tsx +++ b/src/components/PipelineRun/components/InspectPipelineButton.tsx @@ -7,7 +7,10 @@ import { import TooltipButton from "@/components/shared/Buttons/TooltipButton"; import { Icon } from "@/components/ui/icon"; -import { getDefaultEditorPath } from "@/routes/editorRoutes"; +import { + getDefaultEditorHref, + getDefaultEditorTarget, +} from "@/routes/editorRoutes"; type InspectPipelineButtonProps = { pipelineName: string; @@ -30,14 +33,12 @@ export const InspectPipelineButton = ({ const handleInspect = useCallback( (e: MouseEvent) => { - const clickThroughUrl = getDefaultEditorPath(pipelineName); - if (e.ctrlKey || e.metaKey) { - window.open(clickThroughUrl, "_blank"); + window.open(getDefaultEditorHref({ name: pipelineName }), "_blank"); return; } - navigate({ to: clickThroughUrl }); + navigate(getDefaultEditorTarget({ name: pipelineName })); }, [navigate, pipelineName], ); diff --git a/src/components/shared/EditorVersionToggle.tsx b/src/components/shared/EditorVersionToggle.tsx index a9630b0224..bb4434b6de 100644 --- a/src/components/shared/EditorVersionToggle.tsx +++ b/src/components/shared/EditorVersionToggle.tsx @@ -43,6 +43,7 @@ export const EditorVersionToggle = ({ flagName="v2_editor" targetVersion={targetVersion} targetPath={targetPath} + preserveSearch tooltip={tooltip} showWelcomeSpotlight={showWelcomeSpotlight && version === "v2"} welcome={{ diff --git a/src/components/shared/ImportPipeline.tsx b/src/components/shared/ImportPipeline.tsx index 1eaf6ac373..2d968ad264 100644 --- a/src/components/shared/ImportPipeline.tsx +++ b/src/components/shared/ImportPipeline.tsx @@ -20,7 +20,7 @@ import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; import { Textarea } from "@/components/ui/textarea"; import { Heading, Paragraph } from "@/components/ui/typography"; import { useAnalytics } from "@/providers/AnalyticsProvider"; -import { getDefaultEditorPath } from "@/routes/editorRoutes"; +import { getDefaultEditorTarget } from "@/routes/editorRoutes"; import { importPipelineFromFile, importPipelineFromYaml, @@ -63,9 +63,7 @@ const ImportPipeline = ({ if (onImportComplete) { onImportComplete(importedPipeline); } else { - navigate({ - to: getDefaultEditorPath(importedPipeline.name), - }); + navigate(getDefaultEditorTarget({ name: importedPipeline.name })); } }; diff --git a/src/components/shared/NewPipelineButton.tsx b/src/components/shared/NewPipelineButton.tsx index 3721e60b24..4e2d04eeab 100644 --- a/src/components/shared/NewPipelineButton.tsx +++ b/src/components/shared/NewPipelineButton.tsx @@ -3,7 +3,10 @@ import { generate } from "random-words"; import type { MouseEvent, ReactNode } from "react"; import { Button, type ButtonProps } from "@/components/ui/button"; -import { getDefaultEditorPath } from "@/routes/editorRoutes"; +import { + getDefaultEditorHref, + getDefaultEditorTarget, +} from "@/routes/editorRoutes"; import { writeComponentToFileListFromText } from "@/utils/componentStore"; import { defaultPipelineYamlWithName, @@ -32,15 +35,13 @@ const NewPipelineButton = ({ componentText, ); - const clickThroughUrl = getDefaultEditorPath(name); - if (e.ctrlKey || e.metaKey) { - window.open(clickThroughUrl, "_blank"); + window.open(getDefaultEditorHref({ name }), "_blank"); return; } navigate({ - to: clickThroughUrl, + ...getDefaultEditorTarget({ name }), reloadDocument: !IS_GITHUB_PAGES, }); }; diff --git a/src/components/shared/VersionToggle.tsx b/src/components/shared/VersionToggle.tsx index b4d4b0cc55..ac227b9a79 100644 --- a/src/components/shared/VersionToggle.tsx +++ b/src/components/shared/VersionToggle.tsx @@ -28,6 +28,7 @@ interface VersionToggleProps { welcome: VersionToggleWelcome; showWelcomeSpotlight?: boolean; trackingId?: string; + preserveSearch?: boolean; } export function VersionToggle({ @@ -38,6 +39,7 @@ export function VersionToggle({ welcome, showWelcomeSpotlight = false, trackingId, + preserveSearch = false, }: VersionToggleProps) { const navigate = useNavigate(); const isEnabled = useFlagValue(flagName); @@ -61,7 +63,11 @@ export function VersionToggle({ className={cn(showWelcome && "relative z-1001")} onClick={() => { if (showWelcome) dismissWelcome(); - navigate({ to: targetPath }); + navigate( + preserveSearch + ? { to: targetPath, search: (previous) => previous } + : { to: targetPath }, + ); }} variant="header" size="icon" diff --git a/src/routes/Dashboard/TypePill.tsx b/src/routes/Dashboard/TypePill.tsx index d16cad9ab9..889975c5b5 100644 --- a/src/routes/Dashboard/TypePill.tsx +++ b/src/routes/Dashboard/TypePill.tsx @@ -2,7 +2,7 @@ import { Icon, type IconName } from "@/components/ui/icon"; import type { FavoriteItem } from "@/hooks/useFavorites"; import type { RecentItem } from "@/hooks/useRecentlyViewed"; import { cn } from "@/lib/utils"; -import { getDefaultEditorPath } from "@/routes/editorRoutes"; +import { getDefaultEditorHref } from "@/routes/editorRoutes"; import { APP_ROUTES } from "@/routes/router"; import { getDefaultRunPath } from "@/routes/runRoutes"; @@ -59,12 +59,12 @@ export const TypePill = ({ }; export function getFavoriteUrl(item: FavoriteItem): string { - if (item.type === "pipeline") return getDefaultEditorPath(item.id); + if (item.type === "pipeline") return getDefaultEditorHref({ name: item.id }); return getDefaultRunPath(item.id); } export function getRecentlyViewedUrl(item: RecentItem): string { - if (item.type === "pipeline") return getDefaultEditorPath(item.id); + if (item.type === "pipeline") return getDefaultEditorHref({ name: item.id }); if (item.type === "run") return getDefaultRunPath(item.id); if (item.type === "tour") return `${APP_ROUTES.TOUR}/${item.id}`; return APP_ROUTES.DASHBOARD_COMPONENTS; diff --git a/src/routes/Import/Import.test.tsx b/src/routes/Import/Import.test.tsx index a2be0a0b2a..305c660cc2 100644 --- a/src/routes/Import/Import.test.tsx +++ b/src/routes/Import/Import.test.tsx @@ -142,7 +142,9 @@ describe("ImportPage", () => { await waitFor(() => { expect(mockNavigate).toHaveBeenCalledWith({ - to: "/editor-v2/Test%20Pipeline", + to: "/editor-v2/$pipelineName", + params: { pipelineName: "Test Pipeline" }, + search: {}, }); }); }); diff --git a/src/routes/Import/index.tsx b/src/routes/Import/index.tsx index abd552c4fb..1b18cc0bea 100644 --- a/src/routes/Import/index.tsx +++ b/src/routes/Import/index.tsx @@ -6,7 +6,7 @@ import { Button } from "@/components/ui/button"; import { BlockStack, InlineStack } from "@/components/ui/layout"; import { Spinner } from "@/components/ui/spinner"; import { Paragraph, Text } from "@/components/ui/typography"; -import { getDefaultEditorPath } from "@/routes/editorRoutes"; +import { getDefaultEditorTarget } from "@/routes/editorRoutes"; import { importPipelineFromYaml } from "@/services/pipelineService"; /** @@ -201,9 +201,7 @@ export const ImportPage = () => { importedRef.current = true; setPipelineName(result.name); setStep(Step.Done); - navigate({ - to: getDefaultEditorPath(result.name), - }); + navigate(getDefaultEditorTarget({ name: result.name })); } else { setError(result.errorMessage || "Failed to import pipeline from URL."); } diff --git a/src/routes/editorRoutes.ts b/src/routes/editorRoutes.ts index 347a7c1c2b..edd26fb1af 100644 --- a/src/routes/editorRoutes.ts +++ b/src/routes/editorRoutes.ts @@ -1,17 +1,37 @@ import { isFlagEnabled } from "@/components/shared/Settings/useFlags"; +import type { PipelineRef } from "@/services/pipelineStorage/types"; import { APP_ROUTES, EDITOR_PATH } from "./appRoutes"; -function getLegacyEditorPath(pipelineName: string): string { - return `${EDITOR_PATH}/${encodeURIComponent(pipelineName)}`; +interface EditorSearch { + fileId?: string; } -function getEditorV2Path(pipelineName: string): string { - return `${APP_ROUTES.EDITOR_V2}/${encodeURIComponent(pipelineName)}`; +export interface EditorTarget { + to: string; + params: Record; + search: EditorSearch; } -export function getDefaultEditorPath(pipelineName: string): string { +/** + * A pipeline's identity travels in the search params, not the path: the slug is + * only ever a display name, and two pipelines are allowed to share one. + */ +export function getDefaultEditorTarget(ref: PipelineRef): EditorTarget { + const search: EditorSearch = ref.fileId ? { fileId: ref.fileId } : {}; + return isFlagEnabled("v2_editor") - ? getEditorV2Path(pipelineName) - : getLegacyEditorPath(pipelineName); + ? { + to: APP_ROUTES.EDITOR_V2_PIPELINE, + params: { pipelineName: ref.name }, + search, + } + : { to: APP_ROUTES.PIPELINE_EDITOR, params: { name: ref.name }, search }; +} + +export function getDefaultEditorHref(ref: PipelineRef): string { + const base = isFlagEnabled("v2_editor") ? APP_ROUTES.EDITOR_V2 : EDITOR_PATH; + const path = `${base}/${encodeURIComponent(ref.name)}`; + + return ref.fileId ? `${path}?fileId=${encodeURIComponent(ref.fileId)}` : path; } diff --git a/src/routes/router.ts b/src/routes/router.ts index e7c1655d7e..d948c6da65 100644 --- a/src/routes/router.ts +++ b/src/routes/router.ts @@ -261,10 +261,6 @@ const editorRoute = createRoute({ getParentRoute: () => mainLayout, path: APP_ROUTES.PIPELINE_EDITOR, component: Editor, - beforeLoad: ({ search }: { search: { name?: string } }) => { - const name = search.name || ""; - return { name }; - }, }); const githubAuthCallbackRoute = createRoute({ @@ -326,6 +322,7 @@ const editorV2PipelineRoute = createRoute({ throw redirect({ to: APP_ROUTES.PIPELINE_EDITOR, params: { name: params.pipelineName }, + search: (previous) => previous, }); } }, diff --git a/src/routes/v2/pages/Editor/hooks/useLoadSpec.ts b/src/routes/v2/pages/Editor/hooks/useLoadSpec.ts index 3fb03661d3..ec71d573fd 100644 --- a/src/routes/v2/pages/Editor/hooks/useLoadSpec.ts +++ b/src/routes/v2/pages/Editor/hooks/useLoadSpec.ts @@ -83,20 +83,18 @@ export function useLoadSpec(ref: PipelineRef) { return useSuspenseQuery({ queryKey, queryFn: async (): Promise => { - const filePromise = storage.resolve(ref); + const file = await storage.resolve(ref); + loadedStorageKey.current = file.storageKey; - const [{ file, specData }, undoHistory] = await Promise.all([ - filePromise.then(async (file) => { - loadedStorageKey.current = file.storageKey; - return { - file, - specData: yaml.load(await file.read(), PIPELINE_YAML_LOAD_OPTIONS), - }; - }), - loadUndoHistory(ref.name).catch(() => null), + const [yamlText, undoHistory] = await Promise.all([ + file.read(), + loadUndoHistory(file.id).catch(() => null), ]); - const loadedSpec = deserializeSpecData(specData, undoHistory); + const loadedSpec = deserializeSpecData( + yaml.load(yamlText, PIPELINE_YAML_LOAD_OPTIONS), + undoHistory, + ); await hydrateLoadedSpecRefs(loadedSpec.spec); return { ...loadedSpec, file }; diff --git a/src/routes/v2/pages/Editor/store/autoSaveStore.ts b/src/routes/v2/pages/Editor/store/autoSaveStore.ts index 703cadbdcc..c38d470de5 100644 --- a/src/routes/v2/pages/Editor/store/autoSaveStore.ts +++ b/src/routes/v2/pages/Editor/store/autoSaveStore.ts @@ -146,13 +146,14 @@ export class AutoSaveStore { } private async persistUndoHistory() { - if (!this.spec || !this.pipelineName) return; + const fileId = this.pipelineFileStore.activePipelineFile?.id; + if (!this.spec || !fileId) return; const manager = this.undoStore.undoManager; if (!manager) return; try { const idStack = collectIdStack(this.spec); - await saveUndoHistory(this.pipelineName, idStack, manager); + await saveUndoHistory(fileId, idStack, manager); } catch (error) { console.error("Failed to persist undo history:", error); } diff --git a/src/routes/v2/pages/Editor/utils/undoHistoryStorage.ts b/src/routes/v2/pages/Editor/utils/undoHistoryStorage.ts index 3c2495630d..92d1a35c44 100644 --- a/src/routes/v2/pages/Editor/utils/undoHistoryStorage.ts +++ b/src/routes/v2/pages/Editor/utils/undoHistoryStorage.ts @@ -9,20 +9,31 @@ const CURRENT_VERSION = 2; const MAX_UNDO_EVENTS = 10; interface StoredUndoHistory { - pipelineName: string; + fileId: string; version: number; idStack: string[]; undoEvents: UndoEvent[]; } const UndoHistoryDB = new Dexie("undo-history") as Dexie & { - entries: EntityTable; + history: EntityTable; }; UndoHistoryDB.version(1).stores({ entries: "pipelineName", }); +/** + * The old table keyed on the pipeline's name, which two pipelines are allowed + * to share once storage keys are opaque — and a collision here replays one + * pipeline's undo events onto another. There is nothing to migrate: the events + * only make sense against the ids of the spec they were recorded from. + */ +UndoHistoryDB.version(2).stores({ + entries: null, + history: "fileId", +}); + /** * Saves undo history for a pipeline. * @@ -33,15 +44,15 @@ UndoHistoryDB.version(1).stores({ * to strip MobX observables while preserving the original format. */ export async function saveUndoHistory( - pipelineName: string, + fileId: string, idStack: string[], undoManager: UndoManager, ): Promise { const rawEvents = undoManager.undoQueue.slice(-MAX_UNDO_EVENTS); const clonedEvents: UndoEvent[] = JSON.parse(JSON.stringify(rawEvents)); - await UndoHistoryDB.entries.put({ - pipelineName, + await UndoHistoryDB.history.put({ + fileId, version: CURRENT_VERSION, idStack, undoEvents: clonedEvents, @@ -49,9 +60,9 @@ export async function saveUndoHistory( } export async function loadUndoHistory( - pipelineName: string, + fileId: string, ): Promise { - const data = await UndoHistoryDB.entries.get(pipelineName); + const data = await UndoHistoryDB.history.get(fileId); if (!data) return null; if (data.version !== CURRENT_VERSION) return null; diff --git a/src/routes/v2/pages/RunView/hooks/useRunViewActions.ts b/src/routes/v2/pages/RunView/hooks/useRunViewActions.ts index 7ac2be45d3..a38c886537 100644 --- a/src/routes/v2/pages/RunView/hooks/useRunViewActions.ts +++ b/src/routes/v2/pages/RunView/hooks/useRunViewActions.ts @@ -8,7 +8,7 @@ import { useCheckComponentSpecFromPath } from "@/hooks/useCheckComponentSpecFrom import { useUserDetails } from "@/hooks/useUserDetails"; import { useComponentSpec } from "@/providers/ComponentSpecProvider"; import { useExecutionData } from "@/providers/ExecutionDataProvider"; -import { getDefaultEditorPath } from "@/routes/editorRoutes"; +import { getDefaultEditorHref } from "@/routes/editorRoutes"; import { extractCanonicalName } from "@/utils/canonicalPipelineName"; import type { ComponentSpec } from "@/utils/componentSpec"; import { @@ -81,7 +81,7 @@ export function useRunViewActions(): RunViewActions { const { data: currentUserDetails } = useUserDetails(); const editorRoute = componentSpec?.name - ? getDefaultEditorPath(componentSpec.name) + ? getDefaultEditorHref({ name: componentSpec.name }) : ""; const canAccessEditorSpec = useCheckComponentSpecFromPath( diff --git a/src/services/pipelineRunService.ts b/src/services/pipelineRunService.ts index e148dbc6ea..995c39e1a5 100644 --- a/src/services/pipelineRunService.ts +++ b/src/services/pipelineRunService.ts @@ -4,7 +4,7 @@ import type { BodyCreateApiPipelineRunsPost, ListAnnotationsApiPipelineRunsIdAnnotationsGetResponse, } from "@/api/types.gen"; -import { getDefaultEditorPath } from "@/routes/editorRoutes"; +import { getDefaultEditorHref } from "@/routes/editorRoutes"; import type { PipelineRun } from "@/types/pipelineRun"; import { EDITOR_FLOW_DIRECTION_ANNOTATION } from "@/utils/annotations"; import { removeCachingStrategyFromSpec } from "@/utils/cache"; @@ -162,7 +162,7 @@ export const copyRunToPipeline = async ( ); return { - url: getDefaultEditorPath(newName), + url: getDefaultEditorHref({ name: newName }), name: newName, }; } catch (error) { From 420c67823eb1e528b49dc77f90f88473e5003728 Mon Sep 17 00:00:00 2001 From: Camiel van Schoonhoven Date: Tue, 8 Sep 2026 15:30:35 -0700 Subject: [PATCH 11/36] refactor(pipeline-storage): read pipelines through the storage service Every read of the pipeline list went straight to the browser-local store, so a host-provided store was invisible to the dashboard, the name dialog, the metadata panels, the session stats and the v1 loader alike. Route them through the storage service instead. `pipelineOperations` gives the non-React callers the same service instance the provider hands to components, and a driver may now `describe` a key so a file resolved by name still carries its timestamps. The dashboard list filters and searches inside a spec that a remote listing does not carry, so a host-backed store gets the folder table. Session stats leave out the node-count distribution against a remote store: counting nodes would mean fetching every pipeline once a day. --- .../Editor/Context/PipelineDetails.tsx | 26 +++++-------- .../shared/Dialogs/PipelineNameDialog.tsx | 4 +- .../FlowSidebar/sections/FileActions.tsx | 6 +-- src/hooks/useLoadUserPipelines.ts | 39 +++++-------------- src/hooks/useSessionPipelineStats.ts | 18 +++++++++ src/routes/router.ts | 9 +++++ .../components/MetadataBlock.tsx | 18 ++++----- src/services/pipelineService.ts | 28 +++++++------ .../pipelineStorage/PipelineFolder.ts | 14 +++++-- .../PipelineStorageProvider.tsx | 7 +++- .../pipelineStorage/PipelineStorageService.ts | 11 ++++++ .../drivers/FolderIndexDbStorageDriver.ts | 8 ++++ .../drivers/RootFolderDbStorageDriver.ts | 13 +++++++ .../pipelineStorage/pipelineOperations.ts | 26 +++++++++++++ src/services/pipelineStorage/types.ts | 1 + 15 files changed, 148 insertions(+), 80 deletions(-) create mode 100644 src/services/pipelineStorage/pipelineOperations.ts diff --git a/src/components/Editor/Context/PipelineDetails.tsx b/src/components/Editor/Context/PipelineDetails.tsx index 453bcee2ea..478a155ac1 100644 --- a/src/components/Editor/Context/PipelineDetails.tsx +++ b/src/components/Editor/Context/PipelineDetails.tsx @@ -13,9 +13,8 @@ import { PipelineRunNameTemplateEditor } from "@/components/shared/PipelineRunNa import { BlockStack } from "@/components/ui/layout"; import useToastNotification from "@/hooks/useToastNotification"; import { useComponentSpec } from "@/providers/ComponentSpecProvider"; +import { findPipelineFile } from "@/services/pipelineStorage/pipelineOperations"; import { SYSTEM_ANNOTATIONS } from "@/utils/annotations"; -import { getComponentFileFromList } from "@/utils/componentStore"; -import { USER_PIPELINES_LIST_NAME } from "@/utils/constants"; import { tracking } from "@/utils/tracking"; import PipelineIO from "../../shared/Execution/PipelineIO"; @@ -32,28 +31,21 @@ const PipelineDetails = () => { globalValidationIssues, ); - // State for file metadata const [fileMeta, setFileMeta] = useState<{ - creationTime?: Date; - modificationTime?: Date; - createdBy?: string; + createdAt?: Date; + modifiedAt?: Date; }>({}); - // Fetch file metadata on mount or when componentSpec.name changes useEffect(() => { const fetchMeta = async () => { if (!componentSpec.name) return; try { - const file = await getComponentFileFromList( - USER_PIPELINES_LIST_NAME, - componentSpec.name, - ); + const file = await findPipelineFile({ name: componentSpec.name }); if (file) { setFileMeta({ - creationTime: file.creationTime, - modificationTime: file.modificationTime, - createdBy: file.componentRef.spec.metadata?.annotations?.author, + createdAt: file.createdAt, + modifiedAt: file.modifiedAt, }); } } catch (error) { @@ -67,15 +59,15 @@ const PipelineDetails = () => { const metadata = [ { label: "Created by", - value: fileMeta.createdBy, + value: componentSpec.metadata?.annotations?.author as string | undefined, }, { label: "Created at", - value: fileMeta.creationTime?.toLocaleString(), + value: fileMeta.createdAt?.toLocaleString(), }, { label: "Last updated", - value: fileMeta.modificationTime?.toLocaleString(), + value: fileMeta.modifiedAt?.toLocaleString(), }, ]; diff --git a/src/components/shared/Dialogs/PipelineNameDialog.tsx b/src/components/shared/Dialogs/PipelineNameDialog.tsx index 5d44be1f61..d8d30800b1 100644 --- a/src/components/shared/Dialogs/PipelineNameDialog.tsx +++ b/src/components/shared/Dialogs/PipelineNameDialog.tsx @@ -50,7 +50,7 @@ const PipelineNameDialog = ({ const [touched, setTouched] = useState(false); const { - userPipelines, + pipelineNames, isLoadingUserPipelines, refetch: refetchUserPipelines, } = useLoadUserPipelines(); @@ -59,7 +59,7 @@ const PipelineNameDialog = ({ const excluded = new Set( (excludeNames ?? []).map((n) => n.trim().toLowerCase()), ); - const nameIsTaken = Array.from(userPipelines.keys()).some((n) => { + const nameIsTaken = pipelineNames.some((n) => { const lower = n.toLowerCase(); return lower === normalized && !excluded.has(lower); }); diff --git a/src/components/shared/ReactFlow/FlowSidebar/sections/FileActions.tsx b/src/components/shared/ReactFlow/FlowSidebar/sections/FileActions.tsx index 9267b43577..9d1d8cfb64 100644 --- a/src/components/shared/ReactFlow/FlowSidebar/sections/FileActions.tsx +++ b/src/components/shared/ReactFlow/FlowSidebar/sections/FileActions.tsx @@ -5,7 +5,7 @@ import { Spinner } from "@/components/ui/spinner"; import { Text } from "@/components/ui/typography"; import { useAutoSaveStatus } from "@/providers/AutoSaveProvider"; import { useComponentSpec } from "@/providers/ComponentSpecProvider"; -import { getPipelineFile } from "@/services/pipelineService"; +import { findPipelineFile } from "@/services/pipelineStorage/pipelineOperations"; import { formatRelativeTime } from "@/utils/date"; import { SidebarSection } from "../components/SidebarSection"; @@ -50,8 +50,8 @@ const FileActions = () => { useEffect(() => { const fetchLastSaved = async () => { if (componentSpec?.name) { - const lastSavedPipeline = await getPipelineFile(componentSpec.name); - setLastSavedAt(lastSavedPipeline?.modificationTime ?? null); + const file = await findPipelineFile({ name: componentSpec.name }); + setLastSavedAt(file?.modifiedAt ?? null); } }; fetchLastSaved(); diff --git a/src/hooks/useLoadUserPipelines.ts b/src/hooks/useLoadUserPipelines.ts index eba8ea3d7e..3cd675c242 100644 --- a/src/hooks/useLoadUserPipelines.ts +++ b/src/hooks/useLoadUserPipelines.ts @@ -1,47 +1,28 @@ -import { useEffect, useState } from "react"; +import { useCallback, useEffect, useState } from "react"; -import { - type ComponentFileEntry, - getAllComponentFilesFromList, -} from "@/utils/componentStore"; -import { USER_PIPELINES_LIST_NAME } from "@/utils/constants"; +import { listPipelineFiles } from "@/services/pipelineStorage/pipelineOperations"; const useLoadUserPipelines = () => { const [isLoadingUserPipelines, setIsLoadingUserPipelines] = useState(true); - const [userPipelines, setUserPipelines] = useState< - Map - >(new Map()); + const [pipelineNames, setPipelineNames] = useState([]); - const fetchUserPipelines = async () => { + const refetch = useCallback(async () => { setIsLoadingUserPipelines(true); try { - const pipelines = await getAllComponentFilesFromList( - USER_PIPELINES_LIST_NAME, - ); - setUserPipelines(pipelines); + const files = await listPipelineFiles(); + setPipelineNames(files.map((file) => file.displayName)); } catch (error) { console.error("Failed to load user pipelines:", error); } finally { setIsLoadingUserPipelines(false); } - }; - - const refetch = async () => { - setIsLoadingUserPipelines(true); - try { - await fetchUserPipelines(); - } catch (error) { - console.error("Failed to refetch user pipelines:", error); - } finally { - setIsLoadingUserPipelines(false); - } - }; + }, []); useEffect(() => { - fetchUserPipelines(); - }, []); + void refetch(); + }, [refetch]); - return { userPipelines, isLoadingUserPipelines, refetch }; + return { pipelineNames, isLoadingUserPipelines, refetch }; }; export default useLoadUserPipelines; diff --git a/src/hooks/useSessionPipelineStats.ts b/src/hooks/useSessionPipelineStats.ts index 68ff4e0bb2..cd569b93d3 100644 --- a/src/hooks/useSessionPipelineStats.ts +++ b/src/hooks/useSessionPipelineStats.ts @@ -1,6 +1,8 @@ import { useEffect } from "react"; import { useAnalytics } from "@/providers/AnalyticsProvider"; +import { listPipelineFiles } from "@/services/pipelineStorage/pipelineOperations"; +import { isHostStorage } from "@/services/pipelineStorage/storageMode"; import type { ComponentSpec, TaskSpec } from "@/utils/componentSpec"; import { isGraphImplementation } from "@/utils/componentSpec"; import { getAllComponentFilesFromList } from "@/utils/componentStore"; @@ -109,6 +111,22 @@ export function useSessionPipelineStats(): void { "234_plus": 0, }; + /** + * A remote store lists summaries, so counting nodes would mean fetching + * every pipeline once a day just for analytics. The total is worth that + * much less than the request storm, so the distribution is left out. + */ + if (isHostStorage()) { + try { + total_pipelines = (await listPipelineFiles()).length; + } catch { + return; + } + + track("session.pipeline_stats.start", { total_pipelines }); + return; + } + try { const pipelines = await getAllComponentFilesFromList( USER_PIPELINES_LIST_NAME, diff --git a/src/routes/router.ts b/src/routes/router.ts index d948c6da65..f333ce34cc 100644 --- a/src/routes/router.ts +++ b/src/routes/router.ts @@ -16,6 +16,7 @@ import { AddSecretView } from "@/components/shared/SecretsManagement/components/ import { ReplaceSecretView } from "@/components/shared/SecretsManagement/components/ReplaceSecretView"; import { SecretsListView } from "@/components/shared/SecretsManagement/components/SecretsListView"; import { isFlagEnabled } from "@/components/shared/Settings/useFlags"; +import { isHostStorage } from "@/services/pipelineStorage/storageMode"; import { BASE_URL, IS_GITHUB_PAGES } from "@/utils/constants"; import RootLayout from "../components/layout/RootLayout"; @@ -103,10 +104,18 @@ const dashboardRunsRoute = createRoute({ component: DashboardRunsView, }); +// The dashboard list reads the browser's own pipeline store directly and +// filters on spec contents a host listing does not carry, so a host-backed +// store gets the folder table, which goes through the storage driver. const dashboardPipelinesRoute = createRoute({ getParentRoute: () => dashboardRoute, path: "/pipelines", component: DashboardPipelinesView, + beforeLoad: () => { + if (isHostStorage()) { + throw redirect({ to: APP_ROUTES.PIPELINE_FOLDERS }); + } + }, }); const dashboardComponentsRoute = createRoute({ diff --git a/src/routes/v2/pages/Editor/components/PipelineDetailsContent/components/MetadataBlock.tsx b/src/routes/v2/pages/Editor/components/PipelineDetailsContent/components/MetadataBlock.tsx index 224049fa9c..41df52f565 100644 --- a/src/routes/v2/pages/Editor/components/PipelineDetailsContent/components/MetadataBlock.tsx +++ b/src/routes/v2/pages/Editor/components/PipelineDetailsContent/components/MetadataBlock.tsx @@ -3,33 +3,33 @@ import { useSuspenseQuery } from "@tanstack/react-query"; import { KeyValueList } from "@/components/shared/ContextPanel/Blocks/KeyValueList"; import { withSuspenseWrapper } from "@/components/shared/SuspenseWrapper"; import type { ComponentSpec } from "@/models/componentSpec"; -import { getComponentFileFromList } from "@/utils/componentStore"; -import { USER_PIPELINES_LIST_NAME } from "@/utils/constants"; +import { findPipelineFile } from "@/services/pipelineStorage/pipelineOperations"; export const MetadataBlock = withSuspenseWrapper(function MetadataBlock({ spec, }: { spec: ComponentSpec; }) { - const { data: fileMeta } = useSuspenseQuery({ + const { data: file } = useSuspenseQuery({ queryKey: ["file-meta", spec.name], - queryFn: () => - getComponentFileFromList(USER_PIPELINES_LIST_NAME, spec.name), + queryFn: () => findPipelineFile({ name: spec.name ?? "" }), }); - const metadata = fileMeta + const author = spec.getMetadata("author"); + + const metadata = file ? [ { label: "Created by", - value: fileMeta.componentRef.spec.metadata?.annotations?.author, + value: author === undefined ? undefined : String(author), }, { label: "Created at", - value: fileMeta.creationTime?.toLocaleString(), + value: file.createdAt?.toLocaleString(), }, { label: "Last updated", - value: fileMeta.modificationTime?.toLocaleString(), + value: file.modifiedAt?.toLocaleString(), }, ] : []; diff --git a/src/services/pipelineService.ts b/src/services/pipelineService.ts index f39ca44254..f241d38194 100644 --- a/src/services/pipelineService.ts +++ b/src/services/pipelineService.ts @@ -8,17 +8,18 @@ import { isGraphImplementation, } from "@/utils/componentSpec"; import { - type ComponentFileEntry, deleteComponentFileFromList, fullyLoadComponentRefFromUrl, - getAllComponentFilesFromList, getComponentFileFromList, + loadComponentAsRefFromText, writeComponentToFileListFromText, } from "@/utils/componentStore"; import { USER_PIPELINES_LIST_NAME } from "@/utils/constants"; import { componentSpecToYaml } from "@/utils/yaml"; import { componentSpecFromYaml } from "@/utils/yaml"; +import type { PipelineFile } from "./pipelineStorage/PipelineFile"; +import { findPipelineFile } from "./pipelineStorage/pipelineOperations"; import { deleteEntry, findByStorageKey, @@ -65,12 +66,14 @@ export const loadPipelineByName = async (name: string) => { const appSettings = getAppSettings(); try { - // Fetch user pipelines - let userPipelines: Map; + /** + * A store that cannot answer is reported as such. Falling through to the + * example library would quietly serve a different pipeline that happens to + * share the name, and the editor would then save over the user's own. + */ + let file: PipelineFile | undefined; try { - userPipelines = await getAllComponentFilesFromList( - USER_PIPELINES_LIST_NAME, - ); + file = await findPipelineFile({ name: decodedName }); } catch (error) { console.error("Failed to load user pipelines:", error); return { @@ -80,11 +83,10 @@ export const loadPipelineByName = async (name: string) => { }; } - // Check if pipeline exists in user pipelines - const pipeline = userPipelines.get(decodedName); - if (pipeline) { + if (file) { + const componentRef = await loadComponentAsRefFromText(await file.read()); return { - experiment: pipeline, + experiment: { componentRef, spec: componentRef.spec }, isLoading: false, error: null, }; @@ -291,7 +293,3 @@ export async function importPipelineFromFile( }; } } - -export function getPipelineFile(pipelineName: string) { - return getComponentFileFromList(USER_PIPELINES_LIST_NAME, pipelineName); -} diff --git a/src/services/pipelineStorage/PipelineFolder.ts b/src/services/pipelineStorage/PipelineFolder.ts index 88cd1229d2..77e7d25da6 100644 --- a/src/services/pipelineStorage/PipelineFolder.ts +++ b/src/services/pipelineStorage/PipelineFolder.ts @@ -110,10 +110,18 @@ export class PipelineFolder { } async findFile(storageKey: string): Promise { - const hasKey = await this.driver.hasKey(storageKey); - if (!hasKey) return undefined; + const descriptor = await this.describeKey(storageKey); + if (!descriptor) return undefined; - return resolveOrCreateRegistryEntry({ storageKey }, this); + return resolveOrCreateRegistryEntry(descriptor, this); + } + + private async describeKey( + storageKey: string, + ): Promise { + if (this.driver.describe) return this.driver.describe(storageKey); + + return (await this.driver.hasKey(storageKey)) ? { storageKey } : undefined; } async assignFile(storageKey: string): Promise { diff --git a/src/services/pipelineStorage/PipelineStorageProvider.tsx b/src/services/pipelineStorage/PipelineStorageProvider.tsx index aa319b71ab..c94f7c9445 100644 --- a/src/services/pipelineStorage/PipelineStorageProvider.tsx +++ b/src/services/pipelineStorage/PipelineStorageProvider.tsx @@ -6,14 +6,17 @@ import { useRequiredContext, } from "@/hooks/useRequiredContext"; -import { PipelineStorageService } from "./PipelineStorageService"; +import { + getPipelineStorageService, + type PipelineStorageService, +} from "./PipelineStorageService"; export const PipelineStorageCtx = createRequiredContext( "PipelineStorageContext", ); export function PipelineStorageProvider({ children }: { children: ReactNode }) { - const [service] = useState(() => new PipelineStorageService()); + const [service] = useState(getPipelineStorageService); return ( diff --git a/src/services/pipelineStorage/PipelineStorageService.ts b/src/services/pipelineStorage/PipelineStorageService.ts index 3953568801..4d89037e11 100644 --- a/src/services/pipelineStorage/PipelineStorageService.ts +++ b/src/services/pipelineStorage/PipelineStorageService.ts @@ -160,6 +160,17 @@ async function resolveInFlatStore( return byName[0]; } +let sharedService: PipelineStorageService | undefined; + +/** + * For the callers that are not React — services and plain utilities — which + * still have to reach the same store the provider hands to components. + */ +export function getPipelineStorageService(): PipelineStorageService { + sharedService ??= new PipelineStorageService(); + return sharedService; +} + function createRoot(mode: StorageMode): PipelineFolder { if (mode.kind === "host") { return new PipelineFolder({ diff --git a/src/services/pipelineStorage/drivers/FolderIndexDbStorageDriver.ts b/src/services/pipelineStorage/drivers/FolderIndexDbStorageDriver.ts index 22f1406fc8..c22b3f91d2 100644 --- a/src/services/pipelineStorage/drivers/FolderIndexDbStorageDriver.ts +++ b/src/services/pipelineStorage/drivers/FolderIndexDbStorageDriver.ts @@ -44,4 +44,12 @@ export class FolderIndexDbStorageDriver extends RootFolderDbStorageDriver { const entry = await findByFolderAndStorageKey(this.folderId, storageKey); return entry != null; } + + override async describe( + storageKey: string, + ): Promise { + if (!(await this.hasKey(storageKey))) return undefined; + + return super.describe(storageKey); + } } diff --git a/src/services/pipelineStorage/drivers/RootFolderDbStorageDriver.ts b/src/services/pipelineStorage/drivers/RootFolderDbStorageDriver.ts index 3f958392bd..4f3ee31303 100644 --- a/src/services/pipelineStorage/drivers/RootFolderDbStorageDriver.ts +++ b/src/services/pipelineStorage/drivers/RootFolderDbStorageDriver.ts @@ -63,4 +63,17 @@ export class RootFolderDbStorageDriver implements PipelineStorageDriver { const entry = await getComponentFileFromList(LIST_NAME, storageKey); return entry != null; } + + async describe( + storageKey: string, + ): Promise { + const entry = await getComponentFileFromList(LIST_NAME, storageKey); + if (!entry) return undefined; + + return { + storageKey, + createdAt: entry.creationTime, + modifiedAt: entry.modificationTime, + }; + } } diff --git a/src/services/pipelineStorage/pipelineOperations.ts b/src/services/pipelineStorage/pipelineOperations.ts new file mode 100644 index 0000000000..8186fcc643 --- /dev/null +++ b/src/services/pipelineStorage/pipelineOperations.ts @@ -0,0 +1,26 @@ +import type { PipelineFile } from "./PipelineFile"; +import { + getPipelineStorageService, + PipelineNotFoundError, +} from "./PipelineStorageService"; +import type { PipelineRef } from "./types"; + +export async function listPipelineFiles(): Promise { + return getPipelineStorageService().rootFolder.listPipelines(); +} + +/** + * The read path for callers that treat "no such pipeline" as an ordinary + * answer. Anything else — an unreachable store, an ambiguous name — still + * throws, because those need saying out loud rather than rendering as empty. + */ +export async function findPipelineFile( + ref: PipelineRef, +): Promise { + try { + return await getPipelineStorageService().resolve(ref); + } catch (error) { + if (error instanceof PipelineNotFoundError) return undefined; + throw error; + } +} diff --git a/src/services/pipelineStorage/types.ts b/src/services/pipelineStorage/types.ts index 029f3290b0..cae9eae3d0 100644 --- a/src/services/pipelineStorage/types.ts +++ b/src/services/pipelineStorage/types.ts @@ -35,6 +35,7 @@ export interface PipelineStorageDriver { rename(oldStorageKey: string, newStorageKey: string): Promise; delete(storageKey: string): Promise; hasKey(storageKey: string): Promise; + describe?(storageKey: string): Promise; } export type DriverConfig = From e2f62d82c02bcb52520bf3eef6e692fe0b2f4e91 Mon Sep 17 00:00:00 2001 From: Camiel van Schoonhoven Date: Tue, 8 Sep 2026 16:03:07 -0700 Subject: [PATCH 12/36] refactor(pipeline-storage): write pipelines through the storage service New Pipeline, save, save-as, import, clone-a-run and the v1 editor's save all wrote straight to the browser-local list, so with a host-provided store they created pipelines nobody could see and saved over ones nobody had opened. Route them through the service instead. `savePipelineByName` updates the pipeline that already has the name wherever it lives, rather than laying a second copy in the root store, and creates one when the name is new. Import now reports the identity of what it wrote, so the three places that navigate afterwards open that pipeline rather than whatever shares its name. Deleting is a fix, not a reroute: it removed the legacy row and the registry row but never asked the driver, so a pipeline deleted from the folder table stayed alive in its own store. It also reported success whatever happened; it now says when it failed. A path segment is classified as a run id only on a run path. The old 20-hex guess would mistake an opaque pipeline key for one, and the editor then loaded nothing at all. Co-Authored-By: Claude Opus 5 (1M context) --- .../Home/PipelineSection/BulkActionsBar.tsx | 4 +- .../Home/PipelineSection/PipelineRow.tsx | 17 ++- src/components/Learn/useImportPipeline.ts | 4 +- src/components/shared/ImportPipeline.tsx | 7 +- src/components/shared/NewPipelineButton.tsx | 15 ++- .../components/SavePipelineAsButton.tsx | 8 +- src/providers/ComponentSpecProvider.tsx | 12 +- src/routes/Import/index.tsx | 4 +- src/services/importPipeline.test.ts | 105 +++++++----------- src/services/pipelineRunService.ts | 49 +++----- src/services/pipelineService.ts | 88 +++++---------- src/services/pipelineStorage/PipelineFile.ts | 12 +- .../pipelineStorage/PipelineFolder.ts | 7 +- .../PipelineStorageService.test.ts | 104 +++++++++++++++-- .../pipelineStorage/PipelineStorageService.ts | 41 ++++++- .../pipelineStorage/pipelineOperations.ts | 20 ++++ src/utils/URL.test.ts | 7 ++ src/utils/URL.ts | 15 +-- tests/e2e/fixtures/pipelineStorageHost.ts | 51 ++++++--- tests/e2e/pipeline-storage-host.spec.ts | 36 ++++++ 20 files changed, 376 insertions(+), 230 deletions(-) diff --git a/src/components/Home/PipelineSection/BulkActionsBar.tsx b/src/components/Home/PipelineSection/BulkActionsBar.tsx index 406277a170..fd1238c779 100644 --- a/src/components/Home/PipelineSection/BulkActionsBar.tsx +++ b/src/components/Home/PipelineSection/BulkActionsBar.tsx @@ -3,7 +3,7 @@ import { FloatingSelectionBar } from "@/components/shared/FloatingSelectionBar"; import { Button } from "@/components/ui/button"; import { Icon } from "@/components/ui/icon"; import useToastNotification from "@/hooks/useToastNotification"; -import { deletePipeline } from "@/services/pipelineService"; +import { deletePipelineByName } from "@/services/pipelineStorage/pipelineOperations"; import { getErrorMessage, pluralize } from "@/utils/string"; interface BulkActionsBarProps { @@ -21,7 +21,7 @@ const BulkActionsBar = ({ const handleBulkDelete = async () => { const deletePromises = selectedPipelines.map((pipelineName) => - deletePipeline(pipelineName), + deletePipelineByName(pipelineName), ); try { diff --git a/src/components/Home/PipelineSection/PipelineRow.tsx b/src/components/Home/PipelineSection/PipelineRow.tsx index fb11e0c92c..65f6bd81bb 100644 --- a/src/components/Home/PipelineSection/PipelineRow.tsx +++ b/src/components/Home/PipelineSection/PipelineRow.tsx @@ -28,16 +28,18 @@ import { TooltipTrigger, } from "@/components/ui/tooltip"; import { Paragraph, Text } from "@/components/ui/typography"; +import useToastNotification from "@/hooks/useToastNotification"; import { cn } from "@/lib/utils"; import { useAnalytics } from "@/providers/AnalyticsProvider"; import { getDefaultEditorHref, getDefaultEditorTarget, } from "@/routes/editorRoutes"; -import { deletePipeline } from "@/services/pipelineService"; +import { deletePipelineByName } from "@/services/pipelineStorage/pipelineOperations"; import { getPipelineTagsFromSpec } from "@/utils/annotations"; import type { ComponentReferenceWithSpec } from "@/utils/componentStore"; import { formatDate } from "@/utils/date"; +import { getErrorMessage } from "@/utils/string"; import { tracking } from "@/utils/tracking"; import type { MatchedField } from "./usePipelineFilters"; @@ -89,6 +91,7 @@ const PipelineRow = withSuspenseWrapper( analyticsTrackingPrefix = DEFAULT_PIPELINE_ROW_ANALYTICS_PREFIX, }: PipelineRowProps) => { const navigate = useNavigate(); + const notify = useToastNotification(); const { track } = useAnalytics(); const rowTrack = (suffix: string, metadata?: Record) => { @@ -130,11 +133,15 @@ const PipelineRow = withSuspenseWrapper( const confirmPipelineDelete = async () => { if (!name) return; - const deleteCallback = () => { + try { + await deletePipelineByName(name); onDelete?.(); - }; - - await deletePipeline(name, deleteCallback); + } catch (error) { + notify( + `Failed to delete "${name}": ${getErrorMessage(error)}`, + "error", + ); + } }; const handleClick = (e: MouseEvent) => { diff --git a/src/components/Learn/useImportPipeline.ts b/src/components/Learn/useImportPipeline.ts index 5b11aab3a8..a934172e17 100644 --- a/src/components/Learn/useImportPipeline.ts +++ b/src/components/Learn/useImportPipeline.ts @@ -14,7 +14,9 @@ export function useImportPipeline() { mutationFn: async (url: string) => await importPipelineFromUrl(url), onSuccess: (result) => { notify(`Pipeline "${result.name}" created successfully`, "success"); - navigate(getDefaultEditorTarget({ name: result.name })); + navigate( + getDefaultEditorTarget({ name: result.name, fileId: result.fileId }), + ); }, }); } diff --git a/src/components/shared/ImportPipeline.tsx b/src/components/shared/ImportPipeline.tsx index 2d968ad264..c9f0b44117 100644 --- a/src/components/shared/ImportPipeline.tsx +++ b/src/components/shared/ImportPipeline.tsx @@ -26,7 +26,6 @@ import { importPipelineFromYaml, type ImportResult, } from "@/services/pipelineService"; -import { usePipelineStorage } from "@/services/pipelineStorage/PipelineStorageProvider"; import type { PipelineRef } from "@/services/pipelineStorage/types"; interface ImportPipelineProps { @@ -49,7 +48,6 @@ const ImportPipeline = ({ const [successMessage, setSuccessMessage] = useState(null); const fileInputRef = useRef(null); const navigate = useNavigate(); - const storage = usePipelineStorage(); const navigateToPipeline = () => { if (!importedPipeline) return; @@ -63,7 +61,7 @@ const ImportPipeline = ({ if (onImportComplete) { onImportComplete(importedPipeline); } else { - navigate(getDefaultEditorTarget({ name: importedPipeline.name })); + navigate(getDefaultEditorTarget(importedPipeline)); } }; @@ -81,8 +79,7 @@ const ImportPipeline = ({ setSuccessMessage(`Pipeline "${result.name}" imported successfully.`); } - const file = await storage.rootFolder.assignFile(result.name); - setImportedPipeline({ name: result.name, fileId: file.id }); + setImportedPipeline({ name: result.name, fileId: result.fileId }); }; const handleFileChange = async (e: ChangeEvent) => { diff --git a/src/components/shared/NewPipelineButton.tsx b/src/components/shared/NewPipelineButton.tsx index 4e2d04eeab..a5bc9ff1ad 100644 --- a/src/components/shared/NewPipelineButton.tsx +++ b/src/components/shared/NewPipelineButton.tsx @@ -7,11 +7,10 @@ import { getDefaultEditorHref, getDefaultEditorTarget, } from "@/routes/editorRoutes"; -import { writeComponentToFileListFromText } from "@/utils/componentStore"; +import { usePipelineStorage } from "@/services/pipelineStorage/PipelineStorageProvider"; import { defaultPipelineYamlWithName, IS_GITHUB_PAGES, - USER_PIPELINES_LIST_NAME, } from "@/utils/constants"; const randomName = () => (generate(4) as string[]).join(" "); @@ -25,23 +24,23 @@ const NewPipelineButton = ({ ...buttonProps }: NewPipelineButtonProps) => { const navigate = useNavigate(); + const storage = usePipelineStorage(); const handleCreate = async (e: MouseEvent) => { const name = randomName(); - const componentText = defaultPipelineYamlWithName(name); - await writeComponentToFileListFromText( - USER_PIPELINES_LIST_NAME, + const file = await storage.createPipeline( name, - componentText, + defaultPipelineYamlWithName(name), ); + const ref = { name: file.displayName, fileId: file.id }; if (e.ctrlKey || e.metaKey) { - window.open(getDefaultEditorHref({ name }), "_blank"); + window.open(getDefaultEditorHref(ref), "_blank"); return; } navigate({ - ...getDefaultEditorTarget({ name }), + ...getDefaultEditorTarget(ref), reloadDocument: !IS_GITHUB_PAGES, }); }; diff --git a/src/components/shared/ReactFlow/FlowSidebar/sections/components/SavePipelineAsButton.tsx b/src/components/shared/ReactFlow/FlowSidebar/sections/components/SavePipelineAsButton.tsx index 41e0225e8d..34b2b5cac1 100644 --- a/src/components/shared/ReactFlow/FlowSidebar/sections/components/SavePipelineAsButton.tsx +++ b/src/components/shared/ReactFlow/FlowSidebar/sections/components/SavePipelineAsButton.tsx @@ -6,7 +6,7 @@ import { PipelineNameDialog } from "@/components/shared/Dialogs"; import useToastNotification from "@/hooks/useToastNotification"; import { useAnalytics } from "@/providers/AnalyticsProvider"; import { useComponentSpec } from "@/providers/ComponentSpecProvider"; -import { EDITOR_PATH } from "@/routes/router"; +import { getDefaultEditorTarget } from "@/routes/editorRoutes"; import { useSavePipeline } from "@/services/pipelineService"; import { tracking } from "@/utils/tracking"; @@ -31,14 +31,12 @@ export const SavePipelineAsButton = ({ const handleSavePipelineAs = useCallback( async (name: string) => { - await savePipeline(name); + const file = await savePipeline(name); track("pipeline_editor.pipeline_actions.save_pipeline_as_completed"); notify(`Pipeline saved as "${name}"`, "success"); onSaveComplete?.(name); - navigate({ - to: `${EDITOR_PATH}/${encodeURIComponent(name)}`, - }); + navigate(getDefaultEditorTarget({ name, fileId: file?.id })); }, [navigate, savePipeline, notify, onSaveComplete, track], ); diff --git a/src/providers/ComponentSpecProvider.tsx b/src/providers/ComponentSpecProvider.tsx index 34e52518c7..d4f5a6023f 100644 --- a/src/providers/ComponentSpecProvider.tsx +++ b/src/providers/ComponentSpecProvider.tsx @@ -10,8 +10,7 @@ import { import { type UndoRedo, useUndoRedo } from "@/hooks/useUndoRedo"; import { loadPipelineByName } from "@/services/pipelineService"; -import { emitPipelineFileChanged } from "@/services/pipelineStorage/pipelineFileEvents"; -import { USER_PIPELINES_LIST_NAME } from "@/utils/constants"; +import { savePipeline } from "@/services/pipelineStorage/pipelineOperations"; import { prepareComponentRefForEditor } from "@/utils/prepareComponentRefForEditor"; import { getSubgraphComponentSpec, @@ -37,7 +36,6 @@ import { import { type ComponentReferenceWithSpec, generateDigest, - writeComponentToFileListFromText, } from "../utils/componentStore"; const EMPTY_GRAPH_SPEC: GraphSpec = { @@ -209,13 +207,7 @@ export const ComponentSpecProvider = ({ const specWithName = { ...componentSpec, name }; - const componentText = componentSpecToYaml(specWithName); - await writeComponentToFileListFromText( - USER_PIPELINES_LIST_NAME, - name, - componentText, - ); - emitPipelineFileChanged({ storageKey: name, source: "v1" }); + await savePipeline(name, componentSpecToYaml(specWithName), "v1"); }, [componentSpec, readOnly], ); diff --git a/src/routes/Import/index.tsx b/src/routes/Import/index.tsx index 1b18cc0bea..64f1857ef9 100644 --- a/src/routes/Import/index.tsx +++ b/src/routes/Import/index.tsx @@ -201,7 +201,9 @@ export const ImportPage = () => { importedRef.current = true; setPipelineName(result.name); setStep(Step.Done); - navigate(getDefaultEditorTarget({ name: result.name })); + navigate( + getDefaultEditorTarget({ name: result.name, fileId: result.fileId }), + ); } else { setError(result.errorMessage || "Failed to import pipeline from URL."); } diff --git a/src/services/importPipeline.test.ts b/src/services/importPipeline.test.ts index 6087b093a1..b9109c90d7 100644 --- a/src/services/importPipeline.test.ts +++ b/src/services/importPipeline.test.ts @@ -1,11 +1,21 @@ import yaml from "js-yaml"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import * as componentStore from "@/utils/componentStore"; -import { USER_PIPELINES_LIST_NAME } from "@/utils/constants"; - import { importPipelineFromYaml } from "./pipelineService"; +const { existingNames, savePipeline } = vi.hoisted(() => ({ + existingNames: new Set(), + savePipeline: vi.fn(), +})); + +vi.mock("./pipelineStorage/pipelineOperations", () => ({ + findPipelineFile: vi.fn(async () => undefined), + listPipelineFiles: vi.fn(async () => + [...existingNames].map((displayName) => ({ displayName })), + ), + savePipeline, +})); + describe("importPipelineFromYaml", () => { const validYamlObject = { name: "Test Pipeline", @@ -25,13 +35,12 @@ describe("importPipelineFromYaml", () => { const validYamlContent = yaml.dump(validYamlObject); beforeEach(() => { - vi.clearAllMocks(); vi.spyOn(console, "error").mockImplementation(() => {}); - - vi.spyOn(componentStore, "getComponentFileFromList").mockResolvedValue( - null, - ); - vi.spyOn(componentStore, "writeComponentToFileListFromText"); + existingNames.clear(); + savePipeline.mockImplementation(async (name: string) => ({ + id: `id-for-${name}`, + displayName: name, + })); }); afterEach(() => { @@ -39,32 +48,20 @@ describe("importPipelineFromYaml", () => { }); it("should successfully import a valid pipeline", async () => { - // Mock no existing pipeline with the same name - vi.mocked(componentStore.getComponentFileFromList).mockResolvedValue(null); - const result = await importPipelineFromYaml(validYamlContent); - // Expect writeComponentToFileListFromText to be called with correct parameters - expect(componentStore.writeComponentToFileListFromText).toHaveBeenCalled(); + expect(savePipeline).toHaveBeenCalled(); - // Expect successful result expect(result).toEqual({ name: "Test Pipeline", + fileId: "id-for-Test Pipeline", overwritten: false, successful: true, }); }); it("should generate a unique name when a name collision occurs", async () => { - // Mock existing pipeline with the same name, but not with "(1)" suffix - vi.mocked(componentStore.getComponentFileFromList).mockImplementation( - async (_, name) => { - if (name === "Test Pipeline") { - return {} as any; - } - return null; - }, - ); + existingNames.add("Test Pipeline"); const result = await importPipelineFromYaml(validYamlContent, false); @@ -73,42 +70,23 @@ describe("importPipelineFromYaml", () => { expect(result.name).toBe("Test Pipeline (1)"); expect(result.errorMessage).toContain("was renamed"); - // Expect writeComponentToFileListFromText to be called with the new name and YAML - expect( - componentStore.writeComponentToFileListFromText, - ).toHaveBeenCalledWith( - USER_PIPELINES_LIST_NAME, + expect(savePipeline).toHaveBeenCalledWith( "Test Pipeline (1)", expect.stringContaining("name: Test Pipeline (1)"), ); }); it("should increment counter when multiple name collisions occur", async () => { - // Mock existing pipelines with the name and first two numbered variants - vi.mocked(componentStore.getComponentFileFromList).mockImplementation( - async (_, name) => { - if ( - name === "Test Pipeline" || - name === "Test Pipeline (1)" || - name === "Test Pipeline (2)" - ) { - return {} as any; - } - return null; - }, - ); + existingNames.add("Test Pipeline"); + existingNames.add("Test Pipeline (1)"); + existingNames.add("Test Pipeline (2)"); const result = await importPipelineFromYaml(validYamlContent, false); - // Expect a successful result with the name incremented to (3) expect(result.successful).toBe(true); expect(result.name).toBe("Test Pipeline (3)"); - // Expect writeComponentToFileListFromText to be called with the new name and YAML - expect( - componentStore.writeComponentToFileListFromText, - ).toHaveBeenCalledWith( - USER_PIPELINES_LIST_NAME, + expect(savePipeline).toHaveBeenCalledWith( "Test Pipeline (3)", yaml.dump({ ...validYamlObject, @@ -117,10 +95,22 @@ describe("importPipelineFromYaml", () => { ); }); + it("overwrites the existing pipeline when asked to", async () => { + existingNames.add("Test Pipeline"); + + const result = await importPipelineFromYaml(validYamlContent, true); + + expect(result.overwritten).toBe(true); + expect(result.name).toBe("Test Pipeline"); + expect(savePipeline).toHaveBeenCalledWith( + "Test Pipeline", + expect.stringContaining("name: Test Pipeline"), + ); + }); + it("should handle invalid YAML content", async () => { const result = await importPipelineFromYaml("invalid: yaml: content: -"); - // Expect unsuccessful result expect(result.successful).toBe(false); expect(result.errorMessage).toBeDefined(); expect(result.name).toBe(""); @@ -140,14 +130,10 @@ describe("importPipelineFromYaml", () => { const result = await importPipelineFromYaml(containerPipeline); - // Expect unsuccessful result expect(result.successful).toBe(false); expect(result.errorMessage).toContain("graph-based pipeline"); - // Expect the writing function not to be called - expect( - componentStore.writeComponentToFileListFromText, - ).not.toHaveBeenCalled(); + expect(savePipeline).not.toHaveBeenCalled(); }); it("should use default name for unnamed pipelines", async () => { @@ -167,18 +153,9 @@ describe("importPipelineFromYaml", () => { const unnamedYaml = yaml.dump(unnamedPipelineSpec); - vi.mocked(componentStore.getComponentFileFromList).mockResolvedValue(null); - const result = await importPipelineFromYaml(unnamedYaml); - // Expect writeComponentToFileListFromText to be called with default name - expect( - componentStore.writeComponentToFileListFromText, - ).toHaveBeenCalledWith( - USER_PIPELINES_LIST_NAME, - "Imported Pipeline", - unnamedYaml, - ); + expect(savePipeline).toHaveBeenCalledWith("Imported Pipeline", unnamedYaml); expect(result.name).toBe("Imported Pipeline"); }); diff --git a/src/services/pipelineRunService.ts b/src/services/pipelineRunService.ts index 995c39e1a5..547c160a98 100644 --- a/src/services/pipelineRunService.ts +++ b/src/services/pipelineRunService.ts @@ -12,18 +12,15 @@ import { type ComponentSpec, isGraphImplementation, } from "@/utils/componentSpec"; -import { - getComponentFileFromList, - writeComponentToFileListFromText, -} from "@/utils/componentStore"; -import { - DB_NAME, - PIPELINE_RUNS_STORE_NAME, - USER_PIPELINES_LIST_NAME, -} from "@/utils/constants"; +import { DB_NAME, PIPELINE_RUNS_STORE_NAME } from "@/utils/constants"; import { fetchWithErrorHandling } from "@/utils/fetchWithErrorHandling"; import { componentSpecToYaml } from "@/utils/yaml"; +import { + createPipeline, + listPipelineFiles, +} from "./pipelineStorage/pipelineOperations"; + export const createPipelineRun = async ( payload: BodyCreateApiPipelineRunsPost, backendUrl: string, @@ -131,39 +128,29 @@ export const copyRunToPipeline = async ( // Generate a name for the copied pipeline const originalName = cleanComponentSpec.name || "Unnamed Pipeline"; - let newName = name || originalName; + const taken = new Set( + (await listPipelineFiles()).map((file) => file.displayName), + ); - // Check if the name already exists and append a number if needed - let nameExists = true; + let newName = name || originalName; let counter = 1; - while (nameExists) { - const existingFile = await getComponentFileFromList( - USER_PIPELINES_LIST_NAME, - newName, - ); - - if (existingFile === null) { - nameExists = false; - } else { - const countNumber = counter > 1 ? " " + counter : ""; - newName = `${originalName} (Copy${countNumber})`; - counter++; - } + while (taken.has(newName)) { + const countNumber = counter > 1 ? " " + counter : ""; + newName = `${originalName} (Copy${countNumber})`; + counter++; } cleanComponentSpec.name = newName; - const componentText = componentSpecToYaml(cleanComponentSpec); - await writeComponentToFileListFromText( - USER_PIPELINES_LIST_NAME, + const file = await createPipeline( newName, - componentText, + componentSpecToYaml(cleanComponentSpec), ); return { - url: getDefaultEditorHref({ name: newName }), - name: newName, + url: getDefaultEditorHref({ name: file.displayName, fileId: file.id }), + name: file.displayName, }; } catch (error) { console.error("Error cloning pipeline:", error); diff --git a/src/services/pipelineService.ts b/src/services/pipelineService.ts index f241d38194..f9ac6b3068 100644 --- a/src/services/pipelineService.ts +++ b/src/services/pipelineService.ts @@ -8,38 +8,26 @@ import { isGraphImplementation, } from "@/utils/componentSpec"; import { - deleteComponentFileFromList, fullyLoadComponentRefFromUrl, - getComponentFileFromList, loadComponentAsRefFromText, - writeComponentToFileListFromText, } from "@/utils/componentStore"; -import { USER_PIPELINES_LIST_NAME } from "@/utils/constants"; import { componentSpecToYaml } from "@/utils/yaml"; import { componentSpecFromYaml } from "@/utils/yaml"; import type { PipelineFile } from "./pipelineStorage/PipelineFile"; -import { findPipelineFile } from "./pipelineStorage/pipelineOperations"; import { - deleteEntry, - findByStorageKey, -} from "./pipelineStorage/pipelineRegistry"; - -export const deletePipeline = async (name: string, onDelete?: () => void) => { - try { - await deleteComponentFileFromList(USER_PIPELINES_LIST_NAME, name); - const entry = await findByStorageKey(name); - if (entry) await deleteEntry(entry.id); - onDelete?.(); - } catch (error) { - console.error("Error deleting pipeline:", error); - } -}; + findPipelineFile, + listPipelineFiles, + savePipeline as savePipelineToStore, +} from "./pipelineStorage/pipelineOperations"; +import { usePipelineStorage } from "./pipelineStorage/PipelineStorageProvider"; export const useSavePipeline = (componentSpec: ComponentSpec) => { + const storage = usePipelineStorage(); + const savePipeline = async (name?: string) => { if (!componentSpec) { - return; + return undefined; } const componentSpecWithNewName = { @@ -47,12 +35,9 @@ export const useSavePipeline = (componentSpec: ComponentSpec) => { name: name ?? componentSpec.name ?? "Untitled Pipeline", }; - const componentSpecAsYaml = componentSpecToYaml(componentSpecWithNewName); - - await writeComponentToFileListFromText( - USER_PIPELINES_LIST_NAME, + return storage.savePipelineByName( componentSpecWithNewName.name, - componentSpecAsYaml, + componentSpecToYaml(componentSpecWithNewName), ); }; @@ -142,38 +127,26 @@ export const loadPipelineByName = async (name: string) => { export interface ImportResult { name: string; + fileId?: string; overwritten: boolean; successful: boolean; errorMessage?: string; } -/** - * Generates a unique pipeline name by adding a numbered suffix when a collision occurs - * @param baseName The original pipeline name - * @returns A promise resolving to a unique pipeline name - */ -async function generateUniquePipelineName(baseName: string): Promise { - // First check if the base name is available - const existingPipeline = await getComponentFileFromList( - USER_PIPELINES_LIST_NAME, - baseName, - ); - - if (!existingPipeline) { - return baseName; // Base name is available +function generateUniquePipelineName( + baseName: string, + taken: ReadonlySet, +): string { + if (!taken.has(baseName)) { + return baseName; } - // Base name exists, try adding numbers let counter = 1; - let newName = `${baseName} (${counter})`; - - // Keep checking until we find an available name - while (await getComponentFileFromList(USER_PIPELINES_LIST_NAME, newName)) { + while (taken.has(`${baseName} (${counter})`)) { counter++; - newName = `${baseName} (${counter})`; } - return newName; + return `${baseName} (${counter})`; } /** @@ -210,16 +183,15 @@ export async function importPipelineFromYaml( let pipelineName = componentSpec.name || "Imported Pipeline"; let wasRenamed = false; - // Check if a pipeline with this name already exists - const existingPipeline = await getComponentFileFromList( - USER_PIPELINES_LIST_NAME, - pipelineName, + const taken = new Set( + (await listPipelineFiles()).map((file) => file.displayName), ); + const nameExists = taken.has(pipelineName); // If exists and we're not overwriting, generate a unique name - if (existingPipeline && !overwrite) { + if (nameExists && !overwrite) { const originalName = pipelineName; - pipelineName = await generateUniquePipelineName(pipelineName); + pipelineName = generateUniquePipelineName(pipelineName, taken); wasRenamed = pipelineName !== originalName; // Update the component spec name to match the new name @@ -230,16 +202,12 @@ export async function importPipelineFromYaml( // This also ensures the ComponentSpec is valid const standardizedYaml = componentSpecToYaml(componentSpec); - // Save the pipeline to IndexedDB - await writeComponentToFileListFromText( - USER_PIPELINES_LIST_NAME, - pipelineName, - standardizedYaml, - ); + const file = await savePipelineToStore(pipelineName, standardizedYaml); return { - name: pipelineName, - overwritten: Boolean(existingPipeline && overwrite), + name: file.displayName, + fileId: file.id, + overwritten: nameExists && overwrite, successful: true, errorMessage: wasRenamed ? `Pipeline was renamed to "${pipelineName}" to avoid name conflict.` diff --git a/src/services/pipelineStorage/PipelineFile.ts b/src/services/pipelineStorage/PipelineFile.ts index 2cdc45ca37..2e323c687c 100644 --- a/src/services/pipelineStorage/PipelineFile.ts +++ b/src/services/pipelineStorage/PipelineFile.ts @@ -2,7 +2,10 @@ import { action, makeObservable, observable, runInAction } from "mobx"; import { emitUserPipelineWritten } from "@/utils/userPipelineWriteEvents"; -import { emitPipelineFileChanged } from "./pipelineFileEvents"; +import { + emitPipelineFileChanged, + type PipelineFileSource, +} from "./pipelineFileEvents"; import type { PipelineFolder } from "./PipelineFolder"; import { deleteEntry, updateEntry } from "./pipelineRegistry"; @@ -44,7 +47,10 @@ export class PipelineFile { return this.folder.driver.read(this.storageKey); } - async write(content: string): Promise { + async write( + content: string, + source: PipelineFileSource = "v2", + ): Promise { const descriptor = await this.folder.driver.write(this.storageKey, content); if (descriptor.contentVersion !== undefined) { @@ -53,7 +59,7 @@ export class PipelineFile { }); } - emitPipelineFileChanged({ storageKey: this.storageKey, source: "v2" }); + emitPipelineFileChanged({ storageKey: this.storageKey, source }); emitUserPipelineWritten(); } diff --git a/src/services/pipelineStorage/PipelineFolder.ts b/src/services/pipelineStorage/PipelineFolder.ts index 77e7d25da6..35c9ed074d 100644 --- a/src/services/pipelineStorage/PipelineFolder.ts +++ b/src/services/pipelineStorage/PipelineFolder.ts @@ -129,7 +129,12 @@ export class PipelineFolder { } async addFile(storageKey: string, content: string): Promise { - await assertStorageKeyUnique(storageKey); + /** + * A flat store hands back its own key, so the caller's is a suggestion the + * write upserts on — there is nothing to collide with, and asking would + * cost a round trip to learn that. + */ + if (!this.isFlat) await assertStorageKeyUnique(storageKey); // Writing before registering means a rejected write leaves no registry row // pointing at a file that was never created. diff --git a/src/services/pipelineStorage/PipelineStorageService.test.ts b/src/services/pipelineStorage/PipelineStorageService.test.ts index 615e3279a3..3a4d67f15f 100644 --- a/src/services/pipelineStorage/PipelineStorageService.test.ts +++ b/src/services/pipelineStorage/PipelineStorageService.test.ts @@ -78,19 +78,39 @@ function summary(key: string, displayName: string): HostPipelineSummary { }; } -function installHost(listing: HostPipelineSummary[] = []): void { +/** + * Mirrors the two host behaviours the write paths are built on: `write` upserts + * on the caller's key, and the displayed name comes from the written spec. + */ +function installHost( + listing: HostPipelineSummary[] = [], +): Map { + const summaries = new Map(listing.map((entry) => [entry.key, entry])); + const specs = new Map(); + const host: PipelineStorageHost = { version: 1, label: LABEL, - list: async () => listing, - read: async () => { - throw new Error("not seeded"); + list: async () => [...summaries.values()], + read: async (key) => { + const found = summaries.get(key); + if (!found) throw new Error(`not seeded: ${key}`); + return { ...found, spec: specs.get(key) }; + }, + write: async (key, spec) => { + specs.set(key, spec); + const written = { + ...summary(key, (spec as { name?: string }).name ?? "Untitled"), + contentVersion: String(summaries.size + 1), + }; + summaries.set(key, written); + return written; }, - write: async () => { - throw new Error("not seeded"); + delete: async (key) => { + summaries.delete(key); + specs.delete(key); }, - delete: async () => undefined, - has: async (key) => listing.some((entry) => entry.key === key), + has: async (key) => summaries.has(key), }; Object.defineProperty(window, "__TANGLE_PIPELINE_STORAGE_HOST__", { @@ -98,8 +118,13 @@ function installHost(listing: HostPipelineSummary[] = []): void { configurable: true, writable: true, }); + + return specs; } +const PIPELINE_YAML = (name: string) => + `name: ${name}\nimplementation:\n graph:\n tasks: {}\n`; + beforeEach(() => { registry.clear(); resetStorageModeForTests(); @@ -225,3 +250,66 @@ describe("resolving a route reference against a host", () => { expect(file.storageKey).toBe("opaque-key-1"); }); }); + +describe("writing to a host", () => { + it("creates a pipeline and registers the identity the store reported", async () => { + installHost(); + const service = new PipelineStorageService(); + + const file = await service.createPipeline( + "Churn model", + PIPELINE_YAML("Churn model"), + ); + + expect(file.id).toBe("id-Churn model"); + expect(file.displayName).toBe("Churn model"); + expect(await service.rootFolder.listPipelines()).toHaveLength(1); + }); + + it("saves over the pipeline that already has the name", async () => { + const specs = installHost(); + const service = new PipelineStorageService(); + + await service.createPipeline("Churn model", PIPELINE_YAML("Churn model")); + await service.savePipelineByName( + "Churn model", + PIPELINE_YAML("Churn model"), + ); + + expect(await service.rootFolder.listPipelines()).toHaveLength(1); + expect(specs.size).toBe(1); + }); + + it("creates a pipeline when saving a name the store has never held", async () => { + installHost(); + const service = new PipelineStorageService(); + + const file = await service.savePipelineByName( + "Ranking model", + PIPELINE_YAML("Ranking model"), + ); + + expect(file.displayName).toBe("Ranking model"); + }); + + it("deletes through the driver so the store loses the pipeline too", async () => { + installHost(); + const service = new PipelineStorageService(); + + await service.createPipeline("Churn model", PIPELINE_YAML("Churn model")); + await service.deletePipelineByName("Churn model"); + + expect(await service.rootFolder.listPipelines()).toEqual([]); + expect(registry.size).toBe(0); + }); + + it("leaves the store alone when asked to delete a name it does not hold", async () => { + installHost([summary("opaque-key-1", "Churn model")]); + const service = new PipelineStorageService(); + + await expect( + service.deletePipelineByName("Ranking model"), + ).resolves.toBeUndefined(); + expect(await service.rootFolder.listPipelines()).toHaveLength(1); + }); +}); diff --git a/src/services/pipelineStorage/PipelineStorageService.ts b/src/services/pipelineStorage/PipelineStorageService.ts index 4d89037e11..104f0520f8 100644 --- a/src/services/pipelineStorage/PipelineStorageService.ts +++ b/src/services/pipelineStorage/PipelineStorageService.ts @@ -4,6 +4,7 @@ import { createDriver } from "./createDriver"; import { pipelineStorageDb } from "./db"; import { RootFolderDbStorageDriver } from "./drivers/RootFolderDbStorageDriver"; import { PipelineFile } from "./PipelineFile"; +import type { PipelineFileSource } from "./pipelineFileEvents"; import { PipelineFolder } from "./PipelineFolder"; import { findById, findByStorageKey } from "./pipelineRegistry"; import { resolveStorageMode, type StorageMode } from "./storageMode"; @@ -37,15 +38,45 @@ export class PipelineStorageService { async resolve(ref: PipelineRef): Promise { if (ref.fileId) return this.findPipelineById(ref.fileId); - const byName = await this.resolvePipelineByName(ref.name); - if (byName) return byName; - - const adopted = await this.adoptFromLegacyStore(ref.name); - if (adopted) return adopted; + const found = await this.findPipelineByName(ref.name); + if (found) return found; throw new PipelineNotFoundError(`Pipeline "${ref.name}" not found`); } + async findPipelineByName(name: string): Promise { + return ( + (await this.resolvePipelineByName(name)) ?? + (await this.adoptFromLegacyStore(name)) + ); + } + + async createPipeline(name: string, content: string): Promise { + return this.rootFolder.addFile(name, content); + } + + /** + * The write path for callers that only ever knew a name — the v1 editor, the + * save buttons, import. Saving under an existing name updates that pipeline + * wherever it lives, rather than laying a second copy in the root store. + */ + async savePipelineByName( + name: string, + content: string, + source?: PipelineFileSource, + ): Promise { + const existing = await this.findPipelineByName(name); + if (!existing) return this.createPipeline(name, content); + + await existing.write(content, source); + return existing; + } + + async deletePipelineByName(name: string): Promise { + const existing = await this.findPipelineByName(name); + await existing?.deleteFile(); + } + async findPipelineById(id: string): Promise { const entry = await findById(id); diff --git a/src/services/pipelineStorage/pipelineOperations.ts b/src/services/pipelineStorage/pipelineOperations.ts index 8186fcc643..3db179ab23 100644 --- a/src/services/pipelineStorage/pipelineOperations.ts +++ b/src/services/pipelineStorage/pipelineOperations.ts @@ -1,4 +1,5 @@ import type { PipelineFile } from "./PipelineFile"; +import type { PipelineFileSource } from "./pipelineFileEvents"; import { getPipelineStorageService, PipelineNotFoundError, @@ -24,3 +25,22 @@ export async function findPipelineFile( throw error; } } + +export async function createPipeline( + name: string, + content: string, +): Promise { + return getPipelineStorageService().createPipeline(name, content); +} + +export async function savePipeline( + name: string, + content: string, + source?: PipelineFileSource, +): Promise { + return getPipelineStorageService().savePipelineByName(name, content, source); +} + +export async function deletePipelineByName(name: string): Promise { + return getPipelineStorageService().deletePipelineByName(name); +} diff --git a/src/utils/URL.test.ts b/src/utils/URL.test.ts index 2ff94f3847..466ad13e68 100644 --- a/src/utils/URL.test.ts +++ b/src/utils/URL.test.ts @@ -248,6 +248,13 @@ describe("getIdOrTitleFromPath", () => { expect(id).toBe("some id"); }); + it("treats an opaque hex-looking editor segment as a title", () => { + const path = "/editor/0123456789abcdef0123"; + const { id, title } = getIdOrTitleFromPath(path); + expect(id).toBe(undefined); + expect(title).toBe("0123456789abcdef0123"); + }); + it("returns undefined if path ends with slash", () => { const path = "/foo/bar/runs/"; const { id, title } = getIdOrTitleFromPath(path); diff --git a/src/utils/URL.ts b/src/utils/URL.ts index 24e13a2d97..224cc073f1 100644 --- a/src/utils/URL.ts +++ b/src/utils/URL.ts @@ -157,26 +157,27 @@ const downloadYamlFromComponentText = (text: string, displayName: string) => { downloadStringAsFile(text, `${displayName}.yaml`, "text/yaml"); }; +/** + * A run path names an execution; every other path names a pipeline. Guessing + * from the shape of the segment instead used to mistake any 20-hex pipeline + * name for an id, and the editor then loaded nothing at all. + */ const getIdOrTitleFromPath = ( pathname: string, ): { id?: string; title?: string; } => { - const isRunPath = pathname.includes(RUNS_BASE_PATH); - const lastPathSegment = pathname.split("/").pop() || ""; - const isId = lastPathSegment.match(/^[0-9a-fA-F]{20}$/) || isRunPath; const decodedSegment = decodeURIComponent(lastPathSegment); if (decodedSegment === "") { return { id: undefined, title: undefined }; } - return { - id: isId ? decodedSegment : undefined, - title: isId ? undefined : decodedSegment, - }; + return pathname.includes(RUNS_BASE_PATH) + ? { id: decodedSegment, title: undefined } + : { id: undefined, title: decodedSegment }; }; const MAX_URL_LENGTH = 2048; diff --git a/tests/e2e/fixtures/pipelineStorageHost.ts b/tests/e2e/fixtures/pipelineStorageHost.ts index 54405820d8..e70e94e127 100644 --- a/tests/e2e/fixtures/pipelineStorageHost.ts +++ b/tests/e2e/fixtures/pipelineStorageHost.ts @@ -35,31 +35,51 @@ declare global { const DEFAULT_LABEL = "Shared storage"; +const STATE_KEY = "__tangle_test_host_state__"; + /** * Stands in for the page that embeds this app. It has to be installed with * `addInitScript` rather than `evaluate`, because storage mode is decided while * the app boots and never revisited. + * + * Its contents outlive a reload — creating a pipeline navigates with a document + * load, and a store that forgot everything at that point could not show that + * the write reached it. */ export async function installPipelineStorageHost( page: Page, options: HostStorageOptions = {}, ): Promise { await page.addInitScript( - (config: Required) => { - const store = new Map(); - let revision = 0; - - for (const seeded of config.seed) { - revision += 1; - store.set(seeded.key, { - key: seeded.key, - externalId: `external-${revision}`, - displayName: seeded.displayName, - contentVersion: `v${revision}`, - spec: seeded.spec, - }); + (config: Required & { stateKey: string }) => { + const saved = window.sessionStorage.getItem(config.stateKey); + const store = new Map( + saved ? (JSON.parse(saved) as [string, HostRecord][]) : [], + ); + let revision = store.size; + + if (!saved) { + for (const seeded of config.seed) { + revision += 1; + store.set(seeded.key, { + key: seeded.key, + externalId: `external-${revision}`, + displayName: seeded.displayName, + contentVersion: `v${revision}`, + spec: seeded.spec, + }); + } + } + + function persist(): void { + window.sessionStorage.setItem( + config.stateKey, + JSON.stringify([...store.entries()]), + ); } + persist(); + async function gate(): Promise { if (config.latencyMs > 0) { await new Promise((resolve) => setTimeout(resolve, config.latencyMs)); @@ -112,11 +132,13 @@ export async function installPipelineStorageHost( spec, }; store.set(key, record); + persist(); return summaryOf(record); }, async delete(key: string) { await gate(); store.delete(key); + persist(); }, async has(key: string) { await gate(); @@ -133,7 +155,8 @@ export async function installPipelineStorageHost( seed: options.seed ?? [], failMode: options.failMode ?? "none", latencyMs: options.latencyMs ?? 0, - } satisfies Required, + stateKey: STATE_KEY, + } satisfies Required & { stateKey: string }, ); } diff --git a/tests/e2e/pipeline-storage-host.spec.ts b/tests/e2e/pipeline-storage-host.spec.ts index c0c34084fd..bd870fcf39 100644 --- a/tests/e2e/pipeline-storage-host.spec.ts +++ b/tests/e2e/pipeline-storage-host.spec.ts @@ -63,4 +63,40 @@ test.describe("host-provided pipeline storage", () => { await expect(page.getByText("Churn model")).toBeHidden(); expect(await readLocallyStoredPipelineKeys(page)).toEqual([]); }); + + test("creates a new pipeline in the host and nowhere else", async ({ + page, + }) => { + await installSeededHost(page); + + await page.goto("/pipeline-folders"); + await expect(page.getByText("Churn model")).toBeVisible(); + + await page.getByTestId("new-pipeline-button").click(); + await expect(page.locator('[data-testid="rf__wrapper"]')).toBeVisible({ + timeout: 30_000, + }); + + expect(await readHostRecords(page)).toHaveLength(SEED.length + 1); + expect(await readLocallyStoredPipelineKeys(page)).toEqual([]); + }); + + test("a deleted pipeline does not come back on reload", async ({ page }) => { + await installSeededHost(page); + + await page.goto("/pipeline-folders"); + const row = page.getByRole("row").filter({ hasText: "Churn model" }); + await row.locator("[data-checkbox]").click(); + + await page.getByRole("button", { name: "Delete", exact: true }).click(); + await page.getByRole("button", { name: "Continue" }).click(); + + await expect(page.getByText("Churn model")).toBeHidden(); + + await page.reload(); + await expect(page.getByText("Nightly refresh")).toBeVisible(); + await expect(page.getByText("Churn model")).toBeHidden(); + + expect(await readHostRecords(page)).toHaveLength(SEED.length - 1); + }); }); From 3195cd08ee51dc47054cf3c722d4e00e82f89313 Mon Sep 17 00:00:00 2001 From: Camiel van Schoonhoven Date: Tue, 8 Sep 2026 16:28:11 -0700 Subject: [PATCH 13/36] refactor(pipeline-storage): let a store that names its own pipelines rename them MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Renaming assumed the store keys a pipeline by its name, so renaming was moving the key. A store that hands out its own keys has no key to move — it takes the displayed name from the next spec written — and the driver could only throw. Make renaming optional on the driver instead. A file whose store cannot rename by key leaves the key alone, and the name arrives with the write that follows. Which is why the two are one operation for callers that only know names: split apart, the store would keep the old name and the caller would then create a second pipeline under the new one. The legacy editor's rename was the last writer still going straight to the browser-local list, and it now goes through the service like the rest. That leaves the list's rename with one caller and no use for its find-the-file-from-the-URL fallback. A name dialog no longer refuses a duplicate the store itself accepts, and no longer lists pipelines it has nothing to check them against. Co-Authored-By: Claude Opus 5 (1M context) --- .../Editor/Context/RenamePipeline.tsx | 31 ++++++++-------- .../shared/Dialogs/PipelineNameDialog.tsx | 20 ++++++++--- src/hooks/useLoadUserPipelines.ts | 8 +++-- src/services/pipelineStorage/PipelineFile.ts | 7 ++++ .../pipelineStorage/PipelineFolder.test.ts | 35 ++++++++++++++++++- .../PipelineStorageService.test.ts | 33 +++++++++++++++++ .../pipelineStorage/PipelineStorageService.ts | 20 +++++++++++ .../drivers/HostStorageDriver.test.ts | 4 +-- .../drivers/HostStorageDriver.ts | 6 ---- .../pipelineStorage/pipelineOperations.ts | 14 ++++++++ src/services/pipelineStorage/types.ts | 2 +- src/utils/componentStore.ts | 23 +++--------- tests/e2e/pipeline-rename.spec.ts | 30 ++++++++++++++++ tests/e2e/pipeline-storage-host.spec.ts | 32 ++++++++++++++--- 14 files changed, 208 insertions(+), 57 deletions(-) create mode 100644 tests/e2e/pipeline-rename.spec.ts diff --git a/src/components/Editor/Context/RenamePipeline.tsx b/src/components/Editor/Context/RenamePipeline.tsx index 16ae63327b..535855720b 100644 --- a/src/components/Editor/Context/RenamePipeline.tsx +++ b/src/components/Editor/Context/RenamePipeline.tsx @@ -1,4 +1,4 @@ -import { useLocation, useNavigate } from "@tanstack/react-router"; +import { useNavigate } from "@tanstack/react-router"; import { Edit3 } from "lucide-react"; import TooltipButton from "@/components/shared/Buttons/TooltipButton"; @@ -7,19 +7,17 @@ import useToastNotification from "@/hooks/useToastNotification"; import { useAnalytics } from "@/providers/AnalyticsProvider"; import { useComponentSpec } from "@/providers/ComponentSpecProvider"; import { APP_ROUTES } from "@/routes/router"; -import { renameComponentFileInList } from "@/utils/componentStore"; -import { USER_PIPELINES_LIST_NAME } from "@/utils/constants"; +import { renamePipeline } from "@/services/pipelineStorage/pipelineOperations"; +import { getErrorMessage } from "@/utils/string"; import { tracking } from "@/utils/tracking"; +import { componentSpecToYaml } from "@/utils/yaml"; const RenamePipeline = () => { - const { componentSpec, saveComponentSpec } = useComponentSpec(); + const { componentSpec } = useComponentSpec(); const notify = useToastNotification(); const { track } = useAnalytics(); const navigate = useNavigate(); - const location = useLocation(); - const pathname = location.pathname; - const title = componentSpec?.name; const isSubmitDisabled = (name: string) => { @@ -32,14 +30,17 @@ const RenamePipeline = () => { return; } - await renameComponentFileInList( - USER_PIPELINES_LIST_NAME, - title ?? "", - name, - pathname, - ); - - await saveComponentSpec(name); + try { + await renamePipeline( + title ?? "", + name, + componentSpecToYaml({ ...componentSpec, name }), + "v1", + ); + } catch (error) { + notify(`Rename failed: ${getErrorMessage(error)}`, "error"); + return; + } const urlName = encodeURIComponent(name); const url = APP_ROUTES.PIPELINE_EDITOR.replace("$name", urlName); diff --git a/src/components/shared/Dialogs/PipelineNameDialog.tsx b/src/components/shared/Dialogs/PipelineNameDialog.tsx index d8d30800b1..b79653bb41 100644 --- a/src/components/shared/Dialogs/PipelineNameDialog.tsx +++ b/src/components/shared/Dialogs/PipelineNameDialog.tsx @@ -16,6 +16,7 @@ import { Icon } from "@/components/ui/icon"; import { Input } from "@/components/ui/input"; import { BlockStack } from "@/components/ui/layout"; import useLoadUserPipelines from "@/hooks/useLoadUserPipelines"; +import { isHostStorage } from "@/services/pipelineStorage/storageMode"; interface PipelineNameDialogProps { trigger?: ReactNode; @@ -49,20 +50,29 @@ const PipelineNameDialog = ({ const [name, setName] = useState(initialName); const [touched, setTouched] = useState(false); + /** + * A store that hands out its own keys can hold two pipelines under one name, + * so refusing a duplicate here would block a name the store itself accepts — + * and there is nothing to list for. + */ + const namesMustBeUnique = !isHostStorage(); + const { pipelineNames, isLoadingUserPipelines, refetch: refetchUserPipelines, - } = useLoadUserPipelines(); + } = useLoadUserPipelines(namesMustBeUnique); const normalized = name.trim().toLowerCase(); const excluded = new Set( (excludeNames ?? []).map((n) => n.trim().toLowerCase()), ); - const nameIsTaken = pipelineNames.some((n) => { - const lower = n.toLowerCase(); - return lower === normalized && !excluded.has(lower); - }); + const nameIsTaken = + namesMustBeUnique && + pipelineNames.some((n) => { + const lower = n.toLowerCase(); + return lower === normalized && !excluded.has(lower); + }); let error: string | null = null; if (!isLoadingUserPipelines) { diff --git a/src/hooks/useLoadUserPipelines.ts b/src/hooks/useLoadUserPipelines.ts index 3cd675c242..331ceec590 100644 --- a/src/hooks/useLoadUserPipelines.ts +++ b/src/hooks/useLoadUserPipelines.ts @@ -2,11 +2,13 @@ import { useCallback, useEffect, useState } from "react"; import { listPipelineFiles } from "@/services/pipelineStorage/pipelineOperations"; -const useLoadUserPipelines = () => { - const [isLoadingUserPipelines, setIsLoadingUserPipelines] = useState(true); +const useLoadUserPipelines = (enabled = true) => { + const [isLoadingUserPipelines, setIsLoadingUserPipelines] = useState(enabled); const [pipelineNames, setPipelineNames] = useState([]); const refetch = useCallback(async () => { + if (!enabled) return; + setIsLoadingUserPipelines(true); try { const files = await listPipelineFiles(); @@ -16,7 +18,7 @@ const useLoadUserPipelines = () => { } finally { setIsLoadingUserPipelines(false); } - }, []); + }, [enabled]); useEffect(() => { void refetch(); diff --git a/src/services/pipelineStorage/PipelineFile.ts b/src/services/pipelineStorage/PipelineFile.ts index 2e323c687c..b660b24a0f 100644 --- a/src/services/pipelineStorage/PipelineFile.ts +++ b/src/services/pipelineStorage/PipelineFile.ts @@ -63,8 +63,15 @@ export class PipelineFile { emitUserPipelineWritten(); } + /** + * A store that keys pipelines by their name has to move the key. One that + * keys them opaquely takes the displayed name from the next spec written to + * the same key, so there is nothing here to move. + */ @action async rename(newName: string): Promise { + if (!this.folder.driver.rename) return; + await this.folder.driver.rename(this.storageKey, newName); await updateEntry(this.id, { storageKey: newName }); diff --git a/src/services/pipelineStorage/PipelineFolder.test.ts b/src/services/pipelineStorage/PipelineFolder.test.ts index 78d6a05310..c3d159463d 100644 --- a/src/services/pipelineStorage/PipelineFolder.test.ts +++ b/src/services/pipelineStorage/PipelineFolder.test.ts @@ -57,6 +57,7 @@ interface FakeDriverOptions { storageKey: string, content: string, ) => Promise; + canRename?: boolean; } interface FakeDriver extends PipelineStorageDriver { @@ -87,7 +88,14 @@ function createFakeDriver(options: FakeDriverOptions = {}): FakeDriver { contents.set(storageKey, content); return { storageKey }; }), - async rename() {}, + rename: + options.canRename === false + ? undefined + : async (oldStorageKey: string, newStorageKey: string) => { + const content = contents.get(oldStorageKey); + contents.delete(oldStorageKey); + if (content !== undefined) contents.set(newStorageKey, content); + }, async delete(storageKey: string) { contents.delete(storageKey); }, @@ -348,6 +356,31 @@ describe("PipelineFolder.findFile", () => { }); }); +describe("PipelineFile.rename", () => { + it("moves the key when the store names its own pipelines", async () => { + const driver = createFakeDriver(); + const folder = createFolder(driver); + const file = await folder.addFile("Churn model", "name: Churn model"); + + await file.rename("Churn model v2"); + + expect(file.storageKey).toBe("Churn model v2"); + expect([...driver.contents.keys()]).toEqual(["Churn model v2"]); + expect(await folder.findFile("Churn model")).toBeUndefined(); + }); + + it("leaves the key alone when the store keys pipelines itself", async () => { + const driver = createFakeDriver({ canRename: false }); + const folder = createFolder(driver); + const file = await folder.addFile("opaque-key", "name: Churn model"); + + await file.rename("Churn model v2"); + + expect(file.storageKey).toBe("opaque-key"); + expect([...driver.contents.keys()]).toEqual(["opaque-key"]); + }); +}); + describe("emitted events", () => { it("does not emit a remote change for a local write", async () => { const folder = createFolder(createFakeDriver()); diff --git a/src/services/pipelineStorage/PipelineStorageService.test.ts b/src/services/pipelineStorage/PipelineStorageService.test.ts index 3a4d67f15f..1a0baff505 100644 --- a/src/services/pipelineStorage/PipelineStorageService.test.ts +++ b/src/services/pipelineStorage/PipelineStorageService.test.ts @@ -303,6 +303,39 @@ describe("writing to a host", () => { expect(registry.size).toBe(0); }); + it("renames in place rather than leaving a copy under the old name", async () => { + installHost(); + const service = new PipelineStorageService(); + const created = await service.createPipeline( + "Churn model", + PIPELINE_YAML("Churn model"), + ); + + const renamed = await service.renamePipelineByName( + "Churn model", + "Churn model v2", + PIPELINE_YAML("Churn model v2"), + ); + + expect(renamed.storageKey).toBe(created.storageKey); + const listed = await service.rootFolder.listPipelines(); + expect(listed.map((file) => file.displayName)).toEqual(["Churn model v2"]); + }); + + it("creates the pipeline when renaming one the store does not hold", async () => { + installHost(); + const service = new PipelineStorageService(); + + const file = await service.renamePipelineByName( + "Churn model", + "Churn model v2", + PIPELINE_YAML("Churn model v2"), + ); + + expect(file.displayName).toBe("Churn model v2"); + expect(await service.rootFolder.listPipelines()).toHaveLength(1); + }); + it("leaves the store alone when asked to delete a name it does not hold", async () => { installHost([summary("opaque-key-1", "Churn model")]); const service = new PipelineStorageService(); diff --git a/src/services/pipelineStorage/PipelineStorageService.ts b/src/services/pipelineStorage/PipelineStorageService.ts index 104f0520f8..814c6d2918 100644 --- a/src/services/pipelineStorage/PipelineStorageService.ts +++ b/src/services/pipelineStorage/PipelineStorageService.ts @@ -72,6 +72,26 @@ export class PipelineStorageService { return existing; } + /** + * Renaming and saving are one operation, because a store that does not key + * pipelines by name only learns the new one from the spec being written. + * Splitting them would leave that store holding the old name and the caller + * about to create a second pipeline under the new one. + */ + async renamePipelineByName( + currentName: string, + newName: string, + content: string, + source?: PipelineFileSource, + ): Promise { + const existing = await this.findPipelineByName(currentName); + if (!existing) return this.savePipelineByName(newName, content, source); + + await existing.rename(newName); + await existing.write(content, source); + return existing; + } + async deletePipelineByName(name: string): Promise { const existing = await this.findPipelineByName(name); await existing?.deleteFile(); diff --git a/src/services/pipelineStorage/drivers/HostStorageDriver.test.ts b/src/services/pipelineStorage/drivers/HostStorageDriver.test.ts index cf3ad7c654..0abd376555 100644 --- a/src/services/pipelineStorage/drivers/HostStorageDriver.test.ts +++ b/src/services/pipelineStorage/drivers/HostStorageDriver.test.ts @@ -254,12 +254,12 @@ describe("HostStorageDriver round trip", () => { }); describe("HostStorageDriver.rename", () => { - it("refuses, because a key carries no name for the host", async () => { + it("is not offered, because a key carries no name for the host", () => { const driver: PipelineStorageDriver = new HostStorageDriver( createFakeHost(), ); - await expect(driver.rename("key-1", "key-2")).rejects.toThrow(LABEL); + expect(driver.rename).toBeUndefined(); }); }); diff --git a/src/services/pipelineStorage/drivers/HostStorageDriver.ts b/src/services/pipelineStorage/drivers/HostStorageDriver.ts index 482399f1d7..e9ae7bb81e 100644 --- a/src/services/pipelineStorage/drivers/HostStorageDriver.ts +++ b/src/services/pipelineStorage/drivers/HostStorageDriver.ts @@ -66,12 +66,6 @@ export class HostStorageDriver implements PipelineStorageDriver { return toDescriptor(summary); } - async rename(): Promise { - throw new Error( - `Pipelines in ${this.host.label} cannot be renamed by key. Save the pipeline under a different name instead.`, - ); - } - async delete(storageKey: string): Promise { await this.call(() => this.host.delete(storageKey)); } diff --git a/src/services/pipelineStorage/pipelineOperations.ts b/src/services/pipelineStorage/pipelineOperations.ts index 3db179ab23..026483daa2 100644 --- a/src/services/pipelineStorage/pipelineOperations.ts +++ b/src/services/pipelineStorage/pipelineOperations.ts @@ -41,6 +41,20 @@ export async function savePipeline( return getPipelineStorageService().savePipelineByName(name, content, source); } +export async function renamePipeline( + currentName: string, + newName: string, + content: string, + source?: PipelineFileSource, +): Promise { + return getPipelineStorageService().renamePipelineByName( + currentName, + newName, + content, + source, + ); +} + export async function deletePipelineByName(name: string): Promise { return getPipelineStorageService().deletePipelineByName(name); } diff --git a/src/services/pipelineStorage/types.ts b/src/services/pipelineStorage/types.ts index cae9eae3d0..c3c48445fd 100644 --- a/src/services/pipelineStorage/types.ts +++ b/src/services/pipelineStorage/types.ts @@ -32,7 +32,7 @@ export interface PipelineStorageDriver { list(): Promise; read(storageKey: string): Promise; write(storageKey: string, content: string): Promise; - rename(oldStorageKey: string, newStorageKey: string): Promise; + rename?(oldStorageKey: string, newStorageKey: string): Promise; delete(storageKey: string): Promise; hasKey(storageKey: string): Promise; describe?(storageKey: string): Promise; diff --git a/src/utils/componentStore.ts b/src/utils/componentStore.ts index 7d397a0657..3b7ef71425 100644 --- a/src/utils/componentStore.ts +++ b/src/utils/componentStore.ts @@ -9,7 +9,6 @@ import { USER_COMPONENTS_LIST_NAME, USER_PIPELINES_LIST_NAME, } from "./constants"; -import { getIdOrTitleFromPath } from "./URL"; import { emitUserPipelineWritten } from "./userPipelineWriteEvents"; import { componentSpecFromYaml, componentSpecToYaml } from "./yaml"; @@ -511,7 +510,6 @@ export const renameComponentFileInList = async ( listName: string, oldFileName: string, newFileName: string, - pathname?: string, ) => { await upgradeSingleComponentListDb(listName); const tableName = FILE_STORE_DB_TABLE_NAME_PREFIX + listName; @@ -520,25 +518,12 @@ export const renameComponentFileInList = async ( storeName: tableName, }); - let fileEntry = + const fileEntry = await componentListDb.getItem(oldFileName); if (!fileEntry) { - // If the old file does not exist and a pathanme is provided, check the url for a filename - if (pathname) { - const { title } = getIdOrTitleFromPath(pathname); - if (title) { - fileEntry = await componentListDb.getItem(title); - } - if (!fileEntry) { - throw new Error( - `Backup file "${title}" does not exist in list "${listName}".`, - ); - } - } else { - throw new Error( - `File "${oldFileName}" does not exist in list "${listName}".`, - ); - } + throw new Error( + `File "${oldFileName}" does not exist in list "${listName}".`, + ); } const existingNewFile = diff --git a/tests/e2e/pipeline-rename.spec.ts b/tests/e2e/pipeline-rename.spec.ts new file mode 100644 index 0000000000..0b5554ebc6 --- /dev/null +++ b/tests/e2e/pipeline-rename.spec.ts @@ -0,0 +1,30 @@ +import { expect, test } from "@playwright/test"; + +import { createNewPipeline } from "./helpers"; + +const RENAME_BUTTON = '[data-tracking-id$="rename_pipeline_click"]'; + +test.describe("Renaming a pipeline in the legacy editor", () => { + test("moves the pipeline rather than leaving one under each name", async ({ + page, + }) => { + await createNewPipeline(page); + + const originalName = decodeURIComponent( + new URL(page.url()).pathname.split("/").pop() ?? "", + ); + expect(originalName).not.toBe(""); + + const renamed = `${originalName} renamed`; + + await page.locator(RENAME_BUTTON).click(); + await page.getByRole("textbox").fill(renamed); + await page.getByRole("button", { name: "Update Title" }).click(); + + await expect(page).toHaveURL(new RegExp(encodeURIComponent(renamed))); + + await page.goto("/pipelines"); + await expect(page.getByText(renamed)).toBeVisible(); + await expect(page.getByText(originalName, { exact: true })).toBeHidden(); + }); +}); diff --git a/tests/e2e/pipeline-storage-host.spec.ts b/tests/e2e/pipeline-storage-host.spec.ts index bd870fcf39..f5cb8da082 100644 --- a/tests/e2e/pipeline-storage-host.spec.ts +++ b/tests/e2e/pipeline-storage-host.spec.ts @@ -1,4 +1,4 @@ -import { expect, test } from "@playwright/test"; +import { expect, type Page, test } from "@playwright/test"; import { type HostStorageOptions, @@ -22,10 +22,11 @@ const SEED = [ }, ]; -async function installSeededHost( - page: Parameters[0], - options: HostStorageOptions = {}, -) { +async function installSeededHost(page: Page, options: HostStorageOptions = {}) { + await page.addInitScript(() => { + window.localStorage.setItem("seen-editor-v2-welcome", JSON.stringify(true)); + }); + await installPipelineStorageHost(page, { label: LABEL, seed: SEED, @@ -81,6 +82,27 @@ test.describe("host-provided pipeline storage", () => { expect(await readLocallyStoredPipelineKeys(page)).toEqual([]); }); + test("renaming keeps one pipeline rather than leaving the old name behind", async ({ + page, + }) => { + await installSeededHost(page); + + await page.goto(`/editor-v2/${SEED[0].key}`); + await expect(page.locator('[data-testid="rf__wrapper"]')).toBeVisible({ + timeout: 30_000, + }); + + await page.locator('[data-tracking-id$="rename_pipeline"]').click(); + await page.getByRole("textbox").fill("Churn model v2"); + await page.getByRole("button", { name: "Rename" }).click(); + + await expect + .poll(async () => + (await readHostRecords(page)).map((record) => record.displayName), + ) + .toEqual(["Churn model v2", "Nightly refresh"]); + }); + test("a deleted pipeline does not come back on reload", async ({ page }) => { await installSeededHost(page); From e409eac07f9007ddd40df4b4d8516df4882d5c1a Mon Sep 17 00:00:00 2001 From: Camiel van Schoonhoven Date: Tue, 8 Sep 2026 17:10:16 -0700 Subject: [PATCH 14/36] fix(pipelines): keep the pipeline table when the store is host-provided MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The pipeline table read the browser's own store directly, so a host-provided store had nothing to show it and the route redirected to the folder table instead — swapping the page out from under anyone who clicked Pipelines. It reads through the storage layer now, like the folder table does. The listing says what pipelines exist but not what is in them, and the table searches inside them, so contents are fetched behind the rows a few at a time and kept against the version the store reports for each pipeline. A revisit re-reads only what actually changed; the first visit is the only one that pays. The flat list is the whole store rather than the root folder's share of it, because that is what this page has always shown and folders are the other page's idea. Two listings at once used to race to claim the same registry rows and the unique index failed the loser, taking down the listing over a row that already said what it wanted to say. Claiming is a transaction now. Co-Authored-By: Claude Opus 5 (1M context) --- .../Home/PipelineSection/BulkActionsBar.tsx | 10 +- .../PipelineSection/PipelineFiltersBar.tsx | 3 + .../Home/PipelineSection/PipelineRow.tsx | 10 +- .../Home/PipelineSection/PipelineSection.tsx | 104 +++++++---------- .../PipelineSection/usePipelineFilters.ts | 88 +++++++------- .../PipelineSection/usePipelineListEntries.ts | 110 ++++++++++++++++++ src/routes/router.ts | 9 -- src/services/pipelineStorage/PipelineFile.ts | 3 + .../pipelineStorage/PipelineFolder.test.ts | 9 ++ .../pipelineStorage/PipelineFolder.ts | 16 +-- .../PipelineStorageService.test.ts | 46 ++++++++ .../pipelineStorage/PipelineStorageService.ts | 18 +++ src/services/pipelineStorage/db.ts | 8 ++ .../pipelineStorage/pipelineRegistry.ts | 22 ++++ .../pipelineStorage/pipelineSpecCache.test.ts | 97 +++++++++++++++ .../pipelineStorage/pipelineSpecCache.ts | 69 +++++++++++ src/services/pipelineStorage/types.ts | 9 ++ tests/e2e/fixtures/pipelineStorageHost.ts | 13 +++ tests/e2e/pipeline-storage-host.spec.ts | 38 +++++- 19 files changed, 550 insertions(+), 132 deletions(-) create mode 100644 src/components/Home/PipelineSection/usePipelineListEntries.ts create mode 100644 src/services/pipelineStorage/pipelineSpecCache.test.ts create mode 100644 src/services/pipelineStorage/pipelineSpecCache.ts diff --git a/src/components/Home/PipelineSection/BulkActionsBar.tsx b/src/components/Home/PipelineSection/BulkActionsBar.tsx index fd1238c779..117ede52b6 100644 --- a/src/components/Home/PipelineSection/BulkActionsBar.tsx +++ b/src/components/Home/PipelineSection/BulkActionsBar.tsx @@ -3,11 +3,11 @@ import { FloatingSelectionBar } from "@/components/shared/FloatingSelectionBar"; import { Button } from "@/components/ui/button"; import { Icon } from "@/components/ui/icon"; import useToastNotification from "@/hooks/useToastNotification"; -import { deletePipelineByName } from "@/services/pipelineStorage/pipelineOperations"; +import type { PipelineFile } from "@/services/pipelineStorage/PipelineFile"; import { getErrorMessage, pluralize } from "@/utils/string"; interface BulkActionsBarProps { - selectedPipelines: string[]; + selectedPipelines: PipelineFile[]; onDeleteSuccess: () => void; onClearSelection: () => void; } @@ -20,12 +20,8 @@ const BulkActionsBar = ({ const notify = useToastNotification(); const handleBulkDelete = async () => { - const deletePromises = selectedPipelines.map((pipelineName) => - deletePipelineByName(pipelineName), - ); - try { - await Promise.all(deletePromises); + await Promise.all(selectedPipelines.map((file) => file.deleteFile())); onDeleteSuccess(); notify( `${selectedPipelines.length} pipelines successfully deleted`, diff --git a/src/components/Home/PipelineSection/PipelineFiltersBar.tsx b/src/components/Home/PipelineSection/PipelineFiltersBar.tsx index a80efc4eb2..605362e373 100644 --- a/src/components/Home/PipelineSection/PipelineFiltersBar.tsx +++ b/src/components/Home/PipelineSection/PipelineFiltersBar.tsx @@ -58,6 +58,7 @@ export function PipelineFiltersBar({ clearFilters, totalCount, filteredCount, + pendingCount, } = filters; const [isAdvancedOpen, setIsAdvancedOpen] = useState(false); @@ -226,6 +227,8 @@ export function PipelineFiltersBar({ Showing {filteredCount} of {totalCount} pipelines + {pendingCount > 0 && + ` — still reading ${pendingCount}, so search results may grow`}
diff --git a/src/components/Home/PipelineSection/PipelineRow.tsx b/src/components/Home/PipelineSection/PipelineRow.tsx index 65f6bd81bb..4e41edc47c 100644 --- a/src/components/Home/PipelineSection/PipelineRow.tsx +++ b/src/components/Home/PipelineSection/PipelineRow.tsx @@ -37,7 +37,7 @@ import { } from "@/routes/editorRoutes"; import { deletePipelineByName } from "@/services/pipelineStorage/pipelineOperations"; import { getPipelineTagsFromSpec } from "@/utils/annotations"; -import type { ComponentReferenceWithSpec } from "@/utils/componentStore"; +import type { ComponentSpec } from "@/utils/componentSpec"; import { formatDate } from "@/utils/date"; import { getErrorMessage } from "@/utils/string"; import { tracking } from "@/utils/tracking"; @@ -51,7 +51,7 @@ const DEFAULT_PIPELINE_ROW_ANALYTICS_PREFIX = "pipeline_home.table"; interface PipelineRowProps { url?: string; - componentRef?: ComponentReferenceWithSpec; + spec?: ComponentSpec; name?: string; modificationTime?: Date; onDelete?: () => void; @@ -73,7 +73,7 @@ interface PipelineRowProps { const PipelineRow = withSuspenseWrapper( ({ name, - componentRef, + spec, modificationTime, onDelete, isSelected = false, @@ -98,9 +98,7 @@ const PipelineRow = withSuspenseWrapper( track(`${analyticsTrackingPrefix}.${suffix}`, metadata); }; - const componentSpec = componentRef?.spec; - - const tags = getPipelineTagsFromSpec(componentSpec); + const tags = getPipelineTagsFromSpec(spec); const handleRowClick = (e: MouseEvent) => { if ((e.target as HTMLElement).closest("[data-popover-trigger]")) { diff --git a/src/components/Home/PipelineSection/PipelineSection.tsx b/src/components/Home/PipelineSection/PipelineSection.tsx index 96b2b56082..2efa7c5b9c 100644 --- a/src/components/Home/PipelineSection/PipelineSection.tsx +++ b/src/components/Home/PipelineSection/PipelineSection.tsx @@ -1,5 +1,5 @@ import { Link } from "@tanstack/react-router"; -import { useEffect, useState } from "react"; +import { useState } from "react"; import { ExamplePipelines } from "@/components/Learn/ExamplePipelines"; import { LoadingScreen } from "@/components/shared/LoadingScreen"; @@ -22,21 +22,15 @@ import { import { Paragraph, Text } from "@/components/ui/typography"; import { usePagination } from "@/hooks/usePagination"; import { APP_ROUTES } from "@/routes/router"; -import { - type ComponentFileEntry, - getAllComponentFilesFromList, -} from "@/utils/componentStore"; -import { USER_PIPELINES_LIST_NAME } from "@/utils/constants"; import BulkActionsBar from "./BulkActionsBar"; import { PipelineFiltersBar } from "./PipelineFiltersBar"; import PipelineRow from "./PipelineRow"; import { usePipelineFilters } from "./usePipelineFilters"; +import { usePipelineListEntries } from "./usePipelineListEntries"; const DEFAULT_PAGE_SIZE = 10; -type Pipelines = Map; - const PipelineSectionSkeleton = () => ( @@ -65,14 +59,15 @@ interface PipelineSectionProps { export const PipelineSection = withSuspenseWrapper( ({ onPipelineClick }: PipelineSectionProps) => { - const [pipelines, setPipelines] = useState(new Map()); - const [isLoading, setIsLoading] = useState(false); - const [selectedPipelines, setSelectedPipelines] = useState>( - new Set(), - ); + const [selectedIds, setSelectedIds] = useState>(new Set()); - const { filteredPipelines, filterBarProps, filterKey } = - usePipelineFilters(pipelines); + const { entries, isLoading, pendingCount, refetch } = + usePipelineListEntries(); + + const { filteredPipelines, filterBarProps, filterKey } = usePipelineFilters( + entries, + pendingCount, + ); const { paginatedItems: paginatedPipelines, @@ -85,39 +80,24 @@ export const PipelineSection = withSuspenseWrapper( resetPage, } = usePagination(filteredPipelines, DEFAULT_PAGE_SIZE, filterKey); - const fetchUserPipelines = async () => { - setIsLoading(true); - try { - setPipelines( - await getAllComponentFilesFromList(USER_PIPELINES_LIST_NAME), - ); - } catch (error) { - console.error("Failed to load user pipelines:", error); - } finally { - setIsLoading(false); - } - }; - const handleSelectAll = (checked: boolean) => { - setSelectedPipelines( - checked ? new Set(filteredPipelines.map(([name]) => name)) : new Set(), + setSelectedIds( + checked + ? new Set(filteredPipelines.map(({ entry }) => entry.file.id)) + : new Set(), ); }; - const handleSelectPipeline = (name: string, checked: boolean) => { - const next = new Set(selectedPipelines); - if (checked) next.add(name); - else next.delete(name); - setSelectedPipelines(next); + const handleSelectPipeline = (id: string, checked: boolean) => { + const next = new Set(selectedIds); + if (checked) next.add(id); + else next.delete(id); + setSelectedIds(next); }; - useEffect(() => { - fetchUserPipelines(); - }, []); - if (isLoading) return ; - if (pipelines.size === 0) { + if (entries.length === 0) { return ( @@ -135,9 +115,13 @@ export const PipelineSection = withSuspenseWrapper( ); } + const selectedFiles = entries + .filter(({ file }) => selectedIds.has(file.id)) + .map(({ file }) => file); + const isAllSelected = filteredPipelines.length > 0 && - filteredPipelines.every(([name]) => selectedPipelines.has(name)); + filteredPipelines.every(({ entry }) => selectedIds.has(entry.file.id)); return ( @@ -171,19 +155,21 @@ export const PipelineSection = withSuspenseWrapper( )} - {paginatedPipelines.map(([name, fileEntry, matchMetadata]) => ( + {paginatedPipelines.map(({ entry, match }) => ( handleSelectPipeline(name, checked)} - searchQuery={matchMetadata.searchQuery} - matchedFields={matchMetadata.matchedFields} - componentQuery={matchMetadata.componentQuery} - matchedComponentNames={matchMetadata.matchedComponentNames} + key={entry.file.id} + name={entry.file.displayName} + spec={entry.spec} + modificationTime={entry.file.modifiedAt} + onDelete={refetch} + isSelected={selectedIds.has(entry.file.id)} + onSelect={(checked) => + handleSelectPipeline(entry.file.id, checked) + } + searchQuery={match.searchQuery} + matchedFields={match.matchedFields} + componentQuery={match.componentQuery} + matchedComponentNames={match.matchedComponentNames} onPipelineClick={onPipelineClick} /> ))} @@ -200,18 +186,18 @@ export const PipelineSection = withSuspenseWrapper( onReset={resetPage} /> - - {selectedPipelines.size > 0 && ( + {selectedFiles.length > 0 && ( { - setSelectedPipelines(new Set()); - fetchUserPipelines(); + setSelectedIds(new Set()); + refetch(); }} - onClearSelection={() => setSelectedPipelines(new Set())} + onClearSelection={() => setSelectedIds(new Set())} /> )} diff --git a/src/components/Home/PipelineSection/usePipelineFilters.ts b/src/components/Home/PipelineSection/usePipelineFilters.ts index cd6a136441..0654616bcc 100644 --- a/src/components/Home/PipelineSection/usePipelineFilters.ts +++ b/src/components/Home/PipelineSection/usePipelineFilters.ts @@ -1,8 +1,12 @@ import { useState } from "react"; import type { DateRange } from "react-day-picker"; -import { isGraphImplementation } from "@/utils/componentSpec"; -import type { ComponentFileEntry } from "@/utils/componentStore"; +import { + type ComponentSpec, + isGraphImplementation, +} from "@/utils/componentSpec"; + +import type { PipelineListEntry } from "./usePipelineListEntries"; export type PipelineSortField = "modified_at" | "name"; type PipelineSortDirection = "asc" | "desc"; @@ -19,7 +23,10 @@ interface PipelineMatchMetadata { matchedComponentNames: string[]; } -type PipelineEntry = [string, ComponentFileEntry, PipelineMatchMetadata]; +interface FilteredPipeline { + entry: PipelineListEntry; + match: PipelineMatchMetadata; +} export interface FilterBarProps { searchQuery: string; @@ -37,15 +44,16 @@ export interface FilterBarProps { clearFilters: () => void; totalCount: number; filteredCount: number; + pendingCount: number; } function matchesComponentQuery( - fileEntry: ComponentFileEntry, + spec: ComponentSpec | undefined, query: string, ): boolean { if (!query) return true; - const impl = fileEntry.componentRef.spec.implementation; - if (!isGraphImplementation(impl)) return false; + const impl = spec?.implementation; + if (!impl || !isGraphImplementation(impl)) return false; const normalizedQuery = query.toLowerCase(); return Object.values(impl.graph.tasks).some((task) => { const refName = task.componentRef.name?.toLowerCase() ?? ""; @@ -58,15 +66,14 @@ function matchesComponentQuery( function matchesSearch( name: string, - fileEntry: ComponentFileEntry, + spec: ComponentSpec | undefined, query: string, ): boolean { if (!query) return true; const normalizedQuery = query.toLowerCase(); - const spec = fileEntry.componentRef.spec; - const description = spec.description?.toLowerCase() ?? ""; - const author = spec.metadata?.annotations?.author?.toLowerCase() ?? ""; - const rawNotes = spec.metadata?.annotations?.["notes"]; + const description = spec?.description?.toLowerCase() ?? ""; + const author = spec?.metadata?.annotations?.author?.toLowerCase() ?? ""; + const rawNotes = spec?.metadata?.annotations?.["notes"]; const notes = typeof rawNotes === "string" ? rawNotes.toLowerCase() : ""; return ( name.toLowerCase().includes(normalizedQuery) || @@ -77,23 +84,22 @@ function matchesSearch( } function getMatchMetadata( - fileEntry: ComponentFileEntry, + spec: ComponentSpec | undefined, searchQuery: string, componentQuery: string, ): PipelineMatchMetadata { const matchedFields: MatchedField[] = []; if (searchQuery) { const q = searchQuery.toLowerCase(); - const spec = fileEntry.componentRef.spec; - const desc = spec.description ?? ""; + const desc = spec?.description ?? ""; if (desc.toLowerCase().includes(q)) { matchedFields.push({ label: "Description", value: desc }); } - const author = spec.metadata?.annotations?.author ?? ""; + const author = spec?.metadata?.annotations?.author ?? ""; if (author.toLowerCase().includes(q)) { matchedFields.push({ label: "Author", value: author }); } - const rawNotes = spec.metadata?.annotations?.["notes"]; + const rawNotes = spec?.metadata?.annotations?.["notes"]; const notes = typeof rawNotes === "string" ? rawNotes : ""; if (notes.toLowerCase().includes(q)) { matchedFields.push({ label: "Note", value: notes }); @@ -101,8 +107,8 @@ function getMatchMetadata( } const matchedComponentNames: string[] = []; - const impl = fileEntry.componentRef.spec.implementation; - if (componentQuery && isGraphImplementation(impl)) { + const impl = spec?.implementation; + if (componentQuery && impl && isGraphImplementation(impl)) { const normalizedQuery = componentQuery.toLowerCase(); const seen = new Set(); for (const task of Object.values(impl.graph.tasks)) { @@ -126,25 +132,27 @@ function getMatchMetadata( } function matchesDateRange( - fileEntry: ComponentFileEntry, + modifiedAt: Date | undefined, dateRange: DateRange | undefined, ): boolean { if (!dateRange) return true; + if (!modifiedAt) return false; - const modificationTime = new Date(fileEntry.modificationTime); - - if (dateRange.from && modificationTime < dateRange.from) return false; + if (dateRange.from && modifiedAt < dateRange.from) return false; if (dateRange.to) { const endOfRange = new Date(dateRange.to); endOfRange.setDate(endOfRange.getDate() + 1); - if (modificationTime > endOfRange) return false; + if (modifiedAt > endOfRange) return false; } return true; } -export function usePipelineFilters(pipelines: Map) { +export function usePipelineFilters( + entries: PipelineListEntry[], + pendingCount: number, +) { const [searchQuery, setSearchQuery] = useState(""); const [dateRange, setDateRange] = useState(undefined); const [sortField, setSortField] = useState("modified_at"); @@ -163,27 +171,28 @@ export function usePipelineFilters(pipelines: Map) { setComponentQuery(""); }; - const filteredPipelines: PipelineEntry[] = Array.from(pipelines.entries()) + const filteredPipelines: FilteredPipeline[] = entries .filter( - ([name, fileEntry]) => - matchesSearch(name, fileEntry, searchQuery) && - matchesDateRange(fileEntry, dateRange) && - matchesComponentQuery(fileEntry, componentQuery), + ({ file, spec }) => + matchesSearch(file.displayName, spec, searchQuery) && + matchesDateRange(file.modifiedAt, dateRange) && + matchesComponentQuery(spec, componentQuery), ) - .sort(([nameA, entryA], [nameB, entryB]) => { + .sort((a, b) => { const dir = sortDirection === "asc" ? 1 : -1; - if (sortField === "name") return dir * nameA.localeCompare(nameB); + if (sortField === "name") { + return dir * a.file.displayName.localeCompare(b.file.displayName); + } return ( dir * - (new Date(entryA.modificationTime).getTime() - - new Date(entryB.modificationTime).getTime()) + ((a.file.modifiedAt?.getTime() ?? 0) - + (b.file.modifiedAt?.getTime() ?? 0)) ); }) - .map(([name, fileEntry]): PipelineEntry => [ - name, - fileEntry, - getMatchMetadata(fileEntry, searchQuery, componentQuery), - ]); + .map((entry) => ({ + entry, + match: getMatchMetadata(entry.spec, searchQuery, componentQuery), + })); const filterKey = [ searchQuery, @@ -208,8 +217,9 @@ export function usePipelineFilters(pipelines: Map) { hasActiveFilters, activeFilterCount, clearFilters, - totalCount: pipelines.size, + totalCount: entries.length, filteredCount: filteredPipelines.length, + pendingCount, }; return { filteredPipelines, filterBarProps, filterKey }; diff --git a/src/components/Home/PipelineSection/usePipelineListEntries.ts b/src/components/Home/PipelineSection/usePipelineListEntries.ts new file mode 100644 index 0000000000..96eb7a5cbe --- /dev/null +++ b/src/components/Home/PipelineSection/usePipelineListEntries.ts @@ -0,0 +1,110 @@ +import { useQuery } from "@tanstack/react-query"; +import { useEffect, useRef, useState } from "react"; + +import type { PipelineFile } from "@/services/pipelineStorage/PipelineFile"; +import { + forgetUnlistedSpecs, + readCachedSpecs, + writeCachedSpec, +} from "@/services/pipelineStorage/pipelineSpecCache"; +import { usePipelineStorage } from "@/services/pipelineStorage/PipelineStorageProvider"; +import { FoldersQueryKeys } from "@/services/pipelineStorage/types"; +import type { ComponentSpec } from "@/utils/componentSpec"; +import { componentSpecFromYaml } from "@/utils/yaml"; + +const HYDRATION_CONCURRENCY = 3; + +export interface PipelineListEntry { + file: PipelineFile; + spec?: ComponentSpec; +} + +/** + * The listing carries names and timestamps; searching by description, author or + * component needs the pipeline itself. Rows are shown as soon as the listing + * lands and the contents fill in behind them, a few at a time, so a store that + * answers one pipeline per request is not asked for all of them at once. + */ +export function usePipelineListEntries() { + const storage = usePipelineStorage(); + + const { + data: files, + isPending, + refetch, + } = useQuery({ + queryKey: FoldersQueryKeys.AllPipelines(), + queryFn: () => storage.listAllPipelines(), + }); + + const [specs, setSpecs] = useState>( + new Map(), + ); + const [pendingCount, setPendingCount] = useState(0); + const generation = useRef(0); + + useEffect(() => { + if (!files) return; + + const generationAtStart = ++generation.current; + const isStale = () => generation.current !== generationAtStart; + + void (async () => { + const cached = await readCachedSpecs(files); + if (isStale()) return; + + setSpecs(cached); + void forgetUnlistedSpecs(new Set(files.map((file) => file.storageKey))); + + const missing = files.filter((file) => !cached.has(file.storageKey)); + setPendingCount(missing.length); + + await hydrate(missing, isStale, (file, spec) => { + setSpecs((previous) => new Map(previous).set(file.storageKey, spec)); + setPendingCount((previous) => previous - 1); + }); + + if (!isStale()) setPendingCount(0); + })(); + + return () => { + generation.current += 1; + }; + }, [files]); + + const entries: PipelineListEntry[] = (files ?? []).map((file) => ({ + file, + spec: specs.get(file.storageKey), + })); + + return { entries, isLoading: isPending, pendingCount, refetch }; +} + +async function hydrate( + files: PipelineFile[], + isStale: () => boolean, + onSpec: (file: PipelineFile, spec: ComponentSpec) => void, +): Promise { + const queue = [...files]; + + const worker = async () => { + for (let file = queue.shift(); file && !isStale(); file = queue.shift()) { + try { + const spec = componentSpecFromYaml(await file.read()); + if (isStale()) return; + + onSpec(file, spec); + void writeCachedSpec(file, spec); + } catch (error) { + console.error(`Failed to read pipeline "${file.displayName}":`, error); + } + } + }; + + await Promise.all( + Array.from( + { length: Math.min(HYDRATION_CONCURRENCY, queue.length) }, + worker, + ), + ); +} diff --git a/src/routes/router.ts b/src/routes/router.ts index f333ce34cc..d948c6da65 100644 --- a/src/routes/router.ts +++ b/src/routes/router.ts @@ -16,7 +16,6 @@ import { AddSecretView } from "@/components/shared/SecretsManagement/components/ import { ReplaceSecretView } from "@/components/shared/SecretsManagement/components/ReplaceSecretView"; import { SecretsListView } from "@/components/shared/SecretsManagement/components/SecretsListView"; import { isFlagEnabled } from "@/components/shared/Settings/useFlags"; -import { isHostStorage } from "@/services/pipelineStorage/storageMode"; import { BASE_URL, IS_GITHUB_PAGES } from "@/utils/constants"; import RootLayout from "../components/layout/RootLayout"; @@ -104,18 +103,10 @@ const dashboardRunsRoute = createRoute({ component: DashboardRunsView, }); -// The dashboard list reads the browser's own pipeline store directly and -// filters on spec contents a host listing does not carry, so a host-backed -// store gets the folder table, which goes through the storage driver. const dashboardPipelinesRoute = createRoute({ getParentRoute: () => dashboardRoute, path: "/pipelines", component: DashboardPipelinesView, - beforeLoad: () => { - if (isHostStorage()) { - throw redirect({ to: APP_ROUTES.PIPELINE_FOLDERS }); - } - }, }); const dashboardComponentsRoute = createRoute({ diff --git a/src/services/pipelineStorage/PipelineFile.ts b/src/services/pipelineStorage/PipelineFile.ts index b660b24a0f..2e8b4d855a 100644 --- a/src/services/pipelineStorage/PipelineFile.ts +++ b/src/services/pipelineStorage/PipelineFile.ts @@ -14,12 +14,14 @@ interface PipelineFileInit { storageKey: string; folder: PipelineFolder; displayName?: string; + contentVersion?: string; createdAt?: Date; modifiedAt?: Date; } export class PipelineFile { readonly id: string; + readonly contentVersion?: string; readonly createdAt?: Date; readonly modifiedAt?: Date; @@ -37,6 +39,7 @@ export class PipelineFile { this.storageKey = options.storageKey; this.folder = options.folder; this.assignedDisplayName = options.displayName; + this.contentVersion = options.contentVersion; this.createdAt = options.createdAt; this.modifiedAt = options.modifiedAt; diff --git a/src/services/pipelineStorage/PipelineFolder.test.ts b/src/services/pipelineStorage/PipelineFolder.test.ts index c3d159463d..d55a2e5081 100644 --- a/src/services/pipelineStorage/PipelineFolder.test.ts +++ b/src/services/pipelineStorage/PipelineFolder.test.ts @@ -26,6 +26,15 @@ vi.mock("./pipelineRegistry", () => ({ addEntry: vi.fn(async (entry: PipelineRegistryEntry) => { registry.set(entry.id, entry); }), + claimEntry: vi.fn(async (entry: PipelineRegistryEntry) => { + const existing = [...registry.values()].find( + (candidate) => candidate.storageKey === entry.storageKey, + ); + if (existing) return existing; + + registry.set(entry.id, entry); + return entry; + }), updateEntry: vi.fn( async (id: string, changes: Partial) => { const existing = registry.get(id); diff --git a/src/services/pipelineStorage/PipelineFolder.ts b/src/services/pipelineStorage/PipelineFolder.ts index 35c9ed074d..8d7d166172 100644 --- a/src/services/pipelineStorage/PipelineFolder.ts +++ b/src/services/pipelineStorage/PipelineFolder.ts @@ -7,9 +7,9 @@ import { emitPipelineFileChanged } from "./pipelineFileEvents"; import { addEntry, assertStorageKeyUnique, + claimEntry, deleteEntry, deleteFoldersAndDetachEntries, - findByStorageKey, getAllByFolderId, updateEntry, } from "./pipelineRegistry"; @@ -272,20 +272,14 @@ async function resolveOrCreateRegistryEntry( descriptor: PipelineFileDescriptor, folder: PipelineFolder, ): Promise { - const existing = await findByStorageKey(descriptor.storageKey); - - if (existing) { - return new PipelineFile({ id: existing.id, folder, ...descriptor }); - } - - const id = descriptor.externalId ?? crypto.randomUUID(); - await addEntry({ - id, + const claimed = await claimEntry({ + id: descriptor.externalId ?? crypto.randomUUID(), storageKey: descriptor.storageKey, folderId: folder.id, contentVersion: descriptor.contentVersion, }); - return new PipelineFile({ id, folder, ...descriptor }); + + return new PipelineFile({ id: claimed.id, folder, ...descriptor }); } /** diff --git a/src/services/pipelineStorage/PipelineStorageService.test.ts b/src/services/pipelineStorage/PipelineStorageService.test.ts index 1a0baff505..2f5c7ff3a5 100644 --- a/src/services/pipelineStorage/PipelineStorageService.test.ts +++ b/src/services/pipelineStorage/PipelineStorageService.test.ts @@ -1,5 +1,6 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { RootFolderDbStorageDriver } from "./drivers/RootFolderDbStorageDriver"; import type { HostPipelineSummary, PipelineStorageHost } from "./host/contract"; import { AmbiguousPipelineNameError, @@ -19,6 +20,15 @@ vi.mock("./pipelineRegistry", () => ({ addEntry: async (entry: PipelineRegistryEntry) => { registry.set(entry.id, entry); }, + claimEntry: async (entry: PipelineRegistryEntry) => { + const existing = [...registry.values()].find( + (candidate) => candidate.storageKey === entry.storageKey, + ); + if (existing) return existing; + + registry.set(entry.id, entry); + return entry; + }, updateEntry: async (id: string, updates: Partial) => { const entry = registry.get(id); if (entry) registry.set(id, { ...entry, ...updates }); @@ -133,6 +143,7 @@ beforeEach(() => { afterEach(() => { delete window.__TANGLE_PIPELINE_STORAGE_HOST__; resetStorageModeForTests(); + vi.restoreAllMocks(); }); describe("with no host on the page", () => { @@ -196,6 +207,41 @@ describe("with a host on the page", () => { }); }); +describe("the flat list of everything", () => { + it("reads the whole browser store, not just the pipelines left in the root", async () => { + const listed = vi.fn(async () => [ + { storageKey: "In the root" }, + { storageKey: "Filed away" }, + ]); + vi.spyOn(RootFolderDbStorageDriver.prototype, "list").mockImplementation( + listed, + ); + registry.set("filed", { + id: "filed", + storageKey: "Filed away", + folderId: "folder-1", + }); + + const files = await new PipelineStorageService().listAllPipelines(); + + expect(files.map((file) => file.storageKey)).toEqual([ + "In the root", + "Filed away", + ]); + expect(registry.get("filed")?.folderId).toBe("folder-1"); + }); + + it("asks a host for its listing rather than the browser store", async () => { + installHost([summary("opaque-key-1", "Churn model")]); + const listed = vi.spyOn(RootFolderDbStorageDriver.prototype, "list"); + + const files = await new PipelineStorageService().listAllPipelines(); + + expect(files.map((file) => file.displayName)).toEqual(["Churn model"]); + expect(listed).not.toHaveBeenCalled(); + }); +}); + describe("resolving a route reference against a host", () => { it("opens the pipeline whose key the route carries", async () => { installHost([summary("opaque-key-1", "Churn model")]); diff --git a/src/services/pipelineStorage/PipelineStorageService.ts b/src/services/pipelineStorage/PipelineStorageService.ts index 814c6d2918..d52f5cb860 100644 --- a/src/services/pipelineStorage/PipelineStorageService.ts +++ b/src/services/pipelineStorage/PipelineStorageService.ts @@ -44,6 +44,24 @@ export class PipelineStorageService { throw new PipelineNotFoundError(`Pipeline "${ref.name}" not found`); } + /** + * The flat pipeline list shows everything the user has, wherever they filed + * it. A store that hands out its own keys has no folders and its listing is + * already that; browser storage keeps one list behind however many folders + * point into it, so the whole list is read rather than the root folder's + * share of it. + */ + async listAllPipelines(): Promise { + if (this.rootFolder.isFlat) return this.rootFolder.listPipelines(); + + return new PipelineFolder({ + id: ROOT_FOLDER_ID, + name: this.rootFolder.name, + parentId: null, + driver: new RootFolderDbStorageDriver(), + }).listPipelines(); + } + async findPipelineByName(name: string): Promise { return ( (await this.resolvePipelineByName(name)) ?? diff --git a/src/services/pipelineStorage/db.ts b/src/services/pipelineStorage/db.ts index 6cb1b81a73..65ce723083 100644 --- a/src/services/pipelineStorage/db.ts +++ b/src/services/pipelineStorage/db.ts @@ -4,6 +4,7 @@ import { USER_PIPELINES_LIST_NAME } from "@/utils/constants"; import { isHostStorage } from "./storageMode"; import { + type CachedPipelineSpec, type FolderEntry, type PipelineRegistryEntry, ROOT_FOLDER_ID, @@ -12,6 +13,7 @@ import { export type PipelineStorageDb = Dexie & { pipeline_registry: EntityTable; folders: EntityTable; + pipeline_specs: EntityTable; }; export const pipelineStorageDb = new Dexie( @@ -64,6 +66,12 @@ pipelineStorageDb } }); +pipelineStorageDb.version(4).stores({ + pipeline_registry: "id, &storageKey, folderId, [folderId+storageKey]", + folders: "id, parentId", + pipeline_specs: "storageKey", +}); + pipelineStorageDb.on("ready", async () => { await seedRegistryFromLegacyList(); }); diff --git a/src/services/pipelineStorage/pipelineRegistry.ts b/src/services/pipelineStorage/pipelineRegistry.ts index 6f06b06c62..709000b5c3 100644 --- a/src/services/pipelineStorage/pipelineRegistry.ts +++ b/src/services/pipelineStorage/pipelineRegistry.ts @@ -5,6 +5,28 @@ export async function addEntry(entry: PipelineRegistryEntry): Promise { await pipelineStorageDb.pipeline_registry.add(entry); } +/** + * Two listings running at once both find no row for a storage key and both try + * to add one, and the unique index fails the loser — taking down a whole + * listing over a row that already says what it wanted to say. Claiming inside a + * transaction makes the second one find the first one's row instead. + */ +export async function claimEntry( + entry: PipelineRegistryEntry, +): Promise { + return pipelineStorageDb.transaction( + "rw", + pipelineStorageDb.pipeline_registry, + async () => { + const existing = await findByStorageKey(entry.storageKey); + if (existing) return existing; + + await pipelineStorageDb.pipeline_registry.add(entry); + return entry; + }, + ); +} + export async function updateEntry( id: string, updates: Partial>, diff --git a/src/services/pipelineStorage/pipelineSpecCache.test.ts b/src/services/pipelineStorage/pipelineSpecCache.test.ts new file mode 100644 index 0000000000..8f4bc255b9 --- /dev/null +++ b/src/services/pipelineStorage/pipelineSpecCache.test.ts @@ -0,0 +1,97 @@ +import "fake-indexeddb/auto"; + +import { beforeEach, describe, expect, it } from "vitest"; + +import type { ComponentSpec } from "@/utils/componentSpec"; + +import { pipelineStorageDb } from "./db"; +import type { PipelineFile } from "./PipelineFile"; +import { + forgetUnlistedSpecs, + readCachedSpecs, + writeCachedSpec, +} from "./pipelineSpecCache"; + +function fakeFile(options: { + storageKey: string; + contentVersion?: string; + modifiedAt?: Date; +}): PipelineFile { + return { + storageKey: options.storageKey, + contentVersion: options.contentVersion, + modifiedAt: options.modifiedAt, + } as PipelineFile; +} + +const spec = (name: string): ComponentSpec => ({ + name, + implementation: { graph: { tasks: {} } }, +}); + +beforeEach(async () => { + await pipelineStorageDb.pipeline_specs.clear(); +}); + +describe("the pipeline spec cache", () => { + it("answers a pipeline whose version has not moved", async () => { + const file = fakeFile({ storageKey: "key-1", contentVersion: "v1" }); + await writeCachedSpec(file, spec("Churn model")); + + const cached = await readCachedSpecs([file]); + + expect(cached.get("key-1")).toEqual(spec("Churn model")); + }); + + it("refuses a pipeline the store has since rewritten", async () => { + await writeCachedSpec( + fakeFile({ storageKey: "key-1", contentVersion: "v1" }), + spec("Churn model"), + ); + + const cached = await readCachedSpecs([ + fakeFile({ storageKey: "key-1", contentVersion: "v2" }), + ]); + + expect(cached.has("key-1")).toBe(false); + }); + + it("falls back to the modification time when a store reports no version", async () => { + const modifiedAt = new Date("2026-01-01T00:00:00.000Z"); + const file = fakeFile({ storageKey: "key-1", modifiedAt }); + await writeCachedSpec(file, spec("Churn model")); + + expect((await readCachedSpecs([file])).has("key-1")).toBe(true); + + const touched = fakeFile({ + storageKey: "key-1", + modifiedAt: new Date("2026-01-02T00:00:00.000Z"), + }); + expect((await readCachedSpecs([touched])).has("key-1")).toBe(false); + }); + + it("stores nothing for a pipeline with no version to check it against", async () => { + const file = fakeFile({ storageKey: "key-1" }); + + await writeCachedSpec(file, spec("Churn model")); + + expect(await pipelineStorageDb.pipeline_specs.count()).toBe(0); + }); + + it("drops pipelines the listing no longer reports", async () => { + await writeCachedSpec( + fakeFile({ storageKey: "key-1", contentVersion: "v1" }), + spec("Churn model"), + ); + await writeCachedSpec( + fakeFile({ storageKey: "key-2", contentVersion: "v1" }), + spec("Nightly refresh"), + ); + + await forgetUnlistedSpecs(new Set(["key-2"])); + + expect( + await pipelineStorageDb.pipeline_specs.toCollection().primaryKeys(), + ).toEqual(["key-2"]); + }); +}); diff --git a/src/services/pipelineStorage/pipelineSpecCache.ts b/src/services/pipelineStorage/pipelineSpecCache.ts new file mode 100644 index 0000000000..42b08ca8dd --- /dev/null +++ b/src/services/pipelineStorage/pipelineSpecCache.ts @@ -0,0 +1,69 @@ +import type { ComponentSpec } from "@/utils/componentSpec"; + +import { pipelineStorageDb } from "./db"; +import type { PipelineFile } from "./PipelineFile"; + +/** + * A listing says what pipelines exist but not what is in them, and the pipeline + * table searches inside them. Reading every pipeline on every visit is the cost + * this avoids: a store that reports a `contentVersion` says which ones actually + * changed, so the rest are answered from here. + * + * Nothing is ever served from this cache without the version the store just + * reported agreeing, so it can go stale but cannot be believed when it has. + */ +function specCacheVersion(file: PipelineFile): string | undefined { + return file.contentVersion ?? file.modifiedAt?.toISOString(); +} + +export async function readCachedSpecs( + files: PipelineFile[], +): Promise> { + const wanted = new Map( + files.flatMap((file) => { + const version = specCacheVersion(file); + return version ? [[file.storageKey, version] as const] : []; + }), + ); + + if (wanted.size === 0) return new Map(); + + const cached = await pipelineStorageDb.pipeline_specs.bulkGet([ + ...wanted.keys(), + ]); + + return new Map( + cached.flatMap((entry) => + entry && entry.version === wanted.get(entry.storageKey) + ? [[entry.storageKey, entry.spec] as const] + : [], + ), + ); +} + +export async function writeCachedSpec( + file: PipelineFile, + spec: ComponentSpec, +): Promise { + const version = specCacheVersion(file); + if (!version) return; + + await pipelineStorageDb.pipeline_specs.put({ + storageKey: file.storageKey, + version, + spec, + }); +} + +export async function forgetUnlistedSpecs( + listedKeys: Set, +): Promise { + const stored = await pipelineStorageDb.pipeline_specs + .toCollection() + .primaryKeys(); + const gone = stored.filter((key) => !listedKeys.has(key)); + + if (gone.length > 0) { + await pipelineStorageDb.pipeline_specs.bulkDelete(gone); + } +} diff --git a/src/services/pipelineStorage/types.ts b/src/services/pipelineStorage/types.ts index c3c48445fd..8943ab0e4f 100644 --- a/src/services/pipelineStorage/types.ts +++ b/src/services/pipelineStorage/types.ts @@ -1,3 +1,5 @@ +import type { ComponentSpec } from "@/utils/componentSpec"; + import type { GoogleDriveDriverConfig } from "../googleDrive/types"; // google-drive import type { FolderIndexDbDriverConfig } from "./drivers/FolderIndexDbStorageDriver"; import type { HostDriverConfig } from "./drivers/HostStorageDriver"; @@ -45,6 +47,12 @@ export type DriverConfig = | HostDriverConfig | GoogleDriveDriverConfig; // google-drive +export interface CachedPipelineSpec { + storageKey: string; + version: string; + spec: ComponentSpec; +} + export interface PipelineRegistryEntry { id: string; storageKey: string; @@ -72,5 +80,6 @@ export const FoldersQueryKeys = { ["pipeline-folders", "breadcrumbs", folderId] as const, Pipelines: (folderId: string | null) => ["pipeline-folders", "pipelines", folderId] as const, + AllPipelines: () => ["pipeline-folders", "pipelines", "all"] as const, Favorites: () => ["pipeline-folders", "favorites"] as const, } as const; diff --git a/tests/e2e/fixtures/pipelineStorageHost.ts b/tests/e2e/fixtures/pipelineStorageHost.ts index e70e94e127..f3361abd9d 100644 --- a/tests/e2e/fixtures/pipelineStorageHost.ts +++ b/tests/e2e/fixtures/pipelineStorageHost.ts @@ -25,6 +25,7 @@ interface HostRecord { interface HostTestState { records(): HostRecord[]; + readKeys(): string[]; } declare global { @@ -58,6 +59,12 @@ export async function installPipelineStorageHost( ); let revision = store.size; + /** + * Deliberately not persisted: a test asserting that a reload served the + * pipeline contents from cache needs the count for this page load alone. + */ + const readKeys: string[] = []; + if (!saved) { for (const seeded of config.seed) { revision += 1; @@ -112,6 +119,7 @@ export async function installPipelineStorageHost( }, async read(key: string) { await gate(); + readKeys.push(key); const found = store.get(key); if (!found) { throw Object.assign(new Error(`no pipeline for ${key}`), { @@ -148,6 +156,7 @@ export async function installPipelineStorageHost( window.__TANGLE_TEST_HOST__ = { records: () => [...store.values()], + readKeys: () => [...readKeys], }; }, { @@ -164,6 +173,10 @@ export async function readHostRecords(page: Page): Promise { return page.evaluate(() => window.__TANGLE_TEST_HOST__?.records() ?? []); } +export async function readHostReadKeys(page: Page): Promise { + return page.evaluate(() => window.__TANGLE_TEST_HOST__?.readKeys() ?? []); +} + /** * The negative assertion host mode exists for: with a host present, nothing may * reach the browser's own pipeline store, whatever the host does. diff --git a/tests/e2e/pipeline-storage-host.spec.ts b/tests/e2e/pipeline-storage-host.spec.ts index f5cb8da082..cde4d59298 100644 --- a/tests/e2e/pipeline-storage-host.spec.ts +++ b/tests/e2e/pipeline-storage-host.spec.ts @@ -3,17 +3,24 @@ import { expect, type Page, test } from "@playwright/test"; import { type HostStorageOptions, installPipelineStorageHost, + readHostReadKeys, readHostRecords, readLocallyStoredPipelineKeys, } from "./fixtures/pipelineStorageHost"; const LABEL = "Shared storage"; +const CHURN_TAG = "quarterly"; + const SEED = [ { key: "0f8c1a2b-0000-4000-8000-000000000001", displayName: "Churn model", - spec: { name: "Churn model", implementation: { graph: { tasks: {} } } }, + spec: { + name: "Churn model", + metadata: { annotations: { tags: CHURN_TAG } }, + implementation: { graph: { tasks: {} } }, + }, }, { key: "0f8c1a2b-0000-4000-8000-000000000002", @@ -44,6 +51,35 @@ test.describe("host-provided pipeline storage", () => { await expect(page.getByText("Nightly refresh")).toBeVisible(); }); + test("keeps the pipeline table at /pipelines, contents and all", async ({ + page, + }) => { + await installSeededHost(page); + + await page.goto("/pipelines"); + + await expect(page.getByText("Churn model")).toBeVisible(); + await expect(page.getByText("Nightly refresh")).toBeVisible(); + await expect(page.getByText(CHURN_TAG)).toBeVisible(); + await expect(page.getByPlaceholder(/search/i).first()).toBeVisible(); + }); + + test("reads each pipeline once and serves the next visit from cache", async ({ + page, + }) => { + await installSeededHost(page); + + await page.goto("/pipelines"); + await expect(page.getByText(CHURN_TAG)).toBeVisible(); + expect((await readHostReadKeys(page)).sort()).toEqual( + SEED.map((entry) => entry.key).sort(), + ); + + await page.reload(); + await expect(page.getByText(CHURN_TAG)).toBeVisible(); + expect(await readHostReadKeys(page)).toEqual([]); + }); + test("keeps the browser's own pipeline store empty", async ({ page }) => { await installSeededHost(page); From 84d209172e804a6e0f6497fb2072688104e6c602 Mon Sep 17 00:00:00 2001 From: Camiel van Schoonhoven Date: Tue, 8 Sep 2026 17:50:46 -0700 Subject: [PATCH 15/36] feat(pipeline-storage): copy browser-stored pipelines into a host-provided store MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A user whose pipelines were all in this browser arrived at a host-backed store to find an empty library, with no way to get any of it across. Copy it, once, the first time the app runs against a host. Each pipeline is written under the name it already had: host writes upsert on the key they are given, so a run that dies halfway can simply be run again and a pipeline copied twice is overwritten rather than duplicated. Two tabs opening together must not both copy the library, so the claim is a transaction. A claim is only honoured for as long as its holder keeps making progress — otherwise one crashed tab would leave the migration permanently unrunnable — and a tab that finds a live claim waits rather than duplicating the work. It runs where the list would be, not in front of the app: a store that cannot be reached must not be able to lock anyone out of the editor. What fails is named, retriable, and skippable, because otherwise one pipeline the store refuses would block the page for good. Nothing is deleted. The browser keeps its copy, which is also what a build with no host still has to find. The pipeline table was also dropping a pipeline's id when opening it, falling back to resolving the name — which is not unique in a store that names nothing. Co-Authored-By: Claude Opus 5 (1M context) --- .../PipelineSection/HostMigrationNotice.tsx | 57 ++++++ .../Home/PipelineSection/PipelineRow.tsx | 6 +- .../Home/PipelineSection/PipelineSection.tsx | 20 ++- .../Home/PipelineSection/useHostMigration.ts | 111 ++++++++++++ .../PipelineSection/usePipelineListEntries.ts | 34 ++-- .../pipelineStorage/PipelineFolder.test.ts | 3 - .../pipelineStorage/PipelineFolder.ts | 10 +- .../PipelineStorageService.test.ts | 3 - src/services/pipelineStorage/db.ts | 9 + .../pipelineStorage/hostMigration.test.ts | 170 ++++++++++++++++++ src/services/pipelineStorage/hostMigration.ts | 149 +++++++++++++++ .../pipelineStorage/pipelineRegistry.ts | 4 - src/services/pipelineStorage/types.ts | 9 + src/utils/concurrency.ts | 22 +++ tests/e2e/pipeline-storage-host.spec.ts | 20 +++ 15 files changed, 584 insertions(+), 43 deletions(-) create mode 100644 src/components/Home/PipelineSection/HostMigrationNotice.tsx create mode 100644 src/components/Home/PipelineSection/useHostMigration.ts create mode 100644 src/services/pipelineStorage/hostMigration.test.ts create mode 100644 src/services/pipelineStorage/hostMigration.ts create mode 100644 src/utils/concurrency.ts diff --git a/src/components/Home/PipelineSection/HostMigrationNotice.tsx b/src/components/Home/PipelineSection/HostMigrationNotice.tsx new file mode 100644 index 0000000000..9e80f8f9a2 --- /dev/null +++ b/src/components/Home/PipelineSection/HostMigrationNotice.tsx @@ -0,0 +1,57 @@ +import { Button } from "@/components/ui/button"; +import { Icon } from "@/components/ui/icon"; +import { BlockStack, InlineStack } from "@/components/ui/layout"; +import { Spinner } from "@/components/ui/spinner"; +import { Paragraph, Text } from "@/components/ui/typography"; +import { pluralize } from "@/utils/string"; + +import type { HostMigration } from "./useHostMigration"; + +export function HostMigrationNotice({ + migration, + storageLabel, +}: { + migration: HostMigration; + storageLabel: string; +}) { + const { phase, progress, failed, retry, skip } = migration; + + if (phase === "copying") { + return ( + + + + Moving your pipelines to {storageLabel} + + + {progress.total > 0 + ? `${progress.copied} of ${progress.total} copied` + : "Starting…"} + + + ); + } + + return ( + + + + + {failed.length} {pluralize(failed.length, "pipeline")} could not be + copied to {storageLabel} + + + + {failed.slice(0, 5).join(", ")} + {failed.length > 5 && ` and ${failed.length - 5} more`}. They are still + in this browser and nothing has been deleted. + + + + + + + ); +} diff --git a/src/components/Home/PipelineSection/PipelineRow.tsx b/src/components/Home/PipelineSection/PipelineRow.tsx index 4e41edc47c..8759eedbd4 100644 --- a/src/components/Home/PipelineSection/PipelineRow.tsx +++ b/src/components/Home/PipelineSection/PipelineRow.tsx @@ -53,6 +53,7 @@ interface PipelineRowProps { url?: string; spec?: ComponentSpec; name?: string; + fileId?: string; modificationTime?: Date; onDelete?: () => void; isSelected?: boolean; @@ -73,6 +74,7 @@ interface PipelineRowProps { const PipelineRow = withSuspenseWrapper( ({ name, + fileId, spec, modificationTime, onDelete, @@ -115,11 +117,11 @@ const PipelineRow = withSuspenseWrapper( if (e.ctrlKey || e.metaKey) { rowTrack("pipeline_opened", { open_mode: "editor_new_tab" }); - window.open(getDefaultEditorHref({ name }), "_blank"); + window.open(getDefaultEditorHref({ name, fileId }), "_blank"); return; } rowTrack("pipeline_opened", { open_mode: "editor_same_tab" }); - navigate(getDefaultEditorTarget({ name })); + navigate(getDefaultEditorTarget({ name, fileId })); }; const handleCheckboxChange = (checked: boolean | "indeterminate") => { diff --git a/src/components/Home/PipelineSection/PipelineSection.tsx b/src/components/Home/PipelineSection/PipelineSection.tsx index 2efa7c5b9c..97c431f555 100644 --- a/src/components/Home/PipelineSection/PipelineSection.tsx +++ b/src/components/Home/PipelineSection/PipelineSection.tsx @@ -22,10 +22,13 @@ import { import { Paragraph, Text } from "@/components/ui/typography"; import { usePagination } from "@/hooks/usePagination"; import { APP_ROUTES } from "@/routes/router"; +import { usePipelineStorage } from "@/services/pipelineStorage/PipelineStorageProvider"; import BulkActionsBar from "./BulkActionsBar"; +import { HostMigrationNotice } from "./HostMigrationNotice"; import { PipelineFiltersBar } from "./PipelineFiltersBar"; import PipelineRow from "./PipelineRow"; +import { useHostMigration } from "./useHostMigration"; import { usePipelineFilters } from "./usePipelineFilters"; import { usePipelineListEntries } from "./usePipelineListEntries"; @@ -61,9 +64,12 @@ export const PipelineSection = withSuspenseWrapper( ({ onPipelineClick }: PipelineSectionProps) => { const [selectedIds, setSelectedIds] = useState>(new Set()); + const storage = usePipelineStorage(); const { entries, isLoading, pendingCount, refetch } = usePipelineListEntries(); + const migration = useHostMigration(refetch); + const { filteredPipelines, filterBarProps, filterKey } = usePipelineFilters( entries, pendingCount, @@ -95,7 +101,18 @@ export const PipelineSection = withSuspenseWrapper( setSelectedIds(next); }; - if (isLoading) return ; + if (migration.phase === "copying" || migration.phase === "incomplete") { + return ( + + ); + } + + if (isLoading || migration.phase === "checking") { + return ; + } if (entries.length === 0) { return ( @@ -159,6 +176,7 @@ export const PipelineSection = withSuspenseWrapper( void; + skip: () => void; +} + +const NOTHING: HostMigrationProgress = { copied: 0, failed: 0, total: 0 }; + +const POLL_MS = 1_000; + +/** + * Copies browser-stored pipelines into a host-provided store the first time the + * app runs against one, so a user does not arrive to an empty library. It runs + * where the list would be rather than in front of the whole app: a store that + * cannot be reached must not be able to lock anyone out of the editor. + */ +export function useHostMigration(onFinished: () => void): HostMigration { + const storage = usePipelineStorage(); + const isHost = storage.mode.kind === "host"; + + const [phase, setPhase] = useState( + isHost ? "checking" : "settled", + ); + const [progress, setProgress] = useState(NOTHING); + const [failed, setFailed] = useState([]); + const [attempt, setAttempt] = useState(0); + + useEffect(() => { + if (!isHost) return; + + let watching = true; + let pollTimer: ReturnType | undefined; + + const settle = (failedKeys: string[]) => { + if (!watching) return; + setFailed(failedKeys); + setPhase(failedKeys.length > 0 ? "incomplete" : "settled"); + if (failedKeys.length === 0) onFinished(); + }; + + /** + * Claiming again rather than only reading is what recovers a claim whose + * holder is gone — a crashed tab, or this effect's own first run under + * StrictMode. The copying itself is deliberately not cancelled when the + * component goes away; only what it reports back is. + */ + const pump = async (): Promise => { + const claim = await claimHostMigration(); + + if (claim === "settled") { + settle((await readHostMigration())?.failed ?? []); + return; + } + + if (watching) setPhase("copying"); + + if (claim === "claimed") { + const record = await runHostMigration(storage.rootFolder, (update) => { + if (watching) setProgress(update); + }); + settle(record.failed); + return; + } + + const record = await readHostMigration(); + if (!watching) return; + + setProgress({ + copied: record?.copied.length ?? 0, + failed: record?.failed.length ?? 0, + total: 0, + }); + pollTimer = setTimeout(() => void pump(), POLL_MS); + }; + + void pump(); + + return () => { + watching = false; + clearTimeout(pollTimer); + }; + }, [isHost, storage, attempt, onFinished]); + + return { + phase, + progress, + failed, + retry: () => setAttempt((previous) => previous + 1), + skip: () => { + void dismissHostMigration().then(() => { + setPhase("settled"); + onFinished(); + }); + }, + }; +} diff --git a/src/components/Home/PipelineSection/usePipelineListEntries.ts b/src/components/Home/PipelineSection/usePipelineListEntries.ts index 96eb7a5cbe..1fbc794aa3 100644 --- a/src/components/Home/PipelineSection/usePipelineListEntries.ts +++ b/src/components/Home/PipelineSection/usePipelineListEntries.ts @@ -10,6 +10,7 @@ import { import { usePipelineStorage } from "@/services/pipelineStorage/PipelineStorageProvider"; import { FoldersQueryKeys } from "@/services/pipelineStorage/types"; import type { ComponentSpec } from "@/utils/componentSpec"; +import { runWithConcurrency } from "@/utils/concurrency"; import { componentSpecFromYaml } from "@/utils/yaml"; const HYDRATION_CONCURRENCY = 3; @@ -85,26 +86,17 @@ async function hydrate( isStale: () => boolean, onSpec: (file: PipelineFile, spec: ComponentSpec) => void, ): Promise { - const queue = [...files]; - - const worker = async () => { - for (let file = queue.shift(); file && !isStale(); file = queue.shift()) { - try { - const spec = componentSpecFromYaml(await file.read()); - if (isStale()) return; - - onSpec(file, spec); - void writeCachedSpec(file, spec); - } catch (error) { - console.error(`Failed to read pipeline "${file.displayName}":`, error); - } - } - }; + await runWithConcurrency(files, HYDRATION_CONCURRENCY, async (file) => { + if (isStale()) return; - await Promise.all( - Array.from( - { length: Math.min(HYDRATION_CONCURRENCY, queue.length) }, - worker, - ), - ); + try { + const spec = componentSpecFromYaml(await file.read()); + if (isStale()) return; + + onSpec(file, spec); + void writeCachedSpec(file, spec); + } catch (error) { + console.error(`Failed to read pipeline "${file.displayName}":`, error); + } + }); } diff --git a/src/services/pipelineStorage/PipelineFolder.test.ts b/src/services/pipelineStorage/PipelineFolder.test.ts index d55a2e5081..c869cd2da0 100644 --- a/src/services/pipelineStorage/PipelineFolder.test.ts +++ b/src/services/pipelineStorage/PipelineFolder.test.ts @@ -23,9 +23,6 @@ vi.mock("./createDriver", () => ({ })); vi.mock("./pipelineRegistry", () => ({ - addEntry: vi.fn(async (entry: PipelineRegistryEntry) => { - registry.set(entry.id, entry); - }), claimEntry: vi.fn(async (entry: PipelineRegistryEntry) => { const existing = [...registry.values()].find( (candidate) => candidate.storageKey === entry.storageKey, diff --git a/src/services/pipelineStorage/PipelineFolder.ts b/src/services/pipelineStorage/PipelineFolder.ts index 8d7d166172..c9d25acb41 100644 --- a/src/services/pipelineStorage/PipelineFolder.ts +++ b/src/services/pipelineStorage/PipelineFolder.ts @@ -5,7 +5,6 @@ import { pipelineStorageDb } from "./db"; import { PipelineFile } from "./PipelineFile"; import { emitPipelineFileChanged } from "./pipelineFileEvents"; import { - addEntry, assertStorageKeyUnique, claimEntry, deleteEntry, @@ -139,15 +138,8 @@ export class PipelineFolder { // Writing before registering means a rejected write leaves no registry row // pointing at a file that was never created. const descriptor = await this.driver.write(storageKey, content); - const id = descriptor.externalId ?? crypto.randomUUID(); - await addEntry({ - id, - storageKey: descriptor.storageKey, - folderId: this.id, - contentVersion: descriptor.contentVersion, - }); - return new PipelineFile({ id, folder: this, ...descriptor }); + return resolveOrCreateRegistryEntry(descriptor, this); } async listSubfolders(): Promise { diff --git a/src/services/pipelineStorage/PipelineStorageService.test.ts b/src/services/pipelineStorage/PipelineStorageService.test.ts index 2f5c7ff3a5..e06220033b 100644 --- a/src/services/pipelineStorage/PipelineStorageService.test.ts +++ b/src/services/pipelineStorage/PipelineStorageService.test.ts @@ -17,9 +17,6 @@ import { const registry = new Map(); vi.mock("./pipelineRegistry", () => ({ - addEntry: async (entry: PipelineRegistryEntry) => { - registry.set(entry.id, entry); - }, claimEntry: async (entry: PipelineRegistryEntry) => { const existing = [...registry.values()].find( (candidate) => candidate.storageKey === entry.storageKey, diff --git a/src/services/pipelineStorage/db.ts b/src/services/pipelineStorage/db.ts index 65ce723083..af2068aef6 100644 --- a/src/services/pipelineStorage/db.ts +++ b/src/services/pipelineStorage/db.ts @@ -6,6 +6,7 @@ import { isHostStorage } from "./storageMode"; import { type CachedPipelineSpec, type FolderEntry, + type HostMigrationRecord, type PipelineRegistryEntry, ROOT_FOLDER_ID, } from "./types"; @@ -14,6 +15,7 @@ export type PipelineStorageDb = Dexie & { pipeline_registry: EntityTable; folders: EntityTable; pipeline_specs: EntityTable; + host_migration: EntityTable; }; export const pipelineStorageDb = new Dexie( @@ -72,6 +74,13 @@ pipelineStorageDb.version(4).stores({ pipeline_specs: "storageKey", }); +pipelineStorageDb.version(5).stores({ + pipeline_registry: "id, &storageKey, folderId, [folderId+storageKey]", + folders: "id, parentId", + pipeline_specs: "storageKey", + host_migration: "id", +}); + pipelineStorageDb.on("ready", async () => { await seedRegistryFromLegacyList(); }); diff --git a/src/services/pipelineStorage/hostMigration.test.ts b/src/services/pipelineStorage/hostMigration.test.ts new file mode 100644 index 0000000000..1da2ba5d67 --- /dev/null +++ b/src/services/pipelineStorage/hostMigration.test.ts @@ -0,0 +1,170 @@ +import "fake-indexeddb/auto"; + +import { beforeEach, describe, expect, it, vi } from "vitest"; + +import { pipelineStorageDb } from "./db"; +import { RootFolderDbStorageDriver } from "./drivers/RootFolderDbStorageDriver"; +import { + claimHostMigration, + dismissHostMigration, + readHostMigration, + runHostMigration, +} from "./hostMigration"; +import { PipelineFolder } from "./PipelineFolder"; +import { ROOT_FOLDER_ID } from "./types"; + +const LOCAL = ["Churn model", "Nightly refresh", "Ranking model"]; + +function seedLocalStore(names: string[] = LOCAL) { + vi.spyOn(RootFolderDbStorageDriver.prototype, "list").mockResolvedValue( + names.map((storageKey) => ({ storageKey })), + ); + vi.spyOn(RootFolderDbStorageDriver.prototype, "read").mockImplementation( + async (storageKey: string) => `name: ${storageKey}\n`, + ); +} + +function hostFolder(options: { rejects?: Set } = {}) { + const written = new Map(); + + const folder = new PipelineFolder({ + id: ROOT_FOLDER_ID, + name: "Shared storage", + parentId: null, + isFlat: true, + driver: { + type: "host", + allowsMoveIn: false, + allowsMoveOut: false, + listingIsAuthoritative: true, + list: async () => [], + read: async () => "", + write: async (storageKey: string, content: string) => { + if (options.rejects?.has(storageKey)) { + throw new Error(`refused ${storageKey}`); + } + written.set(storageKey, content); + return { storageKey, externalId: `external-${storageKey}` }; + }, + delete: async () => undefined, + hasKey: async (storageKey: string) => written.has(storageKey), + }, + }); + + return { folder, written }; +} + +beforeEach(async () => { + vi.restoreAllMocks(); + await pipelineStorageDb.host_migration.clear(); + await pipelineStorageDb.pipeline_registry.clear(); +}); + +describe("claiming the migration", () => { + it("lets exactly one of two tabs starting together do the work", async () => { + const claims = await Promise.all([ + claimHostMigration(), + claimHostMigration(), + ]); + + expect(claims.filter((claim) => claim === "claimed")).toHaveLength(1); + expect(claims.filter((claim) => claim === "in-progress")).toHaveLength(1); + }); + + it("takes over from a tab that claimed and then died", async () => { + await claimHostMigration(); + await pipelineStorageDb.host_migration.update("v1", { + startedAt: Date.now() - 10 * 60_000, + }); + + expect(await claimHostMigration()).toBe("claimed"); + }); + + it("does not run again once it has finished", async () => { + seedLocalStore([]); + await claimHostMigration(); + await runHostMigration(hostFolder().folder); + + expect(await claimHostMigration()).toBe("settled"); + }); + + it("does not run again once the user has chosen to go without", async () => { + await claimHostMigration(); + await dismissHostMigration(); + + expect(await claimHostMigration()).toBe("settled"); + }); +}); + +describe("running the migration", () => { + it("copies every browser-stored pipeline under the name it already had", async () => { + seedLocalStore(); + const { folder, written } = hostFolder(); + + const record = await runHostMigration(folder); + + expect([...written.keys()].sort()).toEqual([...LOCAL].sort()); + expect(record.failed).toEqual([]); + expect(record.completedAt).toBeDefined(); + }); + + it("reports what it could not copy and stays unfinished", async () => { + seedLocalStore(); + const { folder, written } = hostFolder({ + rejects: new Set(["Ranking model"]), + }); + + const record = await runHostMigration(folder); + + expect(record.failed).toEqual(["Ranking model"]); + expect(record.completedAt).toBeUndefined(); + expect(written.size).toBe(2); + }); + + it("retries only what failed", async () => { + seedLocalStore(); + await runHostMigration( + hostFolder({ rejects: new Set(["Ranking model"]) }).folder, + ); + + const { folder, written } = hostFolder(); + const record = await runHostMigration(folder); + + expect([...written.keys()]).toEqual(["Ranking model"]); + expect(record.completedAt).toBeDefined(); + }); + + it("leaves the browser's own store untouched", async () => { + seedLocalStore(); + const deleted = vi.spyOn(RootFolderDbStorageDriver.prototype, "delete"); + + await runHostMigration(hostFolder().folder); + + expect(deleted).not.toHaveBeenCalled(); + }); + + it("reports progress as it goes", async () => { + seedLocalStore(); + const seen: number[] = []; + + await runHostMigration(hostFolder().folder, ({ copied }) => + seen.push(copied), + ); + + expect(seen.at(0)).toBe(0); + expect(seen.at(-1)).toBe(LOCAL.length); + }); + + it("survives being run twice without duplicating anything", async () => { + seedLocalStore(); + await runHostMigration(hostFolder().folder); + + const { folder, written } = hostFolder(); + await runHostMigration(folder); + + expect(written.size).toBe(0); + expect((await readHostMigration())?.copied.sort()).toEqual( + [...LOCAL].sort(), + ); + }); +}); diff --git a/src/services/pipelineStorage/hostMigration.ts b/src/services/pipelineStorage/hostMigration.ts new file mode 100644 index 0000000000..35cb87c60a --- /dev/null +++ b/src/services/pipelineStorage/hostMigration.ts @@ -0,0 +1,149 @@ +import { runWithConcurrency } from "@/utils/concurrency"; +import { getErrorMessage } from "@/utils/string"; + +import { pipelineStorageDb } from "./db"; +import { RootFolderDbStorageDriver } from "./drivers/RootFolderDbStorageDriver"; +import type { PipelineFolder } from "./PipelineFolder"; +import type { HostMigrationRecord } from "./types"; + +const RECORD_ID = "v1"; +const COPY_CONCURRENCY = 3; + +/** + * How long a claim is honoured without progress. A tab that crashes mid-copy + * would otherwise hold the claim forever and no tab could ever finish the + * migration; the working tab pushes this forward after every pipeline, so only + * a genuinely dead one loses it. + */ +const CLAIM_STALE_MS = 60_000; + +export type HostMigrationClaim = "claimed" | "in-progress" | "settled"; + +export interface HostMigrationProgress { + copied: number; + failed: number; + total: number; +} + +export async function readHostMigration(): Promise< + HostMigrationRecord | undefined +> { + return pipelineStorageDb.host_migration.get(RECORD_ID); +} + +function isHostMigrationSettled( + record: HostMigrationRecord | undefined, +): boolean { + return record?.completedAt !== undefined || record?.dismissedAt !== undefined; +} + +/** + * Two tabs opening at once must not both copy the whole library. Reading the + * record and claiming it are one transaction so exactly one of them wins. + */ +export async function claimHostMigration(): Promise { + return pipelineStorageDb.transaction( + "rw", + pipelineStorageDb.host_migration, + async () => { + const existing = await pipelineStorageDb.host_migration.get(RECORD_ID); + + if (isHostMigrationSettled(existing)) return "settled"; + + if (existing && Date.now() - existing.startedAt < CLAIM_STALE_MS) { + return "in-progress"; + } + + await pipelineStorageDb.host_migration.put({ + id: RECORD_ID, + startedAt: Date.now(), + copied: existing?.copied ?? [], + failed: [], + }); + + return "claimed"; + }, + ); +} + +export async function dismissHostMigration(): Promise { + await pipelineStorageDb.host_migration.put({ + id: RECORD_ID, + startedAt: Date.now(), + dismissedAt: Date.now(), + copied: (await readHostMigration())?.copied ?? [], + failed: [], + }); +} + +/** + * Copies everything in browser storage into the host, keyed on the name it has + * locally. Host writes upsert on the key they are given, so a pipeline copied + * twice is overwritten rather than duplicated, and a run that died halfway can + * simply be run again. + * + * The local store is left exactly as it was: this is a copy, and a build with + * no host still has to find its pipelines. + */ +export async function runHostMigration( + target: PipelineFolder, + onProgress?: (progress: HostMigrationProgress) => void, +): Promise { + const source = new RootFolderDbStorageDriver(); + const local = await source.list(); + + const record = await readHostMigration(); + const alreadyCopied = new Set(record?.copied ?? []); + const outstanding = local.filter( + (descriptor) => !alreadyCopied.has(descriptor.storageKey), + ); + + const copied = [...alreadyCopied]; + const failed: string[] = []; + const report = () => + onProgress?.({ + copied: copied.length, + failed: failed.length, + total: local.length, + }); + + report(); + + await runWithConcurrency( + outstanding, + COPY_CONCURRENCY, + async (descriptor) => { + try { + await target.addFile( + descriptor.storageKey, + await source.read(descriptor.storageKey), + ); + copied.push(descriptor.storageKey); + } catch (error) { + console.error( + `Could not copy pipeline "${descriptor.storageKey}":`, + getErrorMessage(error), + ); + failed.push(descriptor.storageKey); + } + + await pipelineStorageDb.host_migration.update(RECORD_ID, { + startedAt: Date.now(), + copied: [...copied], + failed: [...failed], + }); + report(); + }, + ); + + const settled: HostMigrationRecord = { + id: RECORD_ID, + startedAt: Date.now(), + completedAt: failed.length === 0 ? Date.now() : undefined, + copied, + failed, + }; + + await pipelineStorageDb.host_migration.put(settled); + return settled; +} diff --git a/src/services/pipelineStorage/pipelineRegistry.ts b/src/services/pipelineStorage/pipelineRegistry.ts index 709000b5c3..161ad3abee 100644 --- a/src/services/pipelineStorage/pipelineRegistry.ts +++ b/src/services/pipelineStorage/pipelineRegistry.ts @@ -1,10 +1,6 @@ import { pipelineStorageDb } from "./db"; import { type PipelineRegistryEntry, ROOT_FOLDER_ID } from "./types"; -export async function addEntry(entry: PipelineRegistryEntry): Promise { - await pipelineStorageDb.pipeline_registry.add(entry); -} - /** * Two listings running at once both find no row for a storage key and both try * to add one, and the unique index fails the loser — taking down a whole diff --git a/src/services/pipelineStorage/types.ts b/src/services/pipelineStorage/types.ts index 8943ab0e4f..38b1c5effe 100644 --- a/src/services/pipelineStorage/types.ts +++ b/src/services/pipelineStorage/types.ts @@ -53,6 +53,15 @@ export interface CachedPipelineSpec { spec: ComponentSpec; } +export interface HostMigrationRecord { + id: string; + startedAt: number; + completedAt?: number; + dismissedAt?: number; + copied: string[]; + failed: string[]; +} + export interface PipelineRegistryEntry { id: string; storageKey: string; diff --git a/src/utils/concurrency.ts b/src/utils/concurrency.ts new file mode 100644 index 0000000000..37d135ae0e --- /dev/null +++ b/src/utils/concurrency.ts @@ -0,0 +1,22 @@ +/** + * Works through `items` with at most `limit` in flight. A store that answers + * one pipeline per request should not be sent the whole list at once, and + * `Promise.all` over a `map` would do exactly that. + */ +export async function runWithConcurrency( + items: T[], + limit: number, + worker: (item: T) => Promise, +): Promise { + const queue = [...items]; + + const drain = async () => { + for (let item = queue.shift(); item !== undefined; item = queue.shift()) { + await worker(item); + } + }; + + await Promise.all( + Array.from({ length: Math.min(limit, queue.length) }, drain), + ); +} diff --git a/tests/e2e/pipeline-storage-host.spec.ts b/tests/e2e/pipeline-storage-host.spec.ts index cde4d59298..0930a2b538 100644 --- a/tests/e2e/pipeline-storage-host.spec.ts +++ b/tests/e2e/pipeline-storage-host.spec.ts @@ -7,6 +7,7 @@ import { readHostRecords, readLocallyStoredPipelineKeys, } from "./fixtures/pipelineStorageHost"; +import { createNewPipeline } from "./helpers"; const LABEL = "Shared storage"; @@ -80,6 +81,25 @@ test.describe("host-provided pipeline storage", () => { expect(await readHostReadKeys(page)).toEqual([]); }); + test("copies pipelines already in the browser into the host", async ({ + page, + }) => { + await createNewPipeline(page); + const localName = decodeURIComponent( + new URL(page.url()).pathname.split("/").pop() ?? "", + ); + + await installSeededHost(page); + await page.goto("/pipelines"); + + await expect(page.getByText(localName)).toBeVisible(); + await expect(page.getByText("Churn model")).toBeVisible(); + + expect( + (await readHostRecords(page)).map((record) => record.displayName), + ).toContain(localName); + }); + test("keeps the browser's own pipeline store empty", async ({ page }) => { await installSeededHost(page); From 2097af0ce69cd80a770d9b5839d1573a9edf7d82 Mon Sep 17 00:00:00 2001 From: Camiel van Schoonhoven Date: Tue, 8 Sep 2026 18:20:48 -0700 Subject: [PATCH 16/36] fix(editor): keep edits a store refused and save them when it recovers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A rejected write set an error and let the edit go. The editor was then the only copy of that work, with a 16px icon as the entire warning, and nothing would try again unless the user happened to keep typing — so anyone who stopped after a failure was quietly left unsaved. Hold the rejected edit instead and retry it: on a backoff, and immediately on coming back online or back to the tab, since either is a cheap sign the store might take it now. The indicator says the changes are still here rather than only that something went wrong. Two saves overlapping used to race to the store and the winner was whichever it finished last. A save that starts mid-write now waits, and the write already running picks up the newer content when it lands. Co-Authored-By: Claude Opus 5 (1M context) --- .../components/AutoSaveIndicator.tsx | 16 ++- .../pages/Editor/store/autoSaveStore.test.ts | 84 +++++++++++++ .../v2/pages/Editor/store/autoSaveStore.ts | 115 +++++++++++++++++- src/services/pipelineStorage/hostMigration.ts | 21 ++++ 4 files changed, 227 insertions(+), 9 deletions(-) diff --git a/src/routes/v2/pages/Editor/components/EditorMenuBar/components/AutoSaveIndicator.tsx b/src/routes/v2/pages/Editor/components/EditorMenuBar/components/AutoSaveIndicator.tsx index 57fb5e680d..9aa6c44494 100644 --- a/src/routes/v2/pages/Editor/components/EditorMenuBar/components/AutoSaveIndicator.tsx +++ b/src/routes/v2/pages/Editor/components/EditorMenuBar/components/AutoSaveIndicator.tsx @@ -45,9 +45,14 @@ function getTooltipText( isSaving: boolean, lastSavedAt: Date | null, saveError: string | null, + hasPendingChanges: boolean, ): string { if (isSaving) return "Saving..."; - if (saveError) return saveError; + if (saveError) { + return hasPendingChanges + ? `${saveError} Your changes are still here and will be saved as soon as it can be reached — click to try now.` + : saveError; + } if (lastSavedAt) { return `Last saved at ${lastSavedAt.toLocaleTimeString()}`; } @@ -56,8 +61,13 @@ function getTooltipText( export const AutoSaveIndicator = observer(function AutoSaveIndicator() { const { autoSave } = useEditorSession(); - const { isSaving, lastSavedAt, saveError } = autoSave; - const tooltipText = getTooltipText(isSaving, lastSavedAt, saveError); + const { isSaving, lastSavedAt, saveError, hasPendingChanges } = autoSave; + const tooltipText = getTooltipText( + isSaving, + lastSavedAt, + saveError, + hasPendingChanges, + ); const handleClick = () => { void autoSave.save(); diff --git a/src/routes/v2/pages/Editor/store/autoSaveStore.test.ts b/src/routes/v2/pages/Editor/store/autoSaveStore.test.ts index 14ecf8d86c..0ecb63761f 100644 --- a/src/routes/v2/pages/Editor/store/autoSaveStore.test.ts +++ b/src/routes/v2/pages/Editor/store/autoSaveStore.test.ts @@ -64,5 +64,89 @@ describe("AutoSaveStore.save", () => { await store.save(); expect(store.saveError).toContain("could not be reached"); + store.dispose(); + }); +}); + +describe("AutoSaveStore when the store cannot be reached", () => { + it("holds on to the rejected edit rather than dropping it", async () => { + const store = createStore({ + write: async () => { + throw new Error("unreachable"); + }, + } as unknown as PipelineFile); + + store.init(createSpec("Churn model"), "Churn model"); + await store.save(); + + expect(store.hasPendingChanges).toBe(true); + store.dispose(); + }); + + it("saves the held edit as soon as the store takes writes again", async () => { + let reachable = false; + const written: string[] = []; + const store = createStore({ + write: async (yamlText: string) => { + if (!reachable) throw new Error("unreachable"); + written.push(yamlText); + }, + } as unknown as PipelineFile); + + store.init(createSpec("Churn model"), "Churn model"); + await store.save(); + expect(written).toEqual([]); + + reachable = true; + window.dispatchEvent(new Event("focus")); + await vi.waitFor(() => expect(store.hasPendingChanges).toBe(false)); + + expect(written).toEqual(["name: Churn model"]); + expect(store.saveError).toBeNull(); + expect(store.lastSavedAt).toBeInstanceOf(Date); + store.dispose(); + }); + + it("stops retrying once the editor is closed", async () => { + const write = vi.fn(async () => { + throw new Error("unreachable"); + }); + const store = createStore({ write } as unknown as PipelineFile); + + store.init(createSpec("Churn model"), "Churn model"); + await store.save(); + store.dispose(); + + write.mockClear(); + window.dispatchEvent(new Event("focus")); + await new Promise((resolve) => setTimeout(resolve, 20)); + + expect(write).not.toHaveBeenCalled(); + }); + + it("lands overlapping saves in order instead of racing them", async () => { + const written: string[] = []; + let release: (() => void) | undefined; + const store = createStore({ + write: async (yamlText: string) => { + if (!release) { + await new Promise((resolve) => (release = resolve)); + } + written.push(yamlText); + }, + } as unknown as PipelineFile); + + const spec = createSpec("Churn model"); + store.init(spec, "Churn model"); + + const first = store.save(); + spec.name = "Churn model v2"; + const second = store.save(); + + release?.(); + await Promise.all([first, second]); + + expect(written).toEqual(["name: Churn model", "name: Churn model v2"]); + store.dispose(); }); }); diff --git a/src/routes/v2/pages/Editor/store/autoSaveStore.ts b/src/routes/v2/pages/Editor/store/autoSaveStore.ts index c38d470de5..bd4653e3e5 100644 --- a/src/routes/v2/pages/Editor/store/autoSaveStore.ts +++ b/src/routes/v2/pages/Editor/store/autoSaveStore.ts @@ -15,10 +15,13 @@ import type { UndoStore } from "./undoStore"; const AUTOSAVE_MIN_SAVING_INDICATOR_MS = 600; +const RETRY_DELAYS_MS = [5_000, 15_000, 30_000, 60_000]; + export class AutoSaveStore { @observable accessor isSaving = false; @observable accessor lastSavedAt: Date | null = null; @observable accessor saveError: string | null = null; + @observable accessor hasPendingChanges = false; private spec: ComponentSpec | null = null; private pipelineName: string | null = null; @@ -26,6 +29,18 @@ export class AutoSaveStore { // Last content written to disk; used to skip a redundant flush on dispose. private lastSavedYaml: string | null = null; + /** + * Edits that storage has not accepted yet. A rejected write leaves them here + * rather than dropping them, because the editor is then the only copy: the + * store is retried until it takes them, and a user who stops editing after a + * failure is not quietly left with unsaved work. + */ + private pendingYaml: string | null = null; + private inFlight: Promise | null = null; + private retryTimer: ReturnType | null = null; + private retryAttempt = 0; + private disposeRecovery: (() => void) | null = null; + private debouncedSave = debounce((yamlText: string) => { void this.performSave(yamlText); }, AUTOSAVE_DEBOUNCE_TIME_MS); @@ -44,6 +59,9 @@ export class AutoSaveStore { this.isSaving = false; this.lastSavedAt = null; this.saveError = null; + this.hasPendingChanges = false; + this.pendingYaml = null; + this.retryAttempt = 0; // The freshly-loaded spec matches what's on disk, so seed the baseline to // avoid flushing an unchanged pipeline on dispose. this.lastSavedYaml = this.serializeSpec(); @@ -53,6 +71,8 @@ export class AutoSaveStore { (yamlText) => this.scheduleAutoSave(yamlText), { fireImmediately: false }, ); + + this.watchForRecovery(); } @action dispose() { @@ -64,6 +84,9 @@ export class AutoSaveStore { }); } this.debouncedSave.cancel(); + this.clearRetry(); + this.disposeRecovery?.(); + this.disposeRecovery = null; this.disposeReaction?.(); this.disposeReaction = null; this.spec = null; @@ -92,6 +115,10 @@ export class AutoSaveStore { this.isSaving = false; } + @action private setPending(value: boolean) { + this.hasPendingChanges = value; + } + private serializeSpec(): string | null { if (!this.spec) return null; try { @@ -110,9 +137,46 @@ export class AutoSaveStore { } private async performSave(yamlText: string) { - const pipelineName = this.pipelineName; - if (!pipelineName) return; + if (!this.pipelineName) return; + + this.pendingYaml = yamlText; + this.setPending(true); + this.clearRetry(); + + // A second save starting mid-write would race the first one to the store, + // and the loser is whichever the store happens to finish last. Later edits + // wait and are picked up by the write already running. + if (this.inFlight) return this.inFlight; + + this.inFlight = this.drainPending(); + try { + await this.inFlight; + } finally { + this.inFlight = null; + } + } + + private async drainPending(): Promise { + while (this.pendingYaml !== null) { + const yamlText = this.pendingYaml; + const outcome = await this.writeOnce(yamlText); + + if (typeof outcome === "string") { + this.setSaveError(outcome); + this.scheduleRetry(); + return; + } + + this.setSaved(outcome); + this.retryAttempt = 0; + if (this.pendingYaml === yamlText) this.pendingYaml = null; + } + + this.setPending(false); + } + private async writeOnce(yamlText: string): Promise { + const pipelineName = this.pipelineName; this.setSaving(true); const savePromise = (async () => { @@ -137,14 +201,53 @@ export class AutoSaveStore { ); const [outcome] = await Promise.all([savePromise, minDisplayPromise]); + return outcome; + } - if (outcome instanceof Date) { - this.setSaved(outcome); - } else { - this.setSaveError(outcome); + private scheduleRetry() { + this.clearRetry(); + + const delay = + RETRY_DELAYS_MS[Math.min(this.retryAttempt, RETRY_DELAYS_MS.length - 1)]; + this.retryAttempt += 1; + this.retryTimer = setTimeout(() => void this.flushPending(), delay); + } + + private clearRetry() { + if (this.retryTimer === null) return; + clearTimeout(this.retryTimer); + this.retryTimer = null; + } + + private async flushPending(): Promise { + if (this.pendingYaml === null || this.inFlight) return; + + this.inFlight = this.drainPending(); + try { + await this.inFlight; + } finally { + this.inFlight = null; } } + /** + * Coming back online or back to the tab is the cheapest signal that a store + * that refused a write a moment ago might take it now, and it beats waiting + * out the backoff. + */ + private watchForRecovery() { + if (typeof window === "undefined") return; + + const retryNow = () => void this.flushPending(); + window.addEventListener("online", retryNow); + window.addEventListener("focus", retryNow); + + this.disposeRecovery = () => { + window.removeEventListener("online", retryNow); + window.removeEventListener("focus", retryNow); + }; + } + private async persistUndoHistory() { const fileId = this.pipelineFileStore.activePipelineFile?.id; if (!this.spec || !fileId) return; diff --git a/src/services/pipelineStorage/hostMigration.ts b/src/services/pipelineStorage/hostMigration.ts index 35cb87c60a..bab9d42679 100644 --- a/src/services/pipelineStorage/hostMigration.ts +++ b/src/services/pipelineStorage/hostMigration.ts @@ -31,6 +31,27 @@ export async function readHostMigration(): Promise< return pipelineStorageDb.host_migration.get(RECORD_ID); } +/** + * Testing the migration means running it more than once, and the whole point of + * the record is that it only runs again if something is wrong. Exposed on the + * window in development so it can be reset without hand-editing IndexedDB — + * pipelines already copied stay in the store, so clear them there too for a + * genuinely clean run. + */ +if (import.meta.env.DEV && typeof window !== "undefined") { + window.resetPipelineStorageMigration = async () => { + await pipelineStorageDb.host_migration.clear(); + await pipelineStorageDb.pipeline_specs.clear(); + console.info("Migration reset. Reload to run it again."); + }; +} + +declare global { + interface Window { + resetPipelineStorageMigration?: () => Promise; + } +} + function isHostMigrationSettled( record: HostMigrationRecord | undefined, ): boolean { From 21de7bb3e02fa07bfad84ad6e6728da2e09f4ccd Mon Sep 17 00:00:00 2001 From: Camiel van Schoonhoven Date: Tue, 8 Sep 2026 18:51:16 -0700 Subject: [PATCH 17/36] feat(pipeline-storage): let a deployment declare which store it runs on MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Storage mode was inferred from whether a host had loaded, which fails open: anything that stopped the host from installing sent the user to browser storage without a word, and their work went somewhere the deployment does not read from. VITE_PIPELINE_STORAGE says it outright. "host" requires one and says so plainly when the page did not provide it, rather than quietly writing elsewhere. "local" ignores one that loads anyway. Left unset it detects as before, which is what an unconfigured build wants and all the open-source build can do — it has no host to find. A store that hands out its own ids can now say so in the url, which carries the id alone: names are not unique there, so the name bought nothing a query param was not already paying for, and a path that is identity cannot go stale when a pipeline is renamed. Where names are the identity the url is unchanged. A single path segment is offered as both, so links written the other way round still open. Co-Authored-By: Claude Opus 5 (1M context) --- .../layout/PipelineStorageUnavailable.tsx | 31 +++++++ src/components/layout/RootLayout.tsx | 6 ++ src/routes/editorRoutes.test.ts | 53 ++++++++++++ src/routes/editorRoutes.ts | 35 ++++++-- src/routes/v2/pages/Editor/EditorV2.tsx | 2 +- .../PipelineStorageService.test.ts | 33 +++++++ .../pipelineStorage/PipelineStorageService.ts | 32 ++++++- .../drivers/UnavailableStorageDriver.ts | 43 ++++++++++ .../pipelineStorage/storageMode.test.ts | 86 +++++++++++++++++++ src/services/pipelineStorage/storageMode.ts | 60 ++++++++++--- tests/e2e/pipeline-storage-host.spec.ts | 18 ++++ 11 files changed, 380 insertions(+), 19 deletions(-) create mode 100644 src/components/layout/PipelineStorageUnavailable.tsx create mode 100644 src/routes/editorRoutes.test.ts create mode 100644 src/services/pipelineStorage/drivers/UnavailableStorageDriver.ts create mode 100644 src/services/pipelineStorage/storageMode.test.ts diff --git a/src/components/layout/PipelineStorageUnavailable.tsx b/src/components/layout/PipelineStorageUnavailable.tsx new file mode 100644 index 0000000000..7eda351377 --- /dev/null +++ b/src/components/layout/PipelineStorageUnavailable.tsx @@ -0,0 +1,31 @@ +import { Button } from "@/components/ui/button"; +import { Icon } from "@/components/ui/icon"; +import { BlockStack } from "@/components/ui/layout"; +import { Heading, Paragraph } from "@/components/ui/typography"; + +/** + * Shown instead of the app when the deployment stores pipelines outside the + * browser and the page did not provide that store. Everything here reads and + * writes pipelines, so carrying on would either show an empty library or save + * work where nobody will look for it. + */ +export function PipelineStorageUnavailable() { + return ( + + + Pipeline storage is not available + + This deployment keeps your pipelines outside the browser, and that store + did not load. Nothing has been lost — reload to try again, and if it + keeps happening, report it rather than working around it. + + + + ); +} diff --git a/src/components/layout/RootLayout.tsx b/src/components/layout/RootLayout.tsx index d4f3a52fc0..a5bdedcc68 100644 --- a/src/components/layout/RootLayout.tsx +++ b/src/components/layout/RootLayout.tsx @@ -13,8 +13,10 @@ import { ComponentSpecProvider } from "@/providers/ComponentSpecProvider"; import { OnboardingProvider } from "@/providers/OnboardingProvider/OnboardingProvider"; import { TourProvider } from "@/providers/TourProvider/TourProvider"; import { PipelineStorageProvider } from "@/services/pipelineStorage/PipelineStorageProvider"; +import { isHostStorageMissing } from "@/services/pipelineStorage/storageMode"; import AppMenu from "./AppMenu"; +import { PipelineStorageUnavailable } from "./PipelineStorageUnavailable"; function SessionPipelineStatsTracker() { useSessionPipelineStats(); @@ -25,6 +27,10 @@ function RootLayoutContent() { usePageViewTracking(); useClickTracking(); + if (isHostStorageMissing()) { + return ; + } + return ( diff --git a/src/routes/editorRoutes.test.ts b/src/routes/editorRoutes.test.ts new file mode 100644 index 0000000000..138df0e978 --- /dev/null +++ b/src/routes/editorRoutes.test.ts @@ -0,0 +1,53 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { getDefaultEditorHref, getDefaultEditorTarget } from "./editorRoutes"; + +const { hostStorage } = vi.hoisted(() => ({ hostStorage: vi.fn(() => false) })); + +vi.mock("@/services/pipelineStorage/storageMode", () => ({ + isHostStorage: hostStorage, +})); + +vi.mock("@/components/shared/Settings/useFlags", () => ({ + isFlagEnabled: () => true, +})); + +const REF = { name: "Churn model", fileId: "ab420234-a05f" }; + +afterEach(() => { + hostStorage.mockReturnValue(false); +}); + +describe("where a store hands out its own ids", () => { + it("puts the id in the path and nothing in the query", () => { + hostStorage.mockReturnValue(true); + + expect(getDefaultEditorTarget(REF)).toEqual({ + to: "/editor-v2/$pipelineName", + params: { pipelineName: "ab420234-a05f" }, + search: {}, + }); + expect(getDefaultEditorHref(REF)).toBe("/editor-v2/ab420234-a05f"); + }); + + it("falls back to the name for a pipeline with no id yet", () => { + hostStorage.mockReturnValue(true); + + expect(getDefaultEditorHref({ name: "Churn model" })).toBe( + "/editor-v2/Churn%20model", + ); + }); +}); + +describe("where names are the identity", () => { + it("keeps the name in the path and the id alongside it", () => { + expect(getDefaultEditorTarget(REF)).toEqual({ + to: "/editor-v2/$pipelineName", + params: { pipelineName: "Churn model" }, + search: { fileId: "ab420234-a05f" }, + }); + expect(getDefaultEditorHref(REF)).toBe( + "/editor-v2/Churn%20model?fileId=ab420234-a05f", + ); + }); +}); diff --git a/src/routes/editorRoutes.ts b/src/routes/editorRoutes.ts index edd26fb1af..7b814fbfb0 100644 --- a/src/routes/editorRoutes.ts +++ b/src/routes/editorRoutes.ts @@ -1,4 +1,5 @@ import { isFlagEnabled } from "@/components/shared/Settings/useFlags"; +import { isHostStorage } from "@/services/pipelineStorage/storageMode"; import type { PipelineRef } from "@/services/pipelineStorage/types"; import { APP_ROUTES, EDITOR_PATH } from "./appRoutes"; @@ -14,24 +15,44 @@ export interface EditorTarget { } /** - * A pipeline's identity travels in the search params, not the path: the slug is - * only ever a display name, and two pipelines are allowed to share one. + * What identifies a pipeline depends on the store. Where the store hands out + * its own ids, the path carries one and nothing else is needed — names are not + * unique there, and an id in the path cannot go stale when the pipeline is + * renamed. Where names are the identity, the path keeps the name and the id + * rides along to settle the cases a name cannot. */ +function editorLocation(ref: PipelineRef): { + segment: string; + search: EditorSearch; +} { + if (isHostStorage() && ref.fileId) { + return { segment: ref.fileId, search: {} }; + } + + return { + segment: ref.name, + search: ref.fileId ? { fileId: ref.fileId } : {}, + }; +} + export function getDefaultEditorTarget(ref: PipelineRef): EditorTarget { - const search: EditorSearch = ref.fileId ? { fileId: ref.fileId } : {}; + const { segment, search } = editorLocation(ref); return isFlagEnabled("v2_editor") ? { to: APP_ROUTES.EDITOR_V2_PIPELINE, - params: { pipelineName: ref.name }, + params: { pipelineName: segment }, search, } - : { to: APP_ROUTES.PIPELINE_EDITOR, params: { name: ref.name }, search }; + : { to: APP_ROUTES.PIPELINE_EDITOR, params: { name: segment }, search }; } export function getDefaultEditorHref(ref: PipelineRef): string { + const { segment, search } = editorLocation(ref); const base = isFlagEnabled("v2_editor") ? APP_ROUTES.EDITOR_V2 : EDITOR_PATH; - const path = `${base}/${encodeURIComponent(ref.name)}`; + const path = `${base}/${encodeURIComponent(segment)}`; - return ref.fileId ? `${path}?fileId=${encodeURIComponent(ref.fileId)}` : path; + return search.fileId + ? `${path}?fileId=${encodeURIComponent(search.fileId)}` + : path; } diff --git a/src/routes/v2/pages/Editor/EditorV2.tsx b/src/routes/v2/pages/Editor/EditorV2.tsx index 9f402a55d0..400c73c48e 100644 --- a/src/routes/v2/pages/Editor/EditorV2.tsx +++ b/src/routes/v2/pages/Editor/EditorV2.tsx @@ -201,7 +201,7 @@ export function EditorV2({ pipelineRefProp !== undefined ? pipelineRefProp : pipelineName - ? { name: pipelineName, fileId } + ? { name: pipelineName, fileId: fileId ?? pipelineName } : null; return ( diff --git a/src/services/pipelineStorage/PipelineStorageService.test.ts b/src/services/pipelineStorage/PipelineStorageService.test.ts index e06220033b..83f36ee208 100644 --- a/src/services/pipelineStorage/PipelineStorageService.test.ts +++ b/src/services/pipelineStorage/PipelineStorageService.test.ts @@ -282,6 +282,39 @@ describe("resolving a route reference against a host", () => { ).rejects.toThrow(PipelineNotFoundError); }); + it("opens a pipeline from a path that carries only its id", async () => { + installHost([summary("opaque-key-1", "Churn model")]); + + const file = await new PipelineStorageService().resolve({ + name: "id-opaque-key-1", + fileId: "id-opaque-key-1", + }); + + expect(file.storageKey).toBe("opaque-key-1"); + }); + + it("still opens an older link whose path carries a name", async () => { + installHost([summary("opaque-key-1", "Churn model")]); + + const file = await new PipelineStorageService().resolve({ + name: "Churn model", + fileId: "Churn model", + }); + + expect(file.storageKey).toBe("opaque-key-1"); + }); + + it("refuses a link whose id is gone rather than opening a namesake", async () => { + installHost([summary("opaque-key-1", "Churn model")]); + + await expect( + new PipelineStorageService().resolve({ + name: "Churn model", + fileId: "id-of-a-deleted-pipeline", + }), + ).rejects.toThrow(PipelineNotFoundError); + }); + it("finds a pipeline the registry has never seen by its id", async () => { installHost([summary("opaque-key-1", "Churn model")]); diff --git a/src/services/pipelineStorage/PipelineStorageService.ts b/src/services/pipelineStorage/PipelineStorageService.ts index d52f5cb860..d1f4257c29 100644 --- a/src/services/pipelineStorage/PipelineStorageService.ts +++ b/src/services/pipelineStorage/PipelineStorageService.ts @@ -3,6 +3,7 @@ import { makeObservable, observable } from "mobx"; import { createDriver } from "./createDriver"; import { pipelineStorageDb } from "./db"; import { RootFolderDbStorageDriver } from "./drivers/RootFolderDbStorageDriver"; +import { UnavailableStorageDriver } from "./drivers/UnavailableStorageDriver"; import { PipelineFile } from "./PipelineFile"; import type { PipelineFileSource } from "./pipelineFileEvents"; import { PipelineFolder } from "./PipelineFolder"; @@ -36,7 +37,26 @@ export class PipelineStorageService { * happens to share it. */ async resolve(ref: PipelineRef): Promise { - if (ref.fileId) return this.findPipelineById(ref.fileId); + if (ref.fileId) { + const found = await this.findPipelineById(ref.fileId).catch( + (error: unknown) => { + /** + * A route that carries one segment offers it as both, because what a + * path means depends on the store and links outlive that. A `fileId` + * given *alongside* a different name is still taken at its word: a + * miss there means the pipeline is gone, not that some namesake + * should be opened in its place. + */ + const isSameSegment = ref.name === ref.fileId; + if (isSameSegment && error instanceof PipelineNotFoundError) { + return undefined; + } + throw error; + }, + ); + + if (found) return found; + } const found = await this.findPipelineByName(ref.name); if (found) return found; @@ -251,6 +271,16 @@ function createRoot(mode: StorageMode): PipelineFolder { }); } + if (mode.kind === "host-missing") { + return new PipelineFolder({ + id: ROOT_FOLDER_ID, + name: "Pipeline storage", + parentId: null, + driver: new UnavailableStorageDriver(), + isFlat: true, + }); + } + return new PipelineFolder({ id: ROOT_FOLDER_ID, name: "Root", diff --git a/src/services/pipelineStorage/drivers/UnavailableStorageDriver.ts b/src/services/pipelineStorage/drivers/UnavailableStorageDriver.ts new file mode 100644 index 0000000000..103f721f99 --- /dev/null +++ b/src/services/pipelineStorage/drivers/UnavailableStorageDriver.ts @@ -0,0 +1,43 @@ +import type { PipelineFileDescriptor, PipelineStorageDriver } from "../types"; + +class PipelineStorageUnavailableError extends Error { + readonly name = "PipelineStorageUnavailableError"; + + constructor() { + super( + "This deployment stores pipelines outside the browser, and that store was not provided by the page.", + ); + } +} + +/** + * Stands in when the deployment requires a host-provided store and the page did + * not supply one. Every operation refuses: falling back to browser storage + * would write the user's work somewhere the deployment does not read from, and + * they would only find out much later. + */ +export class UnavailableStorageDriver implements PipelineStorageDriver { + readonly type = "unavailable"; + readonly allowsMoveIn = false; + readonly allowsMoveOut = false; + + async list(): Promise { + throw new PipelineStorageUnavailableError(); + } + + async read(): Promise { + throw new PipelineStorageUnavailableError(); + } + + async write(): Promise { + throw new PipelineStorageUnavailableError(); + } + + async delete(): Promise { + throw new PipelineStorageUnavailableError(); + } + + async hasKey(): Promise { + throw new PipelineStorageUnavailableError(); + } +} diff --git a/src/services/pipelineStorage/storageMode.test.ts b/src/services/pipelineStorage/storageMode.test.ts new file mode 100644 index 0000000000..70156e9e8a --- /dev/null +++ b/src/services/pipelineStorage/storageMode.test.ts @@ -0,0 +1,86 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; + +import type { PipelineStorageHost } from "./host/contract"; +import { + isHostStorage, + isHostStorageMissing, + resetStorageModeForTests, + resolveStorageMode, +} from "./storageMode"; + +const LABEL = "Shared storage"; + +function installHost(overrides: Partial = {}) { + const host: PipelineStorageHost = { + version: 1, + label: LABEL, + list: async () => [], + read: async () => { + throw new Error("not used"); + }, + write: async () => { + throw new Error("not used"); + }, + delete: async () => undefined, + has: async () => false, + ...overrides, + }; + + Object.defineProperty(window, "__TANGLE_PIPELINE_STORAGE_HOST__", { + value: host, + configurable: true, + writable: true, + }); +} + +afterEach(() => { + delete window.__TANGLE_PIPELINE_STORAGE_HOST__; + vi.unstubAllEnvs(); + resetStorageModeForTests(); +}); + +describe("a deployment that requires a host-provided store", () => { + it("uses it when the page provides one", () => { + vi.stubEnv("VITE_PIPELINE_STORAGE", "host"); + installHost(); + + expect(resolveStorageMode()).toEqual({ kind: "host", label: LABEL }); + }); + + it("refuses to fall back to browser storage when the page provides none", () => { + vi.stubEnv("VITE_PIPELINE_STORAGE", "host"); + + expect(resolveStorageMode()).toEqual({ kind: "host-missing" }); + expect(isHostStorageMissing()).toBe(true); + expect(isHostStorage()).toBe(false); + }); + + it("treats a host it cannot drive as no host at all", () => { + vi.stubEnv("VITE_PIPELINE_STORAGE", "host"); + installHost({ version: 99 }); + + expect(resolveStorageMode()).toEqual({ kind: "host-missing" }); + }); +}); + +describe("a deployment pinned to browser storage", () => { + it("ignores a host the page provides anyway", () => { + vi.stubEnv("VITE_PIPELINE_STORAGE", "local"); + installHost(); + + expect(resolveStorageMode()).toEqual({ kind: "local" }); + }); +}); + +describe("a deployment that says nothing", () => { + it("uses a host if the page provides one", () => { + installHost(); + + expect(resolveStorageMode()).toEqual({ kind: "host", label: LABEL }); + }); + + it("uses browser storage otherwise", () => { + expect(resolveStorageMode()).toEqual({ kind: "local" }); + expect(isHostStorageMissing()).toBe(false); + }); +}); diff --git a/src/services/pipelineStorage/storageMode.ts b/src/services/pipelineStorage/storageMode.ts index 6d1e61136c..30943172ef 100644 --- a/src/services/pipelineStorage/storageMode.ts +++ b/src/services/pipelineStorage/storageMode.ts @@ -1,24 +1,55 @@ import type { PipelineStorageHost } from "./host/contract"; import { getPipelineStorageHost } from "./host/detectHost"; -export type StorageMode = { kind: "local" } | { kind: "host"; label: string }; +export type StorageMode = + | { kind: "local" } + | { kind: "host"; label: string } + | { kind: "host-missing" }; let resolved: StorageMode | undefined; let resolvedHost: PipelineStorageHost | undefined; +/** + * A deployment says which store it runs on; it is not guessed from whether a + * host happens to have loaded. `host` without one is an error rather than a + * quiet fall back to browser storage, which would strand a user's work + * somewhere nobody else can see it. + * + * Unset keeps the old behaviour — a host if one is there, browser storage + * otherwise — because that is what a build nobody configured wants, and the + * open-source build never has a host to find. + */ +function configuredStorage(): "host" | "local" | "detect" { + const configured = import.meta.env.VITE_PIPELINE_STORAGE; + return configured === "host" || configured === "local" + ? configured + : "detect"; +} + /** * Decided once and then frozen for the life of the page, holding on to the host - * itself rather than re-reading the global. Detection is fail-open, so a host - * whose global is removed or whose getter starts throwing would otherwise read - * as "no host" and quietly send the next write to browser storage — the one - * outcome host mode exists to prevent. + * itself rather than re-reading the global — a host whose global is removed or + * whose getter starts throwing must not read as "no host" and quietly send the + * next write to browser storage. */ export function resolveStorageMode(): StorageMode { - if (!resolved) { - resolvedHost = getPipelineStorageHost(); - resolved = resolvedHost - ? { kind: "host", label: resolvedHost.label } - : { kind: "local" }; + if (resolved) return resolved; + + const configured = configuredStorage(); + + if (configured === "local") { + resolvedHost = undefined; + resolved = { kind: "local" }; + return resolved; + } + + resolvedHost = getPipelineStorageHost(); + + if (resolvedHost) { + resolved = { kind: "host", label: resolvedHost.label }; + } else { + resolved = + configured === "host" ? { kind: "host-missing" } : { kind: "local" }; } return resolved; @@ -33,6 +64,15 @@ export function isHostStorage(): boolean { return resolveStorageMode().kind === "host"; } +/** + * The deployment requires a host-provided store and the page did not supply + * one. Nothing can be read or written, so this is worth saying rather than + * rendering an empty library. + */ +export function isHostStorageMissing(): boolean { + return resolveStorageMode().kind === "host-missing"; +} + export function resetStorageModeForTests(): void { resolved = undefined; resolvedHost = undefined; diff --git a/tests/e2e/pipeline-storage-host.spec.ts b/tests/e2e/pipeline-storage-host.spec.ts index 0930a2b538..51783da952 100644 --- a/tests/e2e/pipeline-storage-host.spec.ts +++ b/tests/e2e/pipeline-storage-host.spec.ts @@ -100,6 +100,24 @@ test.describe("host-provided pipeline storage", () => { ).toContain(localName); }); + test("opens a pipeline at a url that is only its id", async ({ page }) => { + await installSeededHost(page); + + await page.goto("/pipelines"); + await page.getByText("Churn model").click(); + + await expect(page.locator('[data-testid="rf__wrapper"]')).toBeVisible({ + timeout: 30_000, + }); + + const url = new URL(page.url()); + const [record] = (await readHostRecords(page)).filter( + (entry) => entry.displayName === "Churn model", + ); + expect(url.pathname).toBe(`/editor-v2/${record.externalId}`); + expect(url.search).toBe(""); + }); + test("keeps the browser's own pipeline store empty", async ({ page }) => { await installSeededHost(page); From 7f1b0c1ec6075c2890c68e321ddf5d31995d4ca1 Mon Sep 17 00:00:00 2001 From: Camiel van Schoonhoven Date: Tue, 8 Sep 2026 19:34:08 -0700 Subject: [PATCH 18/36] refactor(pipeline-storage): make the beta flag the only switch, and the url the identity MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit VITE_PIPELINE_STORAGE_BETA is a plain on/off: on means pipelines live outside the browser, and anything else — false, unset, a typo — means they live in it. Naming it a beta says what it is to anyone reading the source, and leaves the deployment holding the switch rather than inferring one from whether a host happened to load. Browser-stored pipelines go back to a url of just their name. Names are the identity there and each is held once, so the id said nothing the name did not and only made the url longer. The suites now pin the flag rather than inheriting it, because a developer's own .env would otherwise decide what they test. Host storage is settled while the app boots, so those e2e need a server of their own with the flag on, and they seed the browser's store directly — there is no local mode on that page to create a pipeline in, which is the situation the copy exists for. That seeding turned up a migration that could throw and leave the page saying "starting…" for good. It now says what went wrong and offers the same retry as a partial copy. Co-Authored-By: Claude Opus 5 (1M context) --- playwright.config.ci.ts | 44 ++++++++-- playwright.config.ts | 45 +++++++++-- .../PipelineSection/HostMigrationNotice.tsx | 17 ++-- .../Home/PipelineSection/useHostMigration.ts | 17 +++- .../components/InspectPipelineButton.test.tsx | 1 - src/routes/Import/Import.test.tsx | 1 - src/routes/editorRoutes.test.ts | 10 +-- src/routes/editorRoutes.ts | 45 +++-------- .../pages/Editor/hooks/usePipelineRename.ts | 13 +-- .../PipelineStorageService.test.ts | 6 ++ .../pipelineStorage/storageMode.test.ts | 31 +++---- src/services/pipelineStorage/storageMode.ts | 34 +++----- tests/e2e/fixtures/pipelineStorageHost.ts | 81 +++++++++++++++++++ tests/e2e/pipeline-storage-host.spec.ts | 10 +-- vite.config.js | 3 + 15 files changed, 247 insertions(+), 111 deletions(-) diff --git a/playwright.config.ci.ts b/playwright.config.ci.ts index dcf2b90e8a..74dcc0edd1 100644 --- a/playwright.config.ci.ts +++ b/playwright.config.ci.ts @@ -7,6 +7,15 @@ import { defineConfig, devices } from "@playwright/test"; * This config uses CI settings (1 worker, no server reuse) but runs on port 3001 * so it doesn't conflict with your dev server on port 3000 */ + +/** + * Host-provided storage is a build-time switch, so it cannot be turned on per + * test — those specs need a server of their own with the flag set. + */ +const HOST_STORAGE_TESTS = "**/pipeline-storage-host.spec.ts"; +const HOST_STORAGE_PORT = 3011; +const HOST_STORAGE_URL = `http://localhost:${HOST_STORAGE_PORT}`; + export default defineConfig({ testDir: "./tests/e2e", fullyParallel: false, @@ -29,6 +38,7 @@ export default defineConfig({ projects: [ { name: "chromium", + testIgnore: HOST_STORAGE_TESTS, use: { ...devices["Desktop Chrome"], launchOptions: { @@ -39,12 +49,34 @@ export default defineConfig({ }, }, }, + { + name: "chromium-host-storage", + testMatch: HOST_STORAGE_TESTS, + use: { + ...devices["Desktop Chrome"], + baseURL: HOST_STORAGE_URL, + launchOptions: { + args: [ + "--disable-web-security", + "--disable-features=IsolateOrigins,site-per-process", + ], + }, + }, + }, ], - webServer: { - command: "vite --port 3001", - url: "http://localhost:3001", - reuseExistingServer: false, // Force fresh server like CI - timeout: 120 * 1000, - }, + webServer: [ + { + command: "VITE_PIPELINE_STORAGE_BETA=false vite --port 3001", + url: "http://localhost:3001", + reuseExistingServer: false, // Force fresh server like CI + timeout: 120 * 1000, + }, + { + command: `VITE_PIPELINE_STORAGE_BETA=true vite --port ${HOST_STORAGE_PORT}`, + url: HOST_STORAGE_URL, + reuseExistingServer: false, + timeout: 120 * 1000, + }, + ], }); diff --git a/playwright.config.ts b/playwright.config.ts index f4d5175cf7..709bbea611 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -1,5 +1,13 @@ import { defineConfig, devices } from "@playwright/test"; +/** + * Host-provided storage is a build-time switch, so it cannot be turned on per + * test — those specs need a server of their own with the flag set. + */ +const HOST_STORAGE_TESTS = "**/pipeline-storage-host.spec.ts"; +const HOST_STORAGE_PORT = 3010; +const HOST_STORAGE_URL = `http://localhost:${HOST_STORAGE_PORT}`; + /** * @see https://playwright.dev/docs/test-configuration */ @@ -39,6 +47,7 @@ export default defineConfig({ projects: [ { name: "chromium", + testIgnore: HOST_STORAGE_TESTS, use: { ...devices["Desktop Chrome"], launchOptions: { @@ -50,13 +59,37 @@ export default defineConfig({ }, }, }, + { + name: "chromium-host-storage", + testMatch: HOST_STORAGE_TESTS, + use: { + ...devices["Desktop Chrome"], + baseURL: HOST_STORAGE_URL, + launchOptions: { + args: [ + "--disable-web-security", + "--disable-features=IsolateOrigins,site-per-process", + ], + }, + }, + }, ], /* Run your local dev server before starting the tests */ - webServer: { - command: "npm start", - url: "http://localhost:3000", - reuseExistingServer: !process.env.CI, - timeout: 120 * 1000, - }, + webServer: [ + { + // Explicit rather than inherited, so a developer's own .env cannot put + // the whole suite into a storage mode it is not written for. + command: "VITE_PIPELINE_STORAGE_BETA=false npm start", + url: "http://localhost:3000", + reuseExistingServer: !process.env.CI, + timeout: 120 * 1000, + }, + { + command: `VITE_PIPELINE_STORAGE_BETA=true vite --port ${HOST_STORAGE_PORT}`, + url: HOST_STORAGE_URL, + reuseExistingServer: !process.env.CI, + timeout: 120 * 1000, + }, + ], }); diff --git a/src/components/Home/PipelineSection/HostMigrationNotice.tsx b/src/components/Home/PipelineSection/HostMigrationNotice.tsx index 9e80f8f9a2..9a2007b49e 100644 --- a/src/components/Home/PipelineSection/HostMigrationNotice.tsx +++ b/src/components/Home/PipelineSection/HostMigrationNotice.tsx @@ -14,7 +14,7 @@ export function HostMigrationNotice({ migration: HostMigration; storageLabel: string; }) { - const { phase, progress, failed, retry, skip } = migration; + const { phase, progress, failed, error, retry, skip } = migration; if (phase === "copying") { return ( @@ -37,14 +37,19 @@ export function HostMigrationNotice({ - {failed.length} {pluralize(failed.length, "pipeline")} could not be - copied to {storageLabel} + {error + ? `Your pipelines could not be moved to ${storageLabel}` + : `${failed.length} ${pluralize(failed.length, "pipeline")} could not be copied to ${storageLabel}`} - {failed.slice(0, 5).join(", ")} - {failed.length > 5 && ` and ${failed.length - 5} more`}. They are still - in this browser and nothing has been deleted. + {error ?? ( + <> + {failed.slice(0, 5).join(", ")} + {failed.length > 5 && ` and ${failed.length - 5} more`}. + + )}{" "} + They are still in this browser and nothing has been deleted. diff --git a/src/components/Home/PipelineSection/useHostMigration.ts b/src/components/Home/PipelineSection/useHostMigration.ts index 85d590729f..ec0e4c76cd 100644 --- a/src/components/Home/PipelineSection/useHostMigration.ts +++ b/src/components/Home/PipelineSection/useHostMigration.ts @@ -8,6 +8,7 @@ import { runHostMigration, } from "@/services/pipelineStorage/hostMigration"; import { usePipelineStorage } from "@/services/pipelineStorage/PipelineStorageProvider"; +import { getErrorMessage } from "@/utils/string"; type HostMigrationPhase = "checking" | "copying" | "incomplete" | "settled"; @@ -15,6 +16,7 @@ export interface HostMigration { phase: HostMigrationPhase; progress: HostMigrationProgress; failed: string[]; + error: string | null; retry: () => void; skip: () => void; } @@ -38,6 +40,7 @@ export function useHostMigration(onFinished: () => void): HostMigration { ); const [progress, setProgress] = useState(NOTHING); const [failed, setFailed] = useState([]); + const [error, setError] = useState(null); const [attempt, setAttempt] = useState(0); useEffect(() => { @@ -49,6 +52,7 @@ export function useHostMigration(onFinished: () => void): HostMigration { const settle = (failedKeys: string[]) => { if (!watching) return; setFailed(failedKeys); + setError(null); setPhase(failedKeys.length > 0 ? "incomplete" : "settled"); if (failedKeys.length === 0) onFinished(); }; @@ -88,7 +92,17 @@ export function useHostMigration(onFinished: () => void): HostMigration { pollTimer = setTimeout(() => void pump(), POLL_MS); }; - void pump(); + /** + * Whatever goes wrong, this has to stop looking like it is still working. + * A copy that throws — an unreadable local pipeline, a store refusing the + * whole listing — would otherwise leave the page on "starting…" forever. + */ + void pump().catch((thrown: unknown) => { + console.error("Could not copy pipelines into the host store:", thrown); + if (!watching) return; + setError(getErrorMessage(thrown)); + setPhase("incomplete"); + }); return () => { watching = false; @@ -100,6 +114,7 @@ export function useHostMigration(onFinished: () => void): HostMigration { phase, progress, failed, + error, retry: () => setAttempt((previous) => previous + 1), skip: () => { void dismissHostMigration().then(() => { diff --git a/src/components/PipelineRun/components/InspectPipelineButton.test.tsx b/src/components/PipelineRun/components/InspectPipelineButton.test.tsx index 9e6ca046f0..be8a2c18c4 100644 --- a/src/components/PipelineRun/components/InspectPipelineButton.test.tsx +++ b/src/components/PipelineRun/components/InspectPipelineButton.test.tsx @@ -19,7 +19,6 @@ describe("", () => { expect(mockNavigate).toHaveBeenCalledWith({ to: "/editor-v2/$pipelineName", params: { pipelineName: "foo" }, - search: {}, }); }); }); diff --git a/src/routes/Import/Import.test.tsx b/src/routes/Import/Import.test.tsx index 305c660cc2..4b7c73fe15 100644 --- a/src/routes/Import/Import.test.tsx +++ b/src/routes/Import/Import.test.tsx @@ -144,7 +144,6 @@ describe("ImportPage", () => { expect(mockNavigate).toHaveBeenCalledWith({ to: "/editor-v2/$pipelineName", params: { pipelineName: "Test Pipeline" }, - search: {}, }); }); }); diff --git a/src/routes/editorRoutes.test.ts b/src/routes/editorRoutes.test.ts index 138df0e978..8c0033a52c 100644 --- a/src/routes/editorRoutes.test.ts +++ b/src/routes/editorRoutes.test.ts @@ -19,13 +19,12 @@ afterEach(() => { }); describe("where a store hands out its own ids", () => { - it("puts the id in the path and nothing in the query", () => { + it("puts the id in the path and nothing else anywhere", () => { hostStorage.mockReturnValue(true); expect(getDefaultEditorTarget(REF)).toEqual({ to: "/editor-v2/$pipelineName", params: { pipelineName: "ab420234-a05f" }, - search: {}, }); expect(getDefaultEditorHref(REF)).toBe("/editor-v2/ab420234-a05f"); }); @@ -40,14 +39,11 @@ describe("where a store hands out its own ids", () => { }); describe("where names are the identity", () => { - it("keeps the name in the path and the id alongside it", () => { + it("puts the name in the path and never mentions the id", () => { expect(getDefaultEditorTarget(REF)).toEqual({ to: "/editor-v2/$pipelineName", params: { pipelineName: "Churn model" }, - search: { fileId: "ab420234-a05f" }, }); - expect(getDefaultEditorHref(REF)).toBe( - "/editor-v2/Churn%20model?fileId=ab420234-a05f", - ); + expect(getDefaultEditorHref(REF)).toBe("/editor-v2/Churn%20model"); }); }); diff --git a/src/routes/editorRoutes.ts b/src/routes/editorRoutes.ts index 7b814fbfb0..0c116a4cea 100644 --- a/src/routes/editorRoutes.ts +++ b/src/routes/editorRoutes.ts @@ -4,55 +4,32 @@ import type { PipelineRef } from "@/services/pipelineStorage/types"; import { APP_ROUTES, EDITOR_PATH } from "./appRoutes"; -interface EditorSearch { - fileId?: string; -} - export interface EditorTarget { to: string; params: Record; - search: EditorSearch; } /** - * What identifies a pipeline depends on the store. Where the store hands out - * its own ids, the path carries one and nothing else is needed — names are not - * unique there, and an id in the path cannot go stale when the pipeline is - * renamed. Where names are the identity, the path keeps the name and the id - * rides along to settle the cases a name cannot. + * What identifies a pipeline is whatever its store identifies it by, and the + * path carries that and nothing else. Browser storage keys pipelines on their + * name and holds each name once, so the name is the identity there. A store + * that hands out its own ids allows two pipelines the same name, so the id is — + * and being the identity, it cannot go stale when one is renamed. */ -function editorLocation(ref: PipelineRef): { - segment: string; - search: EditorSearch; -} { - if (isHostStorage() && ref.fileId) { - return { segment: ref.fileId, search: {} }; - } - - return { - segment: ref.name, - search: ref.fileId ? { fileId: ref.fileId } : {}, - }; +function editorSegment(ref: PipelineRef): string { + return isHostStorage() && ref.fileId ? ref.fileId : ref.name; } export function getDefaultEditorTarget(ref: PipelineRef): EditorTarget { - const { segment, search } = editorLocation(ref); + const segment = editorSegment(ref); return isFlagEnabled("v2_editor") - ? { - to: APP_ROUTES.EDITOR_V2_PIPELINE, - params: { pipelineName: segment }, - search, - } - : { to: APP_ROUTES.PIPELINE_EDITOR, params: { name: segment }, search }; + ? { to: APP_ROUTES.EDITOR_V2_PIPELINE, params: { pipelineName: segment } } + : { to: APP_ROUTES.PIPELINE_EDITOR, params: { name: segment } }; } export function getDefaultEditorHref(ref: PipelineRef): string { - const { segment, search } = editorLocation(ref); const base = isFlagEnabled("v2_editor") ? APP_ROUTES.EDITOR_V2 : EDITOR_PATH; - const path = `${base}/${encodeURIComponent(segment)}`; - return search.fileId - ? `${path}?fileId=${encodeURIComponent(search.fileId)}` - : path; + return `${base}/${encodeURIComponent(editorSegment(ref))}`; } diff --git a/src/routes/v2/pages/Editor/hooks/usePipelineRename.ts b/src/routes/v2/pages/Editor/hooks/usePipelineRename.ts index d173f632ad..d80be0ac3d 100644 --- a/src/routes/v2/pages/Editor/hooks/usePipelineRename.ts +++ b/src/routes/v2/pages/Editor/hooks/usePipelineRename.ts @@ -1,6 +1,6 @@ import { useNavigate } from "@tanstack/react-router"; -import { APP_ROUTES } from "@/routes/router"; +import { getDefaultEditorTarget } from "@/routes/editorRoutes"; import { usePipelineActions } from "@/routes/v2/pages/Editor/store/actions/usePipelineActions"; import { useEditorSession } from "@/routes/v2/pages/Editor/store/EditorSessionContext"; import { useSharedStores } from "@/routes/v2/shared/store/SharedStoreContext"; @@ -17,10 +17,11 @@ export function usePipelineRename() { await pipelineFileStore.activePipelineFile?.rename(newName); renamePipeline(spec, newName); await autoSave.save(); - await navigate({ - to: APP_ROUTES.EDITOR_V2_PIPELINE, - params: { pipelineName: newName }, - search: { fileId: pipelineFileStore.activePipelineFile?.id }, - }); + await navigate( + getDefaultEditorTarget({ + name: newName, + fileId: pipelineFileStore.activePipelineFile?.id, + }), + ); }; } diff --git a/src/services/pipelineStorage/PipelineStorageService.test.ts b/src/services/pipelineStorage/PipelineStorageService.test.ts index 83f36ee208..f63f411a5c 100644 --- a/src/services/pipelineStorage/PipelineStorageService.test.ts +++ b/src/services/pipelineStorage/PipelineStorageService.test.ts @@ -88,10 +88,15 @@ function summary(key: string, displayName: string): HostPipelineSummary { /** * Mirrors the two host behaviours the write paths are built on: `write` upserts * on the caller's key, and the displayed name comes from the written spec. + * + * A host is only used when the deployment asks for one, so turning the beta on + * is part of installing it. */ function installHost( listing: HostPipelineSummary[] = [], ): Map { + vi.stubEnv("VITE_PIPELINE_STORAGE_BETA", "true"); + const summaries = new Map(listing.map((entry) => [entry.key, entry])); const specs = new Map(); @@ -139,6 +144,7 @@ beforeEach(() => { afterEach(() => { delete window.__TANGLE_PIPELINE_STORAGE_HOST__; + vi.unstubAllEnvs(); resetStorageModeForTests(); vi.restoreAllMocks(); }); diff --git a/src/services/pipelineStorage/storageMode.test.ts b/src/services/pipelineStorage/storageMode.test.ts index 70156e9e8a..8a3bb6bf6c 100644 --- a/src/services/pipelineStorage/storageMode.test.ts +++ b/src/services/pipelineStorage/storageMode.test.ts @@ -39,48 +39,49 @@ afterEach(() => { resetStorageModeForTests(); }); -describe("a deployment that requires a host-provided store", () => { - it("uses it when the page provides one", () => { - vi.stubEnv("VITE_PIPELINE_STORAGE", "host"); +describe("a deployment with the beta on", () => { + it("uses the store the page provides", () => { + vi.stubEnv("VITE_PIPELINE_STORAGE_BETA", "true"); installHost(); expect(resolveStorageMode()).toEqual({ kind: "host", label: LABEL }); }); it("refuses to fall back to browser storage when the page provides none", () => { - vi.stubEnv("VITE_PIPELINE_STORAGE", "host"); + vi.stubEnv("VITE_PIPELINE_STORAGE_BETA", "true"); expect(resolveStorageMode()).toEqual({ kind: "host-missing" }); expect(isHostStorageMissing()).toBe(true); expect(isHostStorage()).toBe(false); }); - it("treats a host it cannot drive as no host at all", () => { - vi.stubEnv("VITE_PIPELINE_STORAGE", "host"); + it("treats a store it cannot drive as none at all", () => { + vi.stubEnv("VITE_PIPELINE_STORAGE_BETA", "true"); installHost({ version: 99 }); expect(resolveStorageMode()).toEqual({ kind: "host-missing" }); }); }); -describe("a deployment pinned to browser storage", () => { - it("ignores a host the page provides anyway", () => { - vi.stubEnv("VITE_PIPELINE_STORAGE", "local"); +describe("a deployment with the beta off", () => { + it("ignores a store the page provides anyway", () => { + vi.stubEnv("VITE_PIPELINE_STORAGE_BETA", "false"); installHost(); expect(resolveStorageMode()).toEqual({ kind: "local" }); }); -}); -describe("a deployment that says nothing", () => { - it("uses a host if the page provides one", () => { + it("is what an unset flag means", () => { installHost(); - expect(resolveStorageMode()).toEqual({ kind: "host", label: LABEL }); + expect(resolveStorageMode()).toEqual({ kind: "local" }); + expect(isHostStorageMissing()).toBe(false); }); - it("uses browser storage otherwise", () => { + it("is what any other value means", () => { + vi.stubEnv("VITE_PIPELINE_STORAGE_BETA", "1"); + installHost(); + expect(resolveStorageMode()).toEqual({ kind: "local" }); - expect(isHostStorageMissing()).toBe(false); }); }); diff --git a/src/services/pipelineStorage/storageMode.ts b/src/services/pipelineStorage/storageMode.ts index 30943172ef..eaeb63946c 100644 --- a/src/services/pipelineStorage/storageMode.ts +++ b/src/services/pipelineStorage/storageMode.ts @@ -10,20 +10,14 @@ let resolved: StorageMode | undefined; let resolvedHost: PipelineStorageHost | undefined; /** - * A deployment says which store it runs on; it is not guessed from whether a - * host happens to have loaded. `host` without one is an error rather than a - * quiet fall back to browser storage, which would strand a user's work - * somewhere nobody else can see it. - * - * Unset keeps the old behaviour — a host if one is there, browser storage - * otherwise — because that is what a build nobody configured wants, and the - * open-source build never has a host to find. + * A deployment says whether it stores pipelines outside the browser; it is not + * guessed from whether a host happens to have loaded. Off is the default and + * the only thing the open-source build can be. On without a host is an error + * rather than a quiet fall back to browser storage, which would strand a user's + * work somewhere nobody else can see it. */ -function configuredStorage(): "host" | "local" | "detect" { - const configured = import.meta.env.VITE_PIPELINE_STORAGE; - return configured === "host" || configured === "local" - ? configured - : "detect"; +function hostStorageEnabled(): boolean { + return import.meta.env.VITE_PIPELINE_STORAGE_BETA === "true"; } /** @@ -35,22 +29,16 @@ function configuredStorage(): "host" | "local" | "detect" { export function resolveStorageMode(): StorageMode { if (resolved) return resolved; - const configured = configuredStorage(); - - if (configured === "local") { + if (!hostStorageEnabled()) { resolvedHost = undefined; resolved = { kind: "local" }; return resolved; } resolvedHost = getPipelineStorageHost(); - - if (resolvedHost) { - resolved = { kind: "host", label: resolvedHost.label }; - } else { - resolved = - configured === "host" ? { kind: "host-missing" } : { kind: "local" }; - } + resolved = resolvedHost + ? { kind: "host", label: resolvedHost.label } + : { kind: "host-missing" }; return resolved; } diff --git a/tests/e2e/fixtures/pipelineStorageHost.ts b/tests/e2e/fixtures/pipelineStorageHost.ts index f3361abd9d..05f2169fed 100644 --- a/tests/e2e/fixtures/pipelineStorageHost.ts +++ b/tests/e2e/fixtures/pipelineStorageHost.ts @@ -169,6 +169,87 @@ export async function installPipelineStorageHost( ); } +/** + * Puts a pipeline in the browser's own store without going through the UI. The + * app cannot be asked to make one here: this page has host storage turned on, + * so there is no local mode to create it in — which is the very situation the + * migration exists for. + */ +export async function seedLocallyStoredPipeline( + page: Page, + name: string, +): Promise { + await page.evaluate(async (pipelineName) => { + const FILES = "file_store_user_pipelines"; + const DATA = "digest_to_component_data"; + const SETTINGS = "component_store_settings"; + const STORES = [FILES, DATA, SETTINGS]; + + const database = await new Promise((resolve, reject) => { + const probe = indexedDB.open("components"); + probe.onerror = () => reject(probe.error); + probe.onsuccess = () => { + const opened = probe.result; + const missing = STORES.filter( + (store) => !opened.objectStoreNames.contains(store), + ); + if (missing.length === 0) return resolve(opened); + + const nextVersion = opened.version + 1; + opened.close(); + const upgrade = indexedDB.open("components", nextVersion); + upgrade.onupgradeneeded = () => { + for (const store of missing) upgrade.result.createObjectStore(store); + }; + upgrade.onerror = () => reject(upgrade.error); + upgrade.onsuccess = () => resolve(upgrade.result); + }; + }); + + const text = `name: ${pipelineName}\nimplementation:\n graph:\n tasks: {}\n`; + const data = new TextEncoder().encode(text).buffer; + const hash = await crypto.subtle.digest("SHA-256", data); + const digest = [...new Uint8Array(hash)] + .map((byte) => byte.toString(16).padStart(2, "0")) + .join(""); + + await new Promise((resolve, reject) => { + const transaction = database.transaction(STORES, "readwrite"); + + transaction.objectStore(DATA).put(data, digest); + /** + * Claims the current format so the legacy store's own upgrade does not + * run over a hand-written entry and reject it as corrupt. + */ + transaction + .objectStore(SETTINGS) + .put(4, "component_list_format_version_user_pipelines"); + transaction.objectStore(FILES).put( + { + name: pipelineName, + data, + componentRef: { + text, + digest, + spec: { + name: pipelineName, + implementation: { graph: { tasks: {} } }, + }, + }, + creationTime: new Date(), + modificationTime: new Date(), + }, + pipelineName, + ); + + transaction.oncomplete = () => resolve(); + transaction.onerror = () => reject(transaction.error); + }); + + database.close(); + }, name); +} + export async function readHostRecords(page: Page): Promise { return page.evaluate(() => window.__TANGLE_TEST_HOST__?.records() ?? []); } diff --git a/tests/e2e/pipeline-storage-host.spec.ts b/tests/e2e/pipeline-storage-host.spec.ts index 51783da952..6d2eb5e5c9 100644 --- a/tests/e2e/pipeline-storage-host.spec.ts +++ b/tests/e2e/pipeline-storage-host.spec.ts @@ -6,8 +6,8 @@ import { readHostReadKeys, readHostRecords, readLocallyStoredPipelineKeys, + seedLocallyStoredPipeline, } from "./fixtures/pipelineStorageHost"; -import { createNewPipeline } from "./helpers"; const LABEL = "Shared storage"; @@ -84,12 +84,12 @@ test.describe("host-provided pipeline storage", () => { test("copies pipelines already in the browser into the host", async ({ page, }) => { - await createNewPipeline(page); - const localName = decodeURIComponent( - new URL(page.url()).pathname.split("/").pop() ?? "", - ); + const localName = "Left behind in the browser"; await installSeededHost(page); + await page.goto("/"); + await seedLocallyStoredPipeline(page, localName); + await page.goto("/pipelines"); await expect(page.getByText(localName)).toBeVisible(); diff --git a/vite.config.js b/vite.config.js index 96f8632d46..4de3b1ef69 100644 --- a/vite.config.js +++ b/vite.config.js @@ -138,6 +138,9 @@ export default defineConfig(({ mode }) => { environment: "jsdom", setupFiles: ["./vitest-setup.js"], include: ["src/**/*.{test,spec}.?(c|m)[jt]s?(x)"], + // Pinned so a developer's own .env cannot change what the suite tests. + // Anything covering the beta turns it on with vi.stubEnv. + env: { VITE_PIPELINE_STORAGE_BETA: "false" }, coverage: { provider: "v8", reporter: ["text", "json", "html"], From e1cb9fbca5ce68ee9d5ec1fa9adff8328374e0e3 Mon Sep 17 00:00:00 2001 From: Camiel van Schoonhoven Date: Tue, 8 Sep 2026 19:55:04 -0700 Subject: [PATCH 19/36] docs(pipeline-storage): declare the storage flag alongside the others Every other flag the app reads is listed here; this one was only discoverable by finding the code that reads it. Co-Authored-By: Claude Opus 5 (1M context) --- .env.example | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.env.example b/.env.example index 133e3b0232..10b200db28 100644 --- a/.env.example +++ b/.env.example @@ -29,6 +29,12 @@ VITE_GITHUB_PAGES= # Hugging Face VITE_HUGGING_FACE_AUTHORIZATION= +# Pipeline storage +# Off (unset or false) keeps pipelines in this browser. On, they are read and +# written through a store the embedding page provides, and the app reports an +# error rather than falling back to the browser when the page provides none. +VITE_PIPELINE_STORAGE_BETA= + # Dev Tools VITE_ENABLE_DEBUG_MODE= VITE_ENABLE_V2_EDITOR_TOGGLE= From 8f812eba31e701eb11351aa3a7317b5983e7c9ea Mon Sep 17 00:00:00 2001 From: Camiel van Schoonhoven Date: Wed, 9 Sep 2026 09:08:07 -0700 Subject: [PATCH 20/36] fix(pipeline-storage): keep one store's cached rows out of the other's MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The registry and the spec cache are keyed on a storage key, which is only unique within one store, and browser storage keys a pipeline on its name — a name a host-provided store may well be holding too. Both stores also used the same root folder id, so nothing distinguished their rows at all. Opening a link to a host pipeline while the app was on browser storage therefore found the host's row, paired its identity with the browser's driver, and — because the key sent to the host is the pipeline's name — read and then saved over whichever browser pipeline happened to share it. The reverse held too, and a listing in one store deleted the other's rows as no longer present. Rows now say which store they describe and every lookup is scoped to the store this page load is using, so the two can hold the same name without meeting. Rows written before this cannot say: only a host-provided store reports a contentVersion, and a row filed in a folder can only be the browser's, which is enough to attribute them. Co-Authored-By: Claude Opus 5 (1M context) --- .../PipelineStorageService.test.ts | 9 +- src/services/pipelineStorage/db.ts | 70 +++++-- .../pipelineStorage/pipelineRegistry.test.ts | 181 ++++++++++++++++++ .../pipelineStorage/pipelineRegistry.ts | 34 +++- .../pipelineStorage/pipelineSpecCache.test.ts | 2 +- .../pipelineStorage/pipelineSpecCache.ts | 17 +- src/services/pipelineStorage/storageMode.ts | 10 + src/services/pipelineStorage/types.ts | 11 ++ 8 files changed, 294 insertions(+), 40 deletions(-) create mode 100644 src/services/pipelineStorage/pipelineRegistry.test.ts diff --git a/src/services/pipelineStorage/PipelineStorageService.test.ts b/src/services/pipelineStorage/PipelineStorageService.test.ts index f63f411a5c..367fef37d3 100644 --- a/src/services/pipelineStorage/PipelineStorageService.test.ts +++ b/src/services/pipelineStorage/PipelineStorageService.test.ts @@ -2,6 +2,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { RootFolderDbStorageDriver } from "./drivers/RootFolderDbStorageDriver"; import type { HostPipelineSummary, PipelineStorageHost } from "./host/contract"; +import type { NewPipelineRegistryEntry } from "./pipelineRegistry"; import { AmbiguousPipelineNameError, PipelineNotFoundError, @@ -17,14 +18,15 @@ import { const registry = new Map(); vi.mock("./pipelineRegistry", () => ({ - claimEntry: async (entry: PipelineRegistryEntry) => { + claimEntry: async (entry: NewPipelineRegistryEntry) => { const existing = [...registry.values()].find( (candidate) => candidate.storageKey === entry.storageKey, ); if (existing) return existing; - registry.set(entry.id, entry); - return entry; + const row = { storage: "local" as const, ...entry }; + registry.set(row.id, row); + return row; }, updateEntry: async (id: string, updates: Partial) => { const entry = registry.get(id); @@ -221,6 +223,7 @@ describe("the flat list of everything", () => { ); registry.set("filed", { id: "filed", + storage: "local", storageKey: "Filed away", folderId: "folder-1", }); diff --git a/src/services/pipelineStorage/db.ts b/src/services/pipelineStorage/db.ts index af2068aef6..258a99a1ec 100644 --- a/src/services/pipelineStorage/db.ts +++ b/src/services/pipelineStorage/db.ts @@ -1,20 +1,21 @@ -import { Dexie, type EntityTable } from "dexie"; +import { Dexie, type EntityTable, type Table } from "dexie"; import { USER_PIPELINES_LIST_NAME } from "@/utils/constants"; -import { isHostStorage } from "./storageMode"; +import { currentStorageKind } from "./storageMode"; import { type CachedPipelineSpec, type FolderEntry, type HostMigrationRecord, type PipelineRegistryEntry, + type PipelineStorageKind, ROOT_FOLDER_ID, } from "./types"; export type PipelineStorageDb = Dexie & { pipeline_registry: EntityTable; folders: EntityTable; - pipeline_specs: EntityTable; + pipeline_specs: Table; host_migration: EntityTable; }; @@ -81,6 +82,41 @@ pipelineStorageDb.version(5).stores({ host_migration: "id", }); +/** + * The spec cache is keyed by store as well, and a primary key cannot be changed + * in place, so the table is dropped here and remade in the next version. It + * holds nothing that is not re-readable. + */ +pipelineStorageDb.version(6).stores({ pipeline_specs: null }); + +pipelineStorageDb + .version(7) + .stores({ + pipeline_registry: + "id, storage, folderId, &[storage+storageKey], [storage+folderId], [storage+folderId+storageKey]", + folders: "id, parentId", + pipeline_specs: "[storage+storageKey]", + host_migration: "id", + }) + .upgrade(async (tx) => { + /** + * Rows predating this version do not say which store they describe, and + * both stores used the same root folder id. Only a host-provided store + * reports a `contentVersion`, which makes the two tellable apart where it + * matters; a row filed in a folder can only be the browser's, because a + * store that keys pipelines itself has no folders. + */ + await tx + .table("pipeline_registry") + .toCollection() + .modify((entry) => { + const isHostRow = + entry.folderId === ROOT_FOLDER_ID && + entry.contentVersion !== undefined; + entry.storage = isHostRow ? "host" : "local"; + }); + }); + pipelineStorageDb.on("ready", async () => { await seedRegistryFromLegacyList(); }); @@ -91,10 +127,13 @@ pipelineStorageDb.on("ready", async () => { * would claim files the host has never heard of. */ async function seedRegistryFromLegacyList() { - if (isHostStorage()) return; + if (currentStorageKind() === "host") return; - const count = await pipelineStorageDb.pipeline_registry.count(); - if (count > 0) return; + const seeded = await pipelineStorageDb.pipeline_registry + .where("storage") + .equals("local") + .count(); + if (seeded > 0) return; const { getAllComponentFilesFromList } = await import("@/utils/componentStore"); @@ -104,23 +143,12 @@ async function seedRegistryFromLegacyList() { if (knownPipelines.size === 0) return; - const pipelineForRegistry = [...knownPipelines.entries()].map( - ([storageKey]) => ({ + await pipelineStorageDb.pipeline_registry.bulkAdd( + [...knownPipelines.keys()].map((storageKey) => ({ id: crypto.randomUUID(), + storage: "local" as const, storageKey, folderId: ROOT_FOLDER_ID, - }), + })), ); - - try { - /** - * This code may be revisited to ensure stability and performance. - */ - pipelineForRegistry.forEach(async (row) => { - await pipelineStorageDb.pipeline_registry.upsert(row.id, row); - }); - } catch (e) { - console.error(e); - throw e; - } } diff --git a/src/services/pipelineStorage/pipelineRegistry.test.ts b/src/services/pipelineStorage/pipelineRegistry.test.ts new file mode 100644 index 0000000000..aa27d6aeb2 --- /dev/null +++ b/src/services/pipelineStorage/pipelineRegistry.test.ts @@ -0,0 +1,181 @@ +import "fake-indexeddb/auto"; + +import { Dexie } from "dexie"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { pipelineStorageDb } from "./db"; +import type { PipelineStorageHost } from "./host/contract"; +import { + assertStorageKeyUnique, + claimEntry, + findById, + findByStorageKey, + getAllByFolderId, +} from "./pipelineRegistry"; +import { resetStorageModeForTests } from "./storageMode"; +import { ROOT_FOLDER_ID } from "./types"; + +const summary = { + key: "", + externalId: "", + displayName: null, + contentVersion: "1", +}; + +const host: PipelineStorageHost = { + version: 1, + label: "Shared storage", + list: async () => [], + read: async () => ({ ...summary, spec: {} }), + write: async () => summary, + delete: async () => undefined, + has: async () => false, +}; + +function useStore(kind: "local" | "host") { + vi.stubEnv("VITE_PIPELINE_STORAGE_BETA", kind === "host" ? "true" : "false"); + if (kind === "host") window.__TANGLE_PIPELINE_STORAGE_HOST__ = host; + else delete window.__TANGLE_PIPELINE_STORAGE_HOST__; + resetStorageModeForTests(); +} + +beforeEach(async () => { + await pipelineStorageDb.pipeline_registry.clear(); +}); + +afterEach(() => { + vi.unstubAllEnvs(); + delete window.__TANGLE_PIPELINE_STORAGE_HOST__; + resetStorageModeForTests(); +}); + +describe("registry rows belong to the store that wrote them", () => { + it("hides a host row from browser storage, and the other way round", async () => { + useStore("host"); + await claimEntry({ + id: "external-1", + storageKey: "Churn model", + folderId: ROOT_FOLDER_ID, + }); + + useStore("local"); + expect(await findById("external-1")).toBeUndefined(); + expect(await findByStorageKey("Churn model")).toBeUndefined(); + expect(await getAllByFolderId(ROOT_FOLDER_ID)).toEqual([]); + + await claimEntry({ + id: "local-1", + storageKey: "Churn model", + folderId: ROOT_FOLDER_ID, + }); + + useStore("host"); + expect(await findById("local-1")).toBeUndefined(); + expect((await findByStorageKey("Churn model"))?.id).toBe("external-1"); + }); + + it("lets both stores hold a pipeline of the same name", async () => { + useStore("local"); + await claimEntry({ + id: "local-1", + storageKey: "Churn model", + folderId: ROOT_FOLDER_ID, + }); + + useStore("host"); + await expect( + claimEntry({ + id: "external-1", + storageKey: "Churn model", + folderId: ROOT_FOLDER_ID, + }), + ).resolves.toMatchObject({ id: "external-1" }); + + expect(await pipelineStorageDb.pipeline_registry.count()).toBe(2); + }); + + it("does not refuse a name only the other store is using", async () => { + useStore("host"); + await claimEntry({ + id: "external-1", + storageKey: "Churn model", + folderId: ROOT_FOLDER_ID, + }); + + useStore("local"); + await expect( + assertStorageKeyUnique("Churn model"), + ).resolves.toBeUndefined(); + + await claimEntry({ + id: "local-1", + storageKey: "Churn model", + folderId: ROOT_FOLDER_ID, + }); + await expect(assertStorageKeyUnique("Churn model")).rejects.toThrow( + /already exists/, + ); + }); + + it("still claims a key once when two listings race", async () => { + useStore("local"); + + const [first, second] = await Promise.all([ + claimEntry({ + id: "first", + storageKey: "Churn model", + folderId: ROOT_FOLDER_ID, + }), + claimEntry({ + id: "second", + storageKey: "Churn model", + folderId: ROOT_FOLDER_ID, + }), + ]); + + expect(first.id).toBe(second.id); + expect(await pipelineStorageDb.pipeline_registry.count()).toBe(1); + }); +}); + +describe("attributing rows written before rows said which store", () => { + const OLD_SCHEMA = { + pipeline_registry: "id, &storageKey, folderId, [folderId+storageKey]", + folders: "id, parentId", + pipeline_specs: "storageKey", + host_migration: "id", + }; + + it("reads a host row by its reported version and a filed row by its folder", async () => { + await pipelineStorageDb.close(); + await Dexie.delete("tangle_pipelines"); + + const old = new Dexie("tangle_pipelines"); + old.version(5).stores(OLD_SCHEMA); + await old.open(); + await old.table("pipeline_registry").bulkAdd([ + { id: "external-1", storageKey: "Churn model", folderId: ROOT_FOLDER_ID }, + { + id: "local-1", + storageKey: "Nightly refresh", + folderId: ROOT_FOLDER_ID, + }, + { id: "filed-1", storageKey: "Ranking model", folderId: "folder-1" }, + ]); + await old + .table("pipeline_registry") + .update("external-1", { contentVersion: "v7" }); + old.close(); + + await pipelineStorageDb.open(); + + const attributed = await pipelineStorageDb.pipeline_registry.toArray(); + expect( + Object.fromEntries(attributed.map((row) => [row.id, row.storage])), + ).toEqual({ + "external-1": "host", + "local-1": "local", + "filed-1": "local", + }); + }); +}); diff --git a/src/services/pipelineStorage/pipelineRegistry.ts b/src/services/pipelineStorage/pipelineRegistry.ts index 161ad3abee..00863ed2bf 100644 --- a/src/services/pipelineStorage/pipelineRegistry.ts +++ b/src/services/pipelineStorage/pipelineRegistry.ts @@ -1,6 +1,19 @@ import { pipelineStorageDb } from "./db"; +import { currentStorageKind } from "./storageMode"; import { type PipelineRegistryEntry, ROOT_FOLDER_ID } from "./types"; +export type NewPipelineRegistryEntry = Omit; + +/** + * Every lookup is scoped to the store this page load is using, so a row written + * against the other one can never answer for a pipeline here. The scope is read + * rather than passed because storage mode is fixed for the life of the page and + * a caller that could get it wrong is a caller that eventually does. + */ +function scoped() { + return { storage: currentStorageKind() }; +} + /** * Two listings running at once both find no row for a storage key and both try * to add one, and the unique index fails the loser — taking down a whole @@ -8,8 +21,10 @@ import { type PipelineRegistryEntry, ROOT_FOLDER_ID } from "./types"; * transaction makes the second one find the first one's row instead. */ export async function claimEntry( - entry: PipelineRegistryEntry, + entry: NewPipelineRegistryEntry, ): Promise { + const row: PipelineRegistryEntry = { ...scoped(), ...entry }; + return pipelineStorageDb.transaction( "rw", pipelineStorageDb.pipeline_registry, @@ -17,15 +32,15 @@ export async function claimEntry( const existing = await findByStorageKey(entry.storageKey); if (existing) return existing; - await pipelineStorageDb.pipeline_registry.add(entry); - return entry; + await pipelineStorageDb.pipeline_registry.add(row); + return row; }, ); } export async function updateEntry( id: string, - updates: Partial>, + updates: Partial>, ): Promise { await pipelineStorageDb.pipeline_registry.update(id, updates); } @@ -37,15 +52,15 @@ export async function deleteEntry(id: string): Promise { export async function findById( id: string, ): Promise { - return pipelineStorageDb.pipeline_registry.get(id); + const entry = await pipelineStorageDb.pipeline_registry.get(id); + return entry?.storage === currentStorageKind() ? entry : undefined; } export async function findByStorageKey( storageKey: string, ): Promise { return pipelineStorageDb.pipeline_registry - .where("storageKey") - .equals(storageKey) + .where({ ...scoped(), storageKey }) .first(); } @@ -53,8 +68,7 @@ export async function getAllByFolderId( folderId: string, ): Promise { return pipelineStorageDb.pipeline_registry - .where("folderId") - .equals(folderId) + .where({ ...scoped(), folderId }) .toArray(); } @@ -63,7 +77,7 @@ export async function findByFolderAndStorageKey( storageKey: string, ): Promise { return pipelineStorageDb.pipeline_registry - .where({ folderId, storageKey }) + .where({ ...scoped(), folderId, storageKey }) .first(); } diff --git a/src/services/pipelineStorage/pipelineSpecCache.test.ts b/src/services/pipelineStorage/pipelineSpecCache.test.ts index 8f4bc255b9..a76b325996 100644 --- a/src/services/pipelineStorage/pipelineSpecCache.test.ts +++ b/src/services/pipelineStorage/pipelineSpecCache.test.ts @@ -92,6 +92,6 @@ describe("the pipeline spec cache", () => { expect( await pipelineStorageDb.pipeline_specs.toCollection().primaryKeys(), - ).toEqual(["key-2"]); + ).toEqual([["local", "key-2"]]); }); }); diff --git a/src/services/pipelineStorage/pipelineSpecCache.ts b/src/services/pipelineStorage/pipelineSpecCache.ts index 42b08ca8dd..ab647a4dfe 100644 --- a/src/services/pipelineStorage/pipelineSpecCache.ts +++ b/src/services/pipelineStorage/pipelineSpecCache.ts @@ -1,7 +1,10 @@ +import { Dexie } from "dexie"; + import type { ComponentSpec } from "@/utils/componentSpec"; import { pipelineStorageDb } from "./db"; import type { PipelineFile } from "./PipelineFile"; +import { currentStorageKind } from "./storageMode"; /** * A listing says what pipelines exist but not what is in them, and the pipeline @@ -28,9 +31,10 @@ export async function readCachedSpecs( if (wanted.size === 0) return new Map(); - const cached = await pipelineStorageDb.pipeline_specs.bulkGet([ - ...wanted.keys(), - ]); + const storage = currentStorageKind(); + const cached = await pipelineStorageDb.pipeline_specs.bulkGet( + [...wanted.keys()].map((storageKey) => [storage, storageKey]), + ); return new Map( cached.flatMap((entry) => @@ -49,6 +53,7 @@ export async function writeCachedSpec( if (!version) return; await pipelineStorageDb.pipeline_specs.put({ + storage: currentStorageKind(), storageKey: file.storageKey, version, spec, @@ -58,10 +63,12 @@ export async function writeCachedSpec( export async function forgetUnlistedSpecs( listedKeys: Set, ): Promise { + const storage = currentStorageKind(); const stored = await pipelineStorageDb.pipeline_specs - .toCollection() + .where("[storage+storageKey]") + .between([storage, Dexie.minKey], [storage, Dexie.maxKey]) .primaryKeys(); - const gone = stored.filter((key) => !listedKeys.has(key)); + const gone = stored.filter(([, storageKey]) => !listedKeys.has(storageKey)); if (gone.length > 0) { await pipelineStorageDb.pipeline_specs.bulkDelete(gone); diff --git a/src/services/pipelineStorage/storageMode.ts b/src/services/pipelineStorage/storageMode.ts index eaeb63946c..93f8d72843 100644 --- a/src/services/pipelineStorage/storageMode.ts +++ b/src/services/pipelineStorage/storageMode.ts @@ -1,5 +1,6 @@ import type { PipelineStorageHost } from "./host/contract"; import { getPipelineStorageHost } from "./host/detectHost"; +import type { PipelineStorageKind } from "./types"; export type StorageMode = | { kind: "local" } @@ -52,6 +53,15 @@ export function isHostStorage(): boolean { return resolveStorageMode().kind === "host"; } +/** + * Which store the cached rows written on this page load describe. A deployment + * that requires a host still belongs to the host's world while that host is + * unreachable — nothing may be written, and browser rows must stay invisible. + */ +export function currentStorageKind(): PipelineStorageKind { + return resolveStorageMode().kind === "local" ? "local" : "host"; +} + /** * The deployment requires a host-provided store and the page did not supply * one. Nothing can be read or written, so this is worth saying rather than diff --git a/src/services/pipelineStorage/types.ts b/src/services/pipelineStorage/types.ts index 38b1c5effe..f7d7c71f5a 100644 --- a/src/services/pipelineStorage/types.ts +++ b/src/services/pipelineStorage/types.ts @@ -47,7 +47,17 @@ export type DriverConfig = | HostDriverConfig | GoogleDriveDriverConfig; // google-drive +/** + * Which store a cached row describes. Storage keys are only unique within one + * store, and browser storage keys a pipeline on its name — a name a + * host-provided store may well be holding too — so a row that did not say which + * store it came from would be found by the other one and answer for a pipeline + * it has never seen. + */ +export type PipelineStorageKind = "local" | "host"; + export interface CachedPipelineSpec { + storage: PipelineStorageKind; storageKey: string; version: string; spec: ComponentSpec; @@ -64,6 +74,7 @@ export interface HostMigrationRecord { export interface PipelineRegistryEntry { id: string; + storage: PipelineStorageKind; storageKey: string; folderId: string; contentVersion?: string; From 5dabde40eca5468f1d56593ff0f7dc7ba3f8a86b Mon Sep 17 00:00:00 2001 From: Camiel van Schoonhoven Date: Wed, 9 Sep 2026 09:16:28 -0700 Subject: [PATCH 21/36] fix(editor): say when the store has refused a save, and keep the order MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A refused save only showed as a tooltip on a toolbar icon, so someone editing when the store went away saw nothing change and had every reason to think their work was being saved. It now says so across the top of the editor, for as long as the risk lasts, with the reason the store gave and a way to try again. The pipeline list said "you don't have any pipelines yet" when it could not read them at all, which invites building the same pipeline twice. It now says what happened and offers a retry, and only retries queries where a second attempt could answer differently. The parting write on close no longer goes straight to the file: sent on its own it raced whatever write was already running, and a slow store finishing them out of order left the older text stored. It goes through the same queue, which already holds only the newest edit — so a retry after a failure can never put back what a later save replaced. Tests now pin all three orderings. Co-Authored-By: Claude Opus 5 (1M context) --- .../Home/PipelineSection/PipelineSection.tsx | 7 +- .../PipelineSection/usePipelineListEntries.ts | 5 +- .../shared/PipelineStorageError.tsx | 43 +++++++++++ src/routes/v2/pages/Editor/EditorV2.tsx | 2 + .../Editor/components/UnsavedWorkBanner.tsx | 49 +++++++++++++ .../pages/Editor/store/autoSaveStore.test.ts | 73 +++++++++++++++++++ .../v2/pages/Editor/store/autoSaveStore.ts | 20 +++-- src/services/pipelineStorage/storageErrors.ts | 33 +++++++++ tests/e2e/fixtures/pipelineStorageHost.ts | 26 ++++++- tests/e2e/pipeline-storage-host.spec.ts | 37 ++++++++++ 10 files changed, 285 insertions(+), 10 deletions(-) create mode 100644 src/components/shared/PipelineStorageError.tsx create mode 100644 src/routes/v2/pages/Editor/components/UnsavedWorkBanner.tsx create mode 100644 src/services/pipelineStorage/storageErrors.ts diff --git a/src/components/Home/PipelineSection/PipelineSection.tsx b/src/components/Home/PipelineSection/PipelineSection.tsx index 97c431f555..156927ad97 100644 --- a/src/components/Home/PipelineSection/PipelineSection.tsx +++ b/src/components/Home/PipelineSection/PipelineSection.tsx @@ -5,6 +5,7 @@ import { ExamplePipelines } from "@/components/Learn/ExamplePipelines"; import { LoadingScreen } from "@/components/shared/LoadingScreen"; import NewPipelineButton from "@/components/shared/NewPipelineButton"; import { PaginationControls } from "@/components/shared/PaginationControls"; +import { PipelineStorageError } from "@/components/shared/PipelineStorageError"; import { withSuspenseWrapper } from "@/components/shared/SuspenseWrapper"; import { Button } from "@/components/ui/button"; import { Checkbox } from "@/components/ui/checkbox"; @@ -65,7 +66,7 @@ export const PipelineSection = withSuspenseWrapper( const [selectedIds, setSelectedIds] = useState>(new Set()); const storage = usePipelineStorage(); - const { entries, isLoading, pendingCount, refetch } = + const { entries, isLoading, error, pendingCount, refetch } = usePipelineListEntries(); const migration = useHostMigration(refetch); @@ -114,6 +115,10 @@ export const PipelineSection = withSuspenseWrapper( return ; } + if (error) { + return refetch()} />; + } + if (entries.length === 0) { return ( diff --git a/src/components/Home/PipelineSection/usePipelineListEntries.ts b/src/components/Home/PipelineSection/usePipelineListEntries.ts index 1fbc794aa3..6825dc5917 100644 --- a/src/components/Home/PipelineSection/usePipelineListEntries.ts +++ b/src/components/Home/PipelineSection/usePipelineListEntries.ts @@ -8,6 +8,7 @@ import { writeCachedSpec, } from "@/services/pipelineStorage/pipelineSpecCache"; import { usePipelineStorage } from "@/services/pipelineStorage/PipelineStorageProvider"; +import { isRetriableStorageError } from "@/services/pipelineStorage/storageErrors"; import { FoldersQueryKeys } from "@/services/pipelineStorage/types"; import type { ComponentSpec } from "@/utils/componentSpec"; import { runWithConcurrency } from "@/utils/concurrency"; @@ -32,10 +33,12 @@ export function usePipelineListEntries() { const { data: files, isPending, + error, refetch, } = useQuery({ queryKey: FoldersQueryKeys.AllPipelines(), queryFn: () => storage.listAllPipelines(), + retry: isRetriableStorageError, }); const [specs, setSpecs] = useState>( @@ -78,7 +81,7 @@ export function usePipelineListEntries() { spec: specs.get(file.storageKey), })); - return { entries, isLoading: isPending, pendingCount, refetch }; + return { entries, isLoading: isPending, error, pendingCount, refetch }; } async function hydrate( diff --git a/src/components/shared/PipelineStorageError.tsx b/src/components/shared/PipelineStorageError.tsx new file mode 100644 index 0000000000..b752b53603 --- /dev/null +++ b/src/components/shared/PipelineStorageError.tsx @@ -0,0 +1,43 @@ +import { Button } from "@/components/ui/button"; +import { Icon } from "@/components/ui/icon"; +import { BlockStack } from "@/components/ui/layout"; +import { Text } from "@/components/ui/typography"; +import { getPipelineStorageService } from "@/services/pipelineStorage/PipelineStorageService"; +import { getErrorMessage } from "@/utils/string"; + +/** + * Shown in place of whatever could not be read. A store that answers nothing + * must not look like a store holding nothing: "you have no pipelines yet" in + * front of a library that is merely unreachable invites someone to build the + * same pipeline a second time. + */ +export function PipelineStorageError({ + error, + onRetry, +}: { + error: unknown; + onRetry?: () => void; +}) { + const { mode } = getPipelineStorageService(); + const storageLabel = mode.kind === "host" ? mode.label : "Pipeline storage"; + + return ( + + + {storageLabel} could not be read + + {getErrorMessage(error)} + + {onRetry && ( + + )} + + ); +} diff --git a/src/routes/v2/pages/Editor/EditorV2.tsx b/src/routes/v2/pages/Editor/EditorV2.tsx index 400c73c48e..fb3d8e222d 100644 --- a/src/routes/v2/pages/Editor/EditorV2.tsx +++ b/src/routes/v2/pages/Editor/EditorV2.tsx @@ -43,6 +43,7 @@ import { EditorMenuBar } from "./components/EditorMenuBar/EditorMenuBar"; import { EditorTourBridge } from "./components/EditorTourBridge/EditorTourBridge"; import { EmptyEditorState } from "./components/EmptyEditorState"; import { FlowCanvas } from "./components/FlowCanvas/FlowCanvas"; +import { UnsavedWorkBanner } from "./components/UnsavedWorkBanner"; import { useAiChatWindow } from "./hooks/useAiChatWindow"; import { useComponentLibraryWindow } from "./hooks/useComponentLibraryWindow"; import { useComponentSearchV2Window } from "./hooks/useComponentSearchV2Window"; @@ -168,6 +169,7 @@ function EditorV2Content({ pipelineRef }: { pipelineRef: PipelineRef | null }) { + diff --git a/src/routes/v2/pages/Editor/components/UnsavedWorkBanner.tsx b/src/routes/v2/pages/Editor/components/UnsavedWorkBanner.tsx new file mode 100644 index 0000000000..957fcf51da --- /dev/null +++ b/src/routes/v2/pages/Editor/components/UnsavedWorkBanner.tsx @@ -0,0 +1,49 @@ +import { observer } from "mobx-react-lite"; + +import { Button } from "@/components/ui/button"; +import { Icon } from "@/components/ui/icon"; +import { InlineStack } from "@/components/ui/layout"; +import { Spinner } from "@/components/ui/spinner"; +import { Text } from "@/components/ui/typography"; +import { useEditorSession } from "@/routes/v2/pages/Editor/store/EditorSessionContext"; + +/** + * A save that the store refuses leaves the editor holding the only copy of the + * work, and a tooltip on a toolbar icon is not where someone finds that out. + * It stays up until a save lands, because the risk lasts exactly that long. + */ +export const UnsavedWorkBanner = observer(function UnsavedWorkBanner() { + const { autoSave } = useEditorSession(); + const { saveError, isSaving } = autoSave; + + if (!saveError) return null; + + return ( + + {isSaving ? ( + + ) : ( + + )} + + Your changes are not saved. They are kept here and will be saved as soon + as they can be. {saveError} + + + + ); +}); diff --git a/src/routes/v2/pages/Editor/store/autoSaveStore.test.ts b/src/routes/v2/pages/Editor/store/autoSaveStore.test.ts index 0ecb63761f..b3dc76d6e7 100644 --- a/src/routes/v2/pages/Editor/store/autoSaveStore.test.ts +++ b/src/routes/v2/pages/Editor/store/autoSaveStore.test.ts @@ -124,6 +124,79 @@ describe("AutoSaveStore when the store cannot be reached", () => { expect(write).not.toHaveBeenCalled(); }); + it("retries with the newest edit, never the one that was refused", async () => { + const written: string[] = []; + let reachable = false; + const store = createStore({ + write: async (yamlText: string) => { + if (!reachable) throw new Error("unreachable"); + written.push(yamlText); + }, + } as unknown as PipelineFile); + + const spec = createSpec("Churn model"); + store.init(spec, "Churn model"); + await store.save(); + + spec.name = "Churn model v2"; + reachable = true; + await store.save(); + + expect(written).toEqual(["name: Churn model v2"]); + store.dispose(); + }); + + it("does not let a retry land on top of a save that already succeeded", async () => { + const written: string[] = []; + let reachable = false; + const store = createStore({ + write: async (yamlText: string) => { + if (!reachable) throw new Error("unreachable"); + written.push(yamlText); + }, + } as unknown as PipelineFile); + + const spec = createSpec("Churn model"); + store.init(spec, "Churn model"); + await store.save(); + + reachable = true; + spec.name = "Churn model v2"; + await store.save(); + + window.dispatchEvent(new Event("focus")); + await new Promise((resolve) => setTimeout(resolve, 20)); + + expect(written).toEqual(["name: Churn model v2"]); + store.dispose(); + }); + + it("waits for the write in flight before the parting one", async () => { + const written: string[] = []; + let release: (() => void) | undefined; + const store = createStore({ + write: async (yamlText: string) => { + if (!release) { + await new Promise((resolve) => (release = resolve)); + } + written.push(yamlText); + }, + } as unknown as PipelineFile); + + const spec = createSpec("Churn model"); + store.init(spec, "Churn model"); + + const saving = store.save(); + spec.name = "Churn model v2"; + store.dispose(); + + release?.(); + await saving; + await vi.waitFor(() => + expect(written).toEqual(["name: Churn model", "name: Churn model v2"]), + ); + }); + it("lands overlapping saves in order instead of racing them", async () => { const written: string[] = []; let release: (() => void) | undefined; diff --git a/src/routes/v2/pages/Editor/store/autoSaveStore.ts b/src/routes/v2/pages/Editor/store/autoSaveStore.ts index bd4653e3e5..373a6f4066 100644 --- a/src/routes/v2/pages/Editor/store/autoSaveStore.ts +++ b/src/routes/v2/pages/Editor/store/autoSaveStore.ts @@ -40,6 +40,7 @@ export class AutoSaveStore { private retryTimer: ReturnType | null = null; private retryAttempt = 0; private disposeRecovery: (() => void) | null = null; + private closed = false; private debouncedSave = debounce((yamlText: string) => { void this.performSave(yamlText); @@ -62,6 +63,7 @@ export class AutoSaveStore { this.hasPendingChanges = false; this.pendingYaml = null; this.retryAttempt = 0; + this.closed = false; // The freshly-loaded spec matches what's on disk, so seed the baseline to // avoid flushing an unchanged pipeline on dispose. this.lastSavedYaml = this.serializeSpec(); @@ -76,13 +78,20 @@ export class AutoSaveStore { } @action dispose() { + /** + * The parting write goes through the queue rather than straight to the + * file. Sent on its own it would race whatever write is already running, + * and a slow store finishing them out of order would leave the older text + * as the stored one. + */ + this.closed = true; + const yaml = this.serializeSpec(); - const file = this.pipelineFileStore.activePipelineFile; - if (yaml && file && yaml !== this.lastSavedYaml) { - void file.write(yaml).catch((error) => { - console.error("Auto-save flush on dispose failed:", error); - }); + if (yaml && yaml !== this.lastSavedYaml) { + this.pendingYaml = yaml; + void this.flushPending(); } + this.debouncedSave.cancel(); this.clearRetry(); this.disposeRecovery?.(); @@ -206,6 +215,7 @@ export class AutoSaveStore { private scheduleRetry() { this.clearRetry(); + if (this.closed) return; const delay = RETRY_DELAYS_MS[Math.min(this.retryAttempt, RETRY_DELAYS_MS.length - 1)]; diff --git a/src/services/pipelineStorage/storageErrors.ts b/src/services/pipelineStorage/storageErrors.ts new file mode 100644 index 0000000000..b162fd11ef --- /dev/null +++ b/src/services/pipelineStorage/storageErrors.ts @@ -0,0 +1,33 @@ +import { HostStorageError } from "./drivers/HostStorageDriver"; +import { + AmbiguousPipelineNameError, + PipelineNotFoundError, +} from "./PipelineStorageService"; + +const NUM_RETRIES = 3; + +/** + * Trying again only helps when the answer might differ next time. A store that + * has said what it holds — no such pipeline, more than one by that name, a + * session that has expired — will say the same thing three more times, and the + * delay before the user is told buys nothing. + */ +export function isRetriableStorageError( + failureCount: number, + error: Error, +): boolean { + if (failureCount >= NUM_RETRIES) return false; + + if ( + error instanceof PipelineNotFoundError || + error instanceof AmbiguousPipelineNameError + ) { + return false; + } + + if (error instanceof HostStorageError) { + return error.code === "unavailable" || error.code === "rate_limited"; + } + + return true; +} diff --git a/tests/e2e/fixtures/pipelineStorageHost.ts b/tests/e2e/fixtures/pipelineStorageHost.ts index 05f2169fed..85e722315b 100644 --- a/tests/e2e/fixtures/pipelineStorageHost.ts +++ b/tests/e2e/fixtures/pipelineStorageHost.ts @@ -26,6 +26,7 @@ interface HostRecord { interface HostTestState { records(): HostRecord[]; readKeys(): string[]; + setFailMode(mode: HostFailMode): void; } declare global { @@ -65,6 +66,8 @@ export async function installPipelineStorageHost( */ const readKeys: string[] = []; + let failMode = config.failMode; + if (!saved) { for (const seeded of config.seed) { revision += 1; @@ -92,9 +95,9 @@ export async function installPipelineStorageHost( await new Promise((resolve) => setTimeout(resolve, config.latencyMs)); } - if (config.failMode !== "none") { - throw Object.assign(new Error(`host is ${config.failMode}`), { - code: config.failMode, + if (failMode !== "none") { + throw Object.assign(new Error(`host is ${failMode}`), { + code: failMode, }); } } @@ -157,6 +160,9 @@ export async function installPipelineStorageHost( window.__TANGLE_TEST_HOST__ = { records: () => [...store.values()], readKeys: () => [...readKeys], + setFailMode: (mode) => { + failMode = mode; + }, }; }, { @@ -250,6 +256,20 @@ export async function seedLocallyStoredPipeline( }, name); } +/** + * Takes the store away, or gives it back, without reloading — the failure that + * matters is the one that arrives while someone is working. + */ +export async function setHostFailMode( + page: Page, + mode: HostFailMode, +): Promise { + await page.evaluate( + (value) => window.__TANGLE_TEST_HOST__?.setFailMode(value), + mode, + ); +} + export async function readHostRecords(page: Page): Promise { return page.evaluate(() => window.__TANGLE_TEST_HOST__?.records() ?? []); } diff --git a/tests/e2e/pipeline-storage-host.spec.ts b/tests/e2e/pipeline-storage-host.spec.ts index 6d2eb5e5c9..174dda8a7a 100644 --- a/tests/e2e/pipeline-storage-host.spec.ts +++ b/tests/e2e/pipeline-storage-host.spec.ts @@ -7,6 +7,7 @@ import { readHostRecords, readLocallyStoredPipelineKeys, seedLocallyStoredPipeline, + setHostFailMode, } from "./fixtures/pipelineStorageHost"; const LABEL = "Shared storage"; @@ -177,6 +178,42 @@ test.describe("host-provided pipeline storage", () => { .toEqual(["Churn model v2", "Nightly refresh"]); }); + test("says the list could not be read rather than showing an empty library", async ({ + page, + }) => { + await installSeededHost(page, { failMode: "unavailable" }); + + await page.goto("/pipelines"); + + await expect(page.getByTestId("pipeline-storage-error")).toBeVisible({ + timeout: 30_000, + }); + await expect(page.getByText(/don't have any pipelines yet/i)).toBeHidden(); + }); + + test("says so in the editor when a save is refused, and stops once it lands", async ({ + page, + }) => { + await installSeededHost(page); + + await page.goto(`/editor-v2/${SEED[0].key}`); + await expect(page.locator('[data-testid="rf__wrapper"]')).toBeVisible({ + timeout: 30_000, + }); + + await setHostFailMode(page, "unavailable"); + await page.getByTestId("auto-save-button").click(); + + const banner = page.getByTestId("unsaved-work-banner"); + await expect(banner).toBeVisible(); + await expect(banner).toContainText(LABEL); + + await setHostFailMode(page, "none"); + await banner.getByRole("button", { name: "Try now" }).click(); + + await expect(banner).toBeHidden(); + }); + test("a deleted pipeline does not come back on reload", async ({ page }) => { await installSeededHost(page); From 993cb342065fbb75ce11c25d9f701267817b1393 Mon Sep 17 00:00:00 2001 From: Camiel van Schoonhoven Date: Wed, 9 Sep 2026 09:21:05 -0700 Subject: [PATCH 22/36] feat(pipeline-storage): copy pipelines in wherever the app opens MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The copy only started on the pipeline list, so someone who landed anywhere else worked against a store that looked empty until they happened to open that tab. It now starts with the app. Nothing is shown from there — progress belongs where the pipelines are, and the list still reports anything that failed and offers to retry it. Co-Authored-By: Claude Opus 5 (1M context) --- .../PipelineStorageProvider.tsx | 11 ++++++++- src/services/pipelineStorage/hostMigration.ts | 14 +++++++++++ tests/e2e/fixtures/pipelineStorageHost.ts | 23 +++++++++++++++++++ tests/e2e/pipeline-storage-host.spec.ts | 21 +++++++++++++++++ 4 files changed, 68 insertions(+), 1 deletion(-) diff --git a/src/services/pipelineStorage/PipelineStorageProvider.tsx b/src/services/pipelineStorage/PipelineStorageProvider.tsx index c94f7c9445..bacd9ac190 100644 --- a/src/services/pipelineStorage/PipelineStorageProvider.tsx +++ b/src/services/pipelineStorage/PipelineStorageProvider.tsx @@ -1,11 +1,12 @@ import type { ReactNode } from "react"; -import { useState } from "react"; +import { useEffect, useState } from "react"; import { createRequiredContext, useRequiredContext, } from "@/hooks/useRequiredContext"; +import { startHostMigration } from "./hostMigration"; import { getPipelineStorageService, type PipelineStorageService, @@ -18,6 +19,14 @@ export const PipelineStorageCtx = createRequiredContext( export function PipelineStorageProvider({ children }: { children: ReactNode }) { const [service] = useState(getPipelineStorageService); + useEffect(() => { + if (service.mode.kind !== "host") return; + + void startHostMigration(service.rootFolder).catch((error: unknown) => { + console.error("Could not copy pipelines into the host store:", error); + }); + }, [service]); + return ( {children} diff --git a/src/services/pipelineStorage/hostMigration.ts b/src/services/pipelineStorage/hostMigration.ts index bab9d42679..cd2ba54089 100644 --- a/src/services/pipelineStorage/hostMigration.ts +++ b/src/services/pipelineStorage/hostMigration.ts @@ -97,6 +97,20 @@ export async function dismissHostMigration(): Promise { }); } +/** + * Starts the copy wherever the app happens to open, so someone who never visits + * the pipeline list still finds their pipelines in the host. Nothing is shown + * from here: progress belongs where the pipelines are, and a copy running in + * the background must not interrupt whatever the user came to do. Anything that + * fails is recorded, and the list offers to retry it when it is next opened. + */ +export async function startHostMigration( + target: PipelineFolder, +): Promise { + if ((await claimHostMigration()) !== "claimed") return; + await runHostMigration(target); +} + /** * Copies everything in browser storage into the host, keyed on the name it has * locally. Host writes upsert on the key they are given, so a pipeline copied diff --git a/tests/e2e/fixtures/pipelineStorageHost.ts b/tests/e2e/fixtures/pipelineStorageHost.ts index 85e722315b..2fc5d64335 100644 --- a/tests/e2e/fixtures/pipelineStorageHost.ts +++ b/tests/e2e/fixtures/pipelineStorageHost.ts @@ -253,6 +253,29 @@ export async function seedLocallyStoredPipeline( }); database.close(); + + /** + * The app copies browser-stored pipelines into the host once, on whichever + * page it first opens, and records that it has. Seeding has to go through a + * running page, which is necessarily after that — so the record goes, which + * puts the app back where a user with pipelines and a new host starts. + */ + const registry = await new Promise((resolve) => { + const request = indexedDB.open("tangle_pipelines"); + request.onsuccess = () => resolve(request.result); + request.onerror = () => resolve(undefined); + }); + + if (registry?.objectStoreNames.contains("host_migration")) { + await new Promise((resolve) => { + const transaction = registry.transaction("host_migration", "readwrite"); + transaction.objectStore("host_migration").clear(); + transaction.oncomplete = () => resolve(); + transaction.onerror = () => resolve(); + }); + } + + registry?.close(); }, name); } diff --git a/tests/e2e/pipeline-storage-host.spec.ts b/tests/e2e/pipeline-storage-host.spec.ts index 174dda8a7a..be78ba140a 100644 --- a/tests/e2e/pipeline-storage-host.spec.ts +++ b/tests/e2e/pipeline-storage-host.spec.ts @@ -101,6 +101,27 @@ test.describe("host-provided pipeline storage", () => { ).toContain(localName); }); + test("copies them even when the pipeline list is never opened", async ({ + page, + }) => { + const localName = "Never opened the list"; + + await installSeededHost(page); + await page.goto("/"); + await seedLocallyStoredPipeline(page, localName); + + await page.goto("/pipeline-folders"); + await expect(page.getByText("Churn model")).toBeVisible(); + + await expect + .poll( + async () => + (await readHostRecords(page)).map((record) => record.displayName), + { timeout: 15_000 }, + ) + .toContain(localName); + }); + test("opens a pipeline at a url that is only its id", async ({ page }) => { await installSeededHost(page); From 254b0454349c0f9616a53fbc4312ed06736a1d69 Mon Sep 17 00:00:00 2001 From: Camiel van Schoonhoven Date: Wed, 9 Sep 2026 09:25:19 -0700 Subject: [PATCH 23/36] fix(pipelines): stop offering folders to a store that has none A store that keys pipelines itself has no folders, and the page still showed New Folder, Connect Folder and a row for a parent that cannot exist. Each was an action with nowhere to go. Co-Authored-By: Claude Opus 5 (1M context) --- .../pages/PipelineFolders/PipelineFolders.tsx | 30 +++++++++++-------- .../FolderPipelineTable.tsx | 2 +- tests/e2e/fixtures/pipelineStorageHost.ts | 26 ++++++++++++++++ tests/e2e/pipeline-storage-host.spec.ts | 16 ++++++++++ 4 files changed, 61 insertions(+), 13 deletions(-) diff --git a/src/routes/v2/pages/PipelineFolders/PipelineFolders.tsx b/src/routes/v2/pages/PipelineFolders/PipelineFolders.tsx index 81fa3b1dd7..3bc5bcce7c 100644 --- a/src/routes/v2/pages/PipelineFolders/PipelineFolders.tsx +++ b/src/routes/v2/pages/PipelineFolders/PipelineFolders.tsx @@ -4,6 +4,7 @@ import { useState } from "react"; import { withSuspenseWrapper } from "@/components/shared/SuspenseWrapper"; import { BlockStack, InlineStack } from "@/components/ui/layout"; import { Skeleton } from "@/components/ui/skeleton"; +import { usePipelineStorage } from "@/services/pipelineStorage/PipelineStorageProvider"; import { type PipelineRef, ROOT_FOLDER_ID, @@ -41,6 +42,7 @@ export const PipelineFolders = withSuspenseWrapper( folderId?: string; }; const [localFolderId, setLocalFolderId] = useState(null); + const hasFolders = !usePipelineStorage().rootFolder.isFlat; const isEmbedded = onPipelineClick !== undefined; const currentFolderId = isEmbedded @@ -55,19 +57,23 @@ export const PipelineFolders = withSuspenseWrapper( inlineAlign="start" align="start" > - + {hasFolders && ( + <> + - - - - - - - + + + + + + + + + )} diff --git a/src/routes/v2/pages/PipelineFolders/components/FolderPipelineTable/FolderPipelineTable.tsx b/src/routes/v2/pages/PipelineFolders/components/FolderPipelineTable/FolderPipelineTable.tsx index 54f7bd54a3..e49b962bdb 100644 --- a/src/routes/v2/pages/PipelineFolders/components/FolderPipelineTable/FolderPipelineTable.tsx +++ b/src/routes/v2/pages/PipelineFolders/components/FolderPipelineTable/FolderPipelineTable.tsx @@ -219,7 +219,7 @@ export const FolderPipelineTable = withSuspenseWrapper( - {folderId !== null && ( + {folderId !== null && !currentFolder.isFlat && ( )} diff --git a/tests/e2e/fixtures/pipelineStorageHost.ts b/tests/e2e/fixtures/pipelineStorageHost.ts index 2fc5d64335..c6a6b61016 100644 --- a/tests/e2e/fixtures/pipelineStorageHost.ts +++ b/tests/e2e/fixtures/pipelineStorageHost.ts @@ -267,6 +267,32 @@ export async function seedLocallyStoredPipeline( }); if (registry?.objectStoreNames.contains("host_migration")) { + const settled = () => + new Promise((resolve) => { + const request = registry + .transaction("host_migration", "readonly") + .objectStore("host_migration") + .get("v1"); + request.onsuccess = () => { + const record = request.result as + { completedAt?: number; dismissedAt?: number } | undefined; + resolve( + record?.completedAt !== undefined || + record?.dismissedAt !== undefined, + ); + }; + request.onerror = () => resolve(false); + }); + + /** + * The copy this page load already started would otherwise write its own + * record back over the cleared one, and the next load would read it as + * done. + */ + for (let attempt = 0; attempt < 50 && !(await settled()); attempt++) { + await new Promise((resolve) => setTimeout(resolve, 100)); + } + await new Promise((resolve) => { const transaction = registry.transaction("host_migration", "readwrite"); transaction.objectStore("host_migration").clear(); diff --git a/tests/e2e/pipeline-storage-host.spec.ts b/tests/e2e/pipeline-storage-host.spec.ts index be78ba140a..a5c682d2d1 100644 --- a/tests/e2e/pipeline-storage-host.spec.ts +++ b/tests/e2e/pipeline-storage-host.spec.ts @@ -53,6 +53,22 @@ test.describe("host-provided pipeline storage", () => { await expect(page.getByText("Nightly refresh")).toBeVisible(); }); + test("offers nothing to file pipelines into a store that has no folders", async ({ + page, + }) => { + await installSeededHost(page); + + await page.goto("/pipeline-folders"); + await expect(page.getByText("Churn model")).toBeVisible(); + + await expect( + page.getByRole("button", { name: /new folder/i }), + ).toBeHidden(); + await expect( + page.getByRole("button", { name: /connect folder/i }), + ).toBeHidden(); + }); + test("keeps the pipeline table at /pipelines, contents and all", async ({ page, }) => { From 6eed5fd834b57e28a80906e4428a0cdba907c258 Mon Sep 17 00:00:00 2001 From: Camiel van Schoonhoven Date: Wed, 9 Sep 2026 09:29:05 -0700 Subject: [PATCH 24/36] feat(editor): offer a copy when the store stops accepting the session MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An expired session was retried on the same ladder as a store that had merely gone away — every five, fifteen, thirty seconds, forever, while the work sat unsaved. It is now recognised as something only the person can resolve: the retrying stops and the editor says so. Signing in again means reloading, which discards whatever the store never took, so the download is offered before the reload rather than after it. Co-Authored-By: Claude Opus 5 (1M context) --- src/routes/v2/pages/Editor/EditorV2.tsx | 2 + .../components/ExpiredSessionDialog.tsx | 51 +++++++++++++++++++ .../v2/pages/Editor/store/autoSaveStore.ts | 20 +++++--- src/services/pipelineStorage/storageErrors.ts | 17 +++++++ tests/e2e/pipeline-storage-host.spec.ts | 20 ++++++++ 5 files changed, 104 insertions(+), 6 deletions(-) create mode 100644 src/routes/v2/pages/Editor/components/ExpiredSessionDialog.tsx diff --git a/src/routes/v2/pages/Editor/EditorV2.tsx b/src/routes/v2/pages/Editor/EditorV2.tsx index fb3d8e222d..9a4de95dfe 100644 --- a/src/routes/v2/pages/Editor/EditorV2.tsx +++ b/src/routes/v2/pages/Editor/EditorV2.tsx @@ -42,6 +42,7 @@ import { DriverPermissionGate } from "./components/DriverPermissionGate"; import { EditorMenuBar } from "./components/EditorMenuBar/EditorMenuBar"; import { EditorTourBridge } from "./components/EditorTourBridge/EditorTourBridge"; import { EmptyEditorState } from "./components/EmptyEditorState"; +import { ExpiredSessionDialog } from "./components/ExpiredSessionDialog"; import { FlowCanvas } from "./components/FlowCanvas/FlowCanvas"; import { UnsavedWorkBanner } from "./components/UnsavedWorkBanner"; import { useAiChatWindow } from "./hooks/useAiChatWindow"; @@ -170,6 +171,7 @@ function EditorV2Content({ pipelineRef }: { pipelineRef: PipelineRef | null }) { + diff --git a/src/routes/v2/pages/Editor/components/ExpiredSessionDialog.tsx b/src/routes/v2/pages/Editor/components/ExpiredSessionDialog.tsx new file mode 100644 index 0000000000..853e85315d --- /dev/null +++ b/src/routes/v2/pages/Editor/components/ExpiredSessionDialog.tsx @@ -0,0 +1,51 @@ +import { observer } from "mobx-react-lite"; + +import { Button } from "@/components/ui/button"; +import { + Dialog, + DialogContent, + DialogDescription, + DialogFooter, + DialogHeader, + DialogTitle, +} from "@/components/ui/dialog"; +import { useEditorSession } from "@/routes/v2/pages/Editor/store/EditorSessionContext"; +import { useSharedStores } from "@/routes/v2/shared/store/SharedStoreContext"; + +import { exportCurrentPipeline } from "./EditorMenuBar/components/fileMenu.actions"; + +/** + * Signing in again means reloading, and reloading throws away edits the store + * never took — so the way out is offered before the reload rather than after + * it. Nothing is dismissable here on purpose: every further edit made in this + * tab is one more thing the export has to carry. + */ +export const ExpiredSessionDialog = observer(function ExpiredSessionDialog() { + const { autoSave } = useEditorSession(); + const { navigation } = useSharedStores(); + + if (!autoSave.sessionExpired) return null; + + return ( + + + + Your session has expired + + {autoSave.saveError} Save a copy first: reloading discards anything + that has not been stored. + + + + + + + + + ); +}); diff --git a/src/routes/v2/pages/Editor/store/autoSaveStore.ts b/src/routes/v2/pages/Editor/store/autoSaveStore.ts index 373a6f4066..789d17b00f 100644 --- a/src/routes/v2/pages/Editor/store/autoSaveStore.ts +++ b/src/routes/v2/pages/Editor/store/autoSaveStore.ts @@ -6,6 +6,10 @@ import { serializePipelineDocumentToText, } from "@/models/componentSpec"; import { saveUndoHistory } from "@/routes/v2/pages/Editor/utils/undoHistoryStorage"; +import { + isExpiredSession, + isWriteWorthRetrying, +} from "@/services/pipelineStorage/storageErrors"; import { AUTOSAVE_DEBOUNCE_TIME_MS } from "@/utils/constants"; import { debounce } from "@/utils/debounce"; import { getErrorMessage } from "@/utils/string"; @@ -22,6 +26,7 @@ export class AutoSaveStore { @observable accessor lastSavedAt: Date | null = null; @observable accessor saveError: string | null = null; @observable accessor hasPendingChanges = false; + @observable accessor sessionExpired = false; private spec: ComponentSpec | null = null; private pipelineName: string | null = null; @@ -61,6 +66,7 @@ export class AutoSaveStore { this.lastSavedAt = null; this.saveError = null; this.hasPendingChanges = false; + this.sessionExpired = false; this.pendingYaml = null; this.retryAttempt = 0; this.closed = false; @@ -117,10 +123,12 @@ export class AutoSaveStore { this.lastSavedAt = date; this.isSaving = false; this.saveError = null; + this.sessionExpired = false; } - @action private setSaveError(message: string) { - this.saveError = message; + @action private setSaveError(error: Error) { + this.saveError = getErrorMessage(error); + this.sessionExpired = isExpiredSession(error); this.isSaving = false; } @@ -170,9 +178,9 @@ export class AutoSaveStore { const yamlText = this.pendingYaml; const outcome = await this.writeOnce(yamlText); - if (typeof outcome === "string") { + if (outcome instanceof Error) { this.setSaveError(outcome); - this.scheduleRetry(); + if (isWriteWorthRetrying(outcome)) this.scheduleRetry(); return; } @@ -184,7 +192,7 @@ export class AutoSaveStore { this.setPending(false); } - private async writeOnce(yamlText: string): Promise { + private async writeOnce(yamlText: string): Promise { const pipelineName = this.pipelineName; this.setSaving(true); @@ -201,7 +209,7 @@ export class AutoSaveStore { return new Date(); } catch (error) { console.error("Auto-save failed:", error); - return getErrorMessage(error); + return error instanceof Error ? error : new Error(String(error)); } })(); diff --git a/src/services/pipelineStorage/storageErrors.ts b/src/services/pipelineStorage/storageErrors.ts index b162fd11ef..c13a5882a2 100644 --- a/src/services/pipelineStorage/storageErrors.ts +++ b/src/services/pipelineStorage/storageErrors.ts @@ -31,3 +31,20 @@ export function isRetriableStorageError( return true; } + +/** + * A write the store will refuse again for the same reason. An expired session + * needs the person, not another attempt, and quietly retrying one until it + * comes back is how work sits unsaved with nobody told why. + */ +export function isWriteWorthRetrying(error: unknown): boolean { + if (error instanceof HostStorageError) { + return error.code !== "unauthenticated" && error.code !== "conflict"; + } + + return true; +} + +export function isExpiredSession(error: unknown): boolean { + return error instanceof HostStorageError && error.code === "unauthenticated"; +} diff --git a/tests/e2e/pipeline-storage-host.spec.ts b/tests/e2e/pipeline-storage-host.spec.ts index a5c682d2d1..bd520f960b 100644 --- a/tests/e2e/pipeline-storage-host.spec.ts +++ b/tests/e2e/pipeline-storage-host.spec.ts @@ -251,6 +251,26 @@ test.describe("host-provided pipeline storage", () => { await expect(banner).toBeHidden(); }); + test("offers a copy before asking for a sign-in that would discard it", async ({ + page, + }) => { + await installSeededHost(page); + + await page.goto(`/editor-v2/${SEED[0].key}`); + await expect(page.locator('[data-testid="rf__wrapper"]')).toBeVisible({ + timeout: 30_000, + }); + + await setHostFailMode(page, "unauthenticated"); + await page.getByTestId("auto-save-button").click(); + + const dialog = page.getByTestId("expired-session"); + await expect(dialog).toBeVisible(); + await expect( + dialog.getByRole("button", { name: "Download a copy" }), + ).toBeVisible(); + }); + test("a deleted pipeline does not come back on reload", async ({ page }) => { await installSeededHost(page); From 81ea649343248ae05be7eab91b30b361cbf39f8d Mon Sep 17 00:00:00 2001 From: Camiel van Schoonhoven Date: Wed, 9 Sep 2026 09:31:41 -0700 Subject: [PATCH 25/36] fix(editor): say which pipeline could not be opened, and why A pipeline the store cannot produce fell to the generic "a UI element failed to render" icon, which names neither the pipeline nor the reason. A link written against a store the app is no longer using now reads as the miss it is. Co-Authored-By: Claude Opus 5 (1M context) --- src/components/shared/PipelineStorageError.tsx | 14 ++++++++++++-- src/routes/v2/pages/Editor/EditorV2.tsx | 10 ++++++++++ tests/e2e/pipeline-storage-host.spec.ts | 13 +++++++++++++ 3 files changed, 35 insertions(+), 2 deletions(-) diff --git a/src/components/shared/PipelineStorageError.tsx b/src/components/shared/PipelineStorageError.tsx index b752b53603..4a6b666e5a 100644 --- a/src/components/shared/PipelineStorageError.tsx +++ b/src/components/shared/PipelineStorageError.tsx @@ -2,7 +2,11 @@ import { Button } from "@/components/ui/button"; import { Icon } from "@/components/ui/icon"; import { BlockStack } from "@/components/ui/layout"; import { Text } from "@/components/ui/typography"; -import { getPipelineStorageService } from "@/services/pipelineStorage/PipelineStorageService"; +import { + AmbiguousPipelineNameError, + getPipelineStorageService, + PipelineNotFoundError, +} from "@/services/pipelineStorage/PipelineStorageService"; import { getErrorMessage } from "@/utils/string"; /** @@ -21,6 +25,12 @@ export function PipelineStorageError({ const { mode } = getPipelineStorageService(); const storageLabel = mode.kind === "host" ? mode.label : "Pipeline storage"; + const headline = + error instanceof PipelineNotFoundError || + error instanceof AmbiguousPipelineNameError + ? "This pipeline could not be opened" + : `${storageLabel} could not be read`; + return ( - {storageLabel} could not be read + {headline} {getErrorMessage(error)} diff --git a/src/routes/v2/pages/Editor/EditorV2.tsx b/src/routes/v2/pages/Editor/EditorV2.tsx index 9a4de95dfe..ec3fe41826 100644 --- a/src/routes/v2/pages/Editor/EditorV2.tsx +++ b/src/routes/v2/pages/Editor/EditorV2.tsx @@ -8,6 +8,7 @@ import { type ReactNode, useEffect } from "react"; import { ComponentEditorProvider } from "@/components/shared/ComponentEditor/ComponentEditorProvider"; import { LoadingScreen } from "@/components/shared/LoadingScreen"; +import { PipelineStorageError } from "@/components/shared/PipelineStorageError"; import { useFlagValue } from "@/components/shared/Settings/useFlags"; import { withSuspenseWrapper } from "@/components/shared/SuspenseWrapper"; import { InlineStack } from "@/components/ui/layout"; @@ -142,6 +143,15 @@ const PipelineEditor = withSuspenseWrapper( ); }), PipelineEditorSkeleton, + /** + * A pipeline that cannot be read is not a component that failed to render, + * and the generic retry icon says neither which pipeline nor why. It matters + * most for a link written against a store the app is no longer using, which + * is a miss rather than a fault. + */ + ({ error, resetErrorBoundary }) => ( + + ), ); function EditorV2Content({ pipelineRef }: { pipelineRef: PipelineRef | null }) { diff --git a/tests/e2e/pipeline-storage-host.spec.ts b/tests/e2e/pipeline-storage-host.spec.ts index bd520f960b..e624277c1e 100644 --- a/tests/e2e/pipeline-storage-host.spec.ts +++ b/tests/e2e/pipeline-storage-host.spec.ts @@ -156,6 +156,19 @@ test.describe("host-provided pipeline storage", () => { expect(url.search).toBe(""); }); + test("says a link to a pipeline it does not hold cannot be opened", async ({ + page, + }) => { + await installSeededHost(page); + + await page.goto("/editor-v2/0f8c1a2b-0000-4000-8000-00000000dead"); + + await expect(page.getByTestId("pipeline-storage-error")).toBeVisible({ + timeout: 30_000, + }); + await expect(page.getByText(/could not be opened/i)).toBeVisible(); + }); + test("keeps the browser's own pipeline store empty", async ({ page }) => { await installSeededHost(page); From 964beb55ba89505cf738600825640b0a827d3925 Mon Sep 17 00:00:00 2001 From: Camiel van Schoonhoven Date: Wed, 9 Sep 2026 09:36:47 -0700 Subject: [PATCH 26/36] docs(pipeline-storage): describe the store the page can provide The architecture notes still described browser storage as the only store, a registry with one global key namespace, and functions that no longer exist. Co-Authored-By: Claude Opus 5 (1M context) --- src/services/pipelineStorage/ARCHITECTURE.md | 217 ++++++++++++++++--- 1 file changed, 188 insertions(+), 29 deletions(-) diff --git a/src/services/pipelineStorage/ARCHITECTURE.md b/src/services/pipelineStorage/ARCHITECTURE.md index c3b0a10dab..edd4c9f428 100644 --- a/src/services/pipelineStorage/ARCHITECTURE.md +++ b/src/services/pipelineStorage/ARCHITECTURE.md @@ -1,6 +1,8 @@ # Pipeline Storage Architecture -Client-side pipeline file storage abstraction that supports multiple storage backends (IndexedDB, local file system) behind a unified driver interface. Every pipeline file lives inside a folder; every folder owns a storage driver that handles the actual I/O. +Pipeline file storage behind one driver interface, over browser storage (IndexedDB, local file system) or over a store the embedding page provides. Every pipeline file lives inside a folder; every folder owns a driver that does the I/O. + +Which store the app uses is decided once, as it boots, and never revisited — see [Storage mode](#storage-mode). ## Module Structure @@ -13,11 +15,20 @@ pipelineStorage/ ├── PipelineFile.ts # Domain model for a single pipeline file ├── PipelineFolder.ts # Domain model for a folder of pipeline files ├── PipelineStorageService.ts # Service layer — entry point for all operations -├── PipelineStorageProvider.tsx # React context provider + usePipelineStorage hook +├── PipelineStorageProvider.tsx # React context provider + usePipelineStorage hook +├── storageMode.ts # Which store this page load uses; decided once +├── storageErrors.ts # Which failures are worth another attempt +├── pipelineSpecCache.ts # Pipeline contents, keyed by the store's own version +├── hostMigration.ts # One-time copy of browser pipelines into a host store +├── host/ +│ ├── contract.ts # The window global an embedding page provides +│ └── detectHost.ts # Reads and version-checks that global └── drivers/ ├── RootFolderDbStorageDriver.ts # Legacy IndexedDB component list ├── FolderIndexDbStorageDriver.ts # Folder-scoped IndexedDB (extends Root driver) - └── LocalFileSystemDriver.ts # File System Access API (local directory) + ├── LocalFileSystemDriver.ts # File System Access API (local directory) + ├── HostStorageDriver.ts # The store the embedding page provides + └── UnavailableStorageDriver.ts # Refuses everything; see host-missing below ``` ```mermaid @@ -98,7 +109,8 @@ Each folder persists a `DriverConfig` in Dexie. The `createDriver` factory resol type DriverConfig = | { driverType: "root-indexdb" } | { driverType: "folder-indexdb"; folderId: string } - | { driverType: "local-fs"; handle: FileSystemDirectoryHandle }; + | { driverType: "local-fs"; handle: FileSystemDirectoryHandle } + | { driverType: "host" }; ``` ### Driver Implementations @@ -108,6 +120,7 @@ type DriverConfig = | `RootFolderDbStorageDriver` | `root-indexdb` | Legacy `localforage` component list | `true` | `true` | none | | `FolderIndexDbStorageDriver` | `folder-indexdb` | Same backing store, scoped via `pipeline_registry` | `true` | `true` | none | | `LocalFileSystemDriver` | `local-fs` | File System Access API directory handle | `false` | `false` | `DriverPermissions` | +| `HostStorageDriver` | `host` | `window.__TANGLE_PIPELINE_STORAGE_HOST__` | `false` | `false` | none | ### Class Hierarchy @@ -168,6 +181,68 @@ classDiagram --- +## Storage mode + +A deployment declares whether pipelines live outside the browser. It is not +inferred from whether a host happens to be present, because a page that failed +to install one would otherwise read as "browser storage" and quietly strand a +user's work where nobody else can see it. + +``` +VITE_PIPELINE_STORAGE_BETA = "true" → a host is required +anything else, or unset → browser storage (the open-source build) +``` + +`storageMode.ts` resolves this once per page load and freezes the answer, +holding the host object itself rather than re-reading the global: + +| `StorageMode` | When | Root folder | +| -------------------------- | --------------------- | --------------------------------------------- | +| `{ kind: "local" }` | flag off | `folder-indexdb` on `ROOT_FOLDER_ID`, folders | +| `{ kind: "host", label }` | flag on, host present | `HostStorageDriver`, `isFlat`, no folders | +| `{ kind: "host-missing" }` | flag on, no host | `UnavailableStorageDriver`; the app blocks | + +In host mode the root folder **is** the host-backed folder, so every existing +`folderId === null ? rootFolder : findFolderById(id)` branch lands on the host +unchanged. `isFlat` is the property to test for "this store has no folders" — +it is what hides folder creation, the parent row, and move-to-folder. + +`host-missing` renders `PipelineStorageUnavailable` in place of the whole app. +Nothing can be read or written, and a blank library would be a lie. + +### The host contract + +`window.__TANGLE_PIPELINE_STORAGE_HOST__`, version-checked against +`PIPELINE_STORAGE_HOST_VERSION`. Five methods: `list`, `read`, `write`, +`delete`, `has`. Three properties of it shape the code above: + +- **`write(key, spec)` upserts on the caller's key.** The app chooses the key — + a new pipeline is written under its name — and the host returns its own + `externalId`, which becomes the pipeline's id and the editor url. +- **Renaming is a write.** A store that does not key on the name only learns a + new one from the spec, so `rename` and `write` are one operation + (`renamePipelineByName`). +- **Writes are last-write-wins**; the host does not reject on conflict. + `contentVersion` still detects "changed since we listed it" on the next + listing, which is what invalidates the spec cache. + +Errors cross a window boundary, where `instanceof` does not survive, so +`HostStorageDriver` duck-types a `code` off the rejection and turns it into a +`HostStorageError` carrying prose that names the store. `storageErrors.ts` +decides what is worth another attempt: an expired session or an ambiguous name +will answer the same way next time; an unreachable store may not. + +### Copying browser pipelines in + +`hostMigration.ts` copies everything in browser storage into the host once, +keyed on each pipeline's local name. Nothing local is deleted — a build with no +host still has to find its pipelines. The claim is a Dexie transaction, honoured +only while its holder keeps making progress, so two tabs cannot both copy and a +tab that died does not block the next one. It starts wherever the app opens; +the pipeline list reports progress and offers to retry anything that failed. + +--- + ## Database Schema Dexie database name: `tangle_pipelines` @@ -176,8 +251,26 @@ Dexie database name: `tangle_pipelines` erDiagram pipeline_registry { string id PK - string storageKey UK "globally unique" + string storage "local | host" + string storageKey UK "unique within one store" string folderId FK + string contentVersion "optional, host only" + } + + pipeline_specs { + string storage PK + string storageKey PK + string version + json spec + } + + host_migration { + string id PK + number startedAt + number completedAt "optional" + number dismissedAt "optional" + json copied + json failed } folders { @@ -195,14 +288,24 @@ erDiagram ### Tables and Indexes -| Table | Primary Key | Indexed Fields | Notes | -| ------------------- | ----------- | ---------------------------------------------------------------------- | -------------------------------------- | -| `pipeline_registry` | `id` | `&storageKey` (unique), `folderId`, `[folderId+storageKey]` (compound) | Maps pipeline names to folders | -| `folders` | `id` | `parentId` | Stores folder tree with `driverConfig` | +| Table | Primary Key | Indexed Fields | Notes | +| ------------------- | ---------------------- | ----------------------------------------------------------------------------------------------------- | -------------------------------------------- | +| `pipeline_registry` | `id` | `storage`, `folderId`, `&[storage+storageKey]`, `[storage+folderId]`, `[storage+folderId+storageKey]` | Maps storage keys to folders, per store | +| `folders` | `id` | `parentId` | Folder tree with `driverConfig`; local only | +| `pipeline_specs` | `[storage+storageKey]` | — | Cached contents, validated by `version` | +| `host_migration` | `id` | — | One row, `"v1"`: the copy's claim and result | ### Migrations -- **v1**: Creates `pipeline_registry` (with compound index `[folderId+storageKey]`) and `folders`. Runs an upgrade that reads the legacy `RootFolderDbStorageDriver.list()` and seeds `pipeline_registry` with every known pipeline assigned to `ROOT_FOLDER_ID` (`"__root__"`). +- **v1**: Creates `pipeline_registry` and `folders`. +- **v2**: Adds `remoteStorageKey`, for an earlier design where the host was a folder beside browser storage. +- **v3**: Drops it again, and deletes the host folder and its rows — the host is the root now, not a child. +- **v4**: Adds `pipeline_specs`. +- **v5**: Adds `host_migration`. +- **v6**: Drops `pipeline_specs`; a primary key cannot be changed in place and it is about to gain one. +- **v7**: Remakes `pipeline_specs` keyed by store, and scopes `pipeline_registry` the same way. Rows written before this do not say which store they describe: only a host reports a `contentVersion`, and a row filed in a folder can only be the browser's, which is enough to attribute them. + +On open (not in a migration), `seedRegistryFromLegacyList` claims every pipeline in the legacy list that has no local row. It is skipped in host mode, where those names mean nothing. --- @@ -214,20 +317,32 @@ erDiagram | Function | Purpose | | ------------------------------------------ | ------------------------------------------------------------------------------------------------------------- | -| `addEntry(entry)` | Insert a new registry row | +| `claimEntry(entry)` | Insert a row, or return the one already holding that key — in a transaction, so two listings cannot both add | | `updateEntry(id, updates)` | Partial update (e.g., change `folderId` on move, `storageKey` on rename) | | `deleteEntry(id)` | Remove a single row | | `findById(id)` | Lookup by primary key | -| `findByStorageKey(key)` | Lookup by unique `storageKey` index | -| `getAllEntries()` | Return every row | +| `findByStorageKey(key)` | Lookup by key **within the current store** | | `getAllByFolderId(folderId)` | Return all pipelines in a folder | | `findByFolderAndStorageKey(folderId, key)` | Compound index lookup | | `assertStorageKeyUnique(key)` | Throws if `storageKey` already exists | | `deleteFoldersAndDetachEntries(folderIds)` | **Transactional**: moves entries from deleted folders back to `ROOT_FOLDER_ID`, then bulk-deletes the folders | -### Constraint: Global Uniqueness of `storageKey` +### Constraint: one key, one store + +Every lookup is scoped to the store this page load is using, and `storageKey` is +unique **within** that store (`&[storage+storageKey]`). The scope is read from +`currentStorageKind()` rather than passed in, because storage mode is fixed for +the life of the page and a caller that could get it wrong eventually does. + +This matters because the two stores share a namespace by nature: browser storage +keys a pipeline on its name, and the key sent to a host is that same name. An +unscoped row was therefore found by whichever store asked first — a link to a +host pipeline, opened in the browser-storage build, paired the host's identity +with the browser's driver and read, then saved over, whatever pipeline happened +to share the name. -The `storageKey` column has a **unique index** (`&storageKey`). This means pipeline names are globally unique across all folders. The `assertStorageKeyUnique` guard must be called before creating new pipelines. +Within browser storage, pipeline names remain globally unique across folders; +`assertStorageKeyUnique` must still be called before creating a pipeline there. --- @@ -295,21 +410,55 @@ Represents a folder that contains pipeline files and subfolders. ### PipelineStorageService -The top-level entry point. Owns the `rootFolder` (a `PipelineFolder` with `folder-indexdb` driver for `ROOT_FOLDER_ID`). +The top-level entry point. Owns the `rootFolder`, which is whatever the storage +mode says it is. ```typescript class PipelineStorageService { + readonly mode: StorageMode; rootFolder: PipelineFolder; // @observable + resolve(ref: PipelineRef): Promise; + listAllPipelines(): Promise; + findPipelineById(id: string): Promise; - resolvePipelineByName(name: string): Promise; + findPipelineByName(name: string): Promise; + createPipeline(name, content): Promise; + savePipelineByName(name, content, source?): Promise; + renamePipelineByName( + currentName, + newName, + content, + source?, + ): Promise; + deletePipelineByName(name): Promise; + findFolderById(id: string): Promise; getAllFolders(): Promise; getFavoriteFolders(): Promise; } ``` -`resolvePipelineByName` first checks the registry by `storageKey`. If found, it resolves the owning folder. If not, it falls back to `rootFolder.findFile(name)` for legacy compatibility. +**`resolve` is the one way to turn a route's reference into a file.** A +`PipelineRef` is `{ name, fileId? }`, and a route carrying a single segment +offers it as both — what a path means depends on the store, and links outlive +that. A `fileId` given alongside a _different_ name is taken at its word: a miss +there means the pipeline is gone, not that a namesake should be opened instead. + +`findPipelineByName` goes through the registry in browser storage and falls back +to adopting a pipeline that predates it. In a flat store there is no such +history: the listing answers, matching the key first and then a **unique** +`displayName` — two pipelines sharing a name raise `AmbiguousPipelineNameError` +rather than one being guessed at. + +`listAllPipelines` is the flat list behind `/pipelines`. A flat store's listing +is already everything; browser storage keeps one list behind however many +folders point into it, so the whole list is read rather than the root's share. + +Non-React callers reach the same instance through `getPipelineStorageService()`. +`ComponentSpecProvider` is mounted _outside_ `PipelineStorageProvider` and must +use it; components inside the provider must use `usePipelineStorage()`, or the +tour's storage override stops applying to them. ### PipelineStorageProvider @@ -320,7 +469,10 @@ function PipelineStorageProvider({ children }: { children: ReactNode }); function usePipelineStorage(): PipelineStorageService; ``` -The service is instantiated lazily via `useState(() => new PipelineStorageService())` so there is exactly one instance per mount. The provider is mounted in `RootLayout.tsx`, making the service available to the entire application. +The provider hands out the module singleton, so React and non-React callers +share one instance. It is mounted in `RootLayout.tsx`, and is also where the +one-time copy into a host store is started. `TourPipelineStorageProvider` +supplies its own service over the same context for the guided tours. ### Consumer Map @@ -377,9 +529,12 @@ graph LR ## Key Flows -### Load Pipeline by Name +### Load a pipeline from a route -The Editor opens a pipeline by name via the URL. The storage service resolves the name to a `PipelineFile`, then reads its content. +The editor opens whatever the url's one segment identifies — a name in browser +storage, the store's own id where the store hands out ids — via `resolve`, then +reads its content. The flow below is browser storage; a flat store answers the +same question from its listing instead of the registry. ```mermaid sequenceDiagram @@ -391,7 +546,10 @@ sequenceDiagram participant Driver as PipelineStorageDriver participant File as PipelineFile - Editor->>+Service: resolvePipelineByName(name) + Editor->>+Service: resolve({ name, fileId }) + Service->>+Registry: findById(fileId), scoped to this store + Registry-->>-Service: undefined + Note over Service,Registry: A miss on a fileId equal to the name
falls through to the name Service->>+Registry: findByStorageKey(name) Registry-->>-Service: entry or undefined @@ -411,7 +569,7 @@ sequenceDiagram Folder->>+Registry: findByStorageKey(name) Registry-->>-Folder: undefined Note over Folder,Registry: Lazy-creates registry entry
with new UUID - Folder->>Registry: addEntry(newEntry) + Folder->>Registry: claimEntry(newEntry) Folder-->>-Service: PipelineFile end @@ -448,7 +606,7 @@ sequenceDiagram else storageKey is unique Registry-->>-Folder: ok Note over Folder: Generates UUID
via crypto.randomUUID() - Folder->>+Registry: addEntry({ id, storageKey, folderId }) + Folder->>+Registry: claimEntry({ id, storageKey, folderId }) Registry-->>-Folder: ok Folder->>+Driver: write(storageKey, yamlContent) Driver-->>-Folder: ok @@ -609,9 +767,9 @@ sequenceDiagram ### Storage Key Uniqueness -- `storageKey` is **globally unique** across all folders (enforced by the `&storageKey` unique index). -- Always call `assertStorageKeyUnique` before creating a new pipeline. -- Renaming a pipeline changes its `storageKey` in both the driver and the registry. +- `storageKey` is unique **within one store** (enforced by `&[storage+storageKey]`), and every registry lookup is scoped to the store in use. Never query the table without that scope. +- In browser storage, that still makes pipeline names globally unique across folders. Always call `assertStorageKeyUnique` before creating a new pipeline. +- Renaming a pipeline changes its `storageKey` in both the driver and the registry — except in a flat store, where the key never changes and the name lives in the spec. ### Move Permissions @@ -652,6 +810,7 @@ sequenceDiagram ### Database Migrations -- v1 migration reads all pipelines from the legacy `RootFolderDbStorageDriver` and seeds the `pipeline_registry` with `ROOT_FOLDER_ID` as their folder. -- v2 adds the compound index `[folderId+storageKey]` for the `findByFolderAndStorageKey` query used by `FolderIndexDbStorageDriver.hasKey()`. +- See [Migrations](#migrations) for what each version did. - New migrations must follow Dexie's versioning rules: increment the version number and never modify existing version schemas. +- A primary key cannot be changed in place. Drop the table in one version (`{ table: null }`) and remake it in the next, and only for a table that can be rebuilt. +- Everything in this database is a cache of what a store already holds, with one exception: `folders`, and the `folderId` on each registry row, are the **only** record of which folder a pipeline is in. Never clear the registry wholesale to fix something. From 4526cf8372a61e117585d2a5415e098cca6dfb92 Mon Sep 17 00:00:00 2001 From: Camiel van Schoonhoven Date: Wed, 9 Sep 2026 09:57:59 -0700 Subject: [PATCH 27/36] fix(pipelines): say the backend is not available before showing its pipelines MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reacting to a failed listing was not enough. A listing that is slow, still working through its retries, or answered from cache leaves the library on screen saying nothing — so turning the deployment off and opening the pipelines page showed a full list of pipelines that could no longer be read or written. Both pipeline pages now ask whether the backend is answering, and say so in the same info box the runs list uses. The provider pings once at startup and never again, which cannot catch a deployment that goes down while a tab is open, so this re-asks on mount, on return to the tab, and on a slow interval while the page is there. Only in the beta: with the flag off nothing is asked and nothing changes. Co-Authored-By: Claude Opus 5 (1M context) --- .../Home/PipelineSection/PipelineSection.tsx | 9 +++++ src/components/shared/BackendUnavailable.tsx | 9 +++++ src/hooks/useStorageBackendUnavailable.ts | 34 +++++++++++++++++++ .../pages/PipelineFolders/PipelineFolders.tsx | 9 ++++- tests/e2e/pipeline-storage-host.spec.ts | 25 ++++++++++++++ 5 files changed, 85 insertions(+), 1 deletion(-) create mode 100644 src/components/shared/BackendUnavailable.tsx create mode 100644 src/hooks/useStorageBackendUnavailable.ts diff --git a/src/components/Home/PipelineSection/PipelineSection.tsx b/src/components/Home/PipelineSection/PipelineSection.tsx index 156927ad97..b85c5ce5d0 100644 --- a/src/components/Home/PipelineSection/PipelineSection.tsx +++ b/src/components/Home/PipelineSection/PipelineSection.tsx @@ -2,6 +2,7 @@ import { Link } from "@tanstack/react-router"; import { useState } from "react"; import { ExamplePipelines } from "@/components/Learn/ExamplePipelines"; +import { BackendUnavailable } from "@/components/shared/BackendUnavailable"; import { LoadingScreen } from "@/components/shared/LoadingScreen"; import NewPipelineButton from "@/components/shared/NewPipelineButton"; import { PaginationControls } from "@/components/shared/PaginationControls"; @@ -22,6 +23,7 @@ import { } from "@/components/ui/table"; import { Paragraph, Text } from "@/components/ui/typography"; import { usePagination } from "@/hooks/usePagination"; +import { useStorageBackendUnavailable } from "@/hooks/useStorageBackendUnavailable"; import { APP_ROUTES } from "@/routes/router"; import { usePipelineStorage } from "@/services/pipelineStorage/PipelineStorageProvider"; @@ -66,6 +68,7 @@ export const PipelineSection = withSuspenseWrapper( const [selectedIds, setSelectedIds] = useState>(new Set()); const storage = usePipelineStorage(); + const backendUnavailable = useStorageBackendUnavailable(); const { entries, isLoading, error, pendingCount, refetch } = usePipelineListEntries(); @@ -102,6 +105,12 @@ export const PipelineSection = withSuspenseWrapper( setSelectedIds(next); }; + /** + * Ahead of everything else, including a listing already in hand: a store + * that cannot be reached must not be represented by the last answer it gave. + */ + if (backendUnavailable) return ; + if (migration.phase === "copying" || migration.phase === "incomplete") { return ( + The configured backend is currently unavailable. + + ); +} diff --git a/src/hooks/useStorageBackendUnavailable.ts b/src/hooks/useStorageBackendUnavailable.ts new file mode 100644 index 0000000000..4ef5758cc7 --- /dev/null +++ b/src/hooks/useStorageBackendUnavailable.ts @@ -0,0 +1,34 @@ +import { useQuery } from "@tanstack/react-query"; + +import { useBackend } from "@/providers/BackendProvider"; +import { isHostStorage } from "@/services/pipelineStorage/storageMode"; + +const PING_STALE_MS = 10_000; + +const PING_INTERVAL_MS = 30_000; + +/** + * Whether the deployment holding the pipelines is answering. Asked directly + * rather than inferred from a failed listing: a listing that is slow, still + * retrying, or served from cache leaves a stale library on screen saying + * nothing, which is the one thing this must never do. + * + * The provider's flag alone is not enough — it is pinged when the app starts + * and not again — so this re-asks on mount, on return to the tab, and while the + * page is open, which is where a deployment usually goes down. + */ +export function useStorageBackendUnavailable(): boolean { + const { ping, backendUrl } = useBackend(); + const hostStorage = isHostStorage(); + + const { data: reachable } = useQuery({ + queryKey: ["pipeline-storage-backend", backendUrl], + queryFn: () => ping({ notifyResult: false }), + enabled: hostStorage, + staleTime: PING_STALE_MS, + refetchInterval: PING_INTERVAL_MS, + retry: false, + }); + + return hostStorage && reachable === false; +} diff --git a/src/routes/v2/pages/PipelineFolders/PipelineFolders.tsx b/src/routes/v2/pages/PipelineFolders/PipelineFolders.tsx index 3bc5bcce7c..9e6551211e 100644 --- a/src/routes/v2/pages/PipelineFolders/PipelineFolders.tsx +++ b/src/routes/v2/pages/PipelineFolders/PipelineFolders.tsx @@ -1,9 +1,11 @@ import { useSearch } from "@tanstack/react-router"; import { useState } from "react"; +import { BackendUnavailable } from "@/components/shared/BackendUnavailable"; import { withSuspenseWrapper } from "@/components/shared/SuspenseWrapper"; import { BlockStack, InlineStack } from "@/components/ui/layout"; import { Skeleton } from "@/components/ui/skeleton"; +import { useStorageBackendUnavailable } from "@/hooks/useStorageBackendUnavailable"; import { usePipelineStorage } from "@/services/pipelineStorage/PipelineStorageProvider"; import { type PipelineRef, @@ -43,13 +45,18 @@ export const PipelineFolders = withSuspenseWrapper( }; const [localFolderId, setLocalFolderId] = useState(null); const hasFolders = !usePipelineStorage().rootFolder.isFlat; + const backendUnavailable = useStorageBackendUnavailable(); const isEmbedded = onPipelineClick !== undefined; const currentFolderId = isEmbedded ? localFolderId : (routeFolderId ?? ROOT_FOLDER_ID); - const content = ( + const content = backendUnavailable ? ( + + + + ) : ( { .toEqual(["Churn model v2", "Nightly refresh"]); }); + test("says the backend is not available, even holding a listing it could show", async ({ + page, + }) => { + await installSeededHost(page); + + await page.goto("/pipelines"); + await expect(page.getByText("Churn model")).toBeVisible(); + + await page.route(/\/services\/ping/, (route) => + route.fulfill({ status: 503, body: "down" }), + ); + await page.reload(); + + await expect(page.getByTestId("info-box-warning")).toContainText( + "Backend not available", + ); + await expect(page.getByText("Churn model")).toBeHidden(); + + await page.goto("/pipeline-folders"); + await expect(page.getByTestId("info-box-warning")).toContainText( + "Backend not available", + ); + await expect(page.getByText("Churn model")).toBeHidden(); + }); + test("says the list could not be read rather than showing an empty library", async ({ page, }) => { From a3dab435522fead298363d72f57699c0e4ea8d21 Mon Sep 17 00:00:00 2001 From: Camiel van Schoonhoven Date: Wed, 9 Sep 2026 10:10:43 -0700 Subject: [PATCH 28/36] fix(editor): show auto-save as off while the backend is away MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The indicator only turned red once a write had already been refused, and read "auto-save enabled" until then — the reassuring state, at exactly the moment nothing could be saved. It now takes the same answer the pipeline list does: if the backend is not answering, the icon is red and the button does not invite a save that cannot happen. It goes back on its own when the next ping lands. Co-Authored-By: Claude Opus 5 (1M context) --- .../components/AutoSaveIndicator.tsx | 21 +++++++++++++++-- tests/e2e/pipeline-storage-host.spec.ts | 23 +++++++++++++++++++ 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/src/routes/v2/pages/Editor/components/EditorMenuBar/components/AutoSaveIndicator.tsx b/src/routes/v2/pages/Editor/components/EditorMenuBar/components/AutoSaveIndicator.tsx index 9aa6c44494..d90d161324 100644 --- a/src/routes/v2/pages/Editor/components/EditorMenuBar/components/AutoSaveIndicator.tsx +++ b/src/routes/v2/pages/Editor/components/EditorMenuBar/components/AutoSaveIndicator.tsx @@ -4,6 +4,7 @@ import type { ReactNode } from "react"; import TooltipButton from "@/components/shared/Buttons/TooltipButton"; import { Icon } from "@/components/ui/icon"; import { Spinner } from "@/components/ui/spinner"; +import { useStorageBackendUnavailable } from "@/hooks/useStorageBackendUnavailable"; import { cn } from "@/lib/utils"; import { useEditorSession } from "@/routes/v2/pages/Editor/store/EditorSessionContext"; import { tracking } from "@/utils/tracking"; @@ -46,7 +47,13 @@ function getTooltipText( lastSavedAt: Date | null, saveError: string | null, hasPendingChanges: boolean, + backendUnavailable: boolean, ): string { + if (backendUnavailable) { + return hasPendingChanges + ? "Backend not available. Your changes are kept here and will be saved when it is back." + : "Backend not available. Nothing can be saved until it is back."; + } if (isSaving) return "Saving..."; if (saveError) { return hasPendingChanges @@ -62,13 +69,23 @@ function getTooltipText( export const AutoSaveIndicator = observer(function AutoSaveIndicator() { const { autoSave } = useEditorSession(); const { isSaving, lastSavedAt, saveError, hasPendingChanges } = autoSave; + const backendUnavailable = useStorageBackendUnavailable(); const tooltipText = getTooltipText( isSaving, lastSavedAt, saveError, hasPendingChanges, + backendUnavailable, ); + /** + * Saying "auto-save enabled" while nothing can be saved is the one thing this + * must not do, so the state is taken from whether the store is answering and + * not only from a write that has already been refused. Clicking it would ask + * for a save that cannot happen, so it does not invite one. + */ + const cannotSave = backendUnavailable || Boolean(saveError); + const handleClick = () => { void autoSave.save(); }; @@ -77,7 +94,7 @@ export const AutoSaveIndicator = observer(function AutoSaveIndicator() { - {saveError ? ( + {cannotSave ? ( ) : ( diff --git a/tests/e2e/pipeline-storage-host.spec.ts b/tests/e2e/pipeline-storage-host.spec.ts index 11bba57045..3356bc668d 100644 --- a/tests/e2e/pipeline-storage-host.spec.ts +++ b/tests/e2e/pipeline-storage-host.spec.ts @@ -266,6 +266,29 @@ test.describe("host-provided pipeline storage", () => { await expect(page.getByText(/don't have any pipelines yet/i)).toBeHidden(); }); + test("shows auto-save as off in the editor while the backend is away", async ({ + page, + }) => { + await installSeededHost(page); + await page.route(/\/services\/ping/, (route) => + route.fulfill({ status: 503, body: "down" }), + ); + + await page.goto(`/editor-v2/${SEED[0].key}`); + await expect(page.locator('[data-testid="rf__wrapper"]')).toBeVisible({ + timeout: 30_000, + }); + + const indicator = page.getByTestId("auto-save-button"); + await expect(indicator).toBeDisabled(); + await expect(indicator.locator(".text-destructive")).toBeVisible(); + + // Recovers on the next ping rather than needing the editor reopened. + await page.unroute(/\/services\/ping/); + await expect(indicator).toBeEnabled({ timeout: 60_000 }); + await expect(indicator.locator(".text-destructive")).toBeHidden(); + }); + test("says so in the editor when a save is refused, and stops once it lands", async ({ page, }) => { From 46cdfe752137389d41ed9eccfe5c6ef9aee98546 Mon Sep 17 00:00:00 2001 From: Camiel van Schoonhoven Date: Wed, 9 Sep 2026 10:27:09 -0700 Subject: [PATCH 29/36] fix(pipeline-storage): take availability from the store, not the runs backend MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The pipeline list and the auto-save indicator were reading the health of the configured execution backend, which is a different system: it can be switched off in Settings while pipelines carry on saving, and it can answer while the store that holds them has gone. Both readings were wrong in one direction or the other. Every call to the store already passes through one funnel, so that is where it is now learned — no health check against anything else, and nothing to go stale between calls. Only a failure to answer counts: "no such pipeline" is an answer. Co-Authored-By: Claude Opus 5 (1M context) --- .../Home/PipelineSection/PipelineSection.tsx | 6 +- src/components/shared/BackendUnavailable.tsx | 6 +- src/hooks/useStorageBackendUnavailable.ts | 34 ------------ .../components/AutoSaveIndicator.tsx | 15 ++--- .../pages/PipelineFolders/PipelineFolders.tsx | 6 +- .../drivers/HostStorageDriver.ts | 14 ++++- src/services/pipelineStorage/storageHealth.ts | 55 +++++++++++++++++++ tests/e2e/pipeline-storage-host.spec.ts | 33 ++++++----- 8 files changed, 106 insertions(+), 63 deletions(-) delete mode 100644 src/hooks/useStorageBackendUnavailable.ts create mode 100644 src/services/pipelineStorage/storageHealth.ts diff --git a/src/components/Home/PipelineSection/PipelineSection.tsx b/src/components/Home/PipelineSection/PipelineSection.tsx index b85c5ce5d0..39397113c2 100644 --- a/src/components/Home/PipelineSection/PipelineSection.tsx +++ b/src/components/Home/PipelineSection/PipelineSection.tsx @@ -23,9 +23,9 @@ import { } from "@/components/ui/table"; import { Paragraph, Text } from "@/components/ui/typography"; import { usePagination } from "@/hooks/usePagination"; -import { useStorageBackendUnavailable } from "@/hooks/useStorageBackendUnavailable"; import { APP_ROUTES } from "@/routes/router"; import { usePipelineStorage } from "@/services/pipelineStorage/PipelineStorageProvider"; +import { useStorageUnavailable } from "@/services/pipelineStorage/storageHealth"; import BulkActionsBar from "./BulkActionsBar"; import { HostMigrationNotice } from "./HostMigrationNotice"; @@ -68,7 +68,7 @@ export const PipelineSection = withSuspenseWrapper( const [selectedIds, setSelectedIds] = useState>(new Set()); const storage = usePipelineStorage(); - const backendUnavailable = useStorageBackendUnavailable(); + const storeUnavailable = useStorageUnavailable(); const { entries, isLoading, error, pendingCount, refetch } = usePipelineListEntries(); @@ -109,7 +109,7 @@ export const PipelineSection = withSuspenseWrapper( * Ahead of everything else, including a listing already in hand: a store * that cannot be reached must not be represented by the last answer it gave. */ - if (backendUnavailable) return ; + if (storeUnavailable) return ; if (migration.phase === "copying" || migration.phase === "incomplete") { return ( diff --git a/src/components/shared/BackendUnavailable.tsx b/src/components/shared/BackendUnavailable.tsx index ada50e739d..e7c3829d99 100644 --- a/src/components/shared/BackendUnavailable.tsx +++ b/src/components/shared/BackendUnavailable.tsx @@ -1,9 +1,13 @@ import { InfoBox } from "@/components/shared/InfoBox"; +import { getPipelineStorageService } from "@/services/pipelineStorage/PipelineStorageService"; export function BackendUnavailable() { + const { mode } = getPipelineStorageService(); + return ( - The configured backend is currently unavailable. + {mode.kind === "host" ? mode.label : "Pipeline storage"} is not answering. + Pipelines cannot be read or saved until it is back. ); } diff --git a/src/hooks/useStorageBackendUnavailable.ts b/src/hooks/useStorageBackendUnavailable.ts deleted file mode 100644 index 4ef5758cc7..0000000000 --- a/src/hooks/useStorageBackendUnavailable.ts +++ /dev/null @@ -1,34 +0,0 @@ -import { useQuery } from "@tanstack/react-query"; - -import { useBackend } from "@/providers/BackendProvider"; -import { isHostStorage } from "@/services/pipelineStorage/storageMode"; - -const PING_STALE_MS = 10_000; - -const PING_INTERVAL_MS = 30_000; - -/** - * Whether the deployment holding the pipelines is answering. Asked directly - * rather than inferred from a failed listing: a listing that is slow, still - * retrying, or served from cache leaves a stale library on screen saying - * nothing, which is the one thing this must never do. - * - * The provider's flag alone is not enough — it is pinged when the app starts - * and not again — so this re-asks on mount, on return to the tab, and while the - * page is open, which is where a deployment usually goes down. - */ -export function useStorageBackendUnavailable(): boolean { - const { ping, backendUrl } = useBackend(); - const hostStorage = isHostStorage(); - - const { data: reachable } = useQuery({ - queryKey: ["pipeline-storage-backend", backendUrl], - queryFn: () => ping({ notifyResult: false }), - enabled: hostStorage, - staleTime: PING_STALE_MS, - refetchInterval: PING_INTERVAL_MS, - retry: false, - }); - - return hostStorage && reachable === false; -} diff --git a/src/routes/v2/pages/Editor/components/EditorMenuBar/components/AutoSaveIndicator.tsx b/src/routes/v2/pages/Editor/components/EditorMenuBar/components/AutoSaveIndicator.tsx index d90d161324..27d459b4f5 100644 --- a/src/routes/v2/pages/Editor/components/EditorMenuBar/components/AutoSaveIndicator.tsx +++ b/src/routes/v2/pages/Editor/components/EditorMenuBar/components/AutoSaveIndicator.tsx @@ -4,9 +4,9 @@ import type { ReactNode } from "react"; import TooltipButton from "@/components/shared/Buttons/TooltipButton"; import { Icon } from "@/components/ui/icon"; import { Spinner } from "@/components/ui/spinner"; -import { useStorageBackendUnavailable } from "@/hooks/useStorageBackendUnavailable"; import { cn } from "@/lib/utils"; import { useEditorSession } from "@/routes/v2/pages/Editor/store/EditorSessionContext"; +import { useStorageUnavailable } from "@/services/pipelineStorage/storageHealth"; import { tracking } from "@/utils/tracking"; const LAYER_BASE_CLASS = @@ -47,13 +47,14 @@ function getTooltipText( lastSavedAt: Date | null, saveError: string | null, hasPendingChanges: boolean, - backendUnavailable: boolean, + storeUnavailable: boolean, ): string { - if (backendUnavailable) { + if (storeUnavailable) { return hasPendingChanges ? "Backend not available. Your changes are kept here and will be saved when it is back." : "Backend not available. Nothing can be saved until it is back."; } + if (isSaving) return "Saving..."; if (saveError) { return hasPendingChanges @@ -69,13 +70,13 @@ function getTooltipText( export const AutoSaveIndicator = observer(function AutoSaveIndicator() { const { autoSave } = useEditorSession(); const { isSaving, lastSavedAt, saveError, hasPendingChanges } = autoSave; - const backendUnavailable = useStorageBackendUnavailable(); + const storeUnavailable = useStorageUnavailable(); const tooltipText = getTooltipText( isSaving, lastSavedAt, saveError, hasPendingChanges, - backendUnavailable, + storeUnavailable, ); /** @@ -84,7 +85,7 @@ export const AutoSaveIndicator = observer(function AutoSaveIndicator() { * not only from a write that has already been refused. Clicking it would ask * for a save that cannot happen, so it does not invite one. */ - const cannotSave = backendUnavailable || Boolean(saveError); + const cannotSave = storeUnavailable || Boolean(saveError); const handleClick = () => { void autoSave.save(); @@ -94,7 +95,7 @@ export const AutoSaveIndicator = observer(function AutoSaveIndicator() { (null); const hasFolders = !usePipelineStorage().rootFolder.isFlat; - const backendUnavailable = useStorageBackendUnavailable(); + const storeUnavailable = useStorageUnavailable(); const isEmbedded = onPipelineClick !== undefined; const currentFolderId = isEmbedded ? localFolderId : (routeFolderId ?? ROOT_FOLDER_ID); - const content = backendUnavailable ? ( + const content = storeUnavailable ? ( diff --git a/src/services/pipelineStorage/drivers/HostStorageDriver.ts b/src/services/pipelineStorage/drivers/HostStorageDriver.ts index e9ae7bb81e..8e14747da4 100644 --- a/src/services/pipelineStorage/drivers/HostStorageDriver.ts +++ b/src/services/pipelineStorage/drivers/HostStorageDriver.ts @@ -7,6 +7,7 @@ import type { HostPipelineSummary, PipelineStorageHost, } from "../host/contract"; +import { reportStorageAnswered, reportStorageFailed } from "../storageHealth"; import { HOST_DRIVER_TYPE, type PipelineFileDescriptor, @@ -74,11 +75,20 @@ export class HostStorageDriver implements PipelineStorageDriver { return this.call(() => this.host.has(storageKey)); } + /** + * Every call to the store passes through here, which makes it the one place + * that knows whether the store is answering — no separate health check, and + * nothing to go stale between calls. + */ private async call(operation: () => Promise): Promise { try { - return await operation(); + const answer = await operation(); + reportStorageAnswered(); + return answer; } catch (error) { - throw this.toStorageError(error); + const failure = this.toStorageError(error); + reportStorageFailed(failure.code); + throw failure; } } diff --git a/src/services/pipelineStorage/storageHealth.ts b/src/services/pipelineStorage/storageHealth.ts new file mode 100644 index 0000000000..09941175c4 --- /dev/null +++ b/src/services/pipelineStorage/storageHealth.ts @@ -0,0 +1,55 @@ +import { useSyncExternalStore } from "react"; + +import type { HostErrorCode } from "./host/contract"; +import { isHostStorage } from "./storageMode"; + +/** + * Whether the store holding the pipelines is answering, learned from the calls + * the app already makes rather than from a health check against something else. + * + * A host-provided store is not the execution backend: it is served by the page + * that embeds this app, from its own endpoint and session, and stays up when + * the configured backend is switched off. Pinging that backend to decide + * whether pipelines can be saved reports an outage while saves are landing, and + * says nothing when the store itself is the thing that has gone. + * + * An error is only an outage if the store failed to answer at all. "No such + * pipeline" is an answer. + */ +let reachable = true; + +const listeners = new Set<() => void>(); + +function set(next: boolean): void { + if (reachable === next) return; + reachable = next; + for (const listener of listeners) listener(); +} + +export function reportStorageAnswered(): void { + set(true); +} + +export function reportStorageFailed(code: HostErrorCode): void { + if (code !== "unavailable") { + set(true); + return; + } + + set(false); +} + +function subscribe(listener: () => void): () => void { + listeners.add(listener); + return () => listeners.delete(listener); +} + +export function useStorageUnavailable(): boolean { + const answering = useSyncExternalStore( + subscribe, + () => reachable, + () => true, + ); + + return isHostStorage() && !answering; +} diff --git a/tests/e2e/pipeline-storage-host.spec.ts b/tests/e2e/pipeline-storage-host.spec.ts index 3356bc668d..9036de7371 100644 --- a/tests/e2e/pipeline-storage-host.spec.ts +++ b/tests/e2e/pipeline-storage-host.spec.ts @@ -236,27 +236,32 @@ test.describe("host-provided pipeline storage", () => { await page.goto("/pipelines"); await expect(page.getByText("Churn model")).toBeVisible(); - await page.route(/\/services\/ping/, (route) => - route.fulfill({ status: 503, body: "down" }), - ); - await page.reload(); + // Not a reload: the listing is already in hand and would otherwise render. + await setHostFailMode(page, "unavailable"); + await page.getByRole("button", { name: "Refresh" }).click(); await expect(page.getByTestId("info-box-warning")).toContainText( "Backend not available", ); await expect(page.getByText("Churn model")).toBeHidden(); + }); + + test("says the backend is not available on the folders page too", async ({ + page, + }) => { + await installSeededHost(page, { failMode: "unavailable" }); await page.goto("/pipeline-folders"); + await expect(page.getByTestId("info-box-warning")).toContainText( "Backend not available", ); - await expect(page.getByText("Churn model")).toBeHidden(); }); - test("says the list could not be read rather than showing an empty library", async ({ + test("says a store that refuses could not be read, not that it is empty", async ({ page, }) => { - await installSeededHost(page, { failMode: "unavailable" }); + await installSeededHost(page, { failMode: "unauthenticated" }); await page.goto("/pipelines"); @@ -270,9 +275,6 @@ test.describe("host-provided pipeline storage", () => { page, }) => { await installSeededHost(page); - await page.route(/\/services\/ping/, (route) => - route.fulfill({ status: 503, body: "down" }), - ); await page.goto(`/editor-v2/${SEED[0].key}`); await expect(page.locator('[data-testid="rf__wrapper"]')).toBeVisible({ @@ -280,12 +282,17 @@ test.describe("host-provided pipeline storage", () => { }); const indicator = page.getByTestId("auto-save-button"); + await expect(indicator).toBeEnabled(); + + await setHostFailMode(page, "unavailable"); + await indicator.click(); + await expect(indicator).toBeDisabled(); await expect(indicator.locator(".text-destructive")).toBeVisible(); - // Recovers on the next ping rather than needing the editor reopened. - await page.unroute(/\/services\/ping/); - await expect(indicator).toBeEnabled({ timeout: 60_000 }); + // Comes back on its own: the held edit is retried and the store answers. + await setHostFailMode(page, "none"); + await expect(indicator).toBeEnabled({ timeout: 30_000 }); await expect(indicator.locator(".text-destructive")).toBeHidden(); }); From 3bd127eddfc431c1af32626856318d5c8420a765 Mon Sep 17 00:00:00 2001 From: Camiel van Schoonhoven Date: Wed, 9 Sep 2026 11:07:58 -0700 Subject: [PATCH 30/36] refactor(pipeline-storage): store pipelines on the configured backend MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pipelines were read and written through a store the embedding page installed on a window global, so this app knew nothing of the API behind it. That put the feature outside the very system built for pointing this editor at a backend: switching backends in Settings did not move the pipelines, switching one off did not stop them saving, and nobody running their own backend could serve them. They now go to `/api/users/me/pipelines` on whichever backend the app is configured against — the same place runs, secrets and components come from. A backend that serves those routes can hold pipelines for this editor, which is the point: the routes are a contract anyone can implement, not a private arrangement with one deployment. The driver interface was the seam, so the rest stands: the flat root, rows scoped to the store that wrote them, the copy out of browser storage, ids in the url, and every error surface. Where the backend is is read per request rather than frozen at startup, because that address is a setting and a setting can change while the app is open. Co-Authored-By: Claude Opus 5 (1M context) --- playwright.config.ci.ts | 6 +- playwright.config.ts | 10 +- .../Home/PipelineSection/useHostMigration.ts | 2 +- .../layout/PipelineStorageUnavailable.tsx | 31 -- src/components/layout/RootLayout.tsx | 6 - src/components/shared/BackendUnavailable.tsx | 4 +- .../shared/Dialogs/PipelineNameDialog.tsx | 4 +- .../shared/PipelineStorageError.tsx | 3 +- src/hooks/useSessionPipelineStats.ts | 4 +- src/providers/BackendProvider.tsx | 10 + src/routes/editorRoutes.test.ts | 12 +- src/routes/editorRoutes.ts | 4 +- .../PipelineStorageProvider.tsx | 2 +- .../PipelineStorageService.test.ts | 180 ++++---- .../pipelineStorage/PipelineStorageService.ts | 15 +- .../pipelineStorage/backendEndpoint.ts | 21 + src/services/pipelineStorage/createDriver.ts | 14 +- src/services/pipelineStorage/db.ts | 8 +- .../drivers/BackendStorageDriver.ts | 302 +++++++++++++ .../drivers/HostStorageDriver.test.ts | 405 ------------------ .../drivers/HostStorageDriver.ts | 155 ------- .../drivers/UnavailableStorageDriver.ts | 43 -- src/services/pipelineStorage/host/contract.ts | 33 -- .../pipelineStorage/host/detectHost.test.ts | 101 ----- .../pipelineStorage/host/detectHost.ts | 40 -- .../pipelineStorage/pipelineRegistry.test.ts | 42 +- src/services/pipelineStorage/storageErrors.ts | 10 +- src/services/pipelineStorage/storageHealth.ts | 16 +- .../pipelineStorage/storageMode.test.ts | 67 +-- src/services/pipelineStorage/storageMode.ts | 68 +-- src/services/pipelineStorage/types.ts | 11 +- ...orageHost.ts => pipelineStorageBackend.ts} | 306 +++++++------ ...ec.ts => pipeline-storage-backend.spec.ts} | 106 ++--- 33 files changed, 744 insertions(+), 1297 deletions(-) delete mode 100644 src/components/layout/PipelineStorageUnavailable.tsx create mode 100644 src/services/pipelineStorage/backendEndpoint.ts create mode 100644 src/services/pipelineStorage/drivers/BackendStorageDriver.ts delete mode 100644 src/services/pipelineStorage/drivers/HostStorageDriver.test.ts delete mode 100644 src/services/pipelineStorage/drivers/HostStorageDriver.ts delete mode 100644 src/services/pipelineStorage/drivers/UnavailableStorageDriver.ts delete mode 100644 src/services/pipelineStorage/host/contract.ts delete mode 100644 src/services/pipelineStorage/host/detectHost.test.ts delete mode 100644 src/services/pipelineStorage/host/detectHost.ts rename tests/e2e/fixtures/{pipelineStorageHost.ts => pipelineStorageBackend.ts} (52%) rename tests/e2e/{pipeline-storage-host.spec.ts => pipeline-storage-backend.spec.ts} (78%) diff --git a/playwright.config.ci.ts b/playwright.config.ci.ts index 74dcc0edd1..09a13d72c0 100644 --- a/playwright.config.ci.ts +++ b/playwright.config.ci.ts @@ -12,7 +12,9 @@ import { defineConfig, devices } from "@playwright/test"; * Host-provided storage is a build-time switch, so it cannot be turned on per * test — those specs need a server of their own with the flag set. */ -const HOST_STORAGE_TESTS = "**/pipeline-storage-host.spec.ts"; +const HOST_STORAGE_TESTS = "**/pipeline-storage-backend.spec.ts"; + +const BACKEND_STUB_URL = "http://backend.test"; const HOST_STORAGE_PORT = 3011; const HOST_STORAGE_URL = `http://localhost:${HOST_STORAGE_PORT}`; @@ -73,7 +75,7 @@ export default defineConfig({ timeout: 120 * 1000, }, { - command: `VITE_PIPELINE_STORAGE_BETA=true vite --port ${HOST_STORAGE_PORT}`, + command: `VITE_PIPELINE_STORAGE_BETA=true VITE_BACKEND_API_URL=${BACKEND_STUB_URL} vite --port ${HOST_STORAGE_PORT}`, url: HOST_STORAGE_URL, reuseExistingServer: false, timeout: 120 * 1000, diff --git a/playwright.config.ts b/playwright.config.ts index 709bbea611..e2ed9db504 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -4,7 +4,13 @@ import { defineConfig, devices } from "@playwright/test"; * Host-provided storage is a build-time switch, so it cannot be turned on per * test — those specs need a server of their own with the flag set. */ -const HOST_STORAGE_TESTS = "**/pipeline-storage-host.spec.ts"; +const HOST_STORAGE_TESTS = "**/pipeline-storage-backend.spec.ts"; + +/** + * Pinned so the pipeline routes are answered by the test and never by whatever + * a developer happens to be running. + */ +const BACKEND_STUB_URL = "http://backend.test"; const HOST_STORAGE_PORT = 3010; const HOST_STORAGE_URL = `http://localhost:${HOST_STORAGE_PORT}`; @@ -86,7 +92,7 @@ export default defineConfig({ timeout: 120 * 1000, }, { - command: `VITE_PIPELINE_STORAGE_BETA=true vite --port ${HOST_STORAGE_PORT}`, + command: `VITE_PIPELINE_STORAGE_BETA=true VITE_BACKEND_API_URL=${BACKEND_STUB_URL} vite --port ${HOST_STORAGE_PORT}`, url: HOST_STORAGE_URL, reuseExistingServer: !process.env.CI, timeout: 120 * 1000, diff --git a/src/components/Home/PipelineSection/useHostMigration.ts b/src/components/Home/PipelineSection/useHostMigration.ts index ec0e4c76cd..35b1002be9 100644 --- a/src/components/Home/PipelineSection/useHostMigration.ts +++ b/src/components/Home/PipelineSection/useHostMigration.ts @@ -33,7 +33,7 @@ const POLL_MS = 1_000; */ export function useHostMigration(onFinished: () => void): HostMigration { const storage = usePipelineStorage(); - const isHost = storage.mode.kind === "host"; + const isHost = storage.mode.kind === "backend"; const [phase, setPhase] = useState( isHost ? "checking" : "settled", diff --git a/src/components/layout/PipelineStorageUnavailable.tsx b/src/components/layout/PipelineStorageUnavailable.tsx deleted file mode 100644 index 7eda351377..0000000000 --- a/src/components/layout/PipelineStorageUnavailable.tsx +++ /dev/null @@ -1,31 +0,0 @@ -import { Button } from "@/components/ui/button"; -import { Icon } from "@/components/ui/icon"; -import { BlockStack } from "@/components/ui/layout"; -import { Heading, Paragraph } from "@/components/ui/typography"; - -/** - * Shown instead of the app when the deployment stores pipelines outside the - * browser and the page did not provide that store. Everything here reads and - * writes pipelines, so carrying on would either show an empty library or save - * work where nobody will look for it. - */ -export function PipelineStorageUnavailable() { - return ( - - - Pipeline storage is not available - - This deployment keeps your pipelines outside the browser, and that store - did not load. Nothing has been lost — reload to try again, and if it - keeps happening, report it rather than working around it. - - - - ); -} diff --git a/src/components/layout/RootLayout.tsx b/src/components/layout/RootLayout.tsx index a5bdedcc68..d4f3a52fc0 100644 --- a/src/components/layout/RootLayout.tsx +++ b/src/components/layout/RootLayout.tsx @@ -13,10 +13,8 @@ import { ComponentSpecProvider } from "@/providers/ComponentSpecProvider"; import { OnboardingProvider } from "@/providers/OnboardingProvider/OnboardingProvider"; import { TourProvider } from "@/providers/TourProvider/TourProvider"; import { PipelineStorageProvider } from "@/services/pipelineStorage/PipelineStorageProvider"; -import { isHostStorageMissing } from "@/services/pipelineStorage/storageMode"; import AppMenu from "./AppMenu"; -import { PipelineStorageUnavailable } from "./PipelineStorageUnavailable"; function SessionPipelineStatsTracker() { useSessionPipelineStats(); @@ -27,10 +25,6 @@ function RootLayoutContent() { usePageViewTracking(); useClickTracking(); - if (isHostStorageMissing()) { - return ; - } - return ( diff --git a/src/components/shared/BackendUnavailable.tsx b/src/components/shared/BackendUnavailable.tsx index e7c3829d99..a4d129ebf3 100644 --- a/src/components/shared/BackendUnavailable.tsx +++ b/src/components/shared/BackendUnavailable.tsx @@ -6,8 +6,8 @@ export function BackendUnavailable() { return ( - {mode.kind === "host" ? mode.label : "Pipeline storage"} is not answering. - Pipelines cannot be read or saved until it is back. + {mode.kind === "backend" ? mode.label : "Pipeline storage"} is not + answering. Pipelines cannot be read or saved until it is back. ); } diff --git a/src/components/shared/Dialogs/PipelineNameDialog.tsx b/src/components/shared/Dialogs/PipelineNameDialog.tsx index b79653bb41..dc5dc9e529 100644 --- a/src/components/shared/Dialogs/PipelineNameDialog.tsx +++ b/src/components/shared/Dialogs/PipelineNameDialog.tsx @@ -16,7 +16,7 @@ import { Icon } from "@/components/ui/icon"; import { Input } from "@/components/ui/input"; import { BlockStack } from "@/components/ui/layout"; import useLoadUserPipelines from "@/hooks/useLoadUserPipelines"; -import { isHostStorage } from "@/services/pipelineStorage/storageMode"; +import { isBackendStorage } from "@/services/pipelineStorage/storageMode"; interface PipelineNameDialogProps { trigger?: ReactNode; @@ -55,7 +55,7 @@ const PipelineNameDialog = ({ * so refusing a duplicate here would block a name the store itself accepts — * and there is nothing to list for. */ - const namesMustBeUnique = !isHostStorage(); + const namesMustBeUnique = !isBackendStorage(); const { pipelineNames, diff --git a/src/components/shared/PipelineStorageError.tsx b/src/components/shared/PipelineStorageError.tsx index 4a6b666e5a..aafc2f0618 100644 --- a/src/components/shared/PipelineStorageError.tsx +++ b/src/components/shared/PipelineStorageError.tsx @@ -23,7 +23,8 @@ export function PipelineStorageError({ onRetry?: () => void; }) { const { mode } = getPipelineStorageService(); - const storageLabel = mode.kind === "host" ? mode.label : "Pipeline storage"; + const storageLabel = + mode.kind === "backend" ? mode.label : "Pipeline storage"; const headline = error instanceof PipelineNotFoundError || diff --git a/src/hooks/useSessionPipelineStats.ts b/src/hooks/useSessionPipelineStats.ts index cd569b93d3..ad309d9342 100644 --- a/src/hooks/useSessionPipelineStats.ts +++ b/src/hooks/useSessionPipelineStats.ts @@ -2,7 +2,7 @@ import { useEffect } from "react"; import { useAnalytics } from "@/providers/AnalyticsProvider"; import { listPipelineFiles } from "@/services/pipelineStorage/pipelineOperations"; -import { isHostStorage } from "@/services/pipelineStorage/storageMode"; +import { isBackendStorage } from "@/services/pipelineStorage/storageMode"; import type { ComponentSpec, TaskSpec } from "@/utils/componentSpec"; import { isGraphImplementation } from "@/utils/componentSpec"; import { getAllComponentFilesFromList } from "@/utils/componentStore"; @@ -116,7 +116,7 @@ export function useSessionPipelineStats(): void { * every pipeline once a day just for analytics. The total is worth that * much less than the request storm, so the distribution is left out. */ - if (isHostStorage()) { + if (isBackendStorage()) { try { total_pipelines = (await listPipelineFiles()).length; } catch { diff --git a/src/providers/BackendProvider.tsx b/src/providers/BackendProvider.tsx index f073112b77..bb28c8797d 100644 --- a/src/providers/BackendProvider.tsx +++ b/src/providers/BackendProvider.tsx @@ -8,6 +8,7 @@ import { import { client } from "@/api/client.gen"; import useToastNotification from "@/hooks/useToastNotification"; +import { setBackendEndpoint } from "@/services/pipelineStorage/backendEndpoint"; import { API_URL } from "@/utils/constants"; import { getUseEnv, @@ -147,6 +148,15 @@ export const BackendProvider = ({ children }: { children: ReactNode }) => { } }, [backendUrl, settingsLoaded]); + /** + * Pipeline storage reaches the same backend from outside React, and this is + * where the setting is resolved. Published whether or not the ping succeeds: + * where to send a request is a different question from whether it will work. + */ + useEffect(() => { + setBackendEndpoint(backendUrl); + }, [backendUrl]); + useEffect(() => { const getSettings = async () => { const url = await getUserBackendUrl(); diff --git a/src/routes/editorRoutes.test.ts b/src/routes/editorRoutes.test.ts index 8c0033a52c..04bacc5f5c 100644 --- a/src/routes/editorRoutes.test.ts +++ b/src/routes/editorRoutes.test.ts @@ -2,10 +2,12 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { getDefaultEditorHref, getDefaultEditorTarget } from "./editorRoutes"; -const { hostStorage } = vi.hoisted(() => ({ hostStorage: vi.fn(() => false) })); +const { backendStorage } = vi.hoisted(() => ({ + backendStorage: vi.fn(() => false), +})); vi.mock("@/services/pipelineStorage/storageMode", () => ({ - isHostStorage: hostStorage, + isBackendStorage: backendStorage, })); vi.mock("@/components/shared/Settings/useFlags", () => ({ @@ -15,12 +17,12 @@ vi.mock("@/components/shared/Settings/useFlags", () => ({ const REF = { name: "Churn model", fileId: "ab420234-a05f" }; afterEach(() => { - hostStorage.mockReturnValue(false); + backendStorage.mockReturnValue(false); }); describe("where a store hands out its own ids", () => { it("puts the id in the path and nothing else anywhere", () => { - hostStorage.mockReturnValue(true); + backendStorage.mockReturnValue(true); expect(getDefaultEditorTarget(REF)).toEqual({ to: "/editor-v2/$pipelineName", @@ -30,7 +32,7 @@ describe("where a store hands out its own ids", () => { }); it("falls back to the name for a pipeline with no id yet", () => { - hostStorage.mockReturnValue(true); + backendStorage.mockReturnValue(true); expect(getDefaultEditorHref({ name: "Churn model" })).toBe( "/editor-v2/Churn%20model", diff --git a/src/routes/editorRoutes.ts b/src/routes/editorRoutes.ts index 0c116a4cea..101e78a2f0 100644 --- a/src/routes/editorRoutes.ts +++ b/src/routes/editorRoutes.ts @@ -1,5 +1,5 @@ import { isFlagEnabled } from "@/components/shared/Settings/useFlags"; -import { isHostStorage } from "@/services/pipelineStorage/storageMode"; +import { isBackendStorage } from "@/services/pipelineStorage/storageMode"; import type { PipelineRef } from "@/services/pipelineStorage/types"; import { APP_ROUTES, EDITOR_PATH } from "./appRoutes"; @@ -17,7 +17,7 @@ export interface EditorTarget { * and being the identity, it cannot go stale when one is renamed. */ function editorSegment(ref: PipelineRef): string { - return isHostStorage() && ref.fileId ? ref.fileId : ref.name; + return isBackendStorage() && ref.fileId ? ref.fileId : ref.name; } export function getDefaultEditorTarget(ref: PipelineRef): EditorTarget { diff --git a/src/services/pipelineStorage/PipelineStorageProvider.tsx b/src/services/pipelineStorage/PipelineStorageProvider.tsx index bacd9ac190..3e958b4aef 100644 --- a/src/services/pipelineStorage/PipelineStorageProvider.tsx +++ b/src/services/pipelineStorage/PipelineStorageProvider.tsx @@ -20,7 +20,7 @@ export function PipelineStorageProvider({ children }: { children: ReactNode }) { const [service] = useState(getPipelineStorageService); useEffect(() => { - if (service.mode.kind !== "host") return; + if (service.mode.kind !== "backend") return; void startHostMigration(service.rootFolder).catch((error: unknown) => { console.error("Could not copy pipelines into the host store:", error); diff --git a/src/services/pipelineStorage/PipelineStorageService.test.ts b/src/services/pipelineStorage/PipelineStorageService.test.ts index 367fef37d3..1bf453c4d4 100644 --- a/src/services/pipelineStorage/PipelineStorageService.test.ts +++ b/src/services/pipelineStorage/PipelineStorageService.test.ts @@ -1,7 +1,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { setBackendEndpoint } from "./backendEndpoint"; import { RootFolderDbStorageDriver } from "./drivers/RootFolderDbStorageDriver"; -import type { HostPipelineSummary, PipelineStorageHost } from "./host/contract"; import type { NewPipelineRegistryEntry } from "./pipelineRegistry"; import { AmbiguousPipelineNameError, @@ -10,7 +10,7 @@ import { } from "./PipelineStorageService"; import { resetStorageModeForTests } from "./storageMode"; import { - HOST_DRIVER_TYPE, + BACKEND_DRIVER_TYPE, type PipelineRegistryEntry, ROOT_FOLDER_ID, } from "./types"; @@ -76,62 +76,96 @@ vi.mock("./db", () => ({ }, })); -const LABEL = "Shared storage"; +const LABEL = "Backend"; -function summary(key: string, displayName: string): HostPipelineSummary { +interface StoredRow { + id: string; + file_path: string; + pipeline_name: string; + current_version: string; + root_pipeline_task: { componentRef: { spec: unknown } }; +} + +function summary(key: string, displayName: string): StoredRow { return { - key, - externalId: `id-${key}`, - displayName, - contentVersion: "1", + id: `id-${key}`, + file_path: key, + pipeline_name: displayName, + current_version: "1", + root_pipeline_task: { componentRef: { spec: {} } }, }; } +const ENDPOINT = "https://backend.test"; + /** - * Mirrors the two host behaviours the write paths are built on: `write` upserts - * on the caller's key, and the displayed name comes from the written spec. + * Answers the pipeline routes out of a map, mirroring the two behaviours the + * write paths are built on: a write upserts on the key it is given, and the + * displayed name comes from the spec that was written. * - * A host is only used when the deployment asks for one, so turning the beta on - * is part of installing it. + * The backend holds pipelines only when the deployment asks it to, so turning + * the beta on is part of installing this. */ -function installHost( - listing: HostPipelineSummary[] = [], -): Map { +function installBackend(listing: StoredRow[] = []): Map { vi.stubEnv("VITE_PIPELINE_STORAGE_BETA", "true"); + setBackendEndpoint(ENDPOINT); + + const rows = new Map(listing.map((row) => [row.file_path, row])); + const specs = new Map( + listing.map((row) => [ + row.file_path, + row.root_pipeline_task.componentRef.spec, + ]), + ); + + const answer = (body: unknown, status = 200) => + Promise.resolve( + new Response(status === 204 ? null : JSON.stringify(body), { status }), + ); - const summaries = new Map(listing.map((entry) => [entry.key, entry])); - const specs = new Map(); - - const host: PipelineStorageHost = { - version: 1, - label: LABEL, - list: async () => [...summaries.values()], - read: async (key) => { - const found = summaries.get(key); - if (!found) throw new Error(`not seeded: ${key}`); - return { ...found, spec: specs.get(key) }; - }, - write: async (key, spec) => { - specs.set(key, spec); - const written = { - ...summary(key, (spec as { name?: string }).name ?? "Untitled"), - contentVersion: String(summaries.size + 1), - }; - summaries.set(key, written); - return written; + vi.spyOn(globalThis, "fetch").mockImplementation( + async (input: RequestInfo | URL, init?: RequestInit) => { + const url = new URL(String(input)); + const key = url.searchParams.get("file_path") ?? ""; + const isList = url.pathname.endsWith("/all"); + + if (isList) { + const matched = [...rows.values()].filter( + (row) => !key || row.file_path === key, + ); + return answer({ pipelines: matched, next_page_token: null }); + } + + switch (init?.method) { + case "PUT": { + const sent = JSON.parse(String(init.body)) as { + root_pipeline_task: { componentRef: { spec: { name?: string } } }; + }; + const spec = sent.root_pipeline_task.componentRef.spec; + specs.set(key, spec); + const written: StoredRow = { + ...summary(key, spec.name ?? "Untitled"), + current_version: String(rows.size + 1), + root_pipeline_task: { componentRef: { spec } }, + }; + rows.set(key, written); + return answer(written); + } + case "DELETE": + rows.delete(key); + specs.delete(key); + return answer(null, 204); + default: { + const found = rows.get(key); + if (!found) return answer({ detail: "not found" }, 404); + return answer({ + ...found, + root_pipeline_task: { componentRef: { spec: specs.get(key) } }, + }); + } + } }, - delete: async (key) => { - summaries.delete(key); - specs.delete(key); - }, - has: async (key) => summaries.has(key), - }; - - Object.defineProperty(window, "__TANGLE_PIPELINE_STORAGE_HOST__", { - value: host, - configurable: true, - writable: true, - }); + ); return specs; } @@ -145,18 +179,18 @@ beforeEach(() => { }); afterEach(() => { - delete window.__TANGLE_PIPELINE_STORAGE_HOST__; + setBackendEndpoint(""); vi.unstubAllEnvs(); resetStorageModeForTests(); vi.restoreAllMocks(); }); -describe("with no host on the page", () => { +describe("with the beta off", () => { it("keeps browser storage as the root", () => { const service = new PipelineStorageService(); expect(service.mode).toEqual({ kind: "local" }); - expect(service.rootFolder.driver.type).not.toBe(HOST_DRIVER_TYPE); + expect(service.rootFolder.driver.type).not.toBe(BACKEND_DRIVER_TYPE); expect(service.rootFolder.isFlat).toBe(false); }); @@ -168,17 +202,17 @@ describe("with no host on the page", () => { }); }); -describe("with a host on the page", () => { +describe("with the beta on", () => { beforeEach(() => { - installHost(); + installBackend(); }); - it("makes the host the only store, under the host's own label", () => { + it("makes the backend the only store", () => { const service = new PipelineStorageService(); - expect(service.mode).toEqual({ kind: "host", label: LABEL }); + expect(service.mode).toEqual({ kind: "backend", label: LABEL }); expect(service.rootFolder.id).toBe(ROOT_FOLDER_ID); - expect(service.rootFolder.driver.type).toBe(HOST_DRIVER_TYPE); + expect(service.rootFolder.driver.type).toBe(BACKEND_DRIVER_TYPE); expect(service.rootFolder.name).toBe(LABEL); }); @@ -206,7 +240,7 @@ describe("with a host on the page", () => { it("stays on the host even if the host global disappears mid-session", () => { const service = new PipelineStorageService(); - delete window.__TANGLE_PIPELINE_STORAGE_HOST__; + setBackendEndpoint(""); expect(new PipelineStorageService().mode).toEqual(service.mode); }); @@ -237,8 +271,8 @@ describe("the flat list of everything", () => { expect(registry.get("filed")?.folderId).toBe("folder-1"); }); - it("asks a host for its listing rather than the browser store", async () => { - installHost([summary("opaque-key-1", "Churn model")]); + it("asks the backend for its listing rather than the browser store", async () => { + installBackend([summary("opaque-key-1", "Churn model")]); const listed = vi.spyOn(RootFolderDbStorageDriver.prototype, "list"); const files = await new PipelineStorageService().listAllPipelines(); @@ -250,7 +284,7 @@ describe("the flat list of everything", () => { describe("resolving a route reference against a host", () => { it("opens the pipeline whose key the route carries", async () => { - installHost([summary("opaque-key-1", "Churn model")]); + installBackend([summary("opaque-key-1", "Churn model")]); const file = await new PipelineStorageService().resolve({ name: "opaque-key-1", @@ -260,7 +294,7 @@ describe("resolving a route reference against a host", () => { }); it("opens a pipeline by its displayed name when only one has it", async () => { - installHost([ + installBackend([ summary("opaque-key-1", "Churn model"), summary("opaque-key-2", "Ranking model"), ]); @@ -273,7 +307,7 @@ describe("resolving a route reference against a host", () => { }); it("refuses to guess between pipelines sharing a name", async () => { - installHost([ + installBackend([ summary("opaque-key-1", "Churn model"), summary("opaque-key-2", "Churn model"), ]); @@ -284,7 +318,7 @@ describe("resolving a route reference against a host", () => { }); it("reports a missing pipeline rather than reaching for browser storage", async () => { - installHost([summary("opaque-key-1", "Churn model")]); + installBackend([summary("opaque-key-1", "Churn model")]); await expect( new PipelineStorageService().resolve({ name: "Churn model v2" }), @@ -292,7 +326,7 @@ describe("resolving a route reference against a host", () => { }); it("opens a pipeline from a path that carries only its id", async () => { - installHost([summary("opaque-key-1", "Churn model")]); + installBackend([summary("opaque-key-1", "Churn model")]); const file = await new PipelineStorageService().resolve({ name: "id-opaque-key-1", @@ -303,7 +337,7 @@ describe("resolving a route reference against a host", () => { }); it("still opens an older link whose path carries a name", async () => { - installHost([summary("opaque-key-1", "Churn model")]); + installBackend([summary("opaque-key-1", "Churn model")]); const file = await new PipelineStorageService().resolve({ name: "Churn model", @@ -314,7 +348,7 @@ describe("resolving a route reference against a host", () => { }); it("refuses a link whose id is gone rather than opening a namesake", async () => { - installHost([summary("opaque-key-1", "Churn model")]); + installBackend([summary("opaque-key-1", "Churn model")]); await expect( new PipelineStorageService().resolve({ @@ -325,7 +359,7 @@ describe("resolving a route reference against a host", () => { }); it("finds a pipeline the registry has never seen by its id", async () => { - installHost([summary("opaque-key-1", "Churn model")]); + installBackend([summary("opaque-key-1", "Churn model")]); const file = await new PipelineStorageService().resolve({ name: "whatever-the-link-said", @@ -338,7 +372,7 @@ describe("resolving a route reference against a host", () => { describe("writing to a host", () => { it("creates a pipeline and registers the identity the store reported", async () => { - installHost(); + installBackend(); const service = new PipelineStorageService(); const file = await service.createPipeline( @@ -352,7 +386,7 @@ describe("writing to a host", () => { }); it("saves over the pipeline that already has the name", async () => { - const specs = installHost(); + const specs = installBackend(); const service = new PipelineStorageService(); await service.createPipeline("Churn model", PIPELINE_YAML("Churn model")); @@ -366,7 +400,7 @@ describe("writing to a host", () => { }); it("creates a pipeline when saving a name the store has never held", async () => { - installHost(); + installBackend(); const service = new PipelineStorageService(); const file = await service.savePipelineByName( @@ -378,7 +412,7 @@ describe("writing to a host", () => { }); it("deletes through the driver so the store loses the pipeline too", async () => { - installHost(); + installBackend(); const service = new PipelineStorageService(); await service.createPipeline("Churn model", PIPELINE_YAML("Churn model")); @@ -389,7 +423,7 @@ describe("writing to a host", () => { }); it("renames in place rather than leaving a copy under the old name", async () => { - installHost(); + installBackend(); const service = new PipelineStorageService(); const created = await service.createPipeline( "Churn model", @@ -408,7 +442,7 @@ describe("writing to a host", () => { }); it("creates the pipeline when renaming one the store does not hold", async () => { - installHost(); + installBackend(); const service = new PipelineStorageService(); const file = await service.renamePipelineByName( @@ -422,7 +456,7 @@ describe("writing to a host", () => { }); it("leaves the store alone when asked to delete a name it does not hold", async () => { - installHost([summary("opaque-key-1", "Churn model")]); + installBackend([summary("opaque-key-1", "Churn model")]); const service = new PipelineStorageService(); await expect( diff --git a/src/services/pipelineStorage/PipelineStorageService.ts b/src/services/pipelineStorage/PipelineStorageService.ts index d1f4257c29..4e020d6b85 100644 --- a/src/services/pipelineStorage/PipelineStorageService.ts +++ b/src/services/pipelineStorage/PipelineStorageService.ts @@ -3,7 +3,6 @@ import { makeObservable, observable } from "mobx"; import { createDriver } from "./createDriver"; import { pipelineStorageDb } from "./db"; import { RootFolderDbStorageDriver } from "./drivers/RootFolderDbStorageDriver"; -import { UnavailableStorageDriver } from "./drivers/UnavailableStorageDriver"; import { PipelineFile } from "./PipelineFile"; import type { PipelineFileSource } from "./pipelineFileEvents"; import { PipelineFolder } from "./PipelineFolder"; @@ -261,22 +260,12 @@ export function getPipelineStorageService(): PipelineStorageService { } function createRoot(mode: StorageMode): PipelineFolder { - if (mode.kind === "host") { + if (mode.kind === "backend") { return new PipelineFolder({ id: ROOT_FOLDER_ID, name: mode.label, parentId: null, - driver: createDriver({ driverType: "host" }), - isFlat: true, - }); - } - - if (mode.kind === "host-missing") { - return new PipelineFolder({ - id: ROOT_FOLDER_ID, - name: "Pipeline storage", - parentId: null, - driver: new UnavailableStorageDriver(), + driver: createDriver({ driverType: "backend" }), isFlat: true, }); } diff --git a/src/services/pipelineStorage/backendEndpoint.ts b/src/services/pipelineStorage/backendEndpoint.ts new file mode 100644 index 0000000000..91b33bd6cc --- /dev/null +++ b/src/services/pipelineStorage/backendEndpoint.ts @@ -0,0 +1,21 @@ +import { API_URL } from "@/utils/constants"; + +/** + * Where the backend is, for the parts of pipeline storage that are not React + * and cannot ask the provider. `BackendProvider` owns the setting — env, + * relative path, or one the user typed — and publishes it here whenever it + * changes, so there is still one place that decides it. + * + * Until it does, the environment's own answer stands: a provider publishes + * from an effect, and effects run child-first, so the first read of the + * pipeline list can happen before the provider above it has said anything. + */ +let published: string | undefined; + +export function setBackendEndpoint(url: string): void { + published = url.trim(); +} + +export function getBackendEndpoint(): string { + return published ?? API_URL ?? window.location.origin; +} diff --git a/src/services/pipelineStorage/createDriver.ts b/src/services/pipelineStorage/createDriver.ts index 058e42d540..a185ea13c7 100644 --- a/src/services/pipelineStorage/createDriver.ts +++ b/src/services/pipelineStorage/createDriver.ts @@ -1,10 +1,9 @@ import { getGoogleDriveAuth } from "../googleDrive/GoogleDriveAuthService"; // google-drive import { GoogleDriveStorageDriver } from "../googleDrive/GoogleDriveStorageDriver"; // google-drive +import { BackendStorageDriver } from "./drivers/BackendStorageDriver"; import { FolderIndexDbStorageDriver } from "./drivers/FolderIndexDbStorageDriver"; -import { HostStorageDriver } from "./drivers/HostStorageDriver"; import { LocalFileSystemDriver } from "./drivers/LocalFileSystemDriver"; import { RootFolderDbStorageDriver } from "./drivers/RootFolderDbStorageDriver"; -import { getStorageHost } from "./storageMode"; import type { DriverConfig, PipelineStorageDriver } from "./types"; export function createDriver(config: DriverConfig): PipelineStorageDriver { @@ -15,15 +14,8 @@ export function createDriver(config: DriverConfig): PipelineStorageDriver { return new FolderIndexDbStorageDriver(config.folderId); case "local-fs": return new LocalFileSystemDriver(config.handle); - case "host": { - const host = getStorageHost(); - if (!host) { - throw new Error( - "Host-provided pipeline storage is not available on this page", - ); - } - return new HostStorageDriver(host); - } + case "backend": + return new BackendStorageDriver(); case "google-drive": // google-drive return new GoogleDriveStorageDriver( config.folderId, diff --git a/src/services/pipelineStorage/db.ts b/src/services/pipelineStorage/db.ts index 258a99a1ec..501ce021a8 100644 --- a/src/services/pipelineStorage/db.ts +++ b/src/services/pipelineStorage/db.ts @@ -56,7 +56,7 @@ pipelineStorageDb */ const hostFolders = await tx .table("folders") - .filter((folder) => folder.driverConfig.driverType === "host") + .filter((folder) => folder.driverConfig.driverType === "backend") .toArray(); for (const folder of hostFolders) { @@ -110,10 +110,10 @@ pipelineStorageDb .table("pipeline_registry") .toCollection() .modify((entry) => { - const isHostRow = + const isBackendRow = entry.folderId === ROOT_FOLDER_ID && entry.contentVersion !== undefined; - entry.storage = isHostRow ? "host" : "local"; + entry.storage = isBackendRow ? "backend" : "local"; }); }); @@ -127,7 +127,7 @@ pipelineStorageDb.on("ready", async () => { * would claim files the host has never heard of. */ async function seedRegistryFromLegacyList() { - if (currentStorageKind() === "host") return; + if (currentStorageKind() === "backend") return; const seeded = await pipelineStorageDb.pipeline_registry .where("storage") diff --git a/src/services/pipelineStorage/drivers/BackendStorageDriver.ts b/src/services/pipelineStorage/drivers/BackendStorageDriver.ts new file mode 100644 index 0000000000..9978a71e78 --- /dev/null +++ b/src/services/pipelineStorage/drivers/BackendStorageDriver.ts @@ -0,0 +1,302 @@ +import { toPortablePipelineSpec } from "@/models/componentSpec/serialization/portablePipelineSpec"; +import { isValidComponentSpec } from "@/utils/componentSpec"; +import { componentSpecFromYaml, componentSpecToYaml } from "@/utils/yaml"; + +import { getBackendEndpoint } from "../backendEndpoint"; +import { reportStorageAnswered, reportStorageFailed } from "../storageHealth"; +import { + BACKEND_DRIVER_TYPE, + type PipelineFileDescriptor, + type PipelineStorageDriver, + type StorageErrorCode, +} from "../types"; + +export interface BackendDriverConfig { + driverType: "backend"; +} + +const PIPELINES_PATH = "/api/users/me/pipelines"; + +const LIST_PATH = "/api/users/me/pipelines/all"; + +const PAGE_SIZE = 100; + +const UNTITLED_PIPELINE_NAME = "Untitled pipeline"; + +export class BackendStorageError extends Error { + readonly name = "BackendStorageError"; + + constructor( + readonly code: StorageErrorCode, + message: string, + options?: ErrorOptions, + ) { + super(message, options); + } +} + +interface StoredPipeline { + id: string; + file_path: string; + pipeline_name?: string | null; + current_version?: string | null; + created_at?: string; + updated_at?: string; + root_pipeline_task?: { componentRef?: { spec?: unknown } }; +} + +interface PipelineListPage { + pipelines?: StoredPipeline[]; + next_page_token?: string | null; +} + +/** + * Pipelines kept by the backend the app is configured against, under the + * routes below. Nothing here is specific to a deployment: a backend that serves + * these routes can hold pipelines for this editor. + */ +export class BackendStorageDriver implements PipelineStorageDriver { + readonly type = BACKEND_DRIVER_TYPE; + readonly allowsMoveIn = false; + readonly allowsMoveOut = false; + readonly listingIsAuthoritative = true; + + async list(): Promise { + const rows: PipelineFileDescriptor[] = []; + const seen = new Set(); + + let token: string | undefined; + do { + const page = await this.request("GET", LIST_PATH, { + page_size: String(PAGE_SIZE), + page_token: token, + }); + + for (const row of page.pipelines ?? []) rows.push(toDescriptor(row)); + + const next = page.next_page_token ?? undefined; + /** + * A token that never advances would loop until the backend rate-limits + * the app and the pipeline list hangs. + */ + token = next && !seen.has(next) ? next : undefined; + if (token) seen.add(token); + } while (token); + + return rows; + } + + async read(storageKey: string): Promise { + const row = await this.request("GET", PIPELINES_PATH, { + file_path: storageKey, + }); + + const spec = row.root_pipeline_task?.componentRef?.spec; + if (!isValidComponentSpec(spec)) { + throw new BackendStorageError( + "unavailable", + `Pipeline "${storageKey}" came back in a format this editor cannot read.`, + ); + } + + return componentSpecToYaml(spec); + } + + /** + * An upsert on the caller's key. Renaming is a write of a differently-named + * spec to the same key: the displayed name is derived from the spec. + */ + async write( + storageKey: string, + content: string, + ): Promise { + const spec = toPortablePipelineSpec(componentSpecFromYaml(content)); + + const row = await this.request( + "PUT", + PIPELINES_PATH, + { file_path: storageKey }, + { root_pipeline_task: { componentRef: { spec } } }, + ); + + return toDescriptor(row); + } + + async delete(storageKey: string): Promise { + await this.request("DELETE", PIPELINES_PATH, { file_path: storageKey }); + } + + /** + * Filtering the listing answers this without transferring a spec. The filter + * is a prefix match, so the exact key is confirmed against what comes back. + */ + async hasKey(storageKey: string): Promise { + const page = await this.request("GET", LIST_PATH, { + file_path: storageKey, + page_size: String(PAGE_SIZE), + }); + + return (page.pipelines ?? []).some((row) => row.file_path === storageKey); + } + + private async request( + method: string, + path: string, + query: Record, + body?: unknown, + ): Promise { + const endpoint = getBackendEndpoint(); + if (!endpoint) { + throw this.reported( + new BackendStorageError( + "unavailable", + "No backend is configured, so pipelines cannot be read or saved. Set one in Settings.", + ), + ); + } + + const url = new URL(path, endpoint); + for (const [key, value] of Object.entries(query)) { + if (value !== undefined) url.searchParams.set(key, value); + } + + let response: Response; + try { + response = await fetch(url.toString(), { + method, + credentials: "include", + /** + * An expired session is answered with a cross-origin redirect, which + * arrives as an opaque response rather than a status code. + */ + redirect: "manual", + headers: + body === undefined ? {} : { "Content-Type": "application/json" }, + body: body === undefined ? undefined : JSON.stringify(body), + }); + } catch (error) { + throw this.reported( + new BackendStorageError( + "unavailable", + `The backend could not be reached. Try again in a moment.`, + { cause: error }, + ), + ); + } + + if (response.type === "opaqueredirect" || response.status === 0) { + throw this.reported( + new BackendStorageError( + "unauthenticated", + "Your session has expired. Reload the page to sign in again.", + ), + ); + } + + const raw = response.status === 204 ? "" : await response.text(); + let payload: unknown = null; + let parsed = false; + if (raw) { + try { + payload = JSON.parse(raw); + parsed = true; + } catch { + parsed = false; + } + } + + /** + * A non-JSON body means the request never reached the API — most likely a + * single-page-app fallback answering with the index document. + */ + if (raw && !parsed) { + throw this.reported( + new BackendStorageError( + "unavailable", + `${path} did not return JSON (HTTP ${response.status}). This backend is not serving the pipeline API.`, + ), + ); + } + + if (!response.ok) { + throw this.reported( + new BackendStorageError( + codeForStatus(response.status), + describeFailure(payload, response), + ), + ); + } + + reportStorageAnswered(); + return payload as T; + } + + private reported(error: BackendStorageError): BackendStorageError { + reportStorageFailed(error.code); + return error; + } +} + +function toDescriptor(row: StoredPipeline): PipelineFileDescriptor { + return { + storageKey: row.file_path, + externalId: row.id, + displayName: row.pipeline_name || UNTITLED_PIPELINE_NAME, + /** + * Deliberately not falling back to a revision counter: read as a content + * version it would report "unchanged" for an edited pipeline and leave a + * stale editor open. + */ + contentVersion: row.current_version || "", + createdAt: toDate(row.created_at), + modifiedAt: toDate(row.updated_at), + }; +} + +function toDate(value: string | undefined): Date | undefined { + if (!value) return undefined; + const date = new Date(value); + return Number.isNaN(date.getTime()) ? undefined : date; +} + +function codeForStatus(status: number): StorageErrorCode { + if (status === 401 || status === 403) return "unauthenticated"; + if (status === 404) return "not_found"; + if (status === 409) return "conflict"; + if (status === 429) return "rate_limited"; + return "unavailable"; +} + +/** + * Three error envelopes are in use: `detail` (a string or a validation array), + * `error`, and `error_message`. + */ +function describeFailure(payload: unknown, response: Response): string { + if (typeof payload === "object" && payload !== null) { + const detail: unknown = Reflect.get(payload, "detail"); + if (typeof detail === "string" && detail) return detail; + + if (Array.isArray(detail) && detail.length > 0) { + return detail.map(describeValidationEntry).join("; "); + } + + for (const field of ["error", "error_message", "message"]) { + const value: unknown = Reflect.get(payload, field); + if (typeof value === "string" && value) return value; + } + } + + return `HTTP ${response.status} ${response.statusText}`; +} + +function describeValidationEntry(entry: unknown): string { + if (typeof entry !== "object" || entry === null) return String(entry); + + const message = String(Reflect.get(entry, "msg") ?? ""); + const location: unknown = Reflect.get(entry, "loc"); + const where = Array.isArray(location) + ? location.filter((part) => part !== "body").join(".") + : ""; + + return where ? `${where}: ${message}` : message; +} diff --git a/src/services/pipelineStorage/drivers/HostStorageDriver.test.ts b/src/services/pipelineStorage/drivers/HostStorageDriver.test.ts deleted file mode 100644 index 0abd376555..0000000000 --- a/src/services/pipelineStorage/drivers/HostStorageDriver.test.ts +++ /dev/null @@ -1,405 +0,0 @@ -import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; - -import type { ComponentSpec } from "@/utils/componentSpec"; -import { componentSpecFromYaml, componentSpecToYaml } from "@/utils/yaml"; - -import { createDriver } from "../createDriver"; -import type { - HostErrorCode, - HostPipeline, - HostPipelineSummary, - PipelineStorageHost, -} from "../host/contract"; -import { resetStorageModeForTests } from "../storageMode"; -import type { PipelineStorageDriver } from "../types"; -import { HostStorageDriver, HostStorageError } from "./HostStorageDriver"; - -const LABEL = "Shared storage"; - -interface FakeHost extends PipelineStorageHost { - readonly writtenSpecs: unknown[]; - readonly readCalls: string[]; -} - -function createFakeHost(seed: Record = {}): FakeHost { - const store = new Map(Object.entries(seed)); - const writtenSpecs: unknown[] = []; - const readCalls: string[] = []; - let revision = 0; - - return { - version: 1, - label: LABEL, - writtenSpecs, - readCalls, - async list(): Promise { - return [...store.values()].map(({ spec: _spec, ...summary }) => summary); - }, - async read(key: string): Promise { - readCalls.push(key); - const pipeline = store.get(key); - if (!pipeline) throw { code: "not_found" }; - return pipeline; - }, - async write(key: string, spec: unknown): Promise { - writtenSpecs.push(spec); - revision += 1; - const existing = store.get(key); - const pipeline: HostPipeline = { - key, - externalId: existing?.externalId ?? `external-${store.size + 1}`, - displayName: readSpecName(spec), - contentVersion: `v${revision}`, - spec, - }; - store.set(key, pipeline); - const { spec: _spec, ...summary } = pipeline; - return summary; - }, - async delete(key: string): Promise { - store.delete(key); - }, - async has(key: string): Promise { - return store.has(key); - }, - }; -} - -function readSpecName(spec: unknown): string | null { - if (typeof spec !== "object" || spec === null) return null; - const name: unknown = Reflect.get(spec, "name"); - return typeof name === "string" ? name : null; -} - -function hostPipeline( - key: string, - overrides: Partial = {}, -): HostPipeline { - return { - key, - externalId: `external-${key}`, - displayName: "Churn model", - contentVersion: "v1", - spec: { name: "Churn model", implementation: { graph: { tasks: {} } } }, - ...overrides, - }; -} - -function rejectingHost(rejection: unknown): PipelineStorageHost { - const reject = () => Promise.reject(rejection); - return { - version: 1, - label: LABEL, - list: reject, - read: reject, - write: reject, - delete: reject, - has: reject, - }; -} - -describe("HostStorageDriver.list", () => { - it("maps summaries to descriptors without reading any pipeline", async () => { - const host = createFakeHost({ - "key-1": hostPipeline("key-1", { - createdAt: "2026-01-01T00:00:00.000Z", - modifiedAt: "2026-02-01T00:00:00.000Z", - }), - }); - - const descriptors = await new HostStorageDriver(host).list(); - - expect(descriptors).toEqual([ - { - storageKey: "key-1", - externalId: "external-key-1", - displayName: "Churn model", - contentVersion: "v1", - createdAt: new Date("2026-01-01T00:00:00.000Z"), - modifiedAt: new Date("2026-02-01T00:00:00.000Z"), - }, - ]); - expect(host.readCalls).toEqual([]); - }); - - it("falls back to a placeholder display name when the host has none", async () => { - const host = createFakeHost({ - "key-1": hostPipeline("key-1", { displayName: null }), - }); - - const [descriptor] = await new HostStorageDriver(host).list(); - - expect(descriptor.displayName).toBe("Untitled pipeline"); - }); - - it("omits timestamps the host reports unparseably", async () => { - const host = createFakeHost({ - "key-1": hostPipeline("key-1", { createdAt: "not-a-date" }), - }); - - const [descriptor] = await new HostStorageDriver(host).list(); - - expect(descriptor.createdAt).toBeUndefined(); - }); -}); - -describe("HostStorageDriver.write", () => { - it("projects the document before handing it to the host", async () => { - const host = createFakeHost(); - const yamlText = componentSpecToYaml({ - name: "Churn model", - inputs: [{ name: "epochs", default: "10", value: "42" }], - implementation: { - graph: { - tasks: { - train: { - componentRef: { - name: "Train", - favorited: true, - owned: true, - spec: { - name: "Train", - implementation: { container: { image: "python:3.11" } }, - }, - }, - }, - }, - }, - }, - }); - - await new HostStorageDriver(host).write("key-1", yamlText); - - const written = JSON.stringify(host.writtenSpecs[0]); - expect(written).not.toContain("favorited"); - expect(written).not.toContain("owned"); - expect(written).not.toContain('"value"'); - expect(written).toContain("epochs"); - }); - - it("returns the descriptor the host reports for the write", async () => { - const host = createFakeHost(); - const yamlText = componentSpecToYaml({ - name: "Churn model", - implementation: { graph: { tasks: {} } }, - }); - - const descriptor = await new HostStorageDriver(host).write( - "key-1", - yamlText, - ); - - expect(descriptor).toMatchObject({ - storageKey: "key-1", - displayName: "Churn model", - contentVersion: "v1", - }); - expect(descriptor.externalId).toBeDefined(); - }); - - it("reports a fresh contentVersion on every write", async () => { - const driver = new HostStorageDriver(createFakeHost()); - const yamlText = componentSpecToYaml({ - name: "Churn model", - implementation: { graph: { tasks: {} } }, - }); - - const first = await driver.write("key-1", yamlText); - const second = await driver.write("key-1", yamlText); - - expect(second.contentVersion).not.toBe(first.contentVersion); - }); -}); - -describe("HostStorageDriver round trip", () => { - it("returns YAML for a pipeline it previously saved", async () => { - const spec: ComponentSpec = { - name: "Churn model", - description: "Predicts churn", - inputs: [{ name: "epochs", type: "Integer", default: "10" }], - implementation: { - graph: { - tasks: { - train: { - componentRef: { - name: "Train", - spec: { - name: "Train", - implementation: { container: { image: "python:3.11" } }, - }, - }, - annotations: { "editor.position": '{"x":10,"y":20}' }, - }, - }, - }, - }, - }; - const driver = new HostStorageDriver(createFakeHost()); - - await driver.write("key-1", componentSpecToYaml(spec)); - const yamlText = await driver.read("key-1"); - - expect(componentSpecFromYaml(yamlText)).toEqual(spec); - }); - - it("rejects a pipeline the host returns in an unreadable shape", async () => { - const host = createFakeHost({ - "key-1": hostPipeline("key-1", { spec: { nope: true } }), - }); - - await expect(new HostStorageDriver(host).read("key-1")).rejects.toThrow( - HostStorageError, - ); - }); -}); - -describe("HostStorageDriver.rename", () => { - it("is not offered, because a key carries no name for the host", () => { - const driver: PipelineStorageDriver = new HostStorageDriver( - createFakeHost(), - ); - - expect(driver.rename).toBeUndefined(); - }); -}); - -describe("HostStorageDriver.delete and hasKey", () => { - it("removes the pipeline from the host", async () => { - const host = createFakeHost({ "key-1": hostPipeline("key-1") }); - const driver = new HostStorageDriver(host); - - expect(await driver.hasKey("key-1")).toBe(true); - await driver.delete("key-1"); - - expect(await driver.hasKey("key-1")).toBe(false); - }); -}); - -describe("HostStorageDriver error mapping", () => { - const cases: [HostErrorCode, RegExp][] = [ - ["unauthenticated", /session has expired/], - ["not_found", /no longer exists/], - ["rate_limited", /is busy/], - ["conflict", /changed in/], - ["unavailable", /could not be reached/], - ]; - - it.each(cases)( - "maps the %s code to its own message", - async (code, matcher) => { - const driver = new HostStorageDriver(rejectingHost({ code })); - - await expect(driver.list()).rejects.toMatchObject({ - code, - message: expect.stringMatching(matcher), - }); - }, - ); - - it.each([ - ["an unrecognised code", { code: "teapot" }], - ["no code at all", new Error("boom")], - ["a non-object rejection", "boom"], - ])("degrades %s to unavailable", async (_case, rejection) => { - const driver = new HostStorageDriver(rejectingHost(rejection)); - - await expect(driver.read("key-1")).rejects.toMatchObject({ - code: "unavailable", - }); - }); - - it("names the host label from the contract rather than a hardcoded name", async () => { - const host = { - ...rejectingHost({ code: "unavailable" }), - label: "Team drive", - }; - - await expect(new HostStorageDriver(host).list()).rejects.toThrow( - /Team drive/, - ); - }); - - it("keeps the original rejection as the error cause", async () => { - const rejection = { code: "conflict", detail: "revision mismatch" }; - const driver = new HostStorageDriver(rejectingHost(rejection)); - - const error = await driver.list().catch((thrown: unknown) => thrown); - - expect(error).toBeInstanceOf(HostStorageError); - expect((error as HostStorageError).cause).toBe(rejection); - }); - - it("maps every failing operation, not just reads", async () => { - const driver = new HostStorageDriver(rejectingHost({ code: "conflict" })); - const yamlText = componentSpecToYaml({ - name: "Churn model", - implementation: { graph: { tasks: {} } }, - }); - - await expect(driver.write("key-1", yamlText)).rejects.toMatchObject({ - code: "conflict", - }); - await expect(driver.delete("key-1")).rejects.toMatchObject({ - code: "conflict", - }); - await expect(driver.hasKey("key-1")).rejects.toMatchObject({ - code: "conflict", - }); - }); -}); - -describe("createDriver without a detected host", () => { - beforeEach(() => { - resetStorageModeForTests(); - }); - - afterEach(() => { - delete window.__TANGLE_PIPELINE_STORAGE_HOST__; - resetStorageModeForTests(); - }); - - it.each([ - ["no host global", undefined], - ["a host with a non-function member", { ...createFakeHost(), write: null }], - [ - "a host declaring an unsupported version", - { ...createFakeHost(), version: 99 }, - ], - ])("refuses to build a host driver with %s", (_case, host) => { - if (host) { - Object.defineProperty(window, "__TANGLE_PIPELINE_STORAGE_HOST__", { - value: host, - configurable: true, - writable: true, - }); - } - - expect(() => createDriver({ driverType: "host" })).toThrow(); - }); - - it("leaves the local drivers untouched", () => { - expect(() => createDriver({ driverType: "root-indexdb" })).not.toThrow(); - expect(() => - createDriver({ driverType: "folder-indexdb", folderId: "folder-1" }), - ).not.toThrow(); - }); -}); - -describe("HostStorageDriver capabilities", () => { - it("owns its listing and accepts no moves", () => { - const driver = new HostStorageDriver(createFakeHost()); - - expect(driver.listingIsAuthoritative).toBe(true); - expect(driver.allowsMoveIn).toBe(false); - expect(driver.allowsMoveOut).toBe(false); - }); - - it("does not touch the host until an operation is called", () => { - const host = createFakeHost(); - const list = vi.spyOn(host, "list"); - - new HostStorageDriver(host); - - expect(list).not.toHaveBeenCalled(); - }); -}); diff --git a/src/services/pipelineStorage/drivers/HostStorageDriver.ts b/src/services/pipelineStorage/drivers/HostStorageDriver.ts deleted file mode 100644 index 8e14747da4..0000000000 --- a/src/services/pipelineStorage/drivers/HostStorageDriver.ts +++ /dev/null @@ -1,155 +0,0 @@ -import { toPortablePipelineSpec } from "@/models/componentSpec/serialization/portablePipelineSpec"; -import { isValidComponentSpec } from "@/utils/componentSpec"; -import { componentSpecFromYaml, componentSpecToYaml } from "@/utils/yaml"; - -import type { - HostErrorCode, - HostPipelineSummary, - PipelineStorageHost, -} from "../host/contract"; -import { reportStorageAnswered, reportStorageFailed } from "../storageHealth"; -import { - HOST_DRIVER_TYPE, - type PipelineFileDescriptor, - type PipelineStorageDriver, -} from "../types"; - -export interface HostDriverConfig { - driverType: "host"; -} - -const UNTITLED_PIPELINE_NAME = "Untitled pipeline"; - -export class HostStorageError extends Error { - readonly name = "HostStorageError"; - - constructor( - readonly code: HostErrorCode, - message: string, - options?: ErrorOptions, - ) { - super(message, options); - } -} - -export class HostStorageDriver implements PipelineStorageDriver { - readonly type = HOST_DRIVER_TYPE; - readonly allowsMoveIn = false; - readonly allowsMoveOut = false; - readonly listingIsAuthoritative = true; - - constructor(private readonly host: PipelineStorageHost) {} - - async list(): Promise { - const summaries = await this.call(() => this.host.list()); - return summaries.map((summary) => toDescriptor(summary)); - } - - async read(storageKey: string): Promise { - const pipeline = await this.call(() => this.host.read(storageKey)); - - if (!isValidComponentSpec(pipeline.spec)) { - throw new HostStorageError( - "unavailable", - `Pipeline "${storageKey}" came back in a format this editor cannot read.`, - ); - } - - return componentSpecToYaml(pipeline.spec); - } - - async write( - storageKey: string, - content: string, - ): Promise { - const spec = toPortablePipelineSpec(componentSpecFromYaml(content)); - const summary = await this.call(() => this.host.write(storageKey, spec)); - return toDescriptor(summary); - } - - async delete(storageKey: string): Promise { - await this.call(() => this.host.delete(storageKey)); - } - - async hasKey(storageKey: string): Promise { - return this.call(() => this.host.has(storageKey)); - } - - /** - * Every call to the store passes through here, which makes it the one place - * that knows whether the store is answering — no separate health check, and - * nothing to go stale between calls. - */ - private async call(operation: () => Promise): Promise { - try { - const answer = await operation(); - reportStorageAnswered(); - return answer; - } catch (error) { - const failure = this.toStorageError(error); - reportStorageFailed(failure.code); - throw failure; - } - } - - private toStorageError(error: unknown): HostStorageError { - if (error instanceof HostStorageError) return error; - - const code = readErrorCode(error); - return new HostStorageError(code, describe(code, this.host.label), { - cause: error, - }); - } -} - -function toDescriptor(summary: HostPipelineSummary): PipelineFileDescriptor { - return { - storageKey: summary.key, - externalId: summary.externalId, - displayName: summary.displayName ?? UNTITLED_PIPELINE_NAME, - contentVersion: summary.contentVersion, - createdAt: toDate(summary.createdAt), - modifiedAt: toDate(summary.modifiedAt), - }; -} - -function toDate(value: string | undefined): Date | undefined { - if (!value) return undefined; - const date = new Date(value); - return Number.isNaN(date.getTime()) ? undefined : date; -} - -/** - * Errors cross a window boundary, where `instanceof` does not survive, so the - * code is duck-typed off the rejection value and anything unrecognised — an - * older or newer host vocabulary included — degrades to "unavailable". - */ -function readErrorCode(error: unknown): HostErrorCode { - if (typeof error !== "object" || error === null) return "unavailable"; - - const code: unknown = Reflect.get(error, "code"); - switch (code) { - case "unauthenticated": - case "not_found": - case "rate_limited": - case "conflict": - return code; - default: - return "unavailable"; - } -} - -function describe(code: HostErrorCode, label: string): string { - switch (code) { - case "unauthenticated": - return `Your ${label} session has expired. Reload the page to sign in again.`; - case "not_found": - return `This pipeline no longer exists in ${label}.`; - case "rate_limited": - return `${label} is busy. Wait a moment before trying again.`; - case "conflict": - return `This pipeline changed in ${label} since it was opened. Reload it before saving again.`; - case "unavailable": - return `${label} could not be reached. Try again in a moment.`; - } -} diff --git a/src/services/pipelineStorage/drivers/UnavailableStorageDriver.ts b/src/services/pipelineStorage/drivers/UnavailableStorageDriver.ts deleted file mode 100644 index 103f721f99..0000000000 --- a/src/services/pipelineStorage/drivers/UnavailableStorageDriver.ts +++ /dev/null @@ -1,43 +0,0 @@ -import type { PipelineFileDescriptor, PipelineStorageDriver } from "../types"; - -class PipelineStorageUnavailableError extends Error { - readonly name = "PipelineStorageUnavailableError"; - - constructor() { - super( - "This deployment stores pipelines outside the browser, and that store was not provided by the page.", - ); - } -} - -/** - * Stands in when the deployment requires a host-provided store and the page did - * not supply one. Every operation refuses: falling back to browser storage - * would write the user's work somewhere the deployment does not read from, and - * they would only find out much later. - */ -export class UnavailableStorageDriver implements PipelineStorageDriver { - readonly type = "unavailable"; - readonly allowsMoveIn = false; - readonly allowsMoveOut = false; - - async list(): Promise { - throw new PipelineStorageUnavailableError(); - } - - async read(): Promise { - throw new PipelineStorageUnavailableError(); - } - - async write(): Promise { - throw new PipelineStorageUnavailableError(); - } - - async delete(): Promise { - throw new PipelineStorageUnavailableError(); - } - - async hasKey(): Promise { - throw new PipelineStorageUnavailableError(); - } -} diff --git a/src/services/pipelineStorage/host/contract.ts b/src/services/pipelineStorage/host/contract.ts deleted file mode 100644 index acdec1be77..0000000000 --- a/src/services/pipelineStorage/host/contract.ts +++ /dev/null @@ -1,33 +0,0 @@ -export const PIPELINE_STORAGE_HOST_VERSION = 1; - -export interface HostPipelineSummary { - key: string; - externalId: string; - displayName: string | null; - contentVersion: string; - createdAt?: string; - modifiedAt?: string; -} - -export interface HostPipeline extends HostPipelineSummary { - spec: unknown; -} - -export type HostErrorCode = - "unauthenticated" | "not_found" | "rate_limited" | "conflict" | "unavailable"; - -export interface PipelineStorageHost { - readonly version: number; - readonly label: string; - list(): Promise; - read(key: string): Promise; - write(key: string, spec: unknown): Promise; - delete(key: string): Promise; - has(key: string): Promise; -} - -declare global { - interface Window { - __TANGLE_PIPELINE_STORAGE_HOST__?: PipelineStorageHost; - } -} diff --git a/src/services/pipelineStorage/host/detectHost.test.ts b/src/services/pipelineStorage/host/detectHost.test.ts deleted file mode 100644 index d9c32929b0..0000000000 --- a/src/services/pipelineStorage/host/detectHost.test.ts +++ /dev/null @@ -1,101 +0,0 @@ -import { afterEach, describe, expect, it } from "vitest"; - -import { - PIPELINE_STORAGE_HOST_VERSION, - type PipelineStorageHost, -} from "./contract"; -import { getPipelineStorageHost } from "./detectHost"; - -function installHost(host: unknown) { - Object.defineProperty(window, "__TANGLE_PIPELINE_STORAGE_HOST__", { - value: host, - configurable: true, - writable: true, - }); -} - -function createFakeHost( - overrides: Partial> = {}, -) { - return { - version: PIPELINE_STORAGE_HOST_VERSION, - label: "Shared storage", - list: async () => [], - read: async () => ({}), - write: async () => ({}), - delete: async () => undefined, - has: async () => false, - ...overrides, - }; -} - -afterEach(() => { - delete window.__TANGLE_PIPELINE_STORAGE_HOST__; -}); - -describe("getPipelineStorageHost", () => { - it("returns the host when the global satisfies the contract", () => { - const host = createFakeHost(); - installHost(host); - - expect(getPipelineStorageHost()).toBe(host); - }); - - it("returns undefined when the global is missing", () => { - expect(getPipelineStorageHost()).toBeUndefined(); - }); - - it.each(["list", "read", "write", "delete", "has"] as const)( - "returns undefined when %s is not a function", - (method) => { - installHost(createFakeHost({ [method]: "not-callable" })); - - expect(getPipelineStorageHost()).toBeUndefined(); - }, - ); - - it("returns undefined when the version is above the supported version", () => { - installHost(createFakeHost({ version: PIPELINE_STORAGE_HOST_VERSION + 1 })); - - expect(getPipelineStorageHost()).toBeUndefined(); - }); - - it("accepts a host declaring an older version", () => { - const host = createFakeHost({ version: PIPELINE_STORAGE_HOST_VERSION - 1 }); - installHost(host); - - expect(getPipelineStorageHost()).toBe(host); - }); - - it("returns undefined when the version is not a number", () => { - installHost(createFakeHost({ version: "1" })); - - expect(getPipelineStorageHost()).toBeUndefined(); - }); - - it.each([undefined, "", " "])( - "returns undefined when the label is %p", - (label) => { - installHost(createFakeHost({ label })); - - expect(getPipelineStorageHost()).toBeUndefined(); - }, - ); - - it("returns undefined when the global is not an object", () => { - installHost("a host, honest"); - - expect(getPipelineStorageHost()).toBeUndefined(); - }); - - it("returns undefined when reading the global throws", () => { - Object.defineProperty(window, "__TANGLE_PIPELINE_STORAGE_HOST__", { - get() { - throw new Error("cross-origin"); - }, - configurable: true, - }); - - expect(getPipelineStorageHost()).toBeUndefined(); - }); -}); diff --git a/src/services/pipelineStorage/host/detectHost.ts b/src/services/pipelineStorage/host/detectHost.ts deleted file mode 100644 index a7479c0552..0000000000 --- a/src/services/pipelineStorage/host/detectHost.ts +++ /dev/null @@ -1,40 +0,0 @@ -import { - PIPELINE_STORAGE_HOST_VERSION, - type PipelineStorageHost, -} from "./contract"; - -const HOST_METHODS = [ - "list", - "read", - "write", - "delete", - "has", -] as const satisfies readonly (keyof PipelineStorageHost)[]; - -/** - * The host page and this app deploy independently, so a contract version we do - * not understand has to read as "no host at all" rather than as a host we can - * half-drive. Every failure mode — missing global, wrong shape, throwing - * getter, newer version — resolves to `undefined` and leaves local storage as - * the only storage. - */ -export function getPipelineStorageHost(): PipelineStorageHost | undefined { - try { - if (typeof window === "undefined") return undefined; - - const host = window.__TANGLE_PIPELINE_STORAGE_HOST__; - if (!host || typeof host !== "object") return undefined; - if (typeof host.version !== "number") return undefined; - if (host.version > PIPELINE_STORAGE_HOST_VERSION) return undefined; - if (typeof host.label !== "string" || host.label.trim() === "") { - return undefined; - } - if (HOST_METHODS.some((method) => typeof host[method] !== "function")) { - return undefined; - } - - return host; - } catch { - return undefined; - } -} diff --git a/src/services/pipelineStorage/pipelineRegistry.test.ts b/src/services/pipelineStorage/pipelineRegistry.test.ts index aa27d6aeb2..bb225c601a 100644 --- a/src/services/pipelineStorage/pipelineRegistry.test.ts +++ b/src/services/pipelineStorage/pipelineRegistry.test.ts @@ -4,7 +4,6 @@ import { Dexie } from "dexie"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { pipelineStorageDb } from "./db"; -import type { PipelineStorageHost } from "./host/contract"; import { assertStorageKeyUnique, claimEntry, @@ -15,27 +14,11 @@ import { import { resetStorageModeForTests } from "./storageMode"; import { ROOT_FOLDER_ID } from "./types"; -const summary = { - key: "", - externalId: "", - displayName: null, - contentVersion: "1", -}; - -const host: PipelineStorageHost = { - version: 1, - label: "Shared storage", - list: async () => [], - read: async () => ({ ...summary, spec: {} }), - write: async () => summary, - delete: async () => undefined, - has: async () => false, -}; - -function useStore(kind: "local" | "host") { - vi.stubEnv("VITE_PIPELINE_STORAGE_BETA", kind === "host" ? "true" : "false"); - if (kind === "host") window.__TANGLE_PIPELINE_STORAGE_HOST__ = host; - else delete window.__TANGLE_PIPELINE_STORAGE_HOST__; +function useStore(kind: "local" | "backend") { + vi.stubEnv( + "VITE_PIPELINE_STORAGE_BETA", + kind === "backend" ? "true" : "false", + ); resetStorageModeForTests(); } @@ -45,13 +28,12 @@ beforeEach(async () => { afterEach(() => { vi.unstubAllEnvs(); - delete window.__TANGLE_PIPELINE_STORAGE_HOST__; resetStorageModeForTests(); }); describe("registry rows belong to the store that wrote them", () => { - it("hides a host row from browser storage, and the other way round", async () => { - useStore("host"); + it("hides a backend row from browser storage, and the other way round", async () => { + useStore("backend"); await claimEntry({ id: "external-1", storageKey: "Churn model", @@ -69,7 +51,7 @@ describe("registry rows belong to the store that wrote them", () => { folderId: ROOT_FOLDER_ID, }); - useStore("host"); + useStore("backend"); expect(await findById("local-1")).toBeUndefined(); expect((await findByStorageKey("Churn model"))?.id).toBe("external-1"); }); @@ -82,7 +64,7 @@ describe("registry rows belong to the store that wrote them", () => { folderId: ROOT_FOLDER_ID, }); - useStore("host"); + useStore("backend"); await expect( claimEntry({ id: "external-1", @@ -95,7 +77,7 @@ describe("registry rows belong to the store that wrote them", () => { }); it("does not refuse a name only the other store is using", async () => { - useStore("host"); + useStore("backend"); await claimEntry({ id: "external-1", storageKey: "Churn model", @@ -146,7 +128,7 @@ describe("attributing rows written before rows said which store", () => { host_migration: "id", }; - it("reads a host row by its reported version and a filed row by its folder", async () => { + it("reads a backend row by its reported version and a filed row by its folder", async () => { await pipelineStorageDb.close(); await Dexie.delete("tangle_pipelines"); @@ -173,7 +155,7 @@ describe("attributing rows written before rows said which store", () => { expect( Object.fromEntries(attributed.map((row) => [row.id, row.storage])), ).toEqual({ - "external-1": "host", + "external-1": "backend", "local-1": "local", "filed-1": "local", }); diff --git a/src/services/pipelineStorage/storageErrors.ts b/src/services/pipelineStorage/storageErrors.ts index c13a5882a2..c6a58f6ff3 100644 --- a/src/services/pipelineStorage/storageErrors.ts +++ b/src/services/pipelineStorage/storageErrors.ts @@ -1,4 +1,4 @@ -import { HostStorageError } from "./drivers/HostStorageDriver"; +import { BackendStorageError } from "./drivers/BackendStorageDriver"; import { AmbiguousPipelineNameError, PipelineNotFoundError, @@ -25,7 +25,7 @@ export function isRetriableStorageError( return false; } - if (error instanceof HostStorageError) { + if (error instanceof BackendStorageError) { return error.code === "unavailable" || error.code === "rate_limited"; } @@ -38,7 +38,7 @@ export function isRetriableStorageError( * comes back is how work sits unsaved with nobody told why. */ export function isWriteWorthRetrying(error: unknown): boolean { - if (error instanceof HostStorageError) { + if (error instanceof BackendStorageError) { return error.code !== "unauthenticated" && error.code !== "conflict"; } @@ -46,5 +46,7 @@ export function isWriteWorthRetrying(error: unknown): boolean { } export function isExpiredSession(error: unknown): boolean { - return error instanceof HostStorageError && error.code === "unauthenticated"; + return ( + error instanceof BackendStorageError && error.code === "unauthenticated" + ); } diff --git a/src/services/pipelineStorage/storageHealth.ts b/src/services/pipelineStorage/storageHealth.ts index 09941175c4..15845dd435 100644 --- a/src/services/pipelineStorage/storageHealth.ts +++ b/src/services/pipelineStorage/storageHealth.ts @@ -1,17 +1,15 @@ import { useSyncExternalStore } from "react"; -import type { HostErrorCode } from "./host/contract"; -import { isHostStorage } from "./storageMode"; +import { isBackendStorage } from "./storageMode"; +import type { StorageErrorCode } from "./types"; /** * Whether the store holding the pipelines is answering, learned from the calls * the app already makes rather than from a health check against something else. * - * A host-provided store is not the execution backend: it is served by the page - * that embeds this app, from its own endpoint and session, and stays up when - * the configured backend is switched off. Pinging that backend to decide - * whether pipelines can be saved reports an outage while saves are landing, and - * says nothing when the store itself is the thing that has gone. + * The configured backend serves more than pipelines, and a health check + * against it says nothing about whether the pipeline routes are answering — + * which is the only thing the pipeline list and the save indicator are about. * * An error is only an outage if the store failed to answer at all. "No such * pipeline" is an answer. @@ -30,7 +28,7 @@ export function reportStorageAnswered(): void { set(true); } -export function reportStorageFailed(code: HostErrorCode): void { +export function reportStorageFailed(code: StorageErrorCode): void { if (code !== "unavailable") { set(true); return; @@ -51,5 +49,5 @@ export function useStorageUnavailable(): boolean { () => true, ); - return isHostStorage() && !answering; + return isBackendStorage() && !answering; } diff --git a/src/services/pipelineStorage/storageMode.test.ts b/src/services/pipelineStorage/storageMode.test.ts index 8a3bb6bf6c..c2ebc70baa 100644 --- a/src/services/pipelineStorage/storageMode.test.ts +++ b/src/services/pipelineStorage/storageMode.test.ts @@ -1,87 +1,52 @@ import { afterEach, describe, expect, it, vi } from "vitest"; -import type { PipelineStorageHost } from "./host/contract"; import { - isHostStorage, - isHostStorageMissing, + currentStorageKind, + isBackendStorage, resetStorageModeForTests, resolveStorageMode, } from "./storageMode"; -const LABEL = "Shared storage"; - -function installHost(overrides: Partial = {}) { - const host: PipelineStorageHost = { - version: 1, - label: LABEL, - list: async () => [], - read: async () => { - throw new Error("not used"); - }, - write: async () => { - throw new Error("not used"); - }, - delete: async () => undefined, - has: async () => false, - ...overrides, - }; - - Object.defineProperty(window, "__TANGLE_PIPELINE_STORAGE_HOST__", { - value: host, - configurable: true, - writable: true, - }); -} - afterEach(() => { - delete window.__TANGLE_PIPELINE_STORAGE_HOST__; vi.unstubAllEnvs(); resetStorageModeForTests(); }); describe("a deployment with the beta on", () => { - it("uses the store the page provides", () => { + it("keeps pipelines on the backend it is configured against", () => { vi.stubEnv("VITE_PIPELINE_STORAGE_BETA", "true"); - installHost(); - expect(resolveStorageMode()).toEqual({ kind: "host", label: LABEL }); + expect(resolveStorageMode().kind).toBe("backend"); + expect(isBackendStorage()).toBe(true); + expect(currentStorageKind()).toBe("backend"); }); - it("refuses to fall back to browser storage when the page provides none", () => { + it("does not change store when the answer is asked for again", () => { vi.stubEnv("VITE_PIPELINE_STORAGE_BETA", "true"); + const first = resolveStorageMode(); - expect(resolveStorageMode()).toEqual({ kind: "host-missing" }); - expect(isHostStorageMissing()).toBe(true); - expect(isHostStorage()).toBe(false); - }); - - it("treats a store it cannot drive as none at all", () => { - vi.stubEnv("VITE_PIPELINE_STORAGE_BETA", "true"); - installHost({ version: 99 }); + vi.stubEnv("VITE_PIPELINE_STORAGE_BETA", "false"); - expect(resolveStorageMode()).toEqual({ kind: "host-missing" }); + expect(resolveStorageMode()).toBe(first); }); }); describe("a deployment with the beta off", () => { - it("ignores a store the page provides anyway", () => { + it("keeps pipelines in the browser", () => { vi.stubEnv("VITE_PIPELINE_STORAGE_BETA", "false"); - installHost(); expect(resolveStorageMode()).toEqual({ kind: "local" }); + expect(isBackendStorage()).toBe(false); + expect(currentStorageKind()).toBe("local"); }); - it("is what an unset flag means", () => { - installHost(); + it("treats anything but a plain yes as off", () => { + vi.stubEnv("VITE_PIPELINE_STORAGE_BETA", "1"); expect(resolveStorageMode()).toEqual({ kind: "local" }); - expect(isHostStorageMissing()).toBe(false); }); - it("is what any other value means", () => { - vi.stubEnv("VITE_PIPELINE_STORAGE_BETA", "1"); - installHost(); - + it("is off when nothing was said at all", () => { expect(resolveStorageMode()).toEqual({ kind: "local" }); }); }); diff --git a/src/services/pipelineStorage/storageMode.ts b/src/services/pipelineStorage/storageMode.ts index 93f8d72843..09e53e3e59 100644 --- a/src/services/pipelineStorage/storageMode.ts +++ b/src/services/pipelineStorage/storageMode.ts @@ -1,77 +1,47 @@ -import type { PipelineStorageHost } from "./host/contract"; -import { getPipelineStorageHost } from "./host/detectHost"; import type { PipelineStorageKind } from "./types"; export type StorageMode = - | { kind: "local" } - | { kind: "host"; label: string } - | { kind: "host-missing" }; + { kind: "local" } | { kind: "backend"; label: string }; + +const BACKEND_STORAGE_LABEL = "Backend"; let resolved: StorageMode | undefined; -let resolvedHost: PipelineStorageHost | undefined; /** - * A deployment says whether it stores pipelines outside the browser; it is not - * guessed from whether a host happens to have loaded. Off is the default and - * the only thing the open-source build can be. On without a host is an error - * rather than a quiet fall back to browser storage, which would strand a user's - * work somewhere nobody else can see it. + * A deployment says whether pipelines are stored outside the browser. Off is + * the default and what a build with no backend of its own can do; on, they are + * read and written through the backend the app is configured against, which + * has to serve the pipeline routes. */ -function hostStorageEnabled(): boolean { +function backendStorageEnabled(): boolean { return import.meta.env.VITE_PIPELINE_STORAGE_BETA === "true"; } /** - * Decided once and then frozen for the life of the page, holding on to the host - * itself rather than re-reading the global — a host whose global is removed or - * whose getter starts throwing must not read as "no host" and quietly send the - * next write to browser storage. + * Decided once and then frozen for the life of the page: which store holds the + * pipelines cannot change under an open editor. *Where* that backend is may + * change — that is a setting, and the driver reads it per request. */ export function resolveStorageMode(): StorageMode { - if (resolved) return resolved; - - if (!hostStorageEnabled()) { - resolvedHost = undefined; - resolved = { kind: "local" }; - return resolved; - } - - resolvedHost = getPipelineStorageHost(); - resolved = resolvedHost - ? { kind: "host", label: resolvedHost.label } - : { kind: "host-missing" }; + resolved ??= backendStorageEnabled() + ? { kind: "backend", label: BACKEND_STORAGE_LABEL } + : { kind: "local" }; return resolved; } -export function getStorageHost(): PipelineStorageHost | undefined { - resolveStorageMode(); - return resolvedHost; -} - -export function isHostStorage(): boolean { - return resolveStorageMode().kind === "host"; +export function isBackendStorage(): boolean { + return resolveStorageMode().kind === "backend"; } /** - * Which store the cached rows written on this page load describe. A deployment - * that requires a host still belongs to the host's world while that host is - * unreachable — nothing may be written, and browser rows must stay invisible. + * Which store the cached rows written on this page load describe. Storage keys + * are only unique within one store, so a row must never be read by the other. */ export function currentStorageKind(): PipelineStorageKind { - return resolveStorageMode().kind === "local" ? "local" : "host"; -} - -/** - * The deployment requires a host-provided store and the page did not supply - * one. Nothing can be read or written, so this is worth saying rather than - * rendering an empty library. - */ -export function isHostStorageMissing(): boolean { - return resolveStorageMode().kind === "host-missing"; + return resolveStorageMode().kind === "local" ? "local" : "backend"; } export function resetStorageModeForTests(): void { resolved = undefined; - resolvedHost = undefined; } diff --git a/src/services/pipelineStorage/types.ts b/src/services/pipelineStorage/types.ts index f7d7c71f5a..abdfc3596e 100644 --- a/src/services/pipelineStorage/types.ts +++ b/src/services/pipelineStorage/types.ts @@ -1,13 +1,16 @@ import type { ComponentSpec } from "@/utils/componentSpec"; import type { GoogleDriveDriverConfig } from "../googleDrive/types"; // google-drive +import type { BackendDriverConfig } from "./drivers/BackendStorageDriver"; import type { FolderIndexDbDriverConfig } from "./drivers/FolderIndexDbStorageDriver"; -import type { HostDriverConfig } from "./drivers/HostStorageDriver"; import type { LocalFileSystemDriverConfig } from "./drivers/LocalFileSystemDriver"; import type { RootFolderDbDriverConfig } from "./drivers/RootFolderDbStorageDriver"; export const ROOT_FOLDER_ID = "__root__"; -export const HOST_DRIVER_TYPE = "host"; +export const BACKEND_DRIVER_TYPE = "backend"; + +export type StorageErrorCode = + "unauthenticated" | "not_found" | "rate_limited" | "conflict" | "unavailable"; export interface PipelineFileDescriptor { storageKey: string; @@ -44,7 +47,7 @@ export type DriverConfig = | RootFolderDbDriverConfig | FolderIndexDbDriverConfig | LocalFileSystemDriverConfig - | HostDriverConfig + | BackendDriverConfig | GoogleDriveDriverConfig; // google-drive /** @@ -54,7 +57,7 @@ export type DriverConfig = * store it came from would be found by the other one and answer for a pipeline * it has never seen. */ -export type PipelineStorageKind = "local" | "host"; +export type PipelineStorageKind = "local" | "backend"; export interface CachedPipelineSpec { storage: PipelineStorageKind; diff --git a/tests/e2e/fixtures/pipelineStorageHost.ts b/tests/e2e/fixtures/pipelineStorageBackend.ts similarity index 52% rename from tests/e2e/fixtures/pipelineStorageHost.ts rename to tests/e2e/fixtures/pipelineStorageBackend.ts index c6a6b61016..5a4d07b944 100644 --- a/tests/e2e/fixtures/pipelineStorageHost.ts +++ b/tests/e2e/fixtures/pipelineStorageBackend.ts @@ -1,21 +1,20 @@ import type { Page } from "@playwright/test"; -type HostFailMode = "none" | "unavailable" | "unauthenticated" | "rate_limited"; +type BackendFailMode = + "none" | "unavailable" | "unauthenticated" | "rate_limited"; -interface HostSeedPipeline { +interface SeedPipeline { key: string; displayName: string; spec: unknown; } -export interface HostStorageOptions { - label?: string; - seed?: HostSeedPipeline[]; - failMode?: HostFailMode; - latencyMs?: number; +export interface BackendStorageOptions { + seed?: SeedPipeline[]; + failMode?: BackendFailMode; } -interface HostRecord { +export interface StoredPipeline { key: string; externalId: string; displayName: string | null; @@ -23,156 +22,161 @@ interface HostRecord { spec: unknown; } -interface HostTestState { - records(): HostRecord[]; - readKeys(): string[]; - setFailMode(mode: HostFailMode): void; +interface BackendStore { + records: StoredPipeline[]; + readKeys: string[]; + failMode: BackendFailMode; + revision: number; } -declare global { - interface Window { - __TANGLE_TEST_HOST__?: HostTestState; - } +const PIPELINES_PATH = "**/api/users/me/pipelines**"; + +const STATUS_FOR: Record, number> = { + unavailable: 503, + unauthenticated: 401, + rate_limited: 429, +}; + +const stores = new WeakMap(); + +function storeFor(page: Page): BackendStore { + const store = stores.get(page); + if (!store) throw new Error("No pipeline backend installed for this page"); + return store; } -const DEFAULT_LABEL = "Shared storage"; +function rowOf(record: StoredPipeline, withSpec: boolean) { + return { + id: record.externalId, + file_path: record.key, + pipeline_name: record.displayName, + current_version: record.contentVersion, + created_at: "2026-01-01T00:00:00Z", + updated_at: "2026-01-02T00:00:00Z", + ...(withSpec + ? { root_pipeline_task: { componentRef: { spec: record.spec } } } + : {}), + }; +} -const STATE_KEY = "__tangle_test_host_state__"; +function nameOf(spec: unknown): string | null { + if (typeof spec !== "object" || spec === null) return null; + const name: unknown = Reflect.get(spec, "name"); + return typeof name === "string" ? name : null; +} /** - * Stands in for the page that embeds this app. It has to be installed with - * `addInitScript` rather than `evaluate`, because storage mode is decided while - * the app boots and never revisited. - * - * Its contents outlive a reload — creating a pipeline navigates with a document - * load, and a store that forgot everything at that point could not show that - * the write reached it. + * Serves the pipeline routes the storage driver calls. The store lives in the + * test process rather than the page, so it survives a reload the way a real + * backend does — creating a pipeline navigates with a document load, and a + * store that forgot everything there could not show that the write landed. */ -export async function installPipelineStorageHost( +export async function installPipelineStorageBackend( page: Page, - options: HostStorageOptions = {}, + options: BackendStorageOptions = {}, ): Promise { - await page.addInitScript( - (config: Required & { stateKey: string }) => { - const saved = window.sessionStorage.getItem(config.stateKey); - const store = new Map( - saved ? (JSON.parse(saved) as [string, HostRecord][]) : [], - ); - let revision = store.size; + const seed = options.seed ?? []; + + stores.set(page, { + records: seed.map((entry, index) => ({ + key: entry.key, + externalId: `external-${index + 1}`, + displayName: entry.displayName, + contentVersion: `v${index + 1}`, + spec: entry.spec, + })), + readKeys: [], + failMode: options.failMode ?? "none", + revision: seed.length, + }); - /** - * Deliberately not persisted: a test asserting that a reload served the - * pipeline contents from cache needs the count for this page load alone. - */ - const readKeys: string[] = []; - - let failMode = config.failMode; - - if (!saved) { - for (const seeded of config.seed) { - revision += 1; - store.set(seeded.key, { - key: seeded.key, - externalId: `external-${revision}`, - displayName: seeded.displayName, - contentVersion: `v${revision}`, - spec: seeded.spec, - }); - } - } + await page.route(PIPELINES_PATH, async (route) => { + const store = storeFor(page); - function persist(): void { - window.sessionStorage.setItem( - config.stateKey, - JSON.stringify([...store.entries()]), - ); - } + if (store.failMode !== "none") { + return route.fulfill({ + status: STATUS_FOR[store.failMode], + contentType: "application/json", + body: JSON.stringify({ detail: `backend is ${store.failMode}` }), + }); + } - persist(); + const url = new URL(route.request().url()); + const key = url.searchParams.get("file_path") ?? ""; + const isListing = url.pathname.endsWith("/all"); + const json = (body: unknown, status = 200) => + route.fulfill({ + status, + contentType: "application/json", + body: JSON.stringify(body), + }); - async function gate(): Promise { - if (config.latencyMs > 0) { - await new Promise((resolve) => setTimeout(resolve, config.latencyMs)); - } + if (isListing) { + const matched = store.records.filter((row) => !key || row.key === key); + return json({ + pipelines: matched.map((row) => rowOf(row, false)), + next_page_token: null, + }); + } - if (failMode !== "none") { - throw Object.assign(new Error(`host is ${failMode}`), { - code: failMode, - }); - } - } + switch (route.request().method()) { + case "PUT": { + const sent = route.request().postDataJSON() as { + root_pipeline_task: { componentRef: { spec: unknown } }; + }; + const spec = sent.root_pipeline_task.componentRef.spec; + const existing = store.records.find((row) => row.key === key); + store.revision += 1; + + const written: StoredPipeline = { + key, + externalId: existing?.externalId ?? `external-${store.revision}`, + displayName: nameOf(spec), + contentVersion: `v${store.revision}`, + spec, + }; - function summaryOf(record: HostRecord) { - const { spec: _spec, ...summary } = record; - return summary; + // Keeps its place in the listing, as a store keyed by id would. + store.records = existing + ? store.records.map((row) => (row.key === key ? written : row)) + : [...store.records, written]; + return json(rowOf(written, true)); } - - function nameOf(spec: unknown): string | null { - if (typeof spec !== "object" || spec === null) return null; - const name: unknown = Reflect.get(spec, "name"); - return typeof name === "string" ? name : null; + case "DELETE": + store.records = store.records.filter((row) => row.key !== key); + return route.fulfill({ status: 204, body: "" }); + default: { + store.readKeys.push(key); + const found = store.records.find((row) => row.key === key); + if (!found) return json({ detail: `no pipeline for ${key}` }, 404); + return json(rowOf(found, true)); } + } + }); +} - window.__TANGLE_PIPELINE_STORAGE_HOST__ = { - version: 1, - label: config.label, - async list() { - await gate(); - return [...store.values()].map(summaryOf); - }, - async read(key: string) { - await gate(); - readKeys.push(key); - const found = store.get(key); - if (!found) { - throw Object.assign(new Error(`no pipeline for ${key}`), { - code: "not_found", - }); - } - return found; - }, - async write(key: string, spec: unknown) { - await gate(); - const existing = store.get(key); - revision += 1; - const record: HostRecord = { - key, - externalId: existing?.externalId ?? `external-${revision}`, - displayName: nameOf(spec), - contentVersion: `v${revision}`, - spec, - }; - store.set(key, record); - persist(); - return summaryOf(record); - }, - async delete(key: string) { - await gate(); - store.delete(key); - persist(); - }, - async has(key: string) { - await gate(); - return store.has(key); - }, - }; +/** + * Takes the backend away, or gives it back, without reloading — the failure + * that matters is the one that arrives while someone is working. + */ +export function setBackendFailMode(page: Page, mode: BackendFailMode): void { + storeFor(page).failMode = mode; +} - window.__TANGLE_TEST_HOST__ = { - records: () => [...store.values()], - readKeys: () => [...readKeys], - setFailMode: (mode) => { - failMode = mode; - }, - }; - }, - { - label: options.label ?? DEFAULT_LABEL, - seed: options.seed ?? [], - failMode: options.failMode ?? "none", - latencyMs: options.latencyMs ?? 0, - stateKey: STATE_KEY, - } satisfies Required & { stateKey: string }, - ); +export function readBackendRecords(page: Page): StoredPipeline[] { + return [...storeFor(page).records]; +} + +/** + * Which pipelines were fetched in full. Reset per install rather than per page + * load, so a test can assert that a revisit read none of them. + */ +export function readBackendReadKeys(page: Page): string[] { + return [...storeFor(page).readKeys]; +} + +export function forgetBackendReadKeys(page: Page): void { + storeFor(page).readKeys = []; } /** @@ -306,30 +310,8 @@ export async function seedLocallyStoredPipeline( } /** - * Takes the store away, or gives it back, without reloading — the failure that - * matters is the one that arrives while someone is working. - */ -export async function setHostFailMode( - page: Page, - mode: HostFailMode, -): Promise { - await page.evaluate( - (value) => window.__TANGLE_TEST_HOST__?.setFailMode(value), - mode, - ); -} - -export async function readHostRecords(page: Page): Promise { - return page.evaluate(() => window.__TANGLE_TEST_HOST__?.records() ?? []); -} - -export async function readHostReadKeys(page: Page): Promise { - return page.evaluate(() => window.__TANGLE_TEST_HOST__?.readKeys() ?? []); -} - -/** - * The negative assertion host mode exists for: with a host present, nothing may - * reach the browser's own pipeline store, whatever the host does. + * The negative assertion backend storage exists for: with the beta on, nothing + * may reach the browser's own pipeline store, whatever the backend does. */ export async function readLocallyStoredPipelineKeys( page: Page, diff --git a/tests/e2e/pipeline-storage-host.spec.ts b/tests/e2e/pipeline-storage-backend.spec.ts similarity index 78% rename from tests/e2e/pipeline-storage-host.spec.ts rename to tests/e2e/pipeline-storage-backend.spec.ts index 9036de7371..bae5feb16c 100644 --- a/tests/e2e/pipeline-storage-host.spec.ts +++ b/tests/e2e/pipeline-storage-backend.spec.ts @@ -1,16 +1,15 @@ import { expect, type Page, test } from "@playwright/test"; import { - type HostStorageOptions, - installPipelineStorageHost, - readHostReadKeys, - readHostRecords, + type BackendStorageOptions, + forgetBackendReadKeys, + installPipelineStorageBackend, + readBackendReadKeys, + readBackendRecords, readLocallyStoredPipelineKeys, seedLocallyStoredPipeline, - setHostFailMode, -} from "./fixtures/pipelineStorageHost"; - -const LABEL = "Shared storage"; + setBackendFailMode, +} from "./fixtures/pipelineStorageBackend"; const CHURN_TAG = "quarterly"; @@ -31,21 +30,20 @@ const SEED = [ }, ]; -async function installSeededHost(page: Page, options: HostStorageOptions = {}) { +async function installSeededBackend( + page: Page, + options: BackendStorageOptions = {}, +) { await page.addInitScript(() => { window.localStorage.setItem("seen-editor-v2-welcome", JSON.stringify(true)); }); - await installPipelineStorageHost(page, { - label: LABEL, - seed: SEED, - ...options, - }); + await installPipelineStorageBackend(page, { seed: SEED, ...options }); } -test.describe("host-provided pipeline storage", () => { +test.describe("backend pipeline storage", () => { test("lists what the host holds", async ({ page }) => { - await installSeededHost(page); + await installSeededBackend(page); await page.goto("/pipeline-folders"); @@ -56,7 +54,7 @@ test.describe("host-provided pipeline storage", () => { test("offers nothing to file pipelines into a store that has no folders", async ({ page, }) => { - await installSeededHost(page); + await installSeededBackend(page); await page.goto("/pipeline-folders"); await expect(page.getByText("Churn model")).toBeVisible(); @@ -72,7 +70,7 @@ test.describe("host-provided pipeline storage", () => { test("keeps the pipeline table at /pipelines, contents and all", async ({ page, }) => { - await installSeededHost(page); + await installSeededBackend(page); await page.goto("/pipelines"); @@ -85,25 +83,26 @@ test.describe("host-provided pipeline storage", () => { test("reads each pipeline once and serves the next visit from cache", async ({ page, }) => { - await installSeededHost(page); + await installSeededBackend(page); await page.goto("/pipelines"); await expect(page.getByText(CHURN_TAG)).toBeVisible(); - expect((await readHostReadKeys(page)).sort()).toEqual( + expect([...readBackendReadKeys(page)].sort()).toEqual( SEED.map((entry) => entry.key).sort(), ); + forgetBackendReadKeys(page); await page.reload(); await expect(page.getByText(CHURN_TAG)).toBeVisible(); - expect(await readHostReadKeys(page)).toEqual([]); + expect(readBackendReadKeys(page)).toEqual([]); }); - test("copies pipelines already in the browser into the host", async ({ + test("copies pipelines already in the browser into the backend", async ({ page, }) => { const localName = "Left behind in the browser"; - await installSeededHost(page); + await installSeededBackend(page); await page.goto("/"); await seedLocallyStoredPipeline(page, localName); @@ -113,7 +112,7 @@ test.describe("host-provided pipeline storage", () => { await expect(page.getByText("Churn model")).toBeVisible(); expect( - (await readHostRecords(page)).map((record) => record.displayName), + readBackendRecords(page).map((record) => record.displayName), ).toContain(localName); }); @@ -122,7 +121,7 @@ test.describe("host-provided pipeline storage", () => { }) => { const localName = "Never opened the list"; - await installSeededHost(page); + await installSeededBackend(page); await page.goto("/"); await seedLocallyStoredPipeline(page, localName); @@ -132,14 +131,14 @@ test.describe("host-provided pipeline storage", () => { await expect .poll( async () => - (await readHostRecords(page)).map((record) => record.displayName), + readBackendRecords(page).map((record) => record.displayName), { timeout: 15_000 }, ) .toContain(localName); }); test("opens a pipeline at a url that is only its id", async ({ page }) => { - await installSeededHost(page); + await installSeededBackend(page); await page.goto("/pipelines"); await page.getByText("Churn model").click(); @@ -149,7 +148,7 @@ test.describe("host-provided pipeline storage", () => { }); const url = new URL(page.url()); - const [record] = (await readHostRecords(page)).filter( + const [record] = readBackendRecords(page).filter( (entry) => entry.displayName === "Churn model", ); expect(url.pathname).toBe(`/editor-v2/${record.externalId}`); @@ -159,7 +158,7 @@ test.describe("host-provided pipeline storage", () => { test("says a link to a pipeline it does not hold cannot be opened", async ({ page, }) => { - await installSeededHost(page); + await installSeededBackend(page); await page.goto("/editor-v2/0f8c1a2b-0000-4000-8000-00000000dead"); @@ -170,19 +169,19 @@ test.describe("host-provided pipeline storage", () => { }); test("keeps the browser's own pipeline store empty", async ({ page }) => { - await installSeededHost(page); + await installSeededBackend(page); await page.goto("/pipeline-folders"); await expect(page.getByText("Churn model")).toBeVisible(); expect(await readLocallyStoredPipelineKeys(page)).toEqual([]); - expect(await readHostRecords(page)).toHaveLength(SEED.length); + expect(readBackendRecords(page)).toHaveLength(SEED.length); }); - test("writes nothing locally when the host cannot be reached", async ({ + test("writes nothing locally when the backend cannot be reached", async ({ page, }) => { - await installSeededHost(page, { failMode: "unavailable" }); + await installSeededBackend(page, { failMode: "unavailable" }); await page.goto("/pipeline-folders"); @@ -190,10 +189,10 @@ test.describe("host-provided pipeline storage", () => { expect(await readLocallyStoredPipelineKeys(page)).toEqual([]); }); - test("creates a new pipeline in the host and nowhere else", async ({ + test("creates a new pipeline in the backend and nowhere else", async ({ page, }) => { - await installSeededHost(page); + await installSeededBackend(page); await page.goto("/pipeline-folders"); await expect(page.getByText("Churn model")).toBeVisible(); @@ -203,14 +202,14 @@ test.describe("host-provided pipeline storage", () => { timeout: 30_000, }); - expect(await readHostRecords(page)).toHaveLength(SEED.length + 1); + expect(readBackendRecords(page)).toHaveLength(SEED.length + 1); expect(await readLocallyStoredPipelineKeys(page)).toEqual([]); }); test("renaming keeps one pipeline rather than leaving the old name behind", async ({ page, }) => { - await installSeededHost(page); + await installSeededBackend(page); await page.goto(`/editor-v2/${SEED[0].key}`); await expect(page.locator('[data-testid="rf__wrapper"]')).toBeVisible({ @@ -223,7 +222,7 @@ test.describe("host-provided pipeline storage", () => { await expect .poll(async () => - (await readHostRecords(page)).map((record) => record.displayName), + readBackendRecords(page).map((record) => record.displayName), ) .toEqual(["Churn model v2", "Nightly refresh"]); }); @@ -231,13 +230,13 @@ test.describe("host-provided pipeline storage", () => { test("says the backend is not available, even holding a listing it could show", async ({ page, }) => { - await installSeededHost(page); + await installSeededBackend(page); await page.goto("/pipelines"); await expect(page.getByText("Churn model")).toBeVisible(); // Not a reload: the listing is already in hand and would otherwise render. - await setHostFailMode(page, "unavailable"); + setBackendFailMode(page, "unavailable"); await page.getByRole("button", { name: "Refresh" }).click(); await expect(page.getByTestId("info-box-warning")).toContainText( @@ -249,7 +248,7 @@ test.describe("host-provided pipeline storage", () => { test("says the backend is not available on the folders page too", async ({ page, }) => { - await installSeededHost(page, { failMode: "unavailable" }); + await installSeededBackend(page, { failMode: "unavailable" }); await page.goto("/pipeline-folders"); @@ -261,7 +260,7 @@ test.describe("host-provided pipeline storage", () => { test("says a store that refuses could not be read, not that it is empty", async ({ page, }) => { - await installSeededHost(page, { failMode: "unauthenticated" }); + await installSeededBackend(page, { failMode: "unauthenticated" }); await page.goto("/pipelines"); @@ -274,7 +273,7 @@ test.describe("host-provided pipeline storage", () => { test("shows auto-save as off in the editor while the backend is away", async ({ page, }) => { - await installSeededHost(page); + await installSeededBackend(page); await page.goto(`/editor-v2/${SEED[0].key}`); await expect(page.locator('[data-testid="rf__wrapper"]')).toBeVisible({ @@ -284,14 +283,14 @@ test.describe("host-provided pipeline storage", () => { const indicator = page.getByTestId("auto-save-button"); await expect(indicator).toBeEnabled(); - await setHostFailMode(page, "unavailable"); + setBackendFailMode(page, "unavailable"); await indicator.click(); await expect(indicator).toBeDisabled(); await expect(indicator.locator(".text-destructive")).toBeVisible(); // Comes back on its own: the held edit is retried and the store answers. - await setHostFailMode(page, "none"); + setBackendFailMode(page, "none"); await expect(indicator).toBeEnabled({ timeout: 30_000 }); await expect(indicator.locator(".text-destructive")).toBeHidden(); }); @@ -299,21 +298,22 @@ test.describe("host-provided pipeline storage", () => { test("says so in the editor when a save is refused, and stops once it lands", async ({ page, }) => { - await installSeededHost(page); + await installSeededBackend(page); await page.goto(`/editor-v2/${SEED[0].key}`); await expect(page.locator('[data-testid="rf__wrapper"]')).toBeVisible({ timeout: 30_000, }); - await setHostFailMode(page, "unavailable"); + setBackendFailMode(page, "unavailable"); await page.getByTestId("auto-save-button").click(); const banner = page.getByTestId("unsaved-work-banner"); await expect(banner).toBeVisible(); - await expect(banner).toContainText(LABEL); + // Carries what the backend said, not a message invented here. + await expect(banner).toContainText("unavailable"); - await setHostFailMode(page, "none"); + setBackendFailMode(page, "none"); await banner.getByRole("button", { name: "Try now" }).click(); await expect(banner).toBeHidden(); @@ -322,14 +322,14 @@ test.describe("host-provided pipeline storage", () => { test("offers a copy before asking for a sign-in that would discard it", async ({ page, }) => { - await installSeededHost(page); + await installSeededBackend(page); await page.goto(`/editor-v2/${SEED[0].key}`); await expect(page.locator('[data-testid="rf__wrapper"]')).toBeVisible({ timeout: 30_000, }); - await setHostFailMode(page, "unauthenticated"); + setBackendFailMode(page, "unauthenticated"); await page.getByTestId("auto-save-button").click(); const dialog = page.getByTestId("expired-session"); @@ -340,7 +340,7 @@ test.describe("host-provided pipeline storage", () => { }); test("a deleted pipeline does not come back on reload", async ({ page }) => { - await installSeededHost(page); + await installSeededBackend(page); await page.goto("/pipeline-folders"); const row = page.getByRole("row").filter({ hasText: "Churn model" }); @@ -355,6 +355,6 @@ test.describe("host-provided pipeline storage", () => { await expect(page.getByText("Nightly refresh")).toBeVisible(); await expect(page.getByText("Churn model")).toBeHidden(); - expect(await readHostRecords(page)).toHaveLength(SEED.length - 1); + expect(readBackendRecords(page)).toHaveLength(SEED.length - 1); }); }); From ecbf7f49c3ff20a6673dea98339f53d95cb17f26 Mon Sep 17 00:00:00 2001 From: Camiel van Schoonhoven Date: Wed, 9 Sep 2026 12:09:09 -0700 Subject: [PATCH 31/36] fix(pipeline-storage): notice a backend that goes away while nobody is typing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two fixes to one gap. Rows written before pipelines moved onto the configured backend still say they belong to a store by its old name. Scoped lookups cannot see them, but their ids still occupy the primary key, so the next listing tried to add the same pipeline again and the collision took the pipeline page down with a raw IndexedDB error. They are renamed, and where a listing had already got part way through and written the row again, the newer one wins. Learning availability only from calls the app makes left an editor nobody is touching showing a healthy store through a restart. Now the backend is also pinged on mount, on return to the tab, and on an interval — which says something about the pipelines again, because they are served by the same backend the ping goes to. Co-Authored-By: Claude Opus 5 (1M context) --- .../Home/PipelineSection/PipelineSection.tsx | 2 +- src/components/shared/BackendUnavailable.tsx | 7 +- src/hooks/useStorageUnavailable.ts | 51 +++++++++++++++ .../components/AutoSaveIndicator.tsx | 2 +- .../pages/PipelineFolders/PipelineFolders.tsx | 2 +- src/services/pipelineStorage/db.ts | 35 ++++++++++ .../pipelineStorage/pipelineRegistry.test.ts | 65 +++++++++++++++++++ src/services/pipelineStorage/storageHealth.ts | 42 ++++-------- tests/e2e/fixtures/pipelineStorageBackend.ts | 13 ++++ tests/e2e/pipeline-storage-backend.spec.ts | 20 ++++++ 10 files changed, 205 insertions(+), 34 deletions(-) create mode 100644 src/hooks/useStorageUnavailable.ts diff --git a/src/components/Home/PipelineSection/PipelineSection.tsx b/src/components/Home/PipelineSection/PipelineSection.tsx index 39397113c2..5d8a85a346 100644 --- a/src/components/Home/PipelineSection/PipelineSection.tsx +++ b/src/components/Home/PipelineSection/PipelineSection.tsx @@ -23,9 +23,9 @@ import { } from "@/components/ui/table"; import { Paragraph, Text } from "@/components/ui/typography"; import { usePagination } from "@/hooks/usePagination"; +import { useStorageUnavailable } from "@/hooks/useStorageUnavailable"; import { APP_ROUTES } from "@/routes/router"; import { usePipelineStorage } from "@/services/pipelineStorage/PipelineStorageProvider"; -import { useStorageUnavailable } from "@/services/pipelineStorage/storageHealth"; import BulkActionsBar from "./BulkActionsBar"; import { HostMigrationNotice } from "./HostMigrationNotice"; diff --git a/src/components/shared/BackendUnavailable.tsx b/src/components/shared/BackendUnavailable.tsx index a4d129ebf3..6d8eb125a2 100644 --- a/src/components/shared/BackendUnavailable.tsx +++ b/src/components/shared/BackendUnavailable.tsx @@ -3,11 +3,14 @@ import { getPipelineStorageService } from "@/services/pipelineStorage/PipelineSt export function BackendUnavailable() { const { mode } = getPipelineStorageService(); + const errorMessage = + mode.kind === "backend" + ? "The configured backend is currently unavailable." + : "Pipeline storage is currently unavailable."; return ( - {mode.kind === "backend" ? mode.label : "Pipeline storage"} is not - answering. Pipelines cannot be read or saved until it is back. + {errorMessage} ); } diff --git a/src/hooks/useStorageUnavailable.ts b/src/hooks/useStorageUnavailable.ts new file mode 100644 index 0000000000..193cdc40bc --- /dev/null +++ b/src/hooks/useStorageUnavailable.ts @@ -0,0 +1,51 @@ +import { useQuery } from "@tanstack/react-query"; +import { useSyncExternalStore } from "react"; + +import { useBackend } from "@/providers/BackendProvider"; +import { + isStorageAnswering, + reportStorageAnswered, + reportStorageFailed, + subscribeStorageHealth, +} from "@/services/pipelineStorage/storageHealth"; +import { isBackendStorage } from "@/services/pipelineStorage/storageMode"; + +const PING_STALE_MS = 10_000; + +const PING_INTERVAL_MS = 30_000; + +/** + * Whether the backend holding the pipelines has gone away. + * + * Two sources, because either alone leaves a hole. What the app already asked + * for is the cheapest and most direct answer, but a page nobody is touching + * asks for nothing — an editor left open through a restart would show a healthy + * store until the next keystroke. So the backend is also pinged on mount, on + * return to the tab, and on a slow interval, and both report to the same place. + */ +export function useStorageUnavailable(): boolean { + const backendStorage = isBackendStorage(); + const { ping, backendUrl } = useBackend(); + + const answering = useSyncExternalStore( + subscribeStorageHealth, + isStorageAnswering, + () => true, + ); + + useQuery({ + queryKey: ["pipeline-storage-reachable", backendUrl], + queryFn: async () => { + const reachable = await ping({ notifyResult: false }); + if (reachable) reportStorageAnswered(); + else reportStorageFailed("unavailable"); + return reachable; + }, + enabled: backendStorage, + staleTime: PING_STALE_MS, + refetchInterval: PING_INTERVAL_MS, + retry: false, + }); + + return backendStorage && !answering; +} diff --git a/src/routes/v2/pages/Editor/components/EditorMenuBar/components/AutoSaveIndicator.tsx b/src/routes/v2/pages/Editor/components/EditorMenuBar/components/AutoSaveIndicator.tsx index 27d459b4f5..b30c9b62e7 100644 --- a/src/routes/v2/pages/Editor/components/EditorMenuBar/components/AutoSaveIndicator.tsx +++ b/src/routes/v2/pages/Editor/components/EditorMenuBar/components/AutoSaveIndicator.tsx @@ -4,9 +4,9 @@ import type { ReactNode } from "react"; import TooltipButton from "@/components/shared/Buttons/TooltipButton"; import { Icon } from "@/components/ui/icon"; import { Spinner } from "@/components/ui/spinner"; +import { useStorageUnavailable } from "@/hooks/useStorageUnavailable"; import { cn } from "@/lib/utils"; import { useEditorSession } from "@/routes/v2/pages/Editor/store/EditorSessionContext"; -import { useStorageUnavailable } from "@/services/pipelineStorage/storageHealth"; import { tracking } from "@/utils/tracking"; const LAYER_BASE_CLASS = diff --git a/src/routes/v2/pages/PipelineFolders/PipelineFolders.tsx b/src/routes/v2/pages/PipelineFolders/PipelineFolders.tsx index c102a19ff8..032a426c62 100644 --- a/src/routes/v2/pages/PipelineFolders/PipelineFolders.tsx +++ b/src/routes/v2/pages/PipelineFolders/PipelineFolders.tsx @@ -5,8 +5,8 @@ import { BackendUnavailable } from "@/components/shared/BackendUnavailable"; import { withSuspenseWrapper } from "@/components/shared/SuspenseWrapper"; import { BlockStack, InlineStack } from "@/components/ui/layout"; import { Skeleton } from "@/components/ui/skeleton"; +import { useStorageUnavailable } from "@/hooks/useStorageUnavailable"; import { usePipelineStorage } from "@/services/pipelineStorage/PipelineStorageProvider"; -import { useStorageUnavailable } from "@/services/pipelineStorage/storageHealth"; import { type PipelineRef, ROOT_FOLDER_ID, diff --git a/src/services/pipelineStorage/db.ts b/src/services/pipelineStorage/db.ts index 501ce021a8..069347b614 100644 --- a/src/services/pipelineStorage/db.ts +++ b/src/services/pipelineStorage/db.ts @@ -117,6 +117,41 @@ pipelineStorageDb }); }); +/** + * The value naming the store changed when pipelines moved onto the configured + * backend. A row still saying "host" is invisible to every lookup, which scopes + * itself to the store in use — but its id still occupies the primary key, so + * the next listing tries to add the same pipeline again and the collision takes + * the whole page down rather than one row. + */ +pipelineStorageDb.version(8).upgrade(async (tx) => { + await tx.table("pipeline_specs").clear(); + + const registry = tx.table("pipeline_registry"); + const rows = await registry.toArray(); + const claimed = new Set( + rows + .filter((row) => row.storage === "backend") + .map((row) => row.storageKey), + ); + + for (const row of rows) { + if ((row.storage as string) !== "host") continue; + + /** + * A listing that got part way through before the collision left rows under + * both names. The one written since is the one the backend just described. + */ + if (claimed.has(row.storageKey)) { + await registry.delete(row.id); + continue; + } + + await registry.update(row.id, { storage: "backend" }); + claimed.add(row.storageKey); + } +}); + pipelineStorageDb.on("ready", async () => { await seedRegistryFromLegacyList(); }); diff --git a/src/services/pipelineStorage/pipelineRegistry.test.ts b/src/services/pipelineStorage/pipelineRegistry.test.ts index bb225c601a..d5fd279f89 100644 --- a/src/services/pipelineStorage/pipelineRegistry.test.ts +++ b/src/services/pipelineStorage/pipelineRegistry.test.ts @@ -120,6 +120,71 @@ describe("registry rows belong to the store that wrote them", () => { }); }); +describe("rows left by an earlier name for the same store", () => { + const V7_SCHEMA = { + pipeline_registry: + "id, storage, folderId, &[storage+storageKey], [storage+folderId], [storage+folderId+storageKey]", + folders: "id, parentId", + pipeline_specs: "[storage+storageKey]", + host_migration: "id", + }; + + async function openAtV7() { + await pipelineStorageDb.close(); + await Dexie.delete("tangle_pipelines"); + const old = new Dexie("tangle_pipelines"); + old.version(7).stores(V7_SCHEMA); + await old.open(); + return old; + } + + it("reads them as the store they always described", async () => { + const old = await openAtV7(); + await old.table("pipeline_registry").bulkAdd([ + { + id: "external-1", + storage: "host", + storageKey: "Churn model", + folderId: ROOT_FOLDER_ID, + }, + ]); + old.close(); + + await pipelineStorageDb.open(); + + expect( + (await pipelineStorageDb.pipeline_registry.get("external-1"))?.storage, + ).toBe("backend"); + }); + + it("keeps the newer row when a listing had already claimed the key", async () => { + const old = await openAtV7(); + await old.table("pipeline_registry").bulkAdd([ + { + id: "stale", + storage: "host", + storageKey: "Churn model", + folderId: ROOT_FOLDER_ID, + }, + { + id: "fresh", + storage: "backend", + storageKey: "Churn model", + folderId: ROOT_FOLDER_ID, + }, + ]); + old.close(); + + await pipelineStorageDb.open(); + + expect( + (await pipelineStorageDb.pipeline_registry.toArray()).map( + (row) => row.id, + ), + ).toEqual(["fresh"]); + }); +}); + describe("attributing rows written before rows said which store", () => { const OLD_SCHEMA = { pipeline_registry: "id, &storageKey, folderId, [folderId+storageKey]", diff --git a/src/services/pipelineStorage/storageHealth.ts b/src/services/pipelineStorage/storageHealth.ts index 15845dd435..6b48b57134 100644 --- a/src/services/pipelineStorage/storageHealth.ts +++ b/src/services/pipelineStorage/storageHealth.ts @@ -1,26 +1,21 @@ -import { useSyncExternalStore } from "react"; - -import { isBackendStorage } from "./storageMode"; import type { StorageErrorCode } from "./types"; /** - * Whether the store holding the pipelines is answering, learned from the calls - * the app already makes rather than from a health check against something else. - * - * The configured backend serves more than pipelines, and a health check - * against it says nothing about whether the pipeline routes are answering — - * which is the only thing the pipeline list and the save indicator are about. + * Whether the backend holding the pipelines is answering, learned from the + * calls the app already makes. An error is only an outage if the backend failed + * to answer at all: "no such pipeline" is an answer. * - * An error is only an outage if the store failed to answer at all. "No such - * pipeline" is an answer. + * Kept free of React and of the provider that knows the backend's address, so + * that the driver reporting into it does not drag either into its own module + * graph. `useStorageUnavailable` is the way to read it. */ -let reachable = true; +let answering = true; const listeners = new Set<() => void>(); function set(next: boolean): void { - if (reachable === next) return; - reachable = next; + if (answering === next) return; + answering = next; for (const listener of listeners) listener(); } @@ -29,25 +24,14 @@ export function reportStorageAnswered(): void { } export function reportStorageFailed(code: StorageErrorCode): void { - if (code !== "unavailable") { - set(true); - return; - } - - set(false); + set(code !== "unavailable"); } -function subscribe(listener: () => void): () => void { +export function subscribeStorageHealth(listener: () => void): () => void { listeners.add(listener); return () => listeners.delete(listener); } -export function useStorageUnavailable(): boolean { - const answering = useSyncExternalStore( - subscribe, - () => reachable, - () => true, - ); - - return isBackendStorage() && !answering; +export function isStorageAnswering(): boolean { + return answering; } diff --git a/tests/e2e/fixtures/pipelineStorageBackend.ts b/tests/e2e/fixtures/pipelineStorageBackend.ts index 5a4d07b944..89abf36456 100644 --- a/tests/e2e/fixtures/pipelineStorageBackend.ts +++ b/tests/e2e/fixtures/pipelineStorageBackend.ts @@ -31,6 +31,8 @@ interface BackendStore { const PIPELINES_PATH = "**/api/users/me/pipelines**"; +const PING_PATH = "**/services/ping"; + const STATUS_FOR: Record, number> = { unavailable: 503, unauthenticated: 401, @@ -90,6 +92,17 @@ export async function installPipelineStorageBackend( revision: seed.length, }); + /** + * One deployment serves both, so the health check answers exactly when the + * pipeline routes do. + */ + await page.route(PING_PATH, (route) => { + const store = storeFor(page); + return store.failMode === "none" + ? route.fulfill({ status: 200, body: "ok" }) + : route.fulfill({ status: STATUS_FOR[store.failMode], body: "down" }); + }); + await page.route(PIPELINES_PATH, async (route) => { const store = storeFor(page); diff --git a/tests/e2e/pipeline-storage-backend.spec.ts b/tests/e2e/pipeline-storage-backend.spec.ts index bae5feb16c..b30b51c71d 100644 --- a/tests/e2e/pipeline-storage-backend.spec.ts +++ b/tests/e2e/pipeline-storage-backend.spec.ts @@ -295,6 +295,26 @@ test.describe("backend pipeline storage", () => { await expect(indicator.locator(".text-destructive")).toBeHidden(); }); + test("turns auto-save red in an open editor nobody is touching", async ({ + page, + }) => { + await installSeededBackend(page); + + await page.goto(`/editor-v2/${SEED[0].key}`); + await expect(page.locator('[data-testid="rf__wrapper"]')).toBeVisible({ + timeout: 30_000, + }); + + const indicator = page.getByTestId("auto-save-button"); + await expect(indicator).toBeEnabled(); + + // No edit, no click: the page has to notice on its own. + setBackendFailMode(page, "unavailable"); + + await expect(indicator).toBeDisabled({ timeout: 60_000 }); + await expect(indicator.locator(".text-destructive")).toBeVisible(); + }); + test("says so in the editor when a save is refused, and stops once it lands", async ({ page, }) => { From 63824c1d2d9b49fa89ed1eb82533c019bad130fc Mon Sep 17 00:00:00 2001 From: Camiel van Schoonhoven Date: Wed, 9 Sep 2026 12:17:00 -0700 Subject: [PATCH 32/36] fix(editor): send the held edit the moment the backend answers again MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The retry was there, but only the backoff ladder and the browser's own online and focus events could trigger it — so an edit refused while the backend was down sat unsent for up to a minute after it came back, with the tab in view the whole time and nothing to click. The ping that turns the indicator red already knows the moment it is answering again. The editor now listens to it. Co-Authored-By: Claude Opus 5 (1M context) --- .../pages/Editor/store/autoSaveStore.test.ts | 29 +++++++++++++++++++ .../v2/pages/Editor/store/autoSaveStore.ts | 20 +++++++++++-- tests/e2e/pipeline-storage-backend.spec.ts | 29 +++++++++++++++++++ 3 files changed, 75 insertions(+), 3 deletions(-) diff --git a/src/routes/v2/pages/Editor/store/autoSaveStore.test.ts b/src/routes/v2/pages/Editor/store/autoSaveStore.test.ts index b3dc76d6e7..cd20aba2d2 100644 --- a/src/routes/v2/pages/Editor/store/autoSaveStore.test.ts +++ b/src/routes/v2/pages/Editor/store/autoSaveStore.test.ts @@ -2,6 +2,10 @@ import { describe, expect, it, vi } from "vitest"; import type { ComponentSpec } from "@/models/componentSpec"; import type { PipelineFile } from "@/services/pipelineStorage/PipelineFile"; +import { + reportStorageAnswered, + reportStorageFailed, +} from "@/services/pipelineStorage/storageHealth"; import { AutoSaveStore } from "./autoSaveStore"; import { PipelineFileStore } from "./pipelineFileStore"; @@ -107,6 +111,31 @@ describe("AutoSaveStore when the store cannot be reached", () => { store.dispose(); }); + it("saves the held edit the moment the store answers again", async () => { + let reachable = false; + const written: string[] = []; + const store = createStore({ + write: async (yamlText: string) => { + if (!reachable) throw new Error("unreachable"); + written.push(yamlText); + }, + } as unknown as PipelineFile); + + store.init(createSpec("Churn model"), "Churn model"); + await store.save(); + reportStorageFailed("unavailable"); + expect(written).toEqual([]); + + reachable = true; + reportStorageAnswered(); + + await vi.waitFor(() => { + expect(written).toEqual(["name: Churn model"]); + expect(store.saveError).toBeNull(); + }); + store.dispose(); + }); + it("stops retrying once the editor is closed", async () => { const write = vi.fn(async () => { throw new Error("unreachable"); diff --git a/src/routes/v2/pages/Editor/store/autoSaveStore.ts b/src/routes/v2/pages/Editor/store/autoSaveStore.ts index 789d17b00f..8eb980cd83 100644 --- a/src/routes/v2/pages/Editor/store/autoSaveStore.ts +++ b/src/routes/v2/pages/Editor/store/autoSaveStore.ts @@ -10,6 +10,10 @@ import { isExpiredSession, isWriteWorthRetrying, } from "@/services/pipelineStorage/storageErrors"; +import { + isStorageAnswering, + subscribeStorageHealth, +} from "@/services/pipelineStorage/storageHealth"; import { AUTOSAVE_DEBOUNCE_TIME_MS } from "@/utils/constants"; import { debounce } from "@/utils/debounce"; import { getErrorMessage } from "@/utils/string"; @@ -249,9 +253,10 @@ export class AutoSaveStore { } /** - * Coming back online or back to the tab is the cheapest signal that a store - * that refused a write a moment ago might take it now, and it beats waiting - * out the backoff. + * The backoff is the floor, not the plan. A store that is answering again is + * the direct signal that a refused write can go now, and waiting out a ladder + * that has grown to a minute leaves work unsaved for no reason. Coming back + * online or back to the tab count for the same reason. */ private watchForRecovery() { if (typeof window === "undefined") return; @@ -260,9 +265,18 @@ export class AutoSaveStore { window.addEventListener("online", retryNow); window.addEventListener("focus", retryNow); + let wasAnswering = isStorageAnswering(); + const unsubscribe = subscribeStorageHealth(() => { + const answering = isStorageAnswering(); + const recovered = answering && !wasAnswering; + wasAnswering = answering; + if (recovered) retryNow(); + }); + this.disposeRecovery = () => { window.removeEventListener("online", retryNow); window.removeEventListener("focus", retryNow); + unsubscribe(); }; } diff --git a/tests/e2e/pipeline-storage-backend.spec.ts b/tests/e2e/pipeline-storage-backend.spec.ts index b30b51c71d..2a872b6f09 100644 --- a/tests/e2e/pipeline-storage-backend.spec.ts +++ b/tests/e2e/pipeline-storage-backend.spec.ts @@ -339,6 +339,35 @@ test.describe("backend pipeline storage", () => { await expect(banner).toBeHidden(); }); + test("sends the edit it was holding as soon as the backend answers", async ({ + page, + }) => { + await installSeededBackend(page); + + await page.goto(`/editor-v2/${SEED[0].key}`); + await expect(page.locator('[data-testid="rf__wrapper"]')).toBeVisible({ + timeout: 30_000, + }); + + setBackendFailMode(page, "unavailable"); + await page.getByTestId("auto-save-button").click(); + await expect(page.getByTestId("unsaved-work-banner")).toBeVisible(); + + const before = readBackendRecords(page).find( + (record) => record.key === SEED[0].key, + )?.contentVersion; + + setBackendFailMode(page, "none"); + + await expect(page.getByTestId("unsaved-work-banner")).toBeHidden({ + timeout: 60_000, + }); + expect( + readBackendRecords(page).find((record) => record.key === SEED[0].key) + ?.contentVersion, + ).not.toBe(before); + }); + test("offers a copy before asking for a sign-in that would discard it", async ({ page, }) => { From 97052d8ee32f9e433cf73473e3ef8b11a0f98828 Mon Sep 17 00:00:00 2001 From: Camiel van Schoonhoven Date: Wed, 9 Sep 2026 12:40:54 -0700 Subject: [PATCH 33/36] fix(editor): hold a refused edit where a reload cannot lose it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An edit the backend would not take lived only in the editor that made it, and the most likely next thing someone does is leave the page to go and turn the backend back on. Closing the editor stopped the retrying; a full page load took the held edit with it. Either way the work was gone by the time the backend came back. Refused edits are now written down and sent when it next answers, whatever happened to the editor in between — including a reload, a different tab, or a session that ended hours earlier. The in-memory retry stays as the fast path. An editor that closes still holding work keeps trying too, and stands down if the same pipeline is reopened and saved elsewhere, so its older text cannot land on top of something newer. Co-Authored-By: Claude Opus 5 (1M context) --- .../pages/Editor/store/autoSaveStore.test.ts | 54 ++++++++++-- .../v2/pages/Editor/store/autoSaveStore.ts | 56 ++++++++++-- .../PipelineStorageProvider.tsx | 5 +- src/services/pipelineStorage/db.ts | 11 +++ src/services/pipelineStorage/pendingWrites.ts | 88 +++++++++++++++++++ src/services/pipelineStorage/types.ts | 7 ++ tests/e2e/pipeline-storage-backend.spec.ts | 32 +++++++ 7 files changed, 238 insertions(+), 15 deletions(-) create mode 100644 src/services/pipelineStorage/pendingWrites.ts diff --git a/src/routes/v2/pages/Editor/store/autoSaveStore.test.ts b/src/routes/v2/pages/Editor/store/autoSaveStore.test.ts index cd20aba2d2..79e946223a 100644 --- a/src/routes/v2/pages/Editor/store/autoSaveStore.test.ts +++ b/src/routes/v2/pages/Editor/store/autoSaveStore.test.ts @@ -136,21 +136,57 @@ describe("AutoSaveStore when the store cannot be reached", () => { store.dispose(); }); - it("stops retrying once the editor is closed", async () => { - const write = vi.fn(async () => { - throw new Error("unreachable"); - }); - const store = createStore({ write } as unknown as PipelineFile); + it("keeps trying after the editor is closed, and sends it when the store returns", async () => { + let reachable = false; + const written: string[] = []; + const file = { + storageKey: "Churn model", + write: async (yamlText: string) => { + if (!reachable) throw new Error("unreachable"); + written.push(yamlText); + }, + } as unknown as PipelineFile; + const store = createStore(file); store.init(createSpec("Churn model"), "Churn model"); await store.save(); + reportStorageFailed("unavailable"); + + // Going to fix the connection closes the editor. store.dispose(); + expect(written).toEqual([]); - write.mockClear(); - window.dispatchEvent(new Event("focus")); - await new Promise((resolve) => setTimeout(resolve, 20)); + reachable = true; + reportStorageAnswered(); + + await vi.waitFor(() => expect(written).toEqual(["name: Churn model"])); + }); + + it("stands down rather than putting its older text over a reopened pipeline", async () => { + const written: string[] = []; + const file = { + storageKey: "Churn model", + write: async (yamlText: string) => { + if (yamlText.includes("v1")) throw new Error("unreachable"); + written.push(yamlText); + }, + } as unknown as PipelineFile; + + const closing = createStore(file); + closing.init(createSpec("Churn model v1"), "Churn model"); + await closing.save(); + reportStorageFailed("unavailable"); + closing.dispose(); + + const reopened = createStore(file); + reopened.init(createSpec("Churn model v2"), "Churn model"); + await reopened.save(); - expect(write).not.toHaveBeenCalled(); + reportStorageAnswered(); + await new Promise((resolve) => setTimeout(resolve, 50)); + + expect(written).toEqual(["name: Churn model v2"]); + reopened.dispose(); }); it("retries with the newest edit, never the one that was refused", async () => { diff --git a/src/routes/v2/pages/Editor/store/autoSaveStore.ts b/src/routes/v2/pages/Editor/store/autoSaveStore.ts index 8eb980cd83..48ae466add 100644 --- a/src/routes/v2/pages/Editor/store/autoSaveStore.ts +++ b/src/routes/v2/pages/Editor/store/autoSaveStore.ts @@ -6,6 +6,10 @@ import { serializePipelineDocumentToText, } from "@/models/componentSpec"; import { saveUndoHistory } from "@/routes/v2/pages/Editor/utils/undoHistoryStorage"; +import { + forgetPendingWrite, + recordPendingWrite, +} from "@/services/pipelineStorage/pendingWrites"; import { isExpiredSession, isWriteWorthRetrying, @@ -25,6 +29,14 @@ const AUTOSAVE_MIN_SAVING_INDICATOR_MS = 600; const RETRY_DELAYS_MS = [5_000, 15_000, 30_000, 60_000]; +/** + * Which editor session speaks for a pipeline right now. A session that closes + * holding work the store refused keeps trying, and this is what stops it + * putting that older text back over a session that has since reopened the same + * pipeline and saved something newer. + */ +const speakingFor = new Map(); + export class AutoSaveStore { @observable accessor isSaving = false; @observable accessor lastSavedAt: Date | null = null; @@ -104,12 +116,23 @@ export class AutoSaveStore { this.debouncedSave.cancel(); this.clearRetry(); - this.disposeRecovery?.(); - this.disposeRecovery = null; this.disposeReaction?.(); this.disposeReaction = null; this.spec = null; this.pipelineName = null; + + /** + * Work the store has not taken outlives the editor that made it: leaving + * the page to go and fix the connection is the most likely thing someone + * does next, and it must not be what loses the edit. + */ + if (this.pendingYaml === null) this.stopWatching(); + } + + private stopWatching() { + this.clearRetry(); + this.disposeRecovery?.(); + this.disposeRecovery = null; } async save() { @@ -182,6 +205,11 @@ export class AutoSaveStore { const yamlText = this.pendingYaml; const outcome = await this.writeOnce(yamlText); + if (outcome === "stood-down") { + this.pendingYaml = null; + break; + } + if (outcome instanceof Error) { this.setSaveError(outcome); if (isWriteWorthRetrying(outcome)) this.scheduleRetry(); @@ -194,25 +222,44 @@ export class AutoSaveStore { } this.setPending(false); + if (this.closed) this.stopWatching(); } - private async writeOnce(yamlText: string): Promise { + private async writeOnce( + yamlText: string, + ): Promise { const pipelineName = this.pipelineName; + const open = this.pipelineFileStore.activePipelineFile; + + if (open) { + if (this.closed && speakingFor.get(open.storageKey) !== this) { + return "stood-down"; + } + speakingFor.set(open.storageKey, this); + } + this.setSaving(true); const savePromise = (async () => { + const file = this.pipelineFileStore.activePipelineFile; + try { - const file = this.pipelineFileStore.activePipelineFile; if (!file) { throw new Error(`No open file to save "${pipelineName}" to.`); } await file.write(yamlText); + await forgetPendingWrite(file); await this.persistUndoHistory(); this.lastSavedYaml = yamlText; return new Date(); } catch (error) { console.error("Auto-save failed:", error); + /** + * Held where a reload cannot lose it. The in-memory retry below is the + * fast path; this is what survives the tab. + */ + if (file) void recordPendingWrite(file, yamlText); return error instanceof Error ? error : new Error(String(error)); } })(); @@ -227,7 +274,6 @@ export class AutoSaveStore { private scheduleRetry() { this.clearRetry(); - if (this.closed) return; const delay = RETRY_DELAYS_MS[Math.min(this.retryAttempt, RETRY_DELAYS_MS.length - 1)]; diff --git a/src/services/pipelineStorage/PipelineStorageProvider.tsx b/src/services/pipelineStorage/PipelineStorageProvider.tsx index 3e958b4aef..bc91802866 100644 --- a/src/services/pipelineStorage/PipelineStorageProvider.tsx +++ b/src/services/pipelineStorage/PipelineStorageProvider.tsx @@ -7,6 +7,7 @@ import { } from "@/hooks/useRequiredContext"; import { startHostMigration } from "./hostMigration"; +import { startPendingWriteFlusher } from "./pendingWrites"; import { getPipelineStorageService, type PipelineStorageService, @@ -23,8 +24,10 @@ export function PipelineStorageProvider({ children }: { children: ReactNode }) { if (service.mode.kind !== "backend") return; void startHostMigration(service.rootFolder).catch((error: unknown) => { - console.error("Could not copy pipelines into the host store:", error); + console.error("Could not copy pipelines into the backend:", error); }); + + return startPendingWriteFlusher(); }, [service]); return ( diff --git a/src/services/pipelineStorage/db.ts b/src/services/pipelineStorage/db.ts index 069347b614..32e13f2e84 100644 --- a/src/services/pipelineStorage/db.ts +++ b/src/services/pipelineStorage/db.ts @@ -7,6 +7,7 @@ import { type CachedPipelineSpec, type FolderEntry, type HostMigrationRecord, + type PendingPipelineWrite, type PipelineRegistryEntry, type PipelineStorageKind, ROOT_FOLDER_ID, @@ -17,6 +18,7 @@ export type PipelineStorageDb = Dexie & { folders: EntityTable; pipeline_specs: Table; host_migration: EntityTable; + pending_writes: Table; }; export const pipelineStorageDb = new Dexie( @@ -152,6 +154,15 @@ pipelineStorageDb.version(8).upgrade(async (tx) => { } }); +/** + * Edits a store would not take have to outlive the editor that made them: the + * most likely next thing someone does is leave the page to go and fix the + * connection, and that must not be what loses the work. + */ +pipelineStorageDb.version(9).stores({ + pending_writes: "[storage+storageKey]", +}); + pipelineStorageDb.on("ready", async () => { await seedRegistryFromLegacyList(); }); diff --git a/src/services/pipelineStorage/pendingWrites.ts b/src/services/pipelineStorage/pendingWrites.ts new file mode 100644 index 0000000000..bfe92949cc --- /dev/null +++ b/src/services/pipelineStorage/pendingWrites.ts @@ -0,0 +1,88 @@ +import { pipelineStorageDb } from "./db"; +import type { PipelineFile } from "./PipelineFile"; +import { getPipelineStorageService } from "./PipelineStorageService"; +import { isStorageAnswering, subscribeStorageHealth } from "./storageHealth"; +import { currentStorageKind, isBackendStorage } from "./storageMode"; + +/** + * Edits the backend refused, kept where a reload cannot lose them and sent + * again when it answers. Browser storage does not need this: it does not go + * away between one write and the next. + */ +export async function recordPendingWrite( + file: PipelineFile, + yaml: string, +): Promise { + if (!isBackendStorage()) return; + + await pipelineStorageDb.pending_writes.put({ + storage: currentStorageKind(), + storageKey: file.storageKey, + yaml, + recordedAt: Date.now(), + }); +} + +export async function forgetPendingWrite(file: PipelineFile): Promise { + if (!isBackendStorage()) return; + + await pipelineStorageDb.pending_writes.delete([ + currentStorageKind(), + file.storageKey, + ]); +} + +async function flushPendingWrites(): Promise { + if (!isBackendStorage()) return; + + const storage = currentStorageKind(); + const held = await pipelineStorageDb.pending_writes + .where({ storage }) + .toArray(); + + const { rootFolder } = getPipelineStorageService(); + + for (const write of held) { + try { + await rootFolder.addFile(write.storageKey, write.yaml); + await pipelineStorageDb.pending_writes.delete([ + storage, + write.storageKey, + ]); + } catch (error) { + /** + * Still not taking writes. The record stays exactly as it is and the next + * recovery tries again; nothing else in the list should be abandoned + * because one of them failed. + */ + console.error( + `Could not send the held edit to "${write.storageKey}":`, + error, + ); + } + } +} + +/** + * Sends whatever is held now, and again each time the backend goes from silent + * to answering. Started with the app rather than with an editor, because the + * editor that made the edit may be long gone. + */ +export function startPendingWriteFlusher(): () => void { + let wasAnswering = isStorageAnswering(); + + void flushPendingWrites().catch((error: unknown) => { + console.error("Could not send held edits:", error); + }); + + return subscribeStorageHealth(() => { + const answering = isStorageAnswering(); + const recovered = answering && !wasAnswering; + wasAnswering = answering; + if (!recovered) return; + + void flushPendingWrites().catch((error: unknown) => { + console.error("Could not send held edits:", error); + }); + }); +} diff --git a/src/services/pipelineStorage/types.ts b/src/services/pipelineStorage/types.ts index abdfc3596e..288269546f 100644 --- a/src/services/pipelineStorage/types.ts +++ b/src/services/pipelineStorage/types.ts @@ -66,6 +66,13 @@ export interface CachedPipelineSpec { spec: ComponentSpec; } +export interface PendingPipelineWrite { + storage: PipelineStorageKind; + storageKey: string; + yaml: string; + recordedAt: number; +} + export interface HostMigrationRecord { id: string; startedAt: number; diff --git a/tests/e2e/pipeline-storage-backend.spec.ts b/tests/e2e/pipeline-storage-backend.spec.ts index 2a872b6f09..96220c070b 100644 --- a/tests/e2e/pipeline-storage-backend.spec.ts +++ b/tests/e2e/pipeline-storage-backend.spec.ts @@ -368,6 +368,38 @@ test.describe("backend pipeline storage", () => { ).not.toBe(before); }); + test("sends it even when the editor was left to go and fix the connection", async ({ + page, + }) => { + await installSeededBackend(page); + + await page.goto(`/editor-v2/${SEED[0].key}`); + await expect(page.locator('[data-testid="rf__wrapper"]')).toBeVisible({ + timeout: 30_000, + }); + + setBackendFailMode(page, "unavailable"); + await page.getByTestId("auto-save-button").click(); + await expect(page.getByTestId("unsaved-work-banner")).toBeVisible(); + + const before = readBackendRecords(page).find( + (record) => record.key === SEED[0].key, + )?.contentVersion; + + // Leaving the editor is exactly what someone does to go and turn it back on. + await page.goto("/settings/backend"); + setBackendFailMode(page, "none"); + + await expect + .poll( + () => + readBackendRecords(page).find((record) => record.key === SEED[0].key) + ?.contentVersion, + { timeout: 60_000 }, + ) + .not.toBe(before); + }); + test("offers a copy before asking for a sign-in that would discard it", async ({ page, }) => { From 665c5a01ffe11192b29b376de402a174098b60e2 Mon Sep 17 00:00:00 2001 From: Camiel van Schoonhoven Date: Wed, 9 Sep 2026 13:48:44 -0700 Subject: [PATCH 34/36] fix(pipeline-storage): index browser pipelines off the database open path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pipelines predating the registry were indexed from Dexie's `ready` handler, which gates `open()`. The handler reads the legacy list out of another database first, and once it has yielded to a non-Dexie promise it can never touch this one again: every later call queues behind the open it is itself holding up. Awaiting the write there left the database opening forever, so the pipeline list never loaded for anyone who already had pipelines. Indexing now runs on its own, awaited by the folder listing that needs the rows, and claims only keys with no row yet inside a transaction — a listing running at the same time can no longer collide on the unique index. Co-Authored-By: Claude Opus 5 (1M context) --- src/services/pipelineStorage/db.ts | 77 +++++++++++++------ .../drivers/FolderIndexDbStorageDriver.ts | 3 + tests/e2e/pipeline-local-storage.spec.ts | 33 ++++++++ 3 files changed, 91 insertions(+), 22 deletions(-) create mode 100644 tests/e2e/pipeline-local-storage.spec.ts diff --git a/src/services/pipelineStorage/db.ts b/src/services/pipelineStorage/db.ts index 32e13f2e84..6e28015e8b 100644 --- a/src/services/pipelineStorage/db.ts +++ b/src/services/pipelineStorage/db.ts @@ -163,38 +163,71 @@ pipelineStorageDb.version(9).stores({ pending_writes: "[storage+storageKey]", }); -pipelineStorageDb.on("ready", async () => { - await seedRegistryFromLegacyList(); -}); +let indexing: Promise | undefined; /** - * The registry indexes storage keys within the one store the app is using. In - * host mode those keys are the host's, so seeding it with local pipeline names - * would claim files the host has never heard of. + * Pipelines predating the registry live in the legacy list and have no row + * saying which folder they are in, so a folder listing cannot see them. + * + * Deliberately not run from `on("ready")`: that handler gates `open()`, and it + * has to read another database first — once it has yielded to anything outside + * Dexie it can never touch this one again, because every call would queue + * behind the open it is itself holding up. */ -async function seedRegistryFromLegacyList() { - if (currentStorageKind() === "backend") return; +export function ensureBrowserPipelinesIndexed(): Promise { + indexing ??= indexPipelinesAlreadyInThisBrowser(); + return indexing; +} - const seeded = await pipelineStorageDb.pipeline_registry - .where("storage") - .equals("local") - .count(); - if (seeded > 0) return; +/** + * The registry indexes storage keys within the one store the page is using, and + * in backend mode those keys are the backend's — indexing local pipeline names + * there would claim files it has never heard of. + */ +async function indexPipelinesAlreadyInThisBrowser() { + if (currentStorageKind() === "backend") return; const { getAllComponentFilesFromList } = await import("@/utils/componentStore"); const knownPipelines = await getAllComponentFilesFromList( USER_PIPELINES_LIST_NAME, ); - if (knownPipelines.size === 0) return; - await pipelineStorageDb.pipeline_registry.bulkAdd( - [...knownPipelines.keys()].map((storageKey) => ({ - id: crypto.randomUUID(), - storage: "local" as const, - storageKey, - folderId: ROOT_FOLDER_ID, - })), - ); + try { + /** + * A listing running at the same time claims the same keys, and the unique + * index would fail whichever got there second. Both claim inside a + * transaction over this table, which the browser will not interleave. + */ + await pipelineStorageDb.transaction( + "rw", + pipelineStorageDb.pipeline_registry, + async () => { + const indexed = new Set( + ( + await pipelineStorageDb.pipeline_registry + .where("storage") + .equals("local") + .toArray() + ).map((row) => row.storageKey), + ); + + const missing = [...knownPipelines.keys()] + .filter((storageKey) => !indexed.has(storageKey)) + .map((storageKey) => ({ + id: crypto.randomUUID(), + storage: "local" as const, + storageKey, + folderId: ROOT_FOLDER_ID, + })); + + if (missing.length > 0) { + await pipelineStorageDb.pipeline_registry.bulkAdd(missing); + } + }, + ); + } catch (error) { + console.error("Could not index the pipelines in this browser", error); + } } diff --git a/src/services/pipelineStorage/drivers/FolderIndexDbStorageDriver.ts b/src/services/pipelineStorage/drivers/FolderIndexDbStorageDriver.ts index c22b3f91d2..bbd1b8ebeb 100644 --- a/src/services/pipelineStorage/drivers/FolderIndexDbStorageDriver.ts +++ b/src/services/pipelineStorage/drivers/FolderIndexDbStorageDriver.ts @@ -1,6 +1,7 @@ import { getComponentFileFromList } from "@/utils/componentStore"; import { USER_PIPELINES_LIST_NAME } from "@/utils/constants"; +import { ensureBrowserPipelinesIndexed } from "../db"; import { findByFolderAndStorageKey, getAllByFolderId, @@ -23,6 +24,8 @@ export class FolderIndexDbStorageDriver extends RootFolderDbStorageDriver { } override async list(): Promise { + await ensureBrowserPipelinesIndexed(); + const entries = await getAllByFolderId(this.folderId); const descriptors: PipelineFileDescriptor[] = []; diff --git a/tests/e2e/pipeline-local-storage.spec.ts b/tests/e2e/pipeline-local-storage.spec.ts new file mode 100644 index 0000000000..5f0604b407 --- /dev/null +++ b/tests/e2e/pipeline-local-storage.spec.ts @@ -0,0 +1,33 @@ +import { expect, test } from "@playwright/test"; + +import { seedLocallyStoredPipeline } from "./fixtures/pipelineStorageBackend"; + +/** + * The open-source path: pipelines the browser already holds, and no backend + * involved in storing them. + */ +test.describe("browser-stored pipelines", () => { + test("lists what the browser already held", async ({ page }) => { + const name = "Kept in the browser"; + + await page.goto("/"); + await seedLocallyStoredPipeline(page, name); + + await page.goto("/pipelines"); + + await expect(page.getByText(name)).toBeVisible(); + }); + + test("files one it has never indexed under the root folder", async ({ + page, + }) => { + const name = "Never indexed"; + + await page.goto("/"); + await seedLocallyStoredPipeline(page, name); + + await page.goto("/pipeline-folders"); + + await expect(page.getByText(name)).toBeVisible(); + }); +}); From b07ddc7753b5d095c48673a1064aec15d4e5ffcb Mon Sep 17 00:00:00 2001 From: Camiel van Schoonhoven Date: Wed, 9 Sep 2026 13:49:18 -0700 Subject: [PATCH 35/36] feat(pipeline-storage): empty the backend from the list in development Trying the first-use copy more than once means emptying the backend first, which no user ever needs to do and there is otherwise no way to do from the app. The button leaves for the dashboard afterwards, because the copy starts from the pipeline list and would begin again the moment it finished. Browser storage is left alone: it is what the copy reads from, and its registry rows are the only record of which folder a pipeline is in. Co-Authored-By: Claude Opus 5 (1M context) --- .../Home/PipelineSection/PipelineSection.tsx | 8 ++- .../ResetBackendPipelinesButton.tsx | 55 +++++++++++++++++++ src/services/pipelineStorage/devReset.ts | 37 +++++++++++++ tests/e2e/pipeline-storage-backend.spec.ts | 27 +++++++++ 4 files changed, 126 insertions(+), 1 deletion(-) create mode 100644 src/components/Home/PipelineSection/ResetBackendPipelinesButton.tsx create mode 100644 src/services/pipelineStorage/devReset.ts diff --git a/src/components/Home/PipelineSection/PipelineSection.tsx b/src/components/Home/PipelineSection/PipelineSection.tsx index 5d8a85a346..fcbf61a846 100644 --- a/src/components/Home/PipelineSection/PipelineSection.tsx +++ b/src/components/Home/PipelineSection/PipelineSection.tsx @@ -31,6 +31,7 @@ import BulkActionsBar from "./BulkActionsBar"; import { HostMigrationNotice } from "./HostMigrationNotice"; import { PipelineFiltersBar } from "./PipelineFiltersBar"; import PipelineRow from "./PipelineRow"; +import { ResetBackendPipelinesButton } from "./ResetBackendPipelinesButton"; import { useHostMigration } from "./useHostMigration"; import { usePipelineFilters } from "./usePipelineFilters"; import { usePipelineListEntries } from "./usePipelineListEntries"; @@ -158,7 +159,12 @@ export const PipelineSection = withSuspenseWrapper( } + actions={ + <> + + + + } /> diff --git a/src/components/Home/PipelineSection/ResetBackendPipelinesButton.tsx b/src/components/Home/PipelineSection/ResetBackendPipelinesButton.tsx new file mode 100644 index 0000000000..18a97b4327 --- /dev/null +++ b/src/components/Home/PipelineSection/ResetBackendPipelinesButton.tsx @@ -0,0 +1,55 @@ +import { useNavigate } from "@tanstack/react-router"; +import { useState } from "react"; + +import { ConfirmationDialog } from "@/components/shared/Dialogs"; +import { Button } from "@/components/ui/button"; +import { Icon } from "@/components/ui/icon"; +import useToastNotification from "@/hooks/useToastNotification"; +import { APP_ROUTES } from "@/routes/appRoutes"; +import { resetBackendPipelines } from "@/services/pipelineStorage/devReset"; +import { usePipelineStorage } from "@/services/pipelineStorage/PipelineStorageProvider"; +import { getErrorMessage, pluralize } from "@/utils/string"; + +/** + * Trying the first-use copy more than once means emptying the backend first, + * which no user ever needs to do. It leaves the pipeline list, because the copy + * starts from there and would begin again the moment it finished. + */ +export function ResetBackendPipelinesButton() { + const storage = usePipelineStorage(); + const navigate = useNavigate(); + const notify = useToastNotification(); + const [isResetting, setIsResetting] = useState(false); + + if (!import.meta.env.DEV || storage.mode.kind !== "backend") return null; + + const handleReset = async () => { + setIsResetting(true); + + try { + const removed = await resetBackendPipelines(storage.rootFolder); + notify( + `Removed ${removed} ${pluralize(removed, "pipeline")}. Reload to copy this browser's pipelines up again.`, + "success", + ); + await navigate({ to: APP_ROUTES.DASHBOARD }); + } catch (error) { + notify(`Could not empty the backend: ${getErrorMessage(error)}`, "error"); + setIsResetting(false); + } + }; + + return ( + + + Reset backend pipelines + + } + title="Delete every pipeline in the backend?" + description="This empties the backend so it looks like you have never used one, and returns you to the dashboard. Pipelines in this browser are left alone, and are copied up again on the next load. Development only." + onConfirm={() => void handleReset()} + /> + ); +} diff --git a/src/services/pipelineStorage/devReset.ts b/src/services/pipelineStorage/devReset.ts new file mode 100644 index 0000000000..3383f2092c --- /dev/null +++ b/src/services/pipelineStorage/devReset.ts @@ -0,0 +1,37 @@ +import { runWithConcurrency } from "@/utils/concurrency"; + +import { pipelineStorageDb } from "./db"; +import type { PipelineFolder } from "./PipelineFolder"; + +const DELETE_CONCURRENCY = 3; + +/** + * Puts the backend store back to how someone who has never used one finds it, + * so the copy that runs on first use can be tried again. Development only. + * + * Browser storage is deliberately untouched: it is what the copy reads from, + * and its registry rows are the only record of which folder a pipeline is in. + */ +export async function resetBackendPipelines( + folder: PipelineFolder, +): Promise { + const files = await folder.listPipelines(); + await runWithConcurrency(files, DELETE_CONCURRENCY, (file) => + file.deleteFile(), + ); + + await pipelineStorageDb.pipeline_registry + .where("storage") + .equals("backend") + .delete(); + + /** + * Both are only ever written against a backend, and both are re-readable. + */ + await pipelineStorageDb.pipeline_specs.clear(); + await pipelineStorageDb.pending_writes.clear(); + + await pipelineStorageDb.host_migration.clear(); + + return files.length; +} diff --git a/tests/e2e/pipeline-storage-backend.spec.ts b/tests/e2e/pipeline-storage-backend.spec.ts index 96220c070b..59c3372027 100644 --- a/tests/e2e/pipeline-storage-backend.spec.ts +++ b/tests/e2e/pipeline-storage-backend.spec.ts @@ -438,4 +438,31 @@ test.describe("backend pipeline storage", () => { expect(readBackendRecords(page)).toHaveLength(SEED.length - 1); }); + + test("emptying the backend leaves the list, and copies up again next load", async ({ + page, + }) => { + const localName = "Waiting in the browser"; + + await installSeededBackend(page); + await page.goto("/"); + await seedLocallyStoredPipeline(page, localName); + + await page.goto("/pipelines"); + await expect(page.getByText(localName)).toBeVisible(); + + await page + .getByRole("button", { name: /reset backend pipelines/i }) + .click(); + await page.getByRole("button", { name: "Continue" }).click(); + + await expect(page).toHaveURL(/\/dashboard$/); + expect(readBackendRecords(page)).toEqual([]); + + await page.goto("/pipelines"); + await expect(page.getByText(localName)).toBeVisible(); + expect( + readBackendRecords(page).map((record) => record.displayName), + ).toEqual([localName]); + }); }); From 66e18afe808d28737d4fea6006396b54091d33fc Mon Sep 17 00:00:00 2001 From: Camiel van Schoonhoven Date: Wed, 9 Sep 2026 14:04:32 -0700 Subject: [PATCH 36/36] docs(pipeline-storage): describe the store as the backend it now is The notes still described a store an embedding page installed through a window global, which no longer exists. They now cover the routes the driver calls, how availability is decided, the outbox that holds a refused edit, and the two database traps this cost hours to find: a `ready` handler that deadlocks its own open, and renaming a value that lookups are scoped by. Co-Authored-By: Claude Opus 5 (1M context) --- src/services/pipelineStorage/ARCHITECTURE.md | 219 +++++++++++++------ 1 file changed, 157 insertions(+), 62 deletions(-) diff --git a/src/services/pipelineStorage/ARCHITECTURE.md b/src/services/pipelineStorage/ARCHITECTURE.md index edd4c9f428..a94ce5773a 100644 --- a/src/services/pipelineStorage/ARCHITECTURE.md +++ b/src/services/pipelineStorage/ARCHITECTURE.md @@ -1,6 +1,6 @@ # Pipeline Storage Architecture -Pipeline file storage behind one driver interface, over browser storage (IndexedDB, local file system) or over a store the embedding page provides. Every pipeline file lives inside a folder; every folder owns a driver that does the I/O. +Pipeline file storage behind one driver interface, over browser storage (IndexedDB, local file system) or over the backend the app is configured against. Every pipeline file lives inside a folder; every folder owns a driver that does the I/O. Which store the app uses is decided once, as it boots, and never revisited — see [Storage mode](#storage-mode). @@ -18,19 +18,25 @@ pipelineStorage/ ├── PipelineStorageProvider.tsx # React context provider + usePipelineStorage hook ├── storageMode.ts # Which store this page load uses; decided once ├── storageErrors.ts # Which failures are worth another attempt +├── storageHealth.ts # Whether the backend is answering; no React, no provider +├── backendEndpoint.ts # Where the backend is, for non-React callers +├── pendingWrites.ts # Edits the backend refused, kept and sent on recovery ├── pipelineSpecCache.ts # Pipeline contents, keyed by the store's own version -├── hostMigration.ts # One-time copy of browser pipelines into a host store -├── host/ -│ ├── contract.ts # The window global an embedding page provides -│ └── detectHost.ts # Reads and version-checks that global +├── pipelineFileEvents.ts # "This file changed" between editors +├── pipelineOperations.ts # Thin non-React binding onto the singleton service +├── hostMigration.ts # One-time copy of browser pipelines into the backend +├── devReset.ts # Empties the backend so the copy can be tried again └── drivers/ ├── RootFolderDbStorageDriver.ts # Legacy IndexedDB component list ├── FolderIndexDbStorageDriver.ts # Folder-scoped IndexedDB (extends Root driver) ├── LocalFileSystemDriver.ts # File System Access API (local directory) - ├── HostStorageDriver.ts # The store the embedding page provides - └── UnavailableStorageDriver.ts # Refuses everything; see host-missing below + └── BackendStorageDriver.ts # The configured backend's pipeline routes ``` +The `host` in `hostMigration.ts`, `HostMigrationNotice` and the `host_migration` table is left over +from an earlier design in which an embedding page provided the store through a window global. There +is no host: it is the configured backend, and the names are the only thing still saying otherwise. + ```mermaid graph TD subgraph react [React Layer] @@ -56,6 +62,7 @@ graph TD RootDriver["RootFolderDbStorageDriver"] FolderDriver["FolderIndexDbStorageDriver"] LocalDriver["LocalFileSystemDriver"] + BackendDriver["BackendStorageDriver"] end Provider --> Service @@ -73,6 +80,7 @@ graph TD Factory --> RootDriver Factory --> FolderDriver Factory --> LocalDriver + Factory --> BackendDriver FolderDriver --> RootDriver FolderDriver --> Registry ``` @@ -110,7 +118,8 @@ type DriverConfig = | { driverType: "root-indexdb" } | { driverType: "folder-indexdb"; folderId: string } | { driverType: "local-fs"; handle: FileSystemDirectoryHandle } - | { driverType: "host" }; + | { driverType: "backend" } + | { driverType: "google-drive"; folderId: string }; ``` ### Driver Implementations @@ -120,7 +129,7 @@ type DriverConfig = | `RootFolderDbStorageDriver` | `root-indexdb` | Legacy `localforage` component list | `true` | `true` | none | | `FolderIndexDbStorageDriver` | `folder-indexdb` | Same backing store, scoped via `pipeline_registry` | `true` | `true` | none | | `LocalFileSystemDriver` | `local-fs` | File System Access API directory handle | `false` | `false` | `DriverPermissions` | -| `HostStorageDriver` | `host` | `window.__TANGLE_PIPELINE_STORAGE_HOST__` | `false` | `false` | none | +| `BackendStorageDriver` | `backend` | The configured backend's pipeline routes | `false` | `false` | none | ### Class Hierarchy @@ -183,63 +192,126 @@ classDiagram ## Storage mode -A deployment declares whether pipelines live outside the browser. It is not -inferred from whether a host happens to be present, because a page that failed -to install one would otherwise read as "browser storage" and quietly strand a -user's work where nobody else can see it. +A deployment declares whether pipelines live outside the browser. It is a +build-time flag rather than something inferred at runtime, because a store that +looked absent for a moment would otherwise read as "browser storage" and quietly +strand a user's work where nobody else can see it. ``` -VITE_PIPELINE_STORAGE_BETA = "true" → a host is required +VITE_PIPELINE_STORAGE_BETA = "true" → pipelines live on the configured backend anything else, or unset → browser storage (the open-source build) ``` -`storageMode.ts` resolves this once per page load and freezes the answer, -holding the host object itself rather than re-reading the global: +`storageMode.ts` resolves this once per page load and freezes the answer: + +| `StorageMode` | When | Root folder | +| ---------------------------- | -------- | --------------------------------------------- | +| `{ kind: "local" }` | flag off | `folder-indexdb` on `ROOT_FOLDER_ID`, folders | +| `{ kind: "backend", label }` | flag on | `BackendStorageDriver`, `isFlat`, no folders | + +Only the _mode_ is fixed for the page. The backend's **address** is read per +request, so changing it in Settings takes effect without a reload. -| `StorageMode` | When | Root folder | -| -------------------------- | --------------------- | --------------------------------------------- | -| `{ kind: "local" }` | flag off | `folder-indexdb` on `ROOT_FOLDER_ID`, folders | -| `{ kind: "host", label }` | flag on, host present | `HostStorageDriver`, `isFlat`, no folders | -| `{ kind: "host-missing" }` | flag on, no host | `UnavailableStorageDriver`; the app blocks | +In backend mode the root folder **is** the backend-backed folder, so every +existing `folderId === null ? rootFolder : findFolderById(id)` branch lands on +the backend unchanged. `isFlat` is the property to test for "this store has no +folders" — it is what hides folder creation, the parent row, and move-to-folder. -In host mode the root folder **is** the host-backed folder, so every existing -`folderId === null ? rootFolder : findFolderById(id)` branch lands on the host -unchanged. `isFlat` is the property to test for "this store has no folders" — -it is what hides folder creation, the parent row, and move-to-folder. +### The backend contract -`host-missing` renders `PipelineStorageUnavailable` in place of the whole app. -Nothing can be read or written, and a blank library would be a lie. +Pipelines are read and written over the configured backend, the same one +`BackendProvider` and `/settings/backend` point at. There is no separate seam +and nothing is injected into the page: the app calls documented routes, and a +deployment that serves them — including one an open-source user runs themselves +— can hold pipelines for this editor. -### The host contract +| Route | Purpose | +| ------------------------------------------------------ | ------------------------------------------ | +| `GET /api/users/me/pipelines/all?page_size&page_token` | List, paged; `file_path` filters by prefix | +| `GET /api/users/me/pipelines?file_path=KEY` | Read one, spec included | +| `PUT /api/users/me/pipelines?file_path=KEY` | Upsert on the caller's key | +| `DELETE /api/users/me/pipelines?file_path=KEY` | Delete one | -`window.__TANGLE_PIPELINE_STORAGE_HOST__`, version-checked against -`PIPELINE_STORAGE_HOST_VERSION`. Five methods: `list`, `read`, `write`, -`delete`, `has`. Three properties of it shape the code above: +A row comes back as `{id, file_path, pipeline_name, current_version, +created_at, updated_at, root_pipeline_task.componentRef.spec}`. Three +properties of the contract shape the code above: -- **`write(key, spec)` upserts on the caller's key.** The app chooses the key — - a new pipeline is written under its name — and the host returns its own - `externalId`, which becomes the pipeline's id and the editor url. +- **`PUT` upserts on the caller's key.** The app chooses the key — a new + pipeline is written under its name — and the backend returns its own `id`, + which becomes the pipeline's identity and the editor url. - **Renaming is a write.** A store that does not key on the name only learns a new one from the spec, so `rename` and `write` are one operation (`renamePipelineByName`). -- **Writes are last-write-wins**; the host does not reject on conflict. - `contentVersion` still detects "changed since we listed it" on the next +- **Writes are last-write-wins**; the backend does not reject on conflict. + `current_version` still detects "changed since we listed it" on the next listing, which is what invalidates the spec cache. -Errors cross a window boundary, where `instanceof` does not survive, so -`HostStorageDriver` duck-types a `code` off the rejection and turns it into a -`HostStorageError` carrying prose that names the store. `storageErrors.ts` -decides what is worth another attempt: an expired session or an ambiguous name -will answer the same way next time; an unreachable store may not. +`BackendStorageDriver` turns every failure into a `BackendStorageError` with a +`StorageErrorCode` — `unauthenticated`, `not_found`, `conflict`, +`rate_limited`, `unavailable`. Two answers are not plain status codes: an +expired session arrives as a cross-origin redirect, read as `opaqueredirect` +because the request sets `redirect: "manual"`; and a body that is not JSON means +the request never reached the API at all, most likely a single-page-app fallback +serving the index document, which is reported as `unavailable` rather than +parsed. `storageErrors.ts` decides what is worth another attempt: an expired +session or an ambiguous name will answer the same way next time; an unreachable +store may not. + +Non-React callers get the address from `backendEndpoint.ts`. `BackendProvider` +publishes into it whenever the setting changes, so there is still one place that +decides. Its fallback is load-bearing rather than defensive: a provider +publishes from an effect and effects run child-first, so the first listing can +happen before the provider above it has said anything. + +### Knowing whether the backend is there + +`storageHealth.ts` holds one boolean and a subscription, and is deliberately +free of React and of the provider — the driver reports into it without dragging +either into its module graph. Only `unavailable` counts as an outage: "no such +pipeline" is an answer. + +`useStorageUnavailable()` is how the UI reads it, and combines two sources +because either alone leaves a hole. The calls the app already makes are the +cheapest and most direct signal, but a page nobody is touching makes none — an +editor left open through a restart would look healthy until the next keystroke — +so the backend is also pinged on mount, on return to the tab, and on a slow +interval. Both report to the same place. + +It is rendered **ahead of a listing already in hand**: a store that cannot be +reached must not be represented by the last answer it gave. `BackendUnavailable` +takes the place of the pipeline list and the folders page; in the editor the +same signal turns the auto-save indicator red, and `UnsavedWorkBanner` says the +work is held. An `unauthenticated` failure is the exception — that needs the +person, not a retry, so `ExpiredSessionDialog` offers a download before the +reload that would discard the work. + +### Edits the backend refused + +`pendingWrites.ts` is a durable outbox in Dexie, keyed by store and storage key. +An edit the backend would not take has to outlive the editor that made it: the +most likely next thing someone does is leave the page to go and fix the +connection, and that must not be what loses the work. The flusher is started +with the app rather than with an editor, and sends what it holds on every +transition from silent to answering. + +Browser storage does not use it — it does not go away between one write and the +next. `autoSaveStore` records into it on a refused write and forgets on a +successful one, and stands down if a reopened editor has since claimed the same +file, so an older held text can never land on top of a newer one. ### Copying browser pipelines in -`hostMigration.ts` copies everything in browser storage into the host once, -keyed on each pipeline's local name. Nothing local is deleted — a build with no -host still has to find its pipelines. The claim is a Dexie transaction, honoured -only while its holder keeps making progress, so two tabs cannot both copy and a -tab that died does not block the next one. It starts wherever the app opens; -the pipeline list reports progress and offers to retry anything that failed. +`hostMigration.ts` copies everything in browser storage into the backend once, +keyed on each pipeline's local name. Nothing local is deleted — a build with the +flag off still has to find its pipelines. The claim is a Dexie transaction, +honoured only while its holder keeps making progress, so two tabs cannot both +copy and a tab that died does not block the next one. It starts wherever the app +opens; the pipeline list reports progress and offers to retry anything failed. + +Trying it more than once means emptying the backend first, which no user ever +needs to do. `devReset.ts` does that, behind a development-only button on the +pipeline list, and leaves for the dashboard afterwards because the copy starts +from the list and would begin again the moment it finished. --- @@ -251,10 +323,10 @@ Dexie database name: `tangle_pipelines` erDiagram pipeline_registry { string id PK - string storage "local | host" + string storage "local | backend" string storageKey UK "unique within one store" string folderId FK - string contentVersion "optional, host only" + string contentVersion "optional, backend only" } pipeline_specs { @@ -264,6 +336,13 @@ erDiagram json spec } + pending_writes { + string storage PK + string storageKey PK + string yaml + number recordedAt + } + host_migration { string id PK number startedAt @@ -294,18 +373,31 @@ erDiagram | `folders` | `id` | `parentId` | Folder tree with `driverConfig`; local only | | `pipeline_specs` | `[storage+storageKey]` | — | Cached contents, validated by `version` | | `host_migration` | `id` | — | One row, `"v1"`: the copy's claim and result | +| `pending_writes` | `[storage+storageKey]` | — | Edits refused by the backend, awaiting it | ### Migrations - **v1**: Creates `pipeline_registry` and `folders`. -- **v2**: Adds `remoteStorageKey`, for an earlier design where the host was a folder beside browser storage. -- **v3**: Drops it again, and deletes the host folder and its rows — the host is the root now, not a child. +- **v2**: Adds `remoteStorageKey`, for an earlier design where the outside store was a folder beside browser storage. +- **v3**: Drops it again, and deletes that folder and its rows — the outside store is the root now, not a child. - **v4**: Adds `pipeline_specs`. - **v5**: Adds `host_migration`. - **v6**: Drops `pipeline_specs`; a primary key cannot be changed in place and it is about to gain one. -- **v7**: Remakes `pipeline_specs` keyed by store, and scopes `pipeline_registry` the same way. Rows written before this do not say which store they describe: only a host reports a `contentVersion`, and a row filed in a folder can only be the browser's, which is enough to attribute them. - -On open (not in a migration), `seedRegistryFromLegacyList` claims every pipeline in the legacy list that has no local row. It is skipped in host mode, where those names mean nothing. +- **v7**: Remakes `pipeline_specs` keyed by store, and scopes `pipeline_registry` the same way. Rows written before this do not say which store they describe: only a store outside the browser reports a `contentVersion`, and a row filed in a folder can only be the browser's, which is enough to attribute them. +- **v8**: Renames the persisted store value `"host"` to `"backend"`. A row under the old name is invisible to every lookup, which scopes itself to the store in use — but its id still occupies the primary key, so the next listing tries to add the same pipeline again and the collision takes the whole page down rather than one row. Where a `"backend"` row already claims the key, the older one is dropped. +- **v9**: Adds `pending_writes`. + +Pipelines predating the registry are indexed separately by +`ensureBrowserPipelinesIndexed()`, awaited by the folder listing that needs the +rows. It claims only keys with no row yet, inside a transaction, so a listing +running at the same time cannot collide with it on the unique index. + +**It must never move back into Dexie's `on("ready")` handler.** That handler +gates `open()`, and it has to read the legacy list out of another database +first; once it has yielded to a non-Dexie promise it can never touch this one +again, because every call queues behind the open it is itself holding up. Not +even a `count()` returns. The symptom is the whole app hanging on a spinner with +no error anywhere. --- @@ -335,11 +427,11 @@ unique **within** that store (`&[storage+storageKey]`). The scope is read from the life of the page and a caller that could get it wrong eventually does. This matters because the two stores share a namespace by nature: browser storage -keys a pipeline on its name, and the key sent to a host is that same name. An -unscoped row was therefore found by whichever store asked first — a link to a -host pipeline, opened in the browser-storage build, paired the host's identity -with the browser's driver and read, then saved over, whatever pipeline happened -to share the name. +keys a pipeline on its name, and the key sent to the backend is that same name. +An unscoped row was therefore found by whichever store asked first — a link to a +backend pipeline, opened in the browser-storage build, paired the backend's +identity with the browser's driver and read, then saved over, whatever pipeline +happened to share the name. Within browser storage, pipeline names remain globally unique across folders; `assertStorageKeyUnique` must still be called before creating a pipeline there. @@ -470,9 +562,10 @@ function usePipelineStorage(): PipelineStorageService; ``` The provider hands out the module singleton, so React and non-React callers -share one instance. It is mounted in `RootLayout.tsx`, and is also where the -one-time copy into a host store is started. `TourPipelineStorageProvider` -supplies its own service over the same context for the guided tours. +share one instance. It is mounted in `RootLayout.tsx`, and in backend mode is +also where the one-time copy and the pending-write flusher are started. +`TourPipelineStorageProvider` supplies its own service over the same context for +the guided tours. ### Consumer Map @@ -813,4 +906,6 @@ sequenceDiagram - See [Migrations](#migrations) for what each version did. - New migrations must follow Dexie's versioning rules: increment the version number and never modify existing version schemas. - A primary key cannot be changed in place. Drop the table in one version (`{ table: null }`) and remake it in the next, and only for a table that can be rebuilt. +- **Never touch this database from `on("ready")` after awaiting anything outside Dexie.** See [Migrations](#migrations) — the open deadlocks and the app hangs with no error. +- Renaming a persisted value that lookups are scoped by is a migration, not an edit. Rows under the old name go invisible while their ids still hold the primary key, and the next write collides — see v8. - Everything in this database is a cache of what a store already holds, with one exception: `folders`, and the `folderId` on each registry row, are the **only** record of which folder a pipeline is in. Never clear the registry wholesale to fix something.