diff --git a/desktop/main.cjs b/desktop/main.cjs index 63aabb6..5ba3e99 100644 --- a/desktop/main.cjs +++ b/desktop/main.cjs @@ -7,8 +7,8 @@ * served as app://readm3 when readm3.com cannot be reached on a first launch. * * Files and folders named on the command line (`readm3 edit .`, like `subl .`), or - * handed over by the desktop's "Open with", are read here and given to the page - * through preload.cjs, which can write back only to files this process gave it. + * handed over by the desktop's "Open with", are listed here and read by the page + * through preload.cjs, which can read and write only files this process listed. * * readm3:// links ("Open in app" on readm3.com) open a readm3.com page in this window, and * "Sign in with your browser" signs the window in through the system browser, where the @@ -24,8 +24,6 @@ const { fileURLToPath, pathToFileURL } = require("node:url"); const ORIGIN = (process.env.READM3_URL || "https://readm3.com").replace(/\/$/, ""); const LOCAL = "app://readm3"; -/** Larger files go to the page without their bytes; it reads them in ranges (read-range). */ -const INLINE_BYTES = 4 * 1024 * 1024; /** The largest file a signed-in account may keep (src/blob-store.ts); the page applies its own limits again. */ const MAX_FILE_BYTES = 5 * 1024 ** 3; const MAX_RANGE = 16 * 1024 * 1024; @@ -46,15 +44,16 @@ let pending = []; let window; /** - * Reads files, and every file under a folder. A folder's files carry `relative`, their + * Lists files, and every file under a folder. A folder's files carry `relative`, their * path from the folder's parent ("notes/todo.md"), so the page opens them as the folder. + * No bytes travel with the list: the page reads each file when it needs it (read-file, + * read-range), so a folder of thousands of files is never one message of gigabytes. */ async function load(paths, cwd = process.cwd()) { const files = []; const add = async (path, info, relative) => { if (info.size > MAX_FILE_BYTES || files.length >= MAX_FILES) return; - const data = info.size > INLINE_BYTES ? undefined : new Uint8Array(await readFile(path)); - files.push({ path, name: basename(path), relative, size: info.size, data }); + files.push({ path, name: basename(path), relative, size: info.size }); opened.add(path); }; const walk = async (directory, prefix) => { diff --git a/package.json b/package.json index 161f3f7..30bfaa9 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@profullstack/readm3", - "version": "0.21.0", + "version": "0.22.0", "description": "Markdown editing and sharing for people and teams. Web/PWA, CLI, API and MCP with version history, view/edit links, and a terminal reader.", "type": "module", "license": "MIT", diff --git a/site/autosync.ts b/site/autosync.ts index dcb8fa8..abb8ef8 100644 --- a/site/autosync.ts +++ b/site/autosync.ts @@ -26,6 +26,28 @@ const OFF = "readm3:autosync"; export const autoSyncEnabled = (): boolean => { try { return localStorage.getItem(OFF) !== "off"; } catch { return true; } }; export function setAutoSync(on: boolean) { try { localStorage.setItem(OFF, on ? "on" : "off"); } catch { /* This device only. */ } if (on) void syncNow(); else host?.status("Sync is off on this device"); } +/** + * Files larger than this stay on this device: a folder of films or a disk image is not + * worth uploading to edit Markdown next to it. 0 syncs every file. Set per device. + */ +const LIMIT = "readm3:sync-limit-mb"; +export const DEFAULT_SYNC_LIMIT_MB = 500; +export function syncLimitMb(): number { + try { + const saved = localStorage.getItem(LIMIT); + const mb = saved === null ? DEFAULT_SYNC_LIMIT_MB : Number(saved); + return Number.isFinite(mb) && mb >= 0 ? mb : DEFAULT_SYNC_LIMIT_MB; + } catch { return DEFAULT_SYNC_LIMIT_MB; } +} +export function setSyncLimitMb(mb: number) { + try { localStorage.setItem(LIMIT, String(Math.max(0, Math.round(mb)))); } catch { /* This device only. */ } + void syncNow(); +} +/** Only stored files can pass the limit: inline text is 4 MB at most. */ +export function keptOnDevice(doc: { blob?: BlobRef }, limitMb = syncLimitMb()): boolean { + return limitMb > 0 && !!doc.blob && doc.blob.size > limitMb * 1024 * 1024; +} + export interface AutoSyncHost { read(): SyncedWorkspace; settings(): ReaderSettings; @@ -233,10 +255,17 @@ async function pass(user: CloudUser) { remoteWorkspace = files["workspace.json"] ? JSON.parse(files["workspace.json"].content) : undefined; remoteSettings = files["settings.json"] ? JSON.parse(files["settings.json"].content) : undefined; } - const local = structuredClone(h.read()); - const localDigests = digests(local); + const all = structuredClone(h.read()); + const localDigests = digests(all); + // Files past the size limit take no part in the merge, on either side: this device never + // uploads its own, and one another device synced (a higher limit there) is left as it is. + const limit = syncLimitMb(); + const deviceOnly = all.documents.filter((doc) => keptOnDevice(doc, limit)); + const accountOnly = remoteWorkspace?.documents.filter((doc) => keptOnDevice(doc, limit)) ?? []; + const local = { ...all, documents: all.documents.filter((doc) => !keptOnDevice(doc, limit)) }; + const remoteSynced = remoteWorkspace && { ...remoteWorkspace, documents: remoteWorkspace.documents.filter((doc) => !keptOnDevice(doc, limit)) }; const merge = mergeWorkspaces({ - local, remote: remoteWorkspace, base: state?.base, starter: await starterDigests(), + local, remote: remoteSynced, base: state?.base, starter: await starterDigests(), conflictLabel: `${h.device} ${new Date().toISOString().slice(0, 10)}`, }); // Settings: this device's choice wins only if it changed since the last sync. @@ -250,9 +279,10 @@ async function pass(user: CloudUser) { const readableOnServer = !!remote && !isSealed(remote.snapshot.files); let spills = state?.spills ?? {}; if (!(onlyStarter && !remote) && (merge.upload || (settingsChanged && remote) || readableOnServer || rekey)) { - const outgoing = await forUpload(merge.workspace, remoteWorkspace, spills); + const outgoing = await forUpload(merge.workspace, remoteSynced, spills); spills = outgoing.spills; - const plain = { "settings.json": { content: serialize(settings) }, "workspace.json": { content: serialize(outgoing.workspace) } }; + const uploaded = { ...outgoing.workspace, documents: [...outgoing.workspace.documents, ...accountOnly].sort((a, b) => a.path.localeCompare(b.path)) }; + const plain = { "settings.json": { content: serialize(settings) }, "workspace.json": { content: serialize(uploaded) } }; validateSyncFiles(plain); const sealed = { [SEALED_FILE]: { content: JSON.stringify(await seal(plain, syncKey)) } }; try { @@ -262,7 +292,7 @@ async function pass(user: CloudUser) { rekey = false; if (saved.userId !== user.id) throw new Error("your account changed"); revision = saved.revision; - void collect(outgoing.workspace, outgoing.account); + void collect(uploaded, outgoing.account); } catch (error) { if (error instanceof ApiError && error.status === 409) continue; // Another device saved first. throw error; @@ -270,12 +300,20 @@ async function pass(user: CloudUser) { } // Edits made while this pass ran are not overwritten: the next pass merges them. if (!sameDigests(digests(h.read()), localDigests)) { again = true; return; } - if (merge.download || serialize(settings) !== localSettings) await h.apply(merge.workspace, settings); + if (merge.download || serialize(settings) !== localSettings) { + // The merge never saw the large files, so they are put back: this device's own first. + const merged = new Set(merge.workspace.documents.map((doc) => doc.path)); + const mine = new Set(deviceOnly.map((doc) => doc.path)); + const documents = [...merge.workspace.documents, ...deviceOnly.filter((doc) => !merged.has(doc.path)), + ...accountOnly.filter((doc) => !merged.has(doc.path) && !mine.has(doc.path))]; + await h.apply({ ...merge.workspace, documents: documents.sort((a, b) => a.path.localeCompare(b.path)) }, settings); + } await store(key, { revision, base: digests(merge.workspace), settings: serialize(settings), spills } satisfies State); known = revision; + const kept = deviceOnly.length ? ` · ${deviceOnly.length} file${deviceOnly.length === 1 ? "" : "s"} over ${limit.toLocaleString()} MB kept on this device` : ""; h.status(merge.conflicts.length - ? `Synced · ${merge.conflicts.length} conflict cop${merge.conflicts.length === 1 ? "y" : "ies"} kept` - : `Synced to your account · ${time()}`); + ? `Synced · ${merge.conflicts.length} conflict cop${merge.conflicts.length === 1 ? "y" : "ies"} kept${kept}` + : `Synced to your account · ${time()}${kept}`); return; } throw new Error("other devices kept saving; trying again shortly"); diff --git a/site/blobs.ts b/site/blobs.ts index 5aa12d0..264857a 100644 --- a/site/blobs.ts +++ b/site/blobs.ts @@ -20,9 +20,19 @@ export interface StorageAccount { } const localKey = (id: string) => `blob:${id}`; -export const local = (ref: Pick) => readKey(localKey(ref.id)).catch(() => undefined); +/** + * Files the desktop app opened from disk: read from there each time, never copied into + * browser storage. A reader lasts for the session; opening the folder again restores it. + */ +const onDisk = new Map Promise>(); +export const fromDisk = (ref: Pick, read: () => Promise) => { onDisk.set(ref.id, read); }; +export async function local(ref: Pick): Promise { + const read = onDisk.get(ref.id); + if (read) { try { return await read(); } catch { /* Moved or deleted on disk: a kept or synced copy may still exist. */ } } + return readKey(localKey(ref.id)).catch(() => undefined); +} export const keep = (ref: Pick, blob: Blob) => writeKey(localKey(ref.id), blob); -export const forget = (ref: Pick) => writeKey(localKey(ref.id), undefined).catch(() => undefined); +export const forget = (ref: Pick) => { onDisk.delete(ref.id); return writeKey(localKey(ref.id), undefined).catch(() => undefined); }; export const account = () => request("blobs"); export const remove = (id: string) => request(`blobs/${id}`, "DELETE"); diff --git a/site/disk.ts b/site/disk.ts index e623c76..b45216e 100644 --- a/site/disk.ts +++ b/site/disk.ts @@ -8,7 +8,13 @@ import { accepts, readKey, writeKey } from "./workspace.ts"; * Browsers without the API keep the old behaviour: files are copied in, Ctrl+S downloads. */ export interface DiskEntry { handle: FileSystemFileHandle; saved: number; } -export interface Picked { file: File; path: string; handle?: FileSystemFileHandle; } +/** + * `onDisk`: opened by the desktop app, so the editor reads it from disk rather than keeping + * a copy. A large one comes as its first 64 KB, enough to tell what it is, with `size` + * saying how large the whole file is. + */ +export interface Picked { file: File; path: string; handle?: FileSystemFileHandle; size?: number; onDisk?: boolean; } +export type Progress = (done: number, total: number) => void; /** The parts of the API that TypeScript's DOM library does not describe yet. */ interface PickerWindow { @@ -32,12 +38,12 @@ export function fingerprint(text: string): number { /** null when the person closes the picker, which is not an error. */ const cancelled = (error: unknown) => error instanceof DOMException && error.name === "AbortError"; -export async function pickFolder(): Promise<{ name: string; files: Picked[] } | null> { - // The desktop app opens folders with the system dialog and reads them itself (desktop/main.cjs). +export async function pickFolder(onProgress?: Progress): Promise<{ name: string; files: Picked[] } | null> { + // The desktop app opens folders with the system dialog and lists them itself (desktop/main.cjs). if (desktop?.pick) { const files = await desktop.pick(true); if (!files) return null; - return { name: files.find((file) => file.relative)?.relative!.split("/")[0] ?? "Your workspace", files: await fromDesktop(files) }; + return { name: files.find((file) => file.relative)?.relative!.split("/")[0] ?? "Your workspace", files: await fromDesktop(files, onProgress) }; } let root: Directory; try { root = await picker().showDirectoryPicker({ id: "readm3", mode: "readwrite" }) as Directory; } @@ -58,10 +64,10 @@ export async function pickFolder(): Promise<{ name: string; files: Picked[] } | return { name: root.name, files }; } -export async function pickFiles(): Promise { +export async function pickFiles(onProgress?: Progress): Promise { if (desktop?.pick) { const files = await desktop.pick(false); - return files && fromDesktop(files); + return files && fromDesktop(files, onProgress); } let handles: FileSystemFileHandle[]; try { handles = await picker().showOpenFilePicker({ id: "readm3", multiple: true }); } @@ -76,9 +82,9 @@ export async function launched(handles: readonly FileSystemHandle[]): Promise { +export async function fromDesktop(files: DesktopFile[], onProgress?: Progress): Promise { const picked: Picked[] = []; - for (const entry of files) { + for (const [i, entry] of files.entries()) { // A large file from a desktop build without ranges cannot be read; it is left out. if (!entry.data && !desktop!.readRange) continue; - const file = entry.data ? new File([entry.data.buffer as ArrayBuffer], entry.name) : await whole(entry); - picked.push({ file, path: entry.relative ?? entry.name, handle: desktopHandle(entry) }); + const handle = desktopHandle(entry); + try { + let file: File; + if (entry.data) file = new File([entry.data.buffer as ArrayBuffer], entry.name); + // Past 4 MB only the start is read now; the rest stays on disk until it is opened. + else if (entry.size! > LARGE) file = new File([await desktop!.readRange!(entry.path, 0, SAMPLE) as BlobPart], entry.name); + else file = await handle.getFile(); + picked.push({ file, path: entry.relative ?? entry.name, handle, size: entry.size ?? file.size, onDisk: true }); + } catch { /* Unreadable now (moved, permissions): the rest still opens. */ } + if (onProgress && (i % 50 === 49 || i === files.length - 1)) onProgress(i + 1, files.length); } return picked; } +const LARGE = 4 * 1024 * 1024; +const SAMPLE = 64 * 1024; const RANGE = 8 * 1024 * 1024; /** A large file, read range by range into Blob parts, which the browser keeps on disk. */ async function whole(entry: DesktopFile): Promise { @@ -126,7 +142,8 @@ function desktopHandle(entry: DesktopFile): FileSystemFileHandle { kind: "file", name: entry.name, // Read from disk each time, so the editor sees a file change under it and a save is not undone. getFile: async () => { - if (!entry.data && entry.size !== undefined) return whole(entry); + // A large file is read a range at a time, never whole through one message. + if (!entry.data && entry.size !== undefined && (entry.size > LARGE || !desktop!.readFile)) return whole(entry); if (!desktop!.readFile) return new File([entry.data!.buffer as ArrayBuffer], entry.name); const { data, modified } = await desktop!.readFile(entry.path); return new File([data.buffer as ArrayBuffer], entry.name, { lastModified: modified }); diff --git a/site/sync.ts b/site/sync.ts index aefe95f..67e774f 100644 --- a/site/sync.ts +++ b/site/sync.ts @@ -14,7 +14,11 @@ function loadMarker(key: string): Marker | undefined { function saveMarker(key: string, marker: Marker) { sessionStorage.setItem(`readm3:${key}`, JSON.stringify(marker)); } export interface SyncWorkspace { /** The automatic sync switch for this device (site/autosync.ts). */ - auto?: { enabled(): boolean; set(on: boolean): void; now(): Promise; changePassphrase(userId: string): void; forgetPassphrase(userId: string): Promise }; + auto?: { + enabled(): boolean; set(on: boolean): void; now(): Promise; changePassphrase(userId: string): void; forgetPassphrase(userId: string): Promise; + /** Files larger than this many MB stay on this device; 0 syncs every file. */ + limit: { get(): number; set(mb: number): void; fallback: number }; + }; read(): SyncedWorkspace; settings(): ReaderSettings; apply(workspace: SyncedWorkspace | undefined, settings: ReaderSettings | undefined): Promise; @@ -88,9 +92,32 @@ export async function showSync(workspace: SyncWorkspace) { auto.checked = workspace.auto.enabled(); auto.onchange = () => workspace.auto!.set(auto.checked); autoLabel.append(auto, " Sync automatically on this device (recommended). Every signed-in device keeps the same files; edits on two devices are merged per file, and a file changed on both keeps a conflict copy."); + // Large files (films, disk images) stay on this device unless the person turns this off. + const { limit } = workspace.auto; + const limitLabel = document.createElement("label"); + const skip = document.createElement("input"); + skip.type = "checkbox"; + skip.id = "sync-limit-on"; + skip.checked = limit.get() > 0; + const size = document.createElement("input"); + size.type = "number"; + size.id = "sync-limit-mb"; + size.min = "1"; + size.step = "1"; + size.style.width = "6em"; + size.value = String(limit.get() || limit.fallback); + size.disabled = !skip.checked; + const apply = () => { + size.disabled = !skip.checked; + const mb = Math.round(Number(size.value)); + limit.set(skip.checked ? (Number.isFinite(mb) && mb >= 1 ? mb : limit.fallback) : 0); + }; + skip.onchange = apply; + size.onchange = apply; + limitLabel.append(skip, " Don't sync files larger than ", size, " MB. They stay on this device."); const e2e = document.createElement("p"); e2e.textContent = "End-to-end encrypted: files are compressed and encrypted on this device with your sync passphrase before upload. readm3.com cannot read them."; - box.body.append(autoLabel, e2e); + box.body.append(autoLabel, limitLabel, e2e); const actions = document.createElement("div"); actions.className = "cloud-actions"; const act = (label: string, run: () => void | Promise) => { diff --git a/site/test/sync.spec.ts b/site/test/sync.spec.ts index 8229eec..a1e1ac9 100644 --- a/site/test/sync.spec.ts +++ b/site/test/sync.spec.ts @@ -151,6 +151,35 @@ test("a signed-in account opens a file over 4 MB read-only and syncs it as an en expect((await stat((await (await downloading).path())!)).size).toBe(big.length); }); +test("files over the sync size limit stay on this device until the limit is turned off", async ({ page, context }) => { + await register(context, username()); + // 500 MB by default; a small limit keeps the test's files small. + await page.addInitScript(() => { if (localStorage.getItem("readm3:sync-limit-mb") === null) localStorage.setItem("readm3:sync-limit-mb", "8"); }); + await page.goto("/viewer"); + await setPassphrase(page); + const line = "a line of a large log\n"; + const big = Buffer.from(line.repeat(Math.ceil(9 * 1024 * 1024 / line.length))); + const medium = Buffer.from(line.repeat(Math.ceil(6 * 1024 * 1024 / line.length))); + await page.locator("#files-input").setInputFiles([ + { name: "big.log", mimeType: "text/plain", buffer: big }, + { name: "medium.log", mimeType: "text/plain", buffer: medium }, + ]); + const uploaded = async () => { + const account = await (await page.request.get("/api/v1/blobs")).json(); + return account.blobs.filter((blob: { complete: boolean }) => blob.complete).map((blob: { size: number }) => blob.size).sort(); + }; + await expect.poll(uploaded, { timeout: 20_000 }).toEqual([medium.length]); + await expect(page.locator("#sync-note")).toContainText("1 file over 8 MB kept on this device"); + await expect(page.locator('#file-list button[title="big.log"]')).toBeVisible(); + await openSync(page); + await expect(page.locator("#sync-limit-on")).toBeChecked(); + await expect(page.locator("#sync-limit-mb")).toHaveValue("8"); + await page.locator("#sync-limit-on").uncheck(); + await expect(page.locator("#sync-limit-mb")).toBeDisabled(); + await expect.poll(uploaded, { timeout: 20_000 }).toEqual([big.length, medium.length].sort()); + expect(await page.evaluate(() => localStorage.getItem("readm3:sync-limit-mb"))).toBe("0"); +}); + test("the header shows usage against the plan's limits, signed out and signed in", async ({ page, context }) => { await page.goto("/viewer"); const usage = page.locator("#usage"); diff --git a/site/test/viewer.spec.ts b/site/test/viewer.spec.ts index cdb4d01..9af5d7b 100644 --- a/site/test/viewer.spec.ts +++ b/site/test/viewer.spec.ts @@ -143,6 +143,60 @@ test("in the desktop app, opens and saves through the system dialogs, never the expect(await page.evaluate(() => (window as unknown as { written: Record }).written["/home/me/Welcome.md"])).toContain("Your docs. Your space."); }); +test("in the desktop app, a folder comes as a list and large files and images are read from disk, not copied", async ({ page }) => { + await page.addInitScript(() => { + const text = new TextEncoder(); + const big = "x".repeat(5 * 1024 * 1024 - 7) + "\nend\n"; + const disk: Record = { + "/d/notes.md": text.encode("# Notes\n"), + "/d/huge.log": text.encode(big), + // A one-pixel PNG. + "/d/dot.png": Uint8Array.from(atob("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg=="), (c) => c.charCodeAt(0)), + }; + const reads = { file: [] as string[], range: [] as number[] }; + Object.assign(window, { + reads, + readm3Desktop: { + takeLaunchFiles: async () => [], + onOpenFiles: () => {}, + // As main.cjs lists them since 0.22.0: names and sizes, no bytes. + pick: async () => Object.entries(disk).map(([path, bytes]) => ({ path, name: path.split("/").at(-1), relative: `d/${path.split("/").at(-1)}`, size: bytes.length })), + readFile: async (path: string) => { reads.file.push(path); return { data: disk[path], modified: 1 }; }, + readRange: async (path: string, start: number, end: number) => { reads.range.push(end - start); return disk[path].slice(start, end); }, + writeFile: async () => {}, + }, + }); + }); + await page.reload(); + await page.locator("#open-folder").click(); + await expect(page.locator("#workspace-name")).toHaveText("d"); + await expect(page.locator('#file-list button[title="d/huge.log"]')).toBeVisible(); + const reads = () => page.evaluate(() => (window as unknown as { reads: { file: string[]; range: number[] } }).reads); + // Opening read the small files and only the first 64 KB of the large one. + expect((await reads()).range).toEqual([64 * 1024]); + expect((await reads()).file).not.toContain("/d/huge.log"); + // Nothing large was copied into browser storage. + const keptBlobs = () => page.evaluate(() => new Promise((done) => { + const open = indexedDB.open("readm3"); + open.onsuccess = () => { + const db = open.result; + const names = [...db.objectStoreNames]; + if (!names.length) { done(0); return; } + let count = 0; + const tx = db.transaction(names, "readonly"); + for (const name of names) tx.objectStore(name).getAllKeys().onsuccess = (event) => { count += ((event.target as IDBRequest).result as IDBValidKey[]).filter((key) => String(key).startsWith("blob:")).length; }; + tx.oncomplete = () => done(count); + }; + open.onerror = () => done(0); + })); + expect(await keptBlobs()).toBe(0); + await page.locator('#file-list button[title="d/huge.log"]').click(); + await expect(page.locator("#code-content .large-preview")).toContainText("xxxx"); + await page.locator('#file-list button[title="d/dot.png"]').click(); + await expect(page.locator("#media-content img")).toBeVisible(); + expect(await keptBlobs()).toBe(0); +}); + test("keeps a rolling history: opened, saved, changed on disk, with snippets and restore", async ({ page }) => { // A fake disk whose file can change under the editor, as a regenerated output.json does. await page.addInitScript(() => { diff --git a/site/viewer.ts b/site/viewer.ts index e55947d..bb799fd 100644 --- a/site/viewer.ts +++ b/site/viewer.ts @@ -4,7 +4,7 @@ import { refreshUsage, showUsage } from "./usage.ts"; import { formatStorage, INLINE_MAX_BYTES, MAX_ACCOUNT_FILES, MAX_BLOB_BYTES, newBlobRef } from "../src/blob-store.ts"; import { pasteDialog, publish, sharing, history as versionHistory } from "./sharing.ts"; import { showSync } from "./sync.ts"; -import { askPassphrase, autoSyncEnabled, forgetPassphrase, schedule as scheduleSync, setAutoSync, startAutoSync, syncNow } from "./autosync.ts"; +import { askPassphrase, autoSyncEnabled, DEFAULT_SYNC_LIMIT_MB, forgetPassphrase, schedule as scheduleSync, setAutoSync, setSyncLimitMb, startAutoSync, syncLimitMb, syncNow } from "./autosync.ts"; import { renderMarkdown } from "../src/markdown.ts"; import type { Flavor } from "../src/flavors.ts"; import type { ReaderSettings } from "../src/sync-schema.ts"; @@ -502,7 +502,10 @@ function renderStored() { }) .catch((error) => { if (!still()) return; - status.textContent = `${error instanceof Error ? error.message : "This file could not be opened."}${signedIn ? "" : " Sign in to fetch it from your account."}`; + // A file the desktop app reads from disk is there only while the app has it open. + status.textContent = diskSynced.has(doc.path) || diskEntries.has(doc.path) + ? "This file is read from disk. Open its folder again (File: Open folder…) to read it." + : `${error instanceof Error ? error.message : "This file could not be opened."}${signedIn ? "" : " Sign in to fetch it from your account."}`; previousRender = ""; }); } @@ -683,23 +686,28 @@ async function importFiles(files: disk.Picked[], name = "Your workspace") { let needAccount = 0; let overCount = 0; let unreadable = 0; - for (const { file, path, handle } of files) { + for (const { file, path, handle, size = file.size, onDisk } of files) { if (!accepts(path)) continue; // Past the account's file count nothing more is read. if (signedIn && incoming.length >= MAX_ACCOUNT_FILES) { overCount++; continue; } // A PDF or image is kept as base64, a third larger than the file; the limits apply to what is stored. - const cost = binaryType(path) ? Math.ceil(file.size * 4 / 3) : file.size; - if (cost > MAX_FILE_BYTES) { + const cost = binaryType(path) ? Math.ceil(size * 4 / 3) : size; + // A large file, PDF or image the desktop app opened stays on disk and is read from there: no copy. + const readFromDisk = !!onDisk && !!handle && (cost > MAX_FILE_BYTES || !!binaryType(path)); + if (cost > MAX_FILE_BYTES || readFromDisk) { // Larger files are stored apart as they are (site/blobs.ts), which needs an account. - if (!signedIn) { needAccount++; continue; } - if (file.size > MAX_BLOB_BYTES) { skipped++; continue; } + if (!signedIn && (!readFromDisk || incoming.length >= MAX_FILES)) { needAccount++; continue; } + if (size > MAX_BLOB_BYTES) { skipped++; continue; } if (!binaryType(path) && looksBinary(await file.slice(0, 8192).text())) { unreadable++; continue; } - const doc: Document = { path, source: "", blob: newBlobRef(file.size) }; + // Opening the folder again keeps a file's stored reference, so sync does not upload it again. + const known = readFromDisk ? workspace.documents.find((doc) => doc.path === path && doc.blob?.size === size && !doc.blob.digest)?.blob : undefined; + const doc: Document = { path, source: "", blob: known ?? newBlobRef(size) }; if (!binaryType(path) && !languageForName(path)) { const language = detectLanguage(path, await file.slice(0, 65536).text()); if (language !== MARKDOWN) doc.language = language; } - await stored.keep(doc.blob!, file); + if (readFromDisk) stored.fromDisk(doc.blob!, () => handle!.getFile()); + else await stored.keep(doc.blob!, file); incoming.push(doc); handles.push(handle); continue; @@ -807,11 +815,16 @@ async function checkDisk() { /** What an input or a drop hands over: files with their relative paths, no way back to disk. */ const copies = (files: Iterable): disk.Picked[] => [...files].map((file) => ({ file, path: file.webkitRelativePath || file.name })); +/** Progress while the desktop app reads a folder of thousands of files. */ +const reading: disk.Progress = (done, total) => { + if (total >= 200) saveStatus.textContent = done < total ? `Reading files from disk… ${done.toLocaleString()} of ${total.toLocaleString()}` : "Opening…"; +}; + async function openFromDisk(folder: boolean) { if (!disk.supported()) { (folder ? folderInput : filesInput).click(); return; } try { - if (folder) { const picked = await disk.pickFolder(); if (picked) await importFiles(picked.files, picked.name); } - else { const picked = await disk.pickFiles(); if (picked) await importFiles(picked); } + if (folder) { const picked = await disk.pickFolder(reading); if (picked) await importFiles(picked.files, picked.name); } + else { const picked = await disk.pickFiles(reading); if (picked) await importFiles(picked); } } catch (error) { notice(error instanceof Error ? error.message : "Could not open from disk."); } } @@ -876,7 +889,10 @@ function loadPreferences() { function bindEvents() { element("sync-workspace").onclick = () => void showSync({ - auto: { enabled: autoSyncEnabled, set: setAutoSync, now: syncNow, changePassphrase: (userId) => void askPassphrase("create", userId), forgetPassphrase }, + auto: { + enabled: autoSyncEnabled, set: setAutoSync, now: syncNow, changePassphrase: (userId) => void askPassphrase("create", userId), forgetPassphrase, + limit: { get: syncLimitMb, set: setSyncLimitMb, fallback: DEFAULT_SYNC_LIMIT_MB }, + }, read: () => workspace, settings: () => ({ theme: theme.value, flavor: flavor.value as Flavor }), apply: (incoming, settings) => applySynced(incoming as Workspace | undefined, settings), @@ -1100,7 +1116,7 @@ function receiveLaunches() { const open = (files: disk.DesktopFile[]) => { if (!files.length) return; const folder = files.find((file) => file.relative)?.relative!.split("/")[0]; - void disk.fromDesktop(files).then((picked) => importFiles(picked, folder)).catch(() => notice("A file could not be read from disk.")); + void disk.fromDesktop(files, reading).then((picked) => importFiles(picked, folder)).catch(() => notice("A file could not be read from disk.")); }; void disk.desktop.takeLaunchFiles().then(open); disk.desktop.onOpenFiles(open); diff --git a/src/cli.ts b/src/cli.ts index 51844b4..2aea740 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -12,7 +12,7 @@ import { run, type ViewerOptions } from "./viewer.ts"; import { isCloudCommand, cloudMain, CLOUD_USAGE } from "./cloud-cli.ts"; import { isDesktopCommand, desktopMain, installedDesktop, prefersDesktop, DESKTOP_USAGE } from "./desktop-cli.ts"; -export const VERSION = "0.21.0"; +export const VERSION = "0.22.0"; /** The help text, exported so a test can hold it to the flags it documents. */ export const USAGE = `readm3 — a plain text editor for Markdown and code