Raise the file cap to 1,000,000; sync up to 250,000; fold big folders (v0.23.0)

README.md

@@ -376,7 +376,7 @@ directly with no credentials or server proxy. Local files are uploaded only when
 | | Without an account | Free account | Pay as you go |
 | --- | --- | --- | --- |
 | Largest file | 4 MB | 5 GB | 5 GB |
-| Number of files | 1,000 (20 MB in all) | 5,000 | 5,000 |
+| Number of files | 1,000 (20 MB in all) | 1,000,000 (250,000 synced) | 1,000,000 (250,000 synced) |
 | Synced storage | none | 5 GB | $0.05 per GB a month over 5 GB |
 
 The meter beside Sync in the editor's header shows storage and files against these
@@ -777,7 +777,7 @@ explicitly overrides a conflict. Replaced local files receive numbered `.bak-NNN
 backups. Browser replacements receive backups in IndexedDB, accessible through
 **Sync → Local backups** for download or restoration. Clearing browser site data
 also removes those local backups. The CLI syncs a readable workspace of up to 4 MB
-per file and 20 MB in total; the browser and desktop app sync up to 5,000 files of
+per file and 20 MB in total; the browser and desktop app sync up to 250,000 files (a workspace holds up to 1,000,000) of
 up to 5 GB each (see File limits and storage).
 
 The authenticated API exposes:

desktop/main.cjs

@@ -28,7 +28,7 @@ const LOCAL = "app://readm3";
 const MAX_FILE_BYTES = 5 * 1024 ** 3;
 const MAX_RANGE = 16 * 1024 * 1024;
 /** A signed-in workspace's file count (src/blob-store.ts); the walk stops there. */
-const MAX_FILES = 5000;
+const MAX_FILES = 1_000_000;
 const IGNORED = new Set(["node_modules", "dist", "build", "out", "target", "vendor", "coverage", "venv", "__pycache__", ".git"]);
 const NEVER = /\.(?:zip|gz|tgz|bz2|xz|7z|rar|tar|exe|dll|so|dylib|o|a|class|jar|war|wasm|bin|dat|db|sqlite|sqlite3|mp3|mp4|m4a|mov|avi|mkv|webm|ogg|wav|flac|woff2?|ttf|otf|eot|psd|ai|doc|docx|xls|xlsx|ppt|pptx|iso|dmg|pkg|deb|rpm|apk|lock)$/i;
 /** The bundled editor: inside the package, or the repo's site build when run with `electron .`. */

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@profullstack/readm3",
-  "version": "0.22.0",
+  "version": "0.23.0",
   "description": "Markdown editing and sharing for people and teams. Web/PWA, CLI, API and MCP with version history, view/edit links, and a terminal reader.",
   "type": "module",
   "license": "MIT",

site/autosync.ts

@@ -20,7 +20,7 @@ import { digestDocument, digests, mergeWorkspaces } from "../src/sync-merge.ts";
 import { isSealed, validateSyncFiles, type ReaderSettings, type SyncedWorkspace } from "../src/sync-schema.ts";
 import { store, stored } from "./sync.ts";
 import * as blobs from "./blobs.ts";
-import { formatStorage, newBlobRef, SNAPSHOT_INLINE_BYTES, type BlobRef } from "../src/blob-store.ts";
+import { formatStorage, MAX_SYNC_FILES, newBlobRef, SNAPSHOT_INLINE_BYTES, type BlobRef } from "../src/blob-store.ts";
 
 const OFF = "readm3:autosync";
 export const autoSyncEnabled = (): boolean => { try { return localStorage.getItem(OFF) !== "off"; } catch { return true; } };
@@ -264,6 +264,10 @@ async function pass(user: CloudUser) {
     const accountOnly = remoteWorkspace?.documents.filter((doc) => keptOnDevice(doc, limit)) ?? [];
     const local = { ...all, documents: all.documents.filter((doc) => !keptOnDevice(doc, limit)) };
     const remoteSynced = remoteWorkspace && { ...remoteWorkspace, documents: remoteWorkspace.documents.filter((doc) => !keptOnDevice(doc, limit)) };
+    // Past what one sealed snapshot carries, sync waits rather than drop files from the account copy.
+    if (local.documents.length + accountOnly.length > MAX_SYNC_FILES) {
+      throw new Paused(`Sync paused · ${local.documents.length.toLocaleString()} files; sync carries up to ${MAX_SYNC_FILES.toLocaleString()}. Your files are safe on this device.`);
+    }
     const merge = mergeWorkspaces({
       local, remote: remoteSynced, base: state?.base, starter: await starterDigests(),
       conflictLabel: `${h.device} ${new Date().toISOString().slice(0, 10)}`,

site/build.ts

@@ -255,7 +255,7 @@ function home(): string {
     <div class="card"><h3>Keep the story of the document.</h3><p>Every online save creates a version. See who changed it, share a specific version, or restore an earlier draft without losing the history.</p></div>
     <div class="card"><h3>A workspace for your team.</h3><p>Organize people into organizations and teams. Share with a whole group, a few collaborators, or anyone with your link.</p></div>
     <div class="card"><h3>A plain text editor.</h3><p>Line numbers, multiple cursors, find and replace, folding, <code>Ctrl+P</code> to any file and <code>Ctrl+Shift+P</code> for every command. Open a folder from disk and <code>Ctrl+S</code> saves back to it. <a href="/viewer">Open the editor</a>.</p></div>
-    <div class="card"><h3>Files up to 5 GB, 5,000 of them.</h3><p>A free account lifts the limits: up to 5,000 files, each up to 5 GB, with 5 GB of end-to-end encrypted storage that syncs to every device. Need more? Pay as you go, $0.05 per GB a month, from a CoinPay balance.</p></div>
+    <div class="card"><h3>Files up to 5 GB, a million of them.</h3><p>A free account lifts the limits: up to 1,000,000 files (250,000 of them synced), each up to 5 GB, with 5 GB of end-to-end encrypted storage that syncs to every device. Need more? Pay as you go, $0.05 per GB a month, from a CoinPay balance.</p></div>
     <div class="card"><h3>Summarize any document.</h3><p>Press Summarize in the editor for a short abstract and the key points of the open file, written by Claude. The first 10 documents a week are free on a free account; after that, pay as you go from a CoinPay balance, usually a fraction of a cent each. <a href="/viewer">Try it</a>.</p></div>
     <div class="card"><h3>Take the editor with you.</h3><p>Install readm3 from your browser, or get the desktop app: Linux <a href="${REPO}/releases/latest/download/readm3-linux-x86_64.AppImage">AppImage</a> or <a href="${REPO}/releases/latest/download/readm3-linux-x86_64.flatpak">Flatpak</a>, macOS for <a href="${REPO}/releases/latest/download/readm3-mac-arm64.dmg">Apple silicon</a> or <a href="${REPO}/releases/latest/download/readm3-mac-x64.dmg">Intel</a>, <a href="${REPO}/releases/latest/download/readm3-windows-x64-setup.exe">Windows</a>. Then <code>readm3 edit .</code> opens a folder from the terminal, like <code>subl .</code>. Local files work offline; shared documents stay connected.</p></div>
     <div class="card"><h3>Use the tools you already use.</h3><p>The web editor, CLI, API, and MCP work with the same documents and permissions. Your terminal and your agents can join the workflow.</p></div>

site/test/sync.spec.ts

@@ -180,6 +180,29 @@ test("files over the sync size limit stay on this device until the limit is turn
   expect(await page.evaluate(() => localStorage.getItem("readm3:sync-limit-mb"))).toBe("0");
 });
 
+test("signed in, a folder past the old 5,000-file cap opens whole with its subfolders folded", async ({ page, context }) => {
+  await register(context, username());
+  await page.addInitScript(() => {
+    const file = (name: string) => ({ kind: "file", name, getFile: async () => new File([`# ${name}\n`], name) });
+    const directory = (name: string, entries: unknown[]) => ({ kind: "directory", name, values: async function* () { yield* entries; } });
+    const sections = Array.from({ length: 60 }, (_, s) => directory(`section-${s}`, Array.from({ length: 100 }, (_, i) => file(`page-${i}.md`))));
+    Object.assign(window, { showDirectoryPicker: async () => directory("site", [file("README.md"), ...sections]) });
+  });
+  await page.goto("/viewer");
+  await setPassphrase(page);
+  await page.locator("#open-folder").click();
+  // Reading 6,001 files takes a few seconds on a slow CI runner.
+  await expect(page.locator("#workspace-name")).toHaveText("site", { timeout: 60_000 });
+  await expect(page.locator("#document-name")).toHaveText("site/README.md");
+  // 6,001 files plus the starter documents, all kept.
+  expect(Number(await page.locator("#file-count").textContent())).toBeGreaterThan(6000);
+  // Folded: the site folder, README and the 60 section folders, plus the starter rows.
+  expect(await page.locator("#file-list button").count()).toBeLessThan(100);
+  await page.locator('#file-list .folder-row[title="site/section-7"]').click();
+  await expect(page.locator('#file-list button[title="site/section-7/page-42.md"]')).toBeVisible();
+  await expect(page.locator("#sync-note")).toContainText("Synced", { timeout: 30_000 });
+});
+
 test("the header shows usage against the plan's limits, signed out and signed in", async ({ page, context }) => {
   await page.goto("/viewer");
   const usage = page.locator("#usage");
@@ -194,7 +217,7 @@ test("the header shows usage against the plan's limits, signed out and signed in
   await register(context, username());
   await page.reload();
   await setPassphrase(page);
-  await expect(usage).toHaveAttribute("title", /\/ 5\.0 GB · \d+ \/ 5,000 files/);
+  await expect(usage).toHaveAttribute("title", /\/ 5\.0 GB · \d+ \/ 1,000,000 files/);
   await usage.click();
   await expect(dialog).toContainText("Free account");
   await expect(dialog).toContainText("$0.05 per GB a month");

site/usage.ts

@@ -1,7 +1,7 @@
 /**
  * The usage meter in the editor's header: files and storage against the plan's limits.
  * Signed out it measures this device's workspace (1,000 files, 20 MB); signed in it
- * measures the account (5,000 files, 5 GB stored free, then pay as you go). A click
+ * measures the account (1,000,000 files, 5 GB stored free, then pay as you go). A click
  * opens the limits, the price, the balance and a way to add funds or sign in.
  */
 import { modal } from "./cloud.ts";
@@ -88,7 +88,7 @@ export function showUsage() {
   const note = document.createElement("p");
   note.textContent = last.signedIn
     ? "Files over 4 MB are stored end-to-end encrypted on your account. Storage over the free allowance is charged hourly from your balance; with no balance, new uploads past it wait, and nothing stored is deleted."
-    : "A free account raises the limits to 5,000 files of up to 5 GB each, with 5 GB of encrypted storage that syncs to every device.";
+    : "A free account raises the limits to 1,000,000 files of up to 5 GB each, with 5 GB of encrypted storage that syncs to every device.";
   const actions = document.createElement("p");
   if (last.signedIn) {
     const add = document.createElement("button");

site/viewer.ts

@@ -68,7 +68,7 @@ const isMarkdown = () => kind().kind === "markdown";
  * signed out, the workspace keeps its small local limits.
  */
 let signedIn = false;
-const ACCOUNT_LIMITS = "Sign in (free) for files up to 5 GB and up to 5,000 files.";
+const ACCOUNT_LIMITS = "Sign in (free) for files up to 5 GB and up to 1,000,000 files.";
 /** A stored file too large for the workspace (not a spilled small one): shown read-only. */
 const isLarge = (doc = current()) => !!doc.blob && !doc.blob.digest;
 const PREVIEW_BYTES = 1024 * 1024;
@@ -167,6 +167,23 @@ let noticeTimer: ReturnType<typeof setTimeout>;
 let importBusy = false;
 let saveFailed = false;
 const collapsed = new Set<string>();
+/**
+ * A workspace of thousands of files (a whole ~/Desktop) shows its top folders with every
+ * folder below them folded, except the way to the open file; a small one shows everything.
+ */
+function foldLarge(open: string) {
+  collapsed.clear();
+  if (workspace.documents.length <= 2000) return;
+  const parts = open.split("/");
+  const keep = new Set(parts.slice(0, -1).map((_, depth) => parts.slice(0, depth + 1).join("/")));
+  for (const doc of workspace.documents) {
+    const folders = doc.path.split("/");
+    for (let depth = 1; depth < folders.length - 1; depth++) {
+      const path = folders.slice(0, depth + 1).join("/");
+      if (!keep.has(path)) collapsed.add(path);
+    }
+  }
+}
 
 function notice(message: string) {
   element("notice").textContent = message;
@@ -211,7 +228,7 @@ async function applySynced(incoming: Workspace | undefined, settings: ReaderSett
       await persist(incoming);
       workspace = incoming;
       filter.value = "";
-      collapsed.clear();
+      foldLarge(workspace.active);
     }
     if (settings?.theme && [...theme.options].some((option) => option.value === settings.theme)) theme.value = settings.theme;
     if (settings?.flavor) flavor.value = settings.flavor;
@@ -349,7 +366,7 @@ function renderFiles(focusPath?: string) {
       }
       if (!query && collapsed.has(path)) { hidden = true; break; }
     }
-    if (!hidden) fileList.append(fileRow(doc.path, parts.at(-1)!, parts.length - 1, false));
+    if (!hidden) fileList.append(fileRow(doc.path, parts.at(-1)!, parts.length - 1, false, doc));
   }
   if (!docs.length) {
     const empty = document.createElement("p");
@@ -362,18 +379,18 @@ function renderFiles(focusPath?: string) {
   if (focusPath) Array.from(fileList.querySelectorAll<HTMLButtonElement>("button")).find((button) => button.dataset.path === focusPath)?.focus();
 }
 
-function fileRow(path: string, name: string, depth: number, directory: boolean) {
+function fileRow(path: string, name: string, depth: number, directory: boolean, doc?: Document) {
   const row = document.createElement("button");
   row.className = `file-row${directory ? " folder-row" : ""}`;
   row.dataset.path = path;
   row.style.paddingLeft = `${10 + depth * 15}px`;
   row.title = path;
   if (!directory && path === workspace.active) row.setAttribute("aria-current", "page");
-  if (!directory && unsavedOnDisk(workspace.documents.find((doc) => doc.path === path)!)) row.classList.add("unsaved");
+  if (doc && unsavedOnDisk(doc)) row.classList.add("unsaved");
   const icon = document.createElement("span");
   icon.className = "file-icon";
   icon.setAttribute("aria-hidden", "true");
-  const fileKind = directory ? null : kindOf(workspace.documents.find((doc) => doc.path === path)!);
+  const fileKind = doc ? kindOf(doc) : null;
   icon.textContent = directory ? (filter.value || !collapsed.has(path) ? "▾" : "▸") : fileKind!.kind === "markdown" ? "M↓" : fileKind!.kind === "code" ? "{}" : "▣";
   const label = document.createElement("span");
   label.className = "row-name";
@@ -669,8 +686,8 @@ function addDocuments(incoming: Document[], name?: string): Document[] {
   workspace.documents = [...map.values()].sort((a, b) => comparePaths(a.path, b.path));
   if (name) workspace.name = name;
   filter.value = "";
-  collapsed.clear();
   const first = added.find((doc) => /^readme\b/i.test(doc.path.split("/").at(-1)!)) ?? added[0];
+  foldLarge(first.path);
   openDocument(first.path);
   return added;
 }
@@ -686,6 +703,7 @@ async function importFiles(files: disk.Picked[], name = "Your workspace") {
     let needAccount = 0;
     let overCount = 0;
     let unreadable = 0;
+    const existing = new Map(workspace.documents.map((doc) => [doc.path, doc]));
     for (const { file, path, handle, size = file.size, onDisk } of files) {
       if (!accepts(path)) continue;
       // Past the account's file count nothing more is read.
@@ -700,7 +718,8 @@ async function importFiles(files: disk.Picked[], name = "Your workspace") {
         if (size > MAX_BLOB_BYTES) { skipped++; continue; }
         if (!binaryType(path) && looksBinary(await file.slice(0, 8192).text())) { unreadable++; continue; }
         // Opening the folder again keeps a file's stored reference, so sync does not upload it again.
-        const known = readFromDisk ? workspace.documents.find((doc) => doc.path === path && doc.blob?.size === size && !doc.blob.digest)?.blob : undefined;
+        const before = readFromDisk ? existing.get(path)?.blob : undefined;
+        const known = before && before.size === size && !before.digest ? before : undefined;
         const doc: Document = { path, source: "", blob: known ?? newBlobRef(size) };
         if (!binaryType(path) && !languageForName(path)) {
           const language = detectLanguage(path, await file.slice(0, 65536).text());

src/blob-store.ts

@@ -21,7 +21,12 @@ export const GiB = 1024 ** 3;
 /** The largest single file an account may keep. */
 export const MAX_BLOB_BYTES = 5 * GiB;
 /** Files in one signed-in workspace (signed out, the workspace keeps 1,000). */
-export const MAX_ACCOUNT_FILES = 5000;
+export const MAX_ACCOUNT_FILES = 1_000_000;
+/**
+ * Files one synced workspace carries. Every file's reference is about 100 bytes sealed
+ * (its own random key does not compress), so this many fit the 32 MB snapshot with room.
+ */
+export const MAX_SYNC_FILES = 250_000;
 /** Storage every signed-in account gets free; more is pay as you go from the prepaid balance. */
 export const FREE_STORAGE_BYTES = 5 * GiB;
 /** Pay as you go: micro-dollars per GiB stored per month, over the free allowance. */

src/cli.ts

@@ -12,7 +12,7 @@ import { run, type ViewerOptions } from "./viewer.ts";
 import { isCloudCommand, cloudMain, CLOUD_USAGE } from "./cloud-cli.ts";
 import { isDesktopCommand, desktopMain, installedDesktop, prefersDesktop, DESKTOP_USAGE } from "./desktop-cli.ts";
 
-export const VERSION = "0.22.0";
+export const VERSION = "0.23.0";
 
 /** The help text, exported so a test can hold it to the flags it documents. */
 export const USAGE = `readm3 — a plain text editor for Markdown and code

src/sync-schema.ts

@@ -1,7 +1,7 @@
 /** Browser-safe account data contract. Only these two files may be synced. */
 import { isFlavor, type Flavor } from "./flavors.ts";
 import { envelopeProblem, SEALED_FILE } from "./sync-crypto.ts";
-import { blobRefProblem, INLINE_MAX_BYTES, MAX_ACCOUNT_FILES, SNAPSHOT_INLINE_BYTES, type BlobRef } from "./blob-store.ts";
+import { blobRefProblem, INLINE_MAX_BYTES, MAX_SYNC_FILES, SNAPSHOT_INLINE_BYTES, type BlobRef } from "./blob-store.ts";
 
 /**
  * A workspace file. `source` is the text, or base64 bytes when the path names a PDF or
@@ -33,7 +33,7 @@ export function validateWorkspace(value: unknown): SyncedWorkspace {
   if (!value || typeof value !== "object") throw new Error("Workspace must be an object.");
   const ws = value as SyncedWorkspace;
   if (Object.keys(ws).some((key) => !["name", "active", "documents"].includes(key))) throw new Error("Workspace can only contain name, active and documents.");
-  if (typeof ws.name !== "string" || ws.name.length > 200 || !Array.isArray(ws.documents) || !ws.documents.length || ws.documents.length > MAX_ACCOUNT_FILES) throw new Error("Workspace needs a name and 1–5,000 files.");
+  if (typeof ws.name !== "string" || ws.name.length > 200 || !Array.isArray(ws.documents) || !ws.documents.length || ws.documents.length > MAX_SYNC_FILES) throw new Error(`Workspace needs a name and 1–${MAX_SYNC_FILES.toLocaleString("en-US")} files.`);
   const paths = new Set<string>();
   let total = 0;
   const encoder = new TextEncoder();

test/blob-store.test.ts

@@ -28,12 +28,12 @@ test("chunk arithmetic covers a 5 GB file exactly", () => {
   assert.match(blobRefProblem({ ...newBlobRef(1), id: "../../etc" })!, /invalid id/);
 });
 
-test("a synced workspace holds up to 5,000 files and carries stored files by reference", () => {
-  const documents = Array.from({ length: 4999 }, (_, i) => ({ path: `notes/${i}.md`, source: `# ${i}` }));
+test("a synced workspace holds up to 250,000 files and carries stored files by reference", () => {
+  const documents = Array.from({ length: 249_999 }, (_, i) => ({ path: `notes/${i}.md`, source: `# ${i}` }));
   documents.push({ path: "big/dump.json", source: "", blob: newBlobRef(3 * 1024 ** 3) } as never);
-  assert.equal(validateWorkspace({ name: "Many", active: "notes/0.md", documents }).documents.length, 5000);
+  assert.equal(validateWorkspace({ name: "Many", active: "notes/0.md", documents }).documents.length, 250_000);
   documents.push({ path: "one/more.md", source: "" });
-  assert.throws(() => validateWorkspace({ name: "Many", active: "notes/0.md", documents }), /1–5,000 files/);
+  assert.throws(() => validateWorkspace({ name: "Many", active: "notes/0.md", documents }), /1–250,000 files/);
   assert.throws(() => validateWorkspace({ name: "x", active: "a.md", documents: [{ path: "a.md", source: "text", blob: newBlobRef(1) }] }), /source must be empty/);
 });