From d38db62e8bdd8307d6baed63a9fa0860b3fbef19 Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Sun, 4 Oct 2026 03:45:25 -0700 Subject: [PATCH] vault: autosync the personal vault to the logicsrc account (#226) `logicsrc vault` (OpenCreds) lived only in ~/.config/logicsrc/opencreds. Lose the machine and the vault went with it, and a second machine had no way to get it. It now syncs to the logged-in account. Server (apps/pwa, /api/opencreds, session or lsk_ bearer): - opencreds_vaults: one per user. meta (key material already wrapped under the master password) + an encrypted folder list, each with a revision. - opencreds_items: one row per item, as the spec asks; envelope NULL is a tombstone so purges propagate; seq for incremental pulls. - Every write names its base revision; a stale one gets 409 with the current row. Writes are conditional and read back (each envelope's random IV identifies our write), so this needs nothing dialect-specific from SQLite or Postgres. - Stores ciphertext only. The server learns item count and type codes, as OpenCreds security.md already accepts. Client (@logicsrc/opencreds sync.ts, key-free except where noted): - Pull before every vault command, push after. Offline, the command still works and the change goes up next time. - Conflicts never lose data: the account's edit keeps the id and this machine's becomes " (conflict copy)" (needs the unlocked key, so it waits otherwise). An edit beats a purge. Byte-identical envelopes are adopted, not split, so a lost sync.json is harmless. - Two different vaults (meta.createdAt differs) are never merged: refused, with `vault sync --use-remote` (backs this machine's up to .bak-NNN) or `--use-local`. - Remote item ids must be plain ids; anything else is ignored and never becomes a path. - Folder names are AES-GCM encrypted under the user key before upload. CLI (0.6.0): `logicsrc vault sync [--status|--use-remote|--use-local]`. Only the default vault syncs, and only when logged in; a --home or OPENCREDS_HOME vault stays local unless LOGICSRC_VAULT_SYNC=on; LOGICSRC_VAULT_SYNC=off disables it. The standalone `opencreds` binary gets no remote and never syncs. `init` on a machine that just downloaded the account's vault says to unlock it instead of suggesting --force. Co-authored-by: Claude Opus 5.5 --- .../src/migrations-pg/004_opencreds_sync.sql | 26 ++ .../pwa/src/migrations/004_opencreds_sync.sql | 32 ++ apps/pwa/src/routes/opencreds.mjs | 154 +++++++ apps/pwa/src/server.mjs | 5 + apps/pwa/test/opencreds-sync.test.mjs | 123 ++++++ bun.lock | 2 +- packages/cli/package.json | 2 +- packages/cli/src/creds.ts | 10 +- packages/cli/src/vault-sync.ts | 94 +++++ packages/opencreds/src/commands.ts | 157 ++++++- packages/opencreds/src/index.ts | 16 + packages/opencreds/src/store.ts | 20 + packages/opencreds/src/sync.test.ts | 277 +++++++++++++ packages/opencreds/src/sync.ts | 382 ++++++++++++++++++ 14 files changed, 1293 insertions(+), 7 deletions(-) create mode 100644 apps/pwa/src/migrations-pg/004_opencreds_sync.sql create mode 100644 apps/pwa/src/migrations/004_opencreds_sync.sql create mode 100644 apps/pwa/src/routes/opencreds.mjs create mode 100644 apps/pwa/test/opencreds-sync.test.mjs create mode 100644 packages/cli/src/vault-sync.ts create mode 100644 packages/opencreds/src/sync.test.ts create mode 100644 packages/opencreds/src/sync.ts diff --git a/apps/pwa/src/migrations-pg/004_opencreds_sync.sql b/apps/pwa/src/migrations-pg/004_opencreds_sync.sql new file mode 100644 index 0000000..426bd87 --- /dev/null +++ b/apps/pwa/src/migrations-pg/004_opencreds_sync.sql @@ -0,0 +1,26 @@ +-- Postgres copy of migrations/004_opencreds_sync.sql (INTEGER -> bigint). +-- The personal vault (`logicsrc vault`, OpenCreds), synced to the account; +-- ciphertext and wrapped key material only. + +create table if not exists opencreds_vaults ( + user_id text PRIMARY KEY REFERENCES users(id) ON DELETE CASCADE, + meta text NOT NULL, + meta_revision bigint NOT NULL, + folders text, + folders_revision bigint NOT NULL DEFAULT 0, + created_at bigint NOT NULL, + updated_at bigint NOT NULL +); + +create table if not exists opencreds_items ( + user_id text NOT NULL REFERENCES users(id) ON DELETE CASCADE, + id text NOT NULL, + type bigint, + envelope text, + revision bigint NOT NULL, + seq bigint NOT NULL, + updated_at bigint NOT NULL, + PRIMARY KEY (user_id, id) +); + +CREATE INDEX IF NOT EXISTS idx_opencreds_items_seq ON opencreds_items(user_id, seq); diff --git a/apps/pwa/src/migrations/004_opencreds_sync.sql b/apps/pwa/src/migrations/004_opencreds_sync.sql new file mode 100644 index 0000000..2bae8d4 --- /dev/null +++ b/apps/pwa/src/migrations/004_opencreds_sync.sql @@ -0,0 +1,32 @@ +-- The personal vault (`logicsrc vault`, OpenCreds), synced to the account. +-- Zero-knowledge like credshare: meta holds only key material wrapped under +-- the master password, folders and items are AES-GCM ciphertext under the +-- vault's user key. The server can count items and read their type code +-- (OpenCreds security.md accepts that), and nothing else. + +-- One vault per user. meta_revision / folders_revision are optimistic locks: +-- a write names the revision it was based on and loses if someone moved it. +CREATE TABLE IF NOT EXISTS opencreds_vaults ( + user_id TEXT PRIMARY KEY REFERENCES users(id) ON DELETE CASCADE, + meta TEXT NOT NULL, + meta_revision INTEGER NOT NULL, + folders TEXT, + folders_revision INTEGER NOT NULL DEFAULT 0, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL +); + +-- One row per item, as the spec asks. envelope NULL is a tombstone (purged), +-- kept so the purge reaches every other machine. seq orders changes per user +-- for incremental pulls. +CREATE TABLE IF NOT EXISTS opencreds_items ( + user_id TEXT NOT NULL REFERENCES users(id) ON DELETE CASCADE, + id TEXT NOT NULL, + type INTEGER, + envelope TEXT, + revision INTEGER NOT NULL, + seq INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + PRIMARY KEY (user_id, id) +); +CREATE INDEX IF NOT EXISTS idx_opencreds_items_seq ON opencreds_items(user_id, seq); diff --git a/apps/pwa/src/routes/opencreds.mjs b/apps/pwa/src/routes/opencreds.mjs new file mode 100644 index 0000000..1a629a6 --- /dev/null +++ b/apps/pwa/src/routes/opencreds.mjs @@ -0,0 +1,154 @@ +// Sync for the personal vault (`logicsrc vault`, OpenCreds) — one per account. +// +// Zero-knowledge: the CLI uploads the vault's meta (key material wrapped under +// the master password), an encrypted folder list, and item envelopes. Nothing +// here can decrypt any of it, and nothing here needs to. +// +// Concurrency is optimistic. Every write names the revision it was based on; +// a write against a revision someone else already moved is answered with the +// current row instead of being applied, and the client resolves it. That is +// the reason the spec stores one row per item: two machines editing two +// different passwords both win, and two editing the same one find out. +// +// Auth: browser session or `Bearer lsk_…` (the logicsrc CLI). Mounted at /api/opencreds. +import { Router } from "express"; +import { get, all, run } from "../db.mjs"; +import { bearer, userForApiKey } from "../lib/apikey.mjs"; + +export const opencredsRouter = Router(); + +/** Upper bound on one push, in items. The client batches below this. */ +export const MAX_ITEMS_PER_PUSH = 500; + +function api(handler) { + return async (req, res) => { + const user = req.user || (await userForApiKey(bearer(req))); + if (!user) return res.status(401).json({ error: "Not authenticated. Run: logicsrc login" }); + try { + await handler(req, res, user); + } catch (e) { + console.error("opencreds:", e); + res.status(500).json({ error: e.message || String(e) }); + } + }; +} + +const isObject = (v) => v !== null && typeof v === "object" && !Array.isArray(v); +const nonNegInt = (v) => Number.isInteger(v) && v >= 0; + +function vaultJson(row) { + return { + meta: JSON.parse(row.meta), + metaRevision: Number(row.meta_revision), + folders: row.folders ? { ...JSON.parse(row.folders), revision: Number(row.folders_revision) } : null, + updatedAt: Number(row.updated_at) + }; +} + +function itemJson(row) { + const envelope = row.envelope ? JSON.parse(row.envelope) : null; + const revision = Number(row.revision); + return { id: row.id, envelope: envelope && { ...envelope, revision }, revision, seq: Number(row.seq) }; +} + +const vaultRow = (userId) => get(`SELECT * FROM opencreds_vaults WHERE user_id = ?`, [userId]); +const itemRow = (userId, id) => get(`SELECT * FROM opencreds_items WHERE user_id = ? AND id = ?`, [userId, id]); +const NEXT_SEQ = `(SELECT COALESCE(MAX(seq), 0) + 1 FROM opencreds_items WHERE user_id = ?)`; + +// ---- the vault: meta + folders ---- +opencredsRouter.get("/api/opencreds/vault", api(async (_req, res, user) => { + const row = await vaultRow(user.id); + if (!row) return res.status(404).json({ vault: null }); + const counts = await get( + `SELECT COUNT(*) AS n, MAX(seq) AS seq FROM opencreds_items WHERE user_id = ? AND envelope IS NOT NULL`, [user.id]); + res.json({ vault: { ...vaultJson(row), itemCount: Number(counts?.n || 0) } }); +})); + +opencredsRouter.put("/api/opencreds/vault/meta", api(async (req, res, user) => { + const { meta, baseRevision } = req.body || {}; + if (!isObject(meta) || typeof meta.protectedUserKey !== "string" || !nonNegInt(baseRevision)) { + return res.status(422).json({ error: "Expected { meta, baseRevision }." }); + } + const text = JSON.stringify(meta), now = Date.now(); + if (baseRevision === 0) { + await run(`INSERT INTO opencreds_vaults (user_id, meta, meta_revision, created_at, updated_at) VALUES (?,?,1,?,?) ON CONFLICT (user_id) DO NOTHING`, + [user.id, text, now, now]); + } else { + await run(`UPDATE opencreds_vaults SET meta = ?, meta_revision = meta_revision + 1, updated_at = ? WHERE user_id = ? AND meta_revision = ?`, + [text, now, user.id, baseRevision]); + } + // Read back: our exact meta at base+1 means this write is the one that landed. + const row = await vaultRow(user.id); + if (row && row.meta === text && Number(row.meta_revision) === baseRevision + 1) { + return res.json({ ok: true, revision: baseRevision + 1 }); + } + res.status(409).json({ ok: false, vault: row ? vaultJson(row) : null }); +})); + +opencredsRouter.put("/api/opencreds/vault/folders", api(async (req, res, user) => { + const { ciphertext, iv, baseRevision } = req.body || {}; + if (typeof ciphertext !== "string" || typeof iv !== "string" || !nonNegInt(baseRevision)) { + return res.status(422).json({ error: "Expected { ciphertext, iv, baseRevision }." }); + } + if (!(await vaultRow(user.id))) return res.status(409).json({ ok: false, error: "Push the vault meta first." }); + const text = JSON.stringify({ ciphertext, iv }); + await run(`UPDATE opencreds_vaults SET folders = ?, folders_revision = folders_revision + 1, updated_at = ? WHERE user_id = ? AND folders_revision = ?`, + [text, Date.now(), user.id, baseRevision]); + const row = await vaultRow(user.id); + if (row.folders === text && Number(row.folders_revision) === baseRevision + 1) { + return res.json({ ok: true, revision: baseRevision + 1 }); + } + res.status(409).json({ ok: false, vault: vaultJson(row) }); +})); + +// Replace the account's vault (`logicsrc vault sync --use-local`): drop it and +// every item so the next push starts from revision 0. +opencredsRouter.delete("/api/opencreds/vault", api(async (_req, res, user) => { + await run(`DELETE FROM opencreds_items WHERE user_id = ?`, [user.id]); + await run(`DELETE FROM opencreds_vaults WHERE user_id = ?`, [user.id]); + res.json({ ok: true }); +})); + +// ---- items ---- +// `since` is inclusive: a seq can repeat under concurrent pushes, and an item +// seen twice at the same revision is a no-op on the client. +opencredsRouter.get("/api/opencreds/items", api(async (req, res, user) => { + const since = Number.parseInt(String(req.query.since ?? "0"), 10) || 0; + const rows = await all(`SELECT * FROM opencreds_items WHERE user_id = ? AND seq >= ? ORDER BY seq`, [user.id, since]); + const items = rows.map(itemJson); + res.json({ items, cursor: items.reduce((max, i) => Math.max(max, i.seq), since) }); +})); + +opencredsRouter.put("/api/opencreds/items", api(async (req, res, user) => { + const changes = Array.isArray(req.body?.changes) ? req.body.changes : null; + if (!changes) return res.status(422).json({ error: "Expected { changes: [{ id, envelope|null, baseRevision }] }." }); + if (changes.length > MAX_ITEMS_PER_PUSH) return res.status(413).json({ error: `At most ${MAX_ITEMS_PER_PUSH} items per push.` }); + for (const c of changes) { + const ok = isObject(c) && typeof c.id === "string" && c.id.length > 0 && c.id.length <= 128 && nonNegInt(c.baseRevision) && + (c.envelope === null || (isObject(c.envelope) && c.envelope.id === c.id && typeof c.envelope.ciphertext === "string" && typeof c.envelope.iv === "string")); + if (!ok) return res.status(422).json({ error: `Malformed change${isObject(c) && c.id ? ` for ${c.id}` : ""}.` }); + } + if (!(await vaultRow(user.id))) return res.status(409).json({ error: "Push the vault meta first." }); + + const applied = [], conflicts = []; + for (const c of changes) { + // The server's revision is authoritative; the client's local counter is not stored. + const envelope = c.envelope ? JSON.stringify((({ revision: _r, ...rest }) => rest)(c.envelope)) : null; + const type = c.envelope && Number.isInteger(c.envelope.type) ? c.envelope.type : null; + const now = Date.now(); + if (c.baseRevision === 0) { + await run(`INSERT INTO opencreds_items (user_id, id, type, envelope, revision, seq, updated_at) VALUES (?,?,?,?,1,${NEXT_SEQ},?) ON CONFLICT (user_id, id) DO NOTHING`, + [user.id, c.id, type, envelope, user.id, now]); + } else { + await run(`UPDATE opencreds_items SET type = ?, envelope = ?, revision = revision + 1, seq = ${NEXT_SEQ}, updated_at = ? WHERE user_id = ? AND id = ? AND revision = ?`, + [type, envelope, user.id, now, user.id, c.id, c.baseRevision]); + } + const row = await itemRow(user.id, c.id); + if (row && (row.envelope ?? null) === envelope && Number(row.revision) === c.baseRevision + 1) { + applied.push({ id: c.id, revision: Number(row.revision), seq: Number(row.seq) }); + } else { + conflicts.push(row ? itemJson(row) : { id: c.id, envelope: null, revision: 0, seq: 0 }); + } + } + res.json({ applied, conflicts }); +})); diff --git a/apps/pwa/src/server.mjs b/apps/pwa/src/server.mjs index c5add0a..6d0bbf5 100644 --- a/apps/pwa/src/server.mjs +++ b/apps/pwa/src/server.mjs @@ -10,6 +10,7 @@ import { authRouter } from "./routes/auth.mjs"; import { passkeyRouter } from "./routes/passkey.mjs"; import { coinpayRouter } from "./routes/coinpay.mjs"; import { credshareRouter } from "./routes/credshare.mjs"; +import { opencredsRouter } from "./routes/opencreds.mjs"; import { cliRouter } from "./routes/cli.mjs"; import { pagesRouter } from "./routes/pages.mjs"; @@ -18,6 +19,9 @@ app.disable("x-powered-by"); if (config.secure) app.set("trust proxy", 1); // Railway terminates TLS // body parsing — keep the raw body for HMAC signature verification +// A vault push carries hundreds of encrypted items; parse it under a larger cap +// first. The global parser below skips a body that is already parsed. +app.use("/api/opencreds", express.json({ limit: "10mb" })); app.use(express.json({ verify: (req, _res, buf) => { req.rawBody = buf.toString("utf8"); } })); app.use(express.urlencoded({ extended: false })); app.use(cookieParser()); @@ -46,6 +50,7 @@ app.use(authRouter); // GET / (+ /auth/login|register|logout) app.use(passkeyRouter); app.use(coinpayRouter); app.use(credshareRouter); // /api/credshare/* (session or lsk_ Bearer) +app.use(opencredsRouter); // /api/opencreds/* — personal vault sync (session or lsk_ Bearer) app.use(cliRouter); // /cli/authorize, /cli/token, /api/me app.use(pagesRouter); // /dashboard, /teams/*, /settings diff --git a/apps/pwa/test/opencreds-sync.test.mjs b/apps/pwa/test/opencreds-sync.test.mjs new file mode 100644 index 0000000..90176e4 --- /dev/null +++ b/apps/pwa/test/opencreds-sync.test.mjs @@ -0,0 +1,123 @@ +// /api/opencreds: the account copy of a personal vault. The server stores +// ciphertext and enforces one rule, optimistic revisions: a write based on a +// revision someone else already moved is refused with the current row. +process.env.DATABASE_URL = process.env.PWA_TEST_DATABASE_URL || ":memory:"; + +import test from "node:test"; +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import { dirname, join } from "node:path"; +import express from "express"; + +const here = dirname(fileURLToPath(import.meta.url)); +const { db, run, isPostgres } = await import("../src/db.mjs"); +const { opencredsRouter } = await import("../src/routes/opencreds.mjs"); + +if (isPostgres) { + await db.execute("DROP SCHEMA public CASCADE"); + await db.execute("CREATE SCHEMA public"); +} +for (const file of ["001_auth.sql", "004_opencreds_sync.sql"]) { + const sql = readFileSync(join(here, "..", "src", isPostgres ? "migrations-pg" : "migrations", file), "utf8"); + for (const statement of sql.split(/;\s*$/m).map((s) => s.trim()).filter(Boolean)) await db.execute(statement); +} +for (const uid of ["u1", "u2"]) await run(`INSERT INTO users (id, email, created_at) VALUES (?,?,?)`, [uid, `${uid}@e.test`, Date.now()]); + +const app = express(); +app.use(express.json({ limit: "10mb" })); +app.use((req, _res, next) => { req.user = req.get("x-user") ? { id: req.get("x-user") } : null; next(); }); +app.use(opencredsRouter); +const server = app.listen(0); +await new Promise((r) => server.once("listening", r)); +const base = `http://127.0.0.1:${server.address().port}`; +test.after(() => new Promise((r) => server.close(r))); + +async function call(user, method, path, body) { + const res = await fetch(base + path, { + method, + headers: { ...(user ? { "x-user": user } : {}), ...(body ? { "content-type": "application/json" } : {}) }, + body: body ? JSON.stringify(body) : undefined + }); + return { status: res.status, body: await res.json() }; +} +const meta = (tag) => ({ opencreds: "0.1", namespace: "opencreds", createdAt: "2026-10-04T00:00:00Z", protectedUserKey: "pk-" + tag, protectedUserKeyIv: "iv" }); +const env = (id, c) => ({ id, type: 1, ciphertext: c, iv: "iv-" + c }); + +test("no session, no key: 401", async () => { + assert.equal((await call(null, "GET", "/api/opencreds/vault")).status, 401); +}); + +test("meta: create once, update on the current revision, refuse a stale one", async () => { + assert.equal((await call("u1", "GET", "/api/opencreds/vault")).status, 404); + assert.deepEqual((await call("u1", "PUT", "/api/opencreds/vault/meta", { meta: meta("a"), baseRevision: 0 })).body, { ok: true, revision: 1 }); + + const again = await call("u1", "PUT", "/api/opencreds/vault/meta", { meta: meta("b"), baseRevision: 0 }); + assert.equal(again.status, 409); + assert.equal(again.body.vault.meta.protectedUserKey, "pk-a"); + + assert.equal((await call("u1", "PUT", "/api/opencreds/vault/meta", { meta: meta("c"), baseRevision: 1 })).body.revision, 2); + const stale = await call("u1", "PUT", "/api/opencreds/vault/meta", { meta: meta("d"), baseRevision: 1 }); + assert.equal(stale.status, 409); + assert.equal(stale.body.vault.metaRevision, 2); +}); + +test("items: insert, update, conflict returns the current row, tombstone, cursor", async () => { + let r = await call("u1", "PUT", "/api/opencreds/items", { changes: [{ id: "i1", envelope: env("i1", "c1"), baseRevision: 0 }, { id: "i2", envelope: env("i2", "x"), baseRevision: 0 }] }); + assert.equal(r.body.applied.length, 2); + assert.equal(r.body.conflicts.length, 0); + + r = await call("u1", "PUT", "/api/opencreds/items", { changes: [{ id: "i1", envelope: env("i1", "c2"), baseRevision: 1 }] }); + assert.deepEqual(r.body.applied.map((a) => a.revision), [2]); + + r = await call("u1", "PUT", "/api/opencreds/items", { changes: [{ id: "i1", envelope: env("i1", "stale"), baseRevision: 1 }] }); + assert.equal(r.body.applied.length, 0); + assert.equal(r.body.conflicts[0].revision, 2); + assert.equal(r.body.conflicts[0].envelope.ciphertext, "c2"); + + r = await call("u1", "PUT", "/api/opencreds/items", { changes: [{ id: "i2", envelope: null, baseRevision: 1 }] }); + assert.equal(r.body.applied[0].revision, 2); + + const all = await call("u1", "GET", "/api/opencreds/items?since=0"); + const byId = Object.fromEntries(all.body.items.map((i) => [i.id, i])); + assert.equal(byId.i1.envelope.ciphertext, "c2"); + assert.equal(byId.i1.envelope.revision, 2); + assert.equal(byId.i2.envelope, null); + + const later = await call("u1", "GET", `/api/opencreds/items?since=${byId.i2.seq}`); + assert.deepEqual(later.body.items.map((i) => i.id), ["i2"]); + + const v = await call("u1", "GET", "/api/opencreds/vault"); + assert.equal(v.body.vault.itemCount, 1); +}); + +test("an update to a row that does not exist is a conflict at revision 0", async () => { + const r = await call("u1", "PUT", "/api/opencreds/items", { changes: [{ id: "ghost", envelope: env("ghost", "g"), baseRevision: 3 }] }); + assert.deepEqual(r.body.conflicts, [{ id: "ghost", envelope: null, revision: 0, seq: 0 }]); +}); + +test("one user's vault is invisible to another", async () => { + assert.equal((await call("u2", "GET", "/api/opencreds/vault")).status, 404); + assert.deepEqual((await call("u2", "GET", "/api/opencreds/items?since=0")).body.items, []); + const r = await call("u2", "PUT", "/api/opencreds/items", { changes: [{ id: "i1", envelope: env("i1", "u2"), baseRevision: 0 }] }); + assert.equal(r.status, 409, "items need the user's own vault first"); +}); + +test("malformed changes are refused before anything is written", async () => { + const mismatch = await call("u1", "PUT", "/api/opencreds/items", { changes: [{ id: "a", envelope: env("b", "x"), baseRevision: 0 }] }); + assert.equal(mismatch.status, 422); + const noBase = await call("u1", "PUT", "/api/opencreds/items", { changes: [{ id: "a", envelope: env("a", "x") }] }); + assert.equal(noBase.status, 422); + assert.equal((await call("u1", "PUT", "/api/opencreds/vault/meta", { meta: "nope", baseRevision: 0 })).status, 422); +}); + +test("folders follow the same revision rule", async () => { + assert.equal((await call("u1", "PUT", "/api/opencreds/vault/folders", { ciphertext: "f", iv: "i", baseRevision: 0 })).body.revision, 1); + assert.equal((await call("u1", "PUT", "/api/opencreds/vault/folders", { ciphertext: "g", iv: "i", baseRevision: 0 })).status, 409); +}); + +test("reset drops the vault and its items", async () => { + await call("u1", "DELETE", "/api/opencreds/vault"); + assert.equal((await call("u1", "GET", "/api/opencreds/vault")).status, 404); + assert.deepEqual((await call("u1", "GET", "/api/opencreds/items?since=0")).body.items, []); +}); diff --git a/bun.lock b/bun.lock index a45bdbf..c964981 100644 --- a/bun.lock +++ b/bun.lock @@ -144,7 +144,7 @@ }, "packages/cli": { "name": "@logicsrc/cli", - "version": "0.5.0", + "version": "0.6.0", "bin": { "logicsrc": "dist/index.js", }, diff --git a/packages/cli/package.json b/packages/cli/package.json index c87958e..a0c9050 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@logicsrc/cli", - "version": "0.5.0", + "version": "0.6.0", "description": "LogicSRC CLI: every LogicSRC standard and tool as one command.", "type": "module", "main": "./dist/index.js", diff --git a/packages/cli/src/creds.ts b/packages/cli/src/creds.ts index 156544f..c96b4c3 100644 --- a/packages/cli/src/creds.ts +++ b/packages/cli/src/creds.ts @@ -1,5 +1,6 @@ import type { Command } from "commander"; import { registerCredsCommands } from "@logicsrc/opencreds/commands"; +import { accountRemoteFor } from "./vault-sync.js"; /** * `logicsrc vault …` @@ -45,6 +46,12 @@ Personal vault: logicsrc vault export --format csv --category db --out db.csv --yes logicsrc vault import bitwarden.csv +Your personal vault syncs to your account (after logicsrc login), end to end +encrypted: every vault command pulls first and pushes after. A new machine +gets it with: logicsrc login, then logicsrc vault sync. + logicsrc vault sync --status what is synced, what is waiting + LOGICSRC_VAULT_SYNC=off turn it off (a --home vault never syncs) + Help for any command: logicsrc vault --help (or: logicsrc vault help ) `; @@ -59,5 +66,6 @@ export function registerOpenCredsCommands(program: Command): void { // under `teams`. Say so first, with commands they can paste. .addHelpText("after", VAULT_GUIDE); - registerCredsCommands(vault); + // Logged in: the vault syncs to the account around every command. + registerCredsCommands(vault, { remote: accountRemoteFor }); } diff --git a/packages/cli/src/vault-sync.ts b/packages/cli/src/vault-sync.ts new file mode 100644 index 0000000..b398c01 --- /dev/null +++ b/packages/cli/src/vault-sync.ts @@ -0,0 +1,94 @@ +import { join, resolve } from "node:path"; +import { homedir } from "node:os"; +import type { Envelope, RemoteItem, RemoteVault, SyncRemote, VaultMeta, VaultStore } from "@logicsrc/opencreds"; +import { readIdentity, resolveApiUrl } from "@logicsrc/plugin-credential-sharing"; + +/** + * `logicsrc vault` syncs to the logged-in account (apps/pwa /api/opencreds), + * so the personal vault survives the machine and follows you to the next one. + * + * Only the default vault syncs. A vault opened with --home or OPENCREDS_HOME is + * a second vault (a test, a scratch copy), and the account holds one; letting + * it sync would replace the real one. LOGICSRC_VAULT_SYNC=on opts such a vault + * in, =off turns sync off everywhere. + */ +export function defaultVaultDir(): string { + const config = process.env.XDG_CONFIG_HOME || join(homedir(), ".config"); + return join(config, "logicsrc", "opencreds"); +} + +export function accountRemoteFor(store: VaultStore): SyncRemote | undefined { + const setting = (process.env.LOGICSRC_VAULT_SYNC || "").toLowerCase(); + if (setting === "off" || setting === "0" || setting === "false") return undefined; + if (setting !== "on" && resolve(store.baseDir) !== resolve(defaultVaultDir())) return undefined; + const identity = readIdentity(); + if (!identity?.apiToken) return undefined; + return createAccountRemote(resolveApiUrl(identity), identity.apiToken, identity.email); +} + +class HttpError extends Error { + constructor(message: string, readonly status: number, readonly body: Record) { + super(message); + } +} + +export function createAccountRemote(apiUrl: string, token: string, email?: string): SyncRemote { + const base = apiUrl.replace(/\/+$/, ""); + const host = (() => { + try { + return new URL(base).host; + } catch { + return base; + } + })(); + + async function call(method: string, path: string, body?: unknown, ok: number[] = []): Promise<{ status: number; body: Record }> { + const res = await fetch(`${base}${path}`, { + method, + headers: { authorization: `Bearer ${token}`, accept: "application/json", ...(body ? { "content-type": "application/json" } : {}) }, + body: body ? JSON.stringify(body) : undefined, + signal: AbortSignal.timeout(15_000), + }); + const json = (await res.json().catch(() => ({}))) as Record; + if (!res.ok && !ok.includes(res.status)) { + throw new HttpError(String(json.error || `${host} answered HTTP ${res.status}`), res.status, json); + } + return { status: res.status, body: json }; + } + + return { + label: email ? `${host} (${email})` : host, + + async getVault() { + const { status, body } = await call("GET", "/api/opencreds/vault", undefined, [404]); + return status === 404 ? null : (body.vault as RemoteVault); + }, + + async putMeta(meta: VaultMeta, baseRevision: number) { + const { status, body } = await call("PUT", "/api/opencreds/vault/meta", { meta, baseRevision }, [409]); + return status === 409 ? { ok: false, vault: (body.vault as RemoteVault) ?? null } : { ok: true, revision: Number(body.revision) }; + }, + + async putFolders(blob: { ciphertext: string; iv: string }, baseRevision: number) { + const { status, body } = await call("PUT", "/api/opencreds/vault/folders", { ...blob, baseRevision }, [409]); + return status === 409 ? { ok: false, vault: (body.vault as RemoteVault) ?? null } : { ok: true, revision: Number(body.revision) }; + }, + + async listItems(since: number) { + const { body } = await call("GET", `/api/opencreds/items?since=${since}`); + return { items: body.items as RemoteItem[], cursor: Number(body.cursor) }; + }, + + async putItems(changes: Array<{ id: string; envelope: Envelope | null; baseRevision: number }>) { + const { body } = await call("PUT", "/api/opencreds/items", { changes }); + return { + applied: body.applied as Array<{ id: string; revision: number; seq: number }>, + conflicts: body.conflicts as RemoteItem[], + }; + }, + + async reset() { + await call("DELETE", "/api/opencreds/vault"); + }, + }; +} diff --git a/packages/opencreds/src/commands.ts b/packages/opencreds/src/commands.ts index a20a0fa..c8b495e 100644 --- a/packages/opencreds/src/commands.ts +++ b/packages/opencreds/src/commands.ts @@ -8,7 +8,7 @@ * contracts. */ -import { mkdirSync, readFileSync, writeFileSync, chmodSync } from "node:fs"; +import { mkdirSync, readFileSync, writeFileSync, chmodSync, existsSync, renameSync } from "node:fs"; import { dirname, join } from "node:path"; import type { Command } from "commander"; @@ -56,6 +56,8 @@ import { } from "./types.js"; import { createVault, resetRecoveryKey, rewrapUserKey, unlockVault, unlockWithRecoveryKey } from "./vault-key.js"; import { formatDiagnostics, hasErrors, validateDocument } from "./validate.js"; +import { DivergedVaultError, describeSync, pullVault, pushVault, readSyncState, syncVault, type SyncRemote } from "./sync.js"; +import type { VaultStore } from "./store.js"; /** Exit codes are part of the contract; see docs/opencreds/cli.md. */ export const EXIT = { @@ -117,13 +119,20 @@ function requireMeta(store: ReturnType) { * A live session is used when there is one; otherwise the master password is * asked for. Nothing else unlocks a vault. */ +/** + * The user key this process unlocked, if any. Sync reads it after a command + * runs, to encrypt the folder list and to split a conflicting edit; it is + * never written anywhere by this module. + */ +let unlockedKey: Uint8Array | undefined; + async function unlock(store: ReturnType): Promise { const meta = requireMeta(store); const session = readSession(store.baseDir); - if (session) return session; + if (session) return (unlockedKey = session); const password = await promptSecret("Master password: "); try { - return await unlockVault(meta, password); + return (unlockedKey = await unlockVault(meta, password)); } catch (err) { store.appendAudit( auditEvent({ action: "vault.unlock_failed", namespace: meta.namespace, profile: meta.profile, outcome: "failed" }), @@ -286,13 +295,27 @@ function printItemLine(item: Item): string { } /** Register every OpenCreds command onto `parent`. */ -export function registerCredsCommands(parent: Command): void { +export interface RegisterOptions { + /** + * Where this vault syncs to, if anywhere. Called per command with the store + * the command will use; return undefined to stay local. The standalone + * `opencreds` binary passes nothing, so it never syncs. + */ + remote?: (store: VaultStore) => SyncRemote | undefined; +} + +// Commands that never touch a vault on disk, plus `sync`, which syncs itself. +const NO_AUTOSYNC = new Set(["validate", "conformance", "manifest", "sync"]); + +export function registerCredsCommands(parent: Command, options: RegisterOptions = {}): void { const names: string[] = []; for (let c: Command | null = parent; c; c = c.parent) names.unshift(c.name()); cli = names.join(" "); const examples = (lines: string) => `\nExamples:\n${lines.replace(/^\n/, "").replace(/\$CLI/g, cli)}\n`; parent.option("--home ", "vault directory (default $OPENCREDS_HOME)"); + if (options.remote) registerSync(parent, options.remote, examples); + // ---------------------------------------------------------------- vault --- parent @@ -312,6 +335,13 @@ export function registerCredsCommands(parent: Command): void { await run(async () => { const store = storeFor(this); if (store.exists() && !opts.force) { + if (readSyncState(store).vaultCreatedAt === store.readMeta()?.createdAt) { + fail( + `This machine already has your vault, synced from your account. Unlock it with your master password: \`${cli} unlock\`. ` + + "(--force would start a separate, empty vault.)", + EXIT.REFUSED, + ); + } fail(`A vault already exists at ${store.baseDir}; pass --force to replace it`, EXIT.REFUSED); } // Scripted provisioning reads one line and skips the confirmation; a @@ -1044,3 +1074,122 @@ export function registerCredsCommands(parent: Command): void { }); }); } + +// ----------------------------------------------------------------- sync --- + +/** + * Autosync plus an explicit `sync` command, registered only when the host CLI + * provides a remote. + * + * Every vault command pulls first, so `list` on a second machine shows what + * the first one added, and pushes after, so a write is on the account before + * the prompt comes back. Sync never fails a command: offline, the vault works + * as before and the change goes up next time. Only `sync` itself reports + * failure as an exit code. + */ +function registerSync( + parent: Command, + remoteFor: (store: VaultStore) => SyncRemote | undefined, + examples: (lines: string) => string, +): void { + const say = (text: string) => { + if (text) process.stderr.write(`${text}\n`); + }; + const warn = (err: unknown) => { + const message = (err as Error).message; + say( + err instanceof DivergedVaultError + ? `vault sync: ${message}` + : `vault sync: skipped (${message}); changes stay on this machine and go up next time.`, + ); + }; + const autosync = (command: Command) => !NO_AUTOSYNC.has(command.name()) && process.env.OPENCREDS_SYNC !== "off"; + + parent.hook("preAction", async (_self, command) => { + if (!autosync(command)) return; + const store = storeFor(command); + const remote = remoteFor(store); + if (!remote) return; + try { + say(describeSync(await pullVault(store, remote), remote.label)); + } catch (err) { + warn(err); + } + }); + + parent.hook("postAction", async (_self, command) => { + if (!autosync(command) || process.exitCode) return; + const store = storeFor(command); + const remote = remoteFor(store); + if (!remote || !store.exists()) return; + try { + say(describeSync(await pushVault(store, remote, unlockedKey), remote.label)); + } catch (err) { + warn(err); + } + }); + + parent + .command("sync") + .description("sync this vault with your account now (it also happens around every command)") + .addHelpText("after", examples(` + $CLI sync pull then push, now + $CLI sync --status what is synced, what is waiting + $CLI sync --use-remote two different vaults: keep the account's (this machine's is backed up first) + $CLI sync --use-local two different vaults: replace the account's with this machine's + OPENCREDS_SYNC=off $CLI list one command without syncing`)) + .option("--status", "show sync state and change nothing") + .option("--use-remote", "replace this machine's vault with the account's (backed up first)") + .option("--use-local", "replace the account's vault with this machine's") + .action(async function (this: Command, opts: { status?: boolean; useRemote?: boolean; useLocal?: boolean }) { + await run(async () => { + let store = storeFor(this); + const remote = remoteFor(store); + if (!remote) fail("This vault is not linked to an account. Log in first (logicsrc login).", EXIT.USAGE); + if (opts.useRemote && opts.useLocal) fail("Pick one of --use-remote and --use-local", EXIT.USAGE); + + if (opts.status) { + const state = readSyncState(store); + const envelopes = store.listEnvelopes(); + const here = new Set(envelopes.map((e) => e.id)); + const waiting = + envelopes.filter((e) => e.revision !== state.items[e.id]).length + + Object.keys(state.items).filter((id) => !here.has(id)).length; + const rv = await remote.getVault(); + process.stdout.write( + `Account ${remote.label}${rv ? "" : " (no vault there yet)"}\n` + + `Local ${store.exists() ? `${envelopes.length} item(s) at ${store.baseDir}` : `no vault at ${store.baseDir}`}\n` + + `Synced ${Object.keys(state.items).length} item(s)${state.lastSyncAt ? `, last ${state.lastSyncAt}` : ", never"}\n` + + `Waiting ${waiting} change(s) to push\n`, + ); + return; + } + + if (opts.useRemote) { + if (!(await remote.getVault())) fail(`There is no vault on ${remote.label} to use.`, EXIT.USAGE); + if (store.exists()) { + let n = 1; + const backup = () => `${store.baseDir}.bak-${String(n).padStart(3, "0")}`; + while (existsSync(backup())) n++; + clearSession(store.baseDir); + renameSync(store.baseDir, backup()); + process.stderr.write(`Moved this machine's vault to ${backup()}\n`); + } + store = createVaultStore(store.baseDir); + } else if (opts.useLocal) { + if (!store.exists()) fail(`No vault at ${store.baseDir} to upload.`, EXIT.USAGE); + const ok = await confirm(`Replace the vault on ${remote.label} with this machine's? Its items there are deleted.`); + if (!ok) fail("Nothing changed.", EXIT.REFUSED); + await remote.reset(); + store.writeSyncState({}); + } + + const key = store.exists() ? (readSession(store.baseDir) ?? undefined) : undefined; + const report = await syncVault(store, remote, key); + process.stdout.write(`${describeSync(report, remote.label) || `vault sync (${remote.label}): up to date`}\n`); + if (!key && store.exists()) { + process.stderr.write(`Folders sync on the next unlocked command (or \`${cli} unlock --persist\` first).\n`); + } + }); + }); +} diff --git a/packages/opencreds/src/index.ts b/packages/opencreds/src/index.ts index 86d10f3..1981e6a 100644 --- a/packages/opencreds/src/index.ts +++ b/packages/opencreds/src/index.ts @@ -159,6 +159,22 @@ export { export { createVaultStore, opencredsHome, type VaultStore } from "./store.js"; +export { + DivergedVaultError, + SyncError, + describeSync, + pullVault, + pushVault, + readSyncState, + syncVault, + type Put, + type RemoteItem, + type RemoteVault, + type SyncRemote, + type SyncReport, + type SyncState, +} from "./sync.js"; + export { auditEvent, fingerprint, type AuditInput } from "./audit.js"; export { diff --git a/packages/opencreds/src/store.ts b/packages/opencreds/src/store.ts index 60ebf9a..09fc689 100644 --- a/packages/opencreds/src/store.ts +++ b/packages/opencreds/src/store.ts @@ -7,6 +7,7 @@ * meta.json vault metadata — key material, all of it wrapped * items/.json one envelope per item * audit.jsonl append-only audit events, values never present + * sync.json what was last exchanged with an account (see sync.ts) * * One file per item rather than one file for the vault, for the same reason * storage-backed implementations use one row per item: two writers editing two @@ -48,11 +49,15 @@ export interface VaultStore { listEnvelopes(): Envelope[]; readEnvelope(id: string): Envelope | undefined; writeEnvelope(envelope: Envelope): void; + /** Write an envelope exactly as given, revision included. For sync only. */ + putEnvelope(envelope: Envelope): void; deleteEnvelope(id: string): void; readFolders(): Folder[]; writeFolders(folders: Folder[]): void; appendAudit(event: AuditEvent): void; readAudit(): AuditEvent[]; + readSyncState(): T | undefined; + writeSyncState(state: T): void; } export function createVaultStore(baseDir = opencredsHome()): VaultStore { @@ -60,6 +65,7 @@ export function createVaultStore(baseDir = opencredsHome()): VaultStore { const metaPath = join(baseDir, "meta.json"); const foldersPath = join(baseDir, "folders.json"); const auditPath = join(baseDir, "audit.jsonl"); + const syncPath = join(baseDir, "sync.json"); function ensureDirs(): void { mkdirSync(itemsDir, { recursive: true, mode: 0o700 }); @@ -120,6 +126,11 @@ export function createVaultStore(baseDir = opencredsHome()): VaultStore { writePrivate(join(itemsDir, `${envelope.id}.json`), `${JSON.stringify(next, null, 2)}\n`); }, + putEnvelope(envelope: Envelope): void { + ensureDirs(); + writePrivate(join(itemsDir, `${envelope.id}.json`), `${JSON.stringify(envelope, null, 2)}\n`); + }, + deleteEnvelope(id: string): void { rmSync(join(itemsDir, `${id}.json`), { force: true }); }, @@ -141,6 +152,15 @@ export function createVaultStore(baseDir = opencredsHome()): VaultStore { writeFileSync(auditPath, line, { encoding: "utf8", flag: "a", mode: 0o600 }); }, + readSyncState(): T | undefined { + return readJson(syncPath); + }, + + writeSyncState(state: T): void { + ensureDirs(); + writePrivate(syncPath, `${JSON.stringify(state, null, 2)}\n`); + }, + readAudit(): AuditEvent[] { if (!existsSync(auditPath)) return []; return readFileSync(auditPath, "utf8") diff --git a/packages/opencreds/src/sync.test.ts b/packages/opencreds/src/sync.test.ts new file mode 100644 index 0000000..ca5e1d7 --- /dev/null +++ b/packages/opencreds/src/sync.test.ts @@ -0,0 +1,277 @@ +/** + * Sync between two machines through one account. + * + * The remote here is an in-memory copy of the server's rules (apps/pwa + * routes/opencreds.mjs): optimistic revisions per row, tombstones for purges, + * a 409 carrying the current row. Each "machine" is its own vault directory. + */ + +import { mkdtempSync, readdirSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; + +import { createItem, decryptItem, encryptItem, updateItem } from "./items.js"; +import { createVaultStore, type VaultStore } from "./store.js"; +import { DivergedVaultError, pullVault, pushVault, syncVault, type RemoteItem, type RemoteVault, type SyncRemote } from "./sync.js"; +import type { Envelope, Item, VaultMeta } from "./types.js"; +import { createVault } from "./vault-key.js"; + +const ITERATIONS = { kdf: "pbkdf2-sha256" as const, iterations: 600_000 }; + +function fakeAccount() { + let vault: { meta: VaultMeta; metaRevision: number; folders: { ciphertext: string; iv: string; revision: number } | null } | null = null; + const rows = new Map(); + let seq = 0; + const snapshot = (): RemoteVault | null => (vault ? JSON.parse(JSON.stringify(vault)) : null); + const item = (id: string): RemoteItem => { + const r = rows.get(id); + if (!r) return { id, envelope: null, revision: 0, seq: 0 }; + return { id, envelope: r.envelope ? { ...JSON.parse(r.envelope), revision: r.revision } : null, revision: r.revision, seq: r.seq }; + }; + const remote: SyncRemote = { + label: "test-account", + async getVault() { + return snapshot(); + }, + async putMeta(meta, baseRevision) { + const current = vault?.metaRevision ?? 0; + if (current !== baseRevision) return { ok: false, vault: snapshot() }; + vault = { meta: JSON.parse(JSON.stringify(meta)), metaRevision: current + 1, folders: vault?.folders ?? null }; + return { ok: true, revision: current + 1 }; + }, + async putFolders(blob, baseRevision) { + if (!vault) return { ok: false, vault: null }; + const current = vault.folders?.revision ?? 0; + if (current !== baseRevision) return { ok: false, vault: snapshot() }; + vault.folders = { ...blob, revision: current + 1 }; + return { ok: true, revision: current + 1 }; + }, + async listItems(since) { + const items = [...rows.keys()].map(item).filter((i) => i.seq >= since).sort((a, b) => a.seq - b.seq); + return { items, cursor: items.reduce((m, i) => Math.max(m, i.seq), since) }; + }, + async putItems(changes) { + const applied: Array<{ id: string; revision: number; seq: number }> = []; + const conflicts: RemoteItem[] = []; + for (const c of changes) { + const current = rows.get(c.id)?.revision ?? 0; + if (current !== c.baseRevision || (c.baseRevision > 0 && !rows.has(c.id))) { + conflicts.push(item(c.id)); + continue; + } + const { revision: _r, ...rest } = c.envelope ?? ({} as Envelope); + rows.set(c.id, { envelope: c.envelope ? JSON.stringify(rest) : null, revision: current + 1, seq: ++seq }); + applied.push({ id: c.id, revision: current + 1, seq }); + } + return { applied, conflicts }; + }, + async reset() { + vault = null; + rows.clear(); + }, + }; + return { remote, rows, vault: () => vault }; +} + +const dirs: string[] = []; +function machine(): VaultStore { + const dir = mkdtempSync(join(tmpdir(), "opencreds-sync-")); + dirs.push(dir); + return createVaultStore(join(dir, "vault")); +} +afterEach(() => { + for (const d of dirs.splice(0)) rmSync(d, { recursive: true, force: true }); +}); + +async function newVault(store: VaultStore) { + const { meta, userKey } = await createVault("pw", { params: ITERATIONS }); + store.writeMeta(meta); + return { meta, userKey }; +} + +async function add(store: VaultStore, key: Uint8Array, name: string, password = "s3cret"): Promise { + const item = createItem("login", { name, login: { username: "me", password, uris: [] } } as Partial); + store.writeEnvelope(await encryptItem(key, item)); + return item; +} + +async function names(store: VaultStore, key: Uint8Array): Promise { + const out: string[] = []; + for (const env of store.listEnvelopes()) out.push((await decryptItem(key, env)).name); + return out.sort(); +} + +describe("vault sync", () => { + it("uploads a vault, and a second machine downloads it with no key", async () => { + const acct = fakeAccount(); + const a = machine(); + const { userKey } = await newVault(a); + await add(a, userKey, "GitHub"); + await add(a, userKey, "Bank"); + + const up = await syncVault(a, acct.remote, userKey); + expect(up.created).toBe(true); + expect(up.pushed).toBe(2); + + const b = machine(); + const down = await pullVault(b, acct.remote); + expect(down.downloaded).toBe(true); + expect(down.pulled).toBe(2); + expect(b.readMeta()).toEqual(a.readMeta()); + expect(await names(b, userKey)).toEqual(["Bank", "GitHub"]); + }); + + it("the account holds only ciphertext", async () => { + const acct = fakeAccount(); + const a = machine(); + const { userKey } = await newVault(a); + await add(a, userKey, "VerySecretName", "hunter2-password"); + a.writeFolders([{ id: "f1", name: "PrivateFolderName" }]); + await syncVault(a, acct.remote, userKey); + + const everything = JSON.stringify({ rows: [...acct.rows.values()], vault: acct.vault() }); + expect(everything).not.toMatch(/VerySecretName|hunter2-password|PrivateFolderName/); + expect(acct.vault()?.folders).not.toBeNull(); + }); + + it("an edit on one machine reaches the other; a quiet second sync moves nothing", async () => { + const acct = fakeAccount(); + const a = machine(); + const { userKey } = await newVault(a); + const item = await add(a, userKey, "GitHub"); + await syncVault(a, acct.remote, userKey); + const b = machine(); + await syncVault(b, acct.remote, userKey); + + a.writeEnvelope(await encryptItem(userKey, updateItem(item, { name: "GitHub (work)" }))); + await syncVault(a, acct.remote, userKey); + const r = await syncVault(b, acct.remote, userKey); + expect(r.pulled).toBe(1); + expect(await names(b, userKey)).toEqual(["GitHub (work)"]); + + const again = await syncVault(b, acct.remote, userKey); + expect([again.pulled, again.pushed, again.conflicts]).toEqual([0, 0, 0]); + }); + + it("the same item edited on both machines keeps both edits", async () => { + const acct = fakeAccount(); + const a = machine(); + const { userKey } = await newVault(a); + const item = await add(a, userKey, "Email"); + await syncVault(a, acct.remote, userKey); + const b = machine(); + await syncVault(b, acct.remote, userKey); + + a.writeEnvelope(await encryptItem(userKey, updateItem(item, { name: "Email A" }))); + b.writeEnvelope(await encryptItem(userKey, updateItem(item, { name: "Email B" }))); + await syncVault(a, acct.remote, userKey); + const r = await syncVault(b, acct.remote, userKey); + expect(r.conflicts).toBe(1); + expect(await names(b, userKey)).toEqual(["Email A", "Email B (conflict copy)"]); + + await syncVault(a, acct.remote, userKey); + expect(await names(a, userKey)).toEqual(["Email A", "Email B (conflict copy)"]); + }); + + it("without the key, a conflict waits instead of guessing", async () => { + const acct = fakeAccount(); + const a = machine(); + const { userKey } = await newVault(a); + const item = await add(a, userKey, "Email"); + await syncVault(a, acct.remote, userKey); + const b = machine(); + await syncVault(b, acct.remote, userKey); + + a.writeEnvelope(await encryptItem(userKey, updateItem(item, { name: "Email A" }))); + b.writeEnvelope(await encryptItem(userKey, updateItem(item, { name: "Email B" }))); + await syncVault(a, acct.remote, userKey); + const r = await pushVault(b, acct.remote); + expect(r.notes.join(" ")).toMatch(/next time the vault is unlocked/); + expect(await names(b, userKey)).toEqual(["Email B"]); + const later = await syncVault(b, acct.remote, userKey); + expect(later.conflicts).toBe(1); + expect(await names(b, userKey)).toEqual(["Email A", "Email B (conflict copy)"]); + }); + + it("a purge reaches the other machine; an edit beats a purge", async () => { + const acct = fakeAccount(); + const a = machine(); + const { userKey } = await newVault(a); + const gone = await add(a, userKey, "Old"); + const kept = await add(a, userKey, "Kept"); + await syncVault(a, acct.remote, userKey); + const b = machine(); + await syncVault(b, acct.remote, userKey); + + a.deleteEnvelope(gone.id); + a.deleteEnvelope(kept.id); + b.writeEnvelope(await encryptItem(userKey, updateItem(kept, { name: "Kept, edited" }))); + await syncVault(a, acct.remote, userKey); + await syncVault(b, acct.remote, userKey); + await syncVault(a, acct.remote, userKey); + + expect(await names(a, userKey)).toEqual(["Kept, edited"]); + expect(await names(b, userKey)).toEqual(["Kept, edited"]); + }); + + it("a lost sync.json does not turn every item into a conflict copy", async () => { + const acct = fakeAccount(); + const a = machine(); + const { userKey } = await newVault(a); + await add(a, userKey, "One"); + await add(a, userKey, "Two"); + await syncVault(a, acct.remote, userKey); + rmSync(join(a.baseDir, "sync.json")); + + const r = await syncVault(a, acct.remote, userKey); + expect(r.conflicts).toBe(0); + expect(await names(a, userKey)).toEqual(["One", "Two"]); + expect(acct.rows.size).toBe(2); + }); + + it("two different vaults are never merged", async () => { + const acct = fakeAccount(); + const a = machine(); + const ka = await newVault(a); + await add(a, ka.userKey, "A's"); + await syncVault(a, acct.remote, ka.userKey); + + const b = machine(); + const kb = await newVault(b); + await new Promise((r) => setTimeout(r, 5)); + b.writeMeta({ ...b.readMeta()!, createdAt: new Date(Date.now() + 1000).toISOString() }); + await add(b, kb.userKey, "B's"); + await expect(syncVault(b, acct.remote, kb.userKey)).rejects.toBeInstanceOf(DivergedVaultError); + expect(acct.rows.size).toBe(1); + }); + + it("folders merge across machines, encrypted", async () => { + const acct = fakeAccount(); + const a = machine(); + const { userKey } = await newVault(a); + a.writeFolders([{ id: "fa", name: "Work" }]); + await syncVault(a, acct.remote, userKey); + const b = machine(); + await syncVault(b, acct.remote, userKey); + expect(b.readFolders()).toEqual([{ id: "fa", name: "Work" }]); + + b.writeFolders([...b.readFolders(), { id: "fb", name: "Home" }]); + await syncVault(b, acct.remote, userKey); + await syncVault(a, acct.remote, userKey); + expect(a.readFolders().map((f) => f.name).sort()).toEqual(["Home", "Work"]); + }); + + it("a remote id that is not a plain id never becomes a file", async () => { + const acct = fakeAccount(); + const a = machine(); + const { userKey } = await newVault(a); + await syncVault(a, acct.remote, userKey); + acct.rows.set("../../escape", { envelope: JSON.stringify({ id: "../../escape", type: 1, ciphertext: "x", iv: "y" }), revision: 1, seq: 99 }); + + const r = await pullVault(a, acct.remote); + expect(r.notes.join(" ")).toMatch(/unusable id/); + expect(readdirSync(join(a.baseDir, "items"))).toEqual([]); + expect(() => readFileSync(join(a.baseDir, "..", "..", "escape.json"))).toThrow(); + }); +}); diff --git a/packages/opencreds/src/sync.ts b/packages/opencreds/src/sync.ts new file mode 100644 index 0000000..e63519d --- /dev/null +++ b/packages/opencreds/src/sync.ts @@ -0,0 +1,382 @@ +/** + * Two-way sync between a file vault and a remote copy of the same vault. + * + * Everything exchanged is already ciphertext: the meta carries key material + * wrapped under the master password, items are envelopes under the user key, + * and the folder list is encrypted here before it leaves. The remote only + * stores and orders blobs; it never needs a key, and sync never needs one + * either except to encrypt folders and to split a conflicting edit into two + * items. Without the key those two steps wait for the next unlocked run. + * + * Model: optimistic concurrency per row. `sync.json` records, per item, the + * remote revision this machine last agreed with. An item whose local revision + * differs from it changed here; a pulled item whose remote revision differs + * from it changed there. Both at once is a conflict, resolved without losing + * either side: the remote edit keeps the id, the local edit becomes a copy. + * + * A vault is identified by `meta.createdAt`, which a password change keeps and + * `init` replaces. Two different vaults never merge: that is refused, with the + * commands that pick one. + */ + +import { createHash } from "node:crypto"; + +import { decryptItem, encryptItem } from "./items.js"; +import { aesGcmDecrypt, aesGcmEncrypt, fromBase64, toBase64, utf8Decode, utf8Encode, uuid } from "./primitives.js"; +import type { VaultStore } from "./store.js"; +import type { Envelope, Folder, VaultMeta } from "./types.js"; + +export interface RemoteItem { + id: string; + /** null is a tombstone: the item was purged. */ + envelope: Envelope | null; + revision: number; + seq: number; +} + +export interface RemoteVault { + meta: VaultMeta; + metaRevision: number; + folders: { ciphertext: string; iv: string; revision: number } | null; +} + +export type Put = { ok: true; revision: number } | { ok: false; vault: T | null }; + +/** Where a vault syncs to. The logicsrc CLI provides one backed by the account. */ +export interface SyncRemote { + /** Shown to people: "app.logicsrc.com (you@example.com)". */ + label: string; + getVault(): Promise; + putMeta(meta: VaultMeta, baseRevision: number): Promise>; + putFolders(blob: { ciphertext: string; iv: string }, baseRevision: number): Promise>; + listItems(since: number): Promise<{ items: RemoteItem[]; cursor: number }>; + putItems( + changes: Array<{ id: string; envelope: Envelope | null; baseRevision: number }>, + ): Promise<{ applied: Array<{ id: string; revision: number; seq: number }>; conflicts: RemoteItem[] }>; + /** Drop the remote vault entirely (`sync --use-local`). */ + reset(): Promise; +} + +export interface SyncState { + /** createdAt of the vault this state belongs to. */ + vaultCreatedAt?: string; + cursor: number; + metaRevision: number; + metaHash?: string; + foldersRevision: number; + foldersHash?: string; + /** item id -> the remote revision this machine last agreed with */ + items: Record; + lastSyncAt?: string; +} + +export interface SyncReport { + pulled: number; + pushed: number; + deleted: number; + conflicts: number; + created: boolean; + downloaded: boolean; + notes: string[]; +} + +export class SyncError extends Error {} + +/** Two different vaults: the account's and this machine's. Never merged. */ +export class DivergedVaultError extends SyncError {} + +// An id becomes a filename. A remote that sends "../meta" must not get to +// write outside items/, so anything but a plain id is refused outright. +const SAFE_ID = /^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$/; +const PUSH_BATCH = 200; + +const emptyState = (): SyncState => ({ cursor: 0, metaRevision: 0, foldersRevision: 0, items: {} }); + +function hash(value: unknown): string { + return createHash("sha256").update(JSON.stringify(value)).digest("hex"); +} + +function sameCiphertext(a: Envelope, b: Envelope | null): boolean { + return b !== null && a.ciphertext === b.ciphertext && a.iv === b.iv; +} + +function foldersAad(namespace: string): Uint8Array { + return utf8Encode(`${namespace}:vault:folders:1`); +} + +export function readSyncState(store: VaultStore): SyncState { + return { ...emptyState(), ...(store.readSyncState() ?? {}) }; +} + +function newReport(): SyncReport { + return { pulled: 0, pushed: 0, deleted: 0, conflicts: 0, created: false, downloaded: false, notes: [] }; +} + +function assertSameVault(local: VaultMeta, remote: VaultMeta, label: string): void { + if (local.createdAt !== remote.createdAt) { + throw new DivergedVaultError( + `This machine's vault and the one on ${label} are different vaults (created ${local.createdAt ?? "?"} and ${remote.createdAt ?? "?"}). ` + + "Nothing was synced. Keep one: `sync --use-remote` replaces this machine's (backed up first), " + + "`sync --use-local` replaces the account's.", + ); + } +} + +/** + * Bring remote changes down. Needs no key. On a machine with no vault yet and + * an account that has one, this is what downloads it. + */ +export async function pullVault(store: VaultStore, remote: SyncRemote, report = newReport()): Promise { + const rv = await remote.getVault(); + if (!rv) return report; + let state = readSyncState(store); + const local = store.readMeta(); + + if (!local) { + store.writeMeta(rv.meta); + state = { ...emptyState(), vaultCreatedAt: rv.meta.createdAt, metaRevision: rv.metaRevision, metaHash: hash(rv.meta) }; + report.downloaded = true; + } else { + assertSameVault(local, rv.meta, remote.label); + if (state.vaultCreatedAt !== local.createdAt) state = { ...emptyState(), vaultCreatedAt: local.createdAt }; + const localChanged = hash(local) !== state.metaHash; + if (rv.metaRevision !== state.metaRevision) { + // A password change elsewhere. The user key is the same, so items are + // unaffected; if this machine changed it too, the account's wins. + if (localChanged && state.metaHash) { + report.notes.push(`The master password was changed both here and on ${remote.label}; keeping the account's.`); + } + if (!localChanged || state.metaHash) { + store.writeMeta(rv.meta); + state.metaHash = hash(rv.meta); + state.metaRevision = rv.metaRevision; + } + } + } + + const { items, cursor } = await remote.listItems(state.cursor); + for (const ri of items) { + if (!SAFE_ID.test(ri.id) || (ri.envelope && ri.envelope.id !== ri.id)) { + report.notes.push(`Ignored a remote item with an unusable id (${JSON.stringify(ri.id).slice(0, 40)}).`); + continue; + } + const synced = state.items[ri.id]; + if (synced === ri.revision) continue; // already have it + const localEnv = store.readEnvelope(ri.id); + const changedHere = localEnv ? localEnv.revision !== synced : synced !== undefined; + if (changedHere) { + // Byte-identical is the same item (a lost sync.json, a copied vault): adopt it. + if (localEnv && sameCiphertext(localEnv, ri.envelope)) { + store.putEnvelope({ ...localEnv, revision: ri.revision }); + state.items[ri.id] = ri.revision; + } + continue; // both sides moved; push resolves it + } + if (ri.envelope) { + store.putEnvelope({ ...ri.envelope, revision: ri.revision }); + state.items[ri.id] = ri.revision; + report.pulled++; + } else { + if (localEnv) report.deleted++; + store.deleteEnvelope(ri.id); + delete state.items[ri.id]; + } + } + state.cursor = Math.max(state.cursor, cursor); + state.lastSyncAt = new Date().toISOString(); + store.writeSyncState(state); + return report; +} + +/** + * Send local changes up, resolving conflicts. `userKey` is optional: without + * it, folders wait and a conflicting edit stays pending until a run that has it. + */ +export async function pushVault( + store: VaultStore, + remote: SyncRemote, + userKey?: Uint8Array, + report = newReport(), +): Promise { + const local = store.readMeta(); + if (!local) return report; + let state = readSyncState(store); + if (state.vaultCreatedAt !== local.createdAt) { + // First sync of this vault from this machine. If the account already has + // a different vault, stop before uploading anything. + const rv = await remote.getVault(); + if (rv) assertSameVault(local, rv.meta, remote.label); + state = { ...emptyState(), vaultCreatedAt: local.createdAt }; + } + + // ---- meta ---- + if (hash(local) !== state.metaHash) { + const put = await remote.putMeta(local, state.metaRevision); + if (put.ok) { + state.metaRevision = put.revision; + if (put.revision === 1) report.created = true; + } else if (put.vault) { + assertSameVault(local, put.vault.meta, remote.label); + store.writeMeta(put.vault.meta); + state.metaRevision = put.vault.metaRevision; + report.notes.push(`The account's vault settings were newer; kept them.`); + } + state.metaHash = hash(store.readMeta()); + } + + // ---- items ---- + const pending = (): Array<{ id: string; envelope: Envelope | null; baseRevision: number }> => { + const out: Array<{ id: string; envelope: Envelope | null; baseRevision: number }> = []; + const present = new Set(); + for (const env of store.listEnvelopes()) { + present.add(env.id); + if (!SAFE_ID.test(env.id)) continue; + if (env.revision !== state.items[env.id]) out.push({ id: env.id, envelope: env, baseRevision: state.items[env.id] ?? 0 }); + } + for (const id of Object.keys(state.items)) { + if (!present.has(id)) out.push({ id, envelope: null, baseRevision: state.items[id] }); + } + return out; + }; + + // Two rounds: the second sends what conflict resolution produced (a copy, + // a resurrected edit). Anything still conflicting after that waits. + for (let round = 0; round < 2; round++) { + const changes = pending(); + if (changes.length === 0) break; + for (let i = 0; i < changes.length; i += PUSH_BATCH) { + const batch = changes.slice(i, i + PUSH_BATCH); + const { applied, conflicts } = await remote.putItems(batch); + const byId = new Map(batch.map((c) => [c.id, c])); + for (const a of applied) { + const sent = byId.get(a.id); + if (sent?.envelope) { + store.putEnvelope({ ...sent.envelope, revision: a.revision }); + state.items[a.id] = a.revision; + report.pushed++; + } else { + delete state.items[a.id]; + report.deleted++; + } + } + for (const theirs of conflicts) { + await resolveConflict(store, state, theirs, byId.get(theirs.id)!, userKey, local.namespace, report); + } + } + } + + // ---- folders ---- + if (userKey) await syncFolders(store, remote, state, userKey, local.namespace, report); + state.lastSyncAt = new Date().toISOString(); + + store.writeSyncState(state); + return report; +} + +async function resolveConflict( + store: VaultStore, + state: SyncState, + theirs: RemoteItem, + mine: { id: string; envelope: Envelope | null; baseRevision: number }, + userKey: Uint8Array | undefined, + namespace: string, + report: SyncReport, +): Promise { + if (mine.envelope && sameCiphertext(mine.envelope, theirs.envelope)) { + store.putEnvelope({ ...mine.envelope, revision: theirs.revision }); + state.items[theirs.id] = theirs.revision; + return; + } + report.conflicts++; + if (theirs.revision === 0) { + // The account has no such row: it was never there, or the vault was reset. + delete state.items[mine.id]; + return; + } + if (!mine.envelope) { + // Purged here, edited there: the edit wins and comes back. + if (theirs.envelope) { + store.putEnvelope({ ...theirs.envelope, revision: theirs.revision }); + state.items[theirs.id] = theirs.revision; + report.notes.push(`${theirs.id} was purged here but edited elsewhere; kept the edit.`); + } else { + delete state.items[theirs.id]; + } + return; + } + if (!theirs.envelope) { + // Purged there, edited here: keep the edit by re-sending it over the tombstone. + // Its local revision is set one past the tombstone so it reads as unsent + // (a local counter can happen to equal the remote one). + store.putEnvelope({ ...mine.envelope, revision: theirs.revision + 1 }); + state.items[mine.id] = theirs.revision; + report.notes.push(`${mine.id} was purged elsewhere but edited here; kept the edit.`); + return; + } + if (!userKey) { + report.notes.push(`${mine.id} was edited here and elsewhere; it will be split into two items the next time the vault is unlocked.`); + return; + } + // Both edited. The account's version keeps the id; this machine's becomes a copy. + const item = await decryptItem(userKey, mine.envelope, namespace); + const copy = { ...item, id: uuid(), name: `${item.name} (conflict copy)`, updatedAt: new Date().toISOString() }; + store.writeEnvelope(await encryptItem(userKey, copy, namespace)); + store.putEnvelope({ ...theirs.envelope, revision: theirs.revision }); + state.items[theirs.id] = theirs.revision; + report.notes.push(`"${item.name}" was edited here and elsewhere; kept both (yours is "${copy.name}").`); +} + +async function syncFolders( + store: VaultStore, + remote: SyncRemote, + state: SyncState, + userKey: Uint8Array, + namespace: string, + report: SyncReport, +): Promise { + const aad = foldersAad(namespace); + for (let attempt = 0; attempt < 2; attempt++) { + const rv = await remote.getVault(); + let folders = store.readFolders(); + if (rv?.folders && rv.folders.revision !== state.foldersRevision) { + const plain = await aesGcmDecrypt(userKey, fromBase64(rv.folders.iv), fromBase64(rv.folders.ciphertext), aad); + const theirs = JSON.parse(utf8Decode(plain)) as Folder[]; + // Union by id; a name edited on both sides keeps this machine's. + const merged = new Map(theirs.map((f) => [f.id, f])); + for (const f of folders) merged.set(f.id, f); + folders = [...merged.values()].sort((a, b) => a.id.localeCompare(b.id)); + store.writeFolders(folders); + state.foldersRevision = rv.folders.revision; + state.foldersHash = hash(theirs.slice().sort((a, b) => a.id.localeCompare(b.id))); + } + const sorted = folders.slice().sort((a, b) => a.id.localeCompare(b.id)); + if (hash(sorted) === state.foldersHash || (sorted.length === 0 && !state.foldersHash)) return; + const { iv, ciphertext } = await aesGcmEncrypt(userKey, utf8Encode(JSON.stringify(sorted)), aad); + const put = await remote.putFolders({ iv: toBase64(iv), ciphertext: toBase64(ciphertext) }, state.foldersRevision); + if (put.ok) { + state.foldersRevision = put.revision; + state.foldersHash = hash(sorted); + return; + } + report.notes.push("Folders changed elsewhere at the same time; merged and retried."); + } +} + +/** Pull then push: what `sync` runs, and what the hooks run around a command. */ +export async function syncVault(store: VaultStore, remote: SyncRemote, userKey?: Uint8Array): Promise { + const report = await pullVault(store, remote); + return pushVault(store, remote, userKey, report); +} + +/** One line for a person, or "" when nothing moved. */ +export function describeSync(report: SyncReport, label: string): string { + const parts: string[] = []; + if (report.downloaded) parts.push("downloaded the vault"); + if (report.created) parts.push("uploaded the vault"); + if (report.pulled) parts.push(`${report.pulled} in`); + if (report.pushed) parts.push(`${report.pushed} out`); + if (report.deleted) parts.push(`${report.deleted} purged`); + if (report.conflicts) parts.push(`${report.conflicts} conflict(s)`); + const head = parts.length ? `vault sync (${label}): ${parts.join(", ")}` : ""; + return [head, ...report.notes.map((n) => `vault sync: ${n}`)].filter(Boolean).join("\n"); +}