feat: F5.1 server F keeps the shared documents

A new package, packages/server: Hocuspocus 3.4.4 on /sync, a project a
document, refused with a reason when the store has not got it; every update
stored in Postgres as it comes and compacted after 500 updates, a minute of
quiet and when the last person leaves (FOR UPDATE on the row written when
the project is made); the janitor runs only here, at loading and 250 ms
after a change that is not its own; readers change nothing; /healthz; JSON
log. On closing it takes no one new and closes the sockets too (Hocuspocus'
close is a message after which a client signs in again on the same socket).
The tests use real clients against a throwaway Postgres in Docker.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

packages/server/test/database.ts

@@ -0,0 +1,42 @@
+// A Postgres of its own for the server's tests: FILIUS_TEST_DATABASE_URL
+// when it is set, else a throwaway container of the postgres image (Docker),
+// stopped and gone after the tests. Never a database of the dashboard.
+
+import { execFileSync } from 'node:child_process';
+import pg from 'pg';
+
+export interface TestDatabase {
+  readonly url: string;
+  stop(): Promise<void>;
+}
+
+export async function testDatabase(): Promise<TestDatabase> {
+  const given = process.env.FILIUS_TEST_DATABASE_URL;
+  if (given !== undefined) return { url: given, stop: async () => {} };
+  const name = `filius-test-pg-${process.pid}-${Date.now()}`;
+  execFileSync('docker', ['run', '-d', '--rm', '--name', name, '-e', 'POSTGRES_PASSWORD=filius', '-e', 'POSTGRES_DB=filius', '-p', '127.0.0.1::5432', 'postgres:16-alpine'], { stdio: 'pipe' });
+  const stop = async () => {
+    execFileSync('docker', ['stop', name], { stdio: 'pipe' });
+  };
+  try {
+    const port = execFileSync('docker', ['port', name, '5432/tcp']).toString().trim().split('\n')[0]!.split(':').pop();
+    const url = `postgres://postgres:filius@127.0.0.1:${port}/filius`;
+    // The image's first start runs a server on its socket only; the TCP port answers once the real one is up.
+    for (let tries = 0; ; tries++) {
+      const client = new pg.Client({ connectionString: url });
+      try {
+        await client.connect();
+        await client.query('SELECT 1');
+        await client.end();
+        return { url, stop };
+      } catch (error) {
+        await client.end().catch(() => {});
+        if (tries > 120) throw error;
+        await new Promise((resolve) => setTimeout(resolve, 250));
+      }
+    }
+  } catch (error) {
+    await stop();
+    throw error;
+  }
+}

packages/server/test/sync.test.ts

@@ -0,0 +1,360 @@
+// Server F's shared documents (F5.1) with real clients (Hocuspocus' own
+// provider over WebSocket) and a real Postgres: a project comes back as it
+// was after the service restarts, two people see each other's changes at
+// once, the updates are compacted (after 500, a minute after the last
+// change, when the last person leaves), the janitor repairs what a merge
+// leaves — once, as a client of its own —, a project the service does not
+// know is refused and not made, read-only people cannot write, /healthz
+// answers, and the log is JSON.
+
+import { randomUUID } from 'node:crypto';
+import { HocuspocusProvider, HocuspocusProviderWebsocket } from '@hocuspocus/provider';
+import pg from 'pg';
+import { afterAll, beforeAll, describe, expect, it } from 'vitest';
+import WebSocket from 'ws';
+import * as Y from 'yjs';
+import { addDevice, connect, emptyProject, readProject, ROOTS, topologyView, writeProject } from '@mdg-filius/doc';
+import { COMPACT_AFTER_IDLE_MILLISECONDS, COMPACT_AFTER_UPDATES, JANITOR_DELAY_MILLISECONDS, PostgresStore, startServer, type Access, type FiliusServer, type Log } from '../src/index.js';
+import { testDatabase, type TestDatabase } from './database.js';
+
+let database: TestDatabase;
+let store: PostgresStore;
+let sql: pg.Pool;
+
+beforeAll(async () => {
+  database = await testDatabase();
+  store = PostgresStore.connect(database.url);
+  await store.migrate();
+  sql = new pg.Pool({ connectionString: database.url });
+}, 60_000);
+
+afterAll(async () => {
+  await sql?.end();
+  await store?.close();
+  await database?.stop();
+});
+
+/** The log's lines, kept. */
+function memoryLog(): Log & { readonly lines: Record<string, unknown>[] } {
+  const lines: Record<string, unknown>[] = [];
+  const entry =
+    (level: string) =>
+    (event: string, fields: Record<string, unknown> = {}) => {
+      lines.push(JSON.parse(JSON.stringify({ level, event, ...fields })) as Record<string, unknown>);
+    };
+  return { lines, info: entry('info'), warn: entry('warn'), error: entry('error') };
+}
+
+async function start(options: { readonly access?: Access; readonly idle?: number } = {}): Promise<FiliusServer & { readonly log: ReturnType<typeof memoryLog> }> {
+  const log = memoryLog();
+  const server = await startServer({
+    port: 0,
+    host: '127.0.0.1',
+    store,
+    log,
+    authenticate: async () => options.access ?? { readOnly: false },
+    ...(options.idle === undefined ? {} : { compactAfterIdleMilliseconds: options.idle }),
+  });
+  return Object.assign(server, { log });
+}
+
+/** A new project as the service makes it: an empty FILIUS project. */
+async function newProject(): Promise<string> {
+  const project = randomUUID();
+  const doc = new Y.Doc();
+  writeProject(doc, emptyProject());
+  expect(await store.create(project, Y.encodeStateAsUpdate(doc))).toBe(true);
+  return project;
+}
+
+interface Client {
+  readonly doc: Y.Doc;
+  readonly provider: HocuspocusProvider;
+  close(): void;
+}
+
+/** A person in a project: synced, or refused. `eager` comes back at once when let go (no delay, as a client may). */
+function join(server: FiliusServer, project: string, eager = false): Promise<Client> {
+  const doc = new Y.Doc();
+  const socket = new HocuspocusProviderWebsocket({ url: `ws://127.0.0.1:${server.port}/sync`, WebSocketPolyfill: WebSocket, ...(eager ? { delay: 0, minDelay: 0, jitter: false } : {}) });
+  return new Promise((resolve, reject) => {
+    const timer = setTimeout(() => {
+      provider.destroy();
+      socket.destroy();
+      reject(new Error('not synced within 3 s'));
+    }, 3_000);
+    const provider = new HocuspocusProvider({
+      websocketProvider: socket,
+      name: project,
+      document: doc,
+      token: 'test',
+      onSynced: () => {
+        clearTimeout(timer);
+        resolve({ doc, provider, close: () => (provider.destroy(), socket.destroy()) });
+      },
+      onAuthenticationFailed: ({ reason }) => {
+        clearTimeout(timer);
+        provider.destroy();
+        socket.destroy();
+        reject(new Error(`refused: ${reason}`));
+      },
+      onClose: () => reject(new Error('closed')),
+    });
+    provider.attach();
+  });
+}
+
+const until = async (condition: () => boolean | Promise<boolean>, milliseconds = 5_000) => {
+  for (let waited = 0; !(await condition()); waited += 25) {
+    if (waited > milliseconds) throw new Error('timed out');
+    await new Promise((resolve) => setTimeout(resolve, 25));
+  }
+};
+const deviceCount = (doc: Y.Doc) => doc.getMap(ROOTS.nodes).size;
+const NAMES = { pc: 'Rechner', networkSwitch: 'Switch' } as const;
+/** A device as the palette puts one down. */
+const place = (doc: Y.Doc, id: string, kind: keyof typeof NAMES, x: number, origin: string) => addDevice(doc, { id, kind, name: NAMES[kind], x, y: 0, newId: randomUUID }, origin);
+
+describe('server F keeps the shared documents', () => {
+  it('gives a project back as it was after the service restarts, from its compacted state', async () => {
+    const project = await newProject();
+    let server = await start();
+    const first = await join(server, project);
+    place(first.doc, 'a', 'pc', 10, 'person');
+    place(first.doc, 'b', 'networkSwitch', 200, 'person');
+    await until(async () => (await store.pending(project)) >= 2);
+    first.close();
+    // The last person left: compacted and unloaded.
+    await until(() => server.sync.getDocumentsCount() === 0);
+    expect(await store.pending(project)).toBe(0);
+    await server.close();
+
+    server = await start();
+    const again = await join(server, project);
+    expect(topologyView(again.doc).nodes.map((node) => [node.id, node.kind, node.x])).toEqual([
+      ['a', 'pc', 10],
+      ['b', 'networkSwitch', 200],
+    ]);
+    // Loading stores nothing anew.
+    expect(await store.pending(project)).toBe(0);
+    again.close();
+    await server.close();
+  });
+
+  it('takes the updates a hard stop left uncompacted, and counts them towards the next compaction', async () => {
+    const project = await newProject();
+    // The service stopped hard: three updates stored, never compacted.
+    const before = new Y.Doc();
+    Y.applyUpdate(before, (await store.load(project))!.state);
+    before.on('update', (update: Uint8Array) => void store.append(project, update));
+    for (const [id, x] of [
+      ['a', 0],
+      ['b', 100],
+      ['c', 200],
+    ] as const)
+      place(before, id, 'pc', x, 'person');
+    await until(async () => (await store.pending(project)) === 3);
+    const server = await start();
+    const person = await join(server, project);
+    expect(topologyView(person.doc).nodes.map((node) => node.id)).toEqual(['a', 'b', 'c']);
+    const notes = person.doc.getMap<number>('test');
+    for (let index = 0; index < COMPACT_AFTER_UPDATES - 3; index++) notes.set('n', index);
+    await until(() => server.log.lines.some((line) => line.event === 'doc.compacted'));
+    expect(server.log.lines.find((line) => line.event === 'doc.compacted')!.updates).toBe(COMPACT_AFTER_UPDATES);
+    person.close();
+    await server.close();
+  });
+
+  it('compacts once after exactly 500 updates, and not again when the document has been quiet since', async () => {
+    const project = await newProject();
+    const server = await start({ idle: 200 });
+    const person = await join(server, project);
+    const notes = person.doc.getMap<number>('test');
+    for (let index = 0; index < COMPACT_AFTER_UPDATES; index++) notes.set('n', index);
+    await until(() => server.log.lines.some((line) => line.event === 'doc.compacted'), 15_000);
+    await new Promise((resolve) => setTimeout(resolve, 600));
+    expect(server.log.lines.filter((line) => line.event === 'doc.compacted').map((line) => line.updates)).toEqual([COMPACT_AFTER_UPDATES]);
+    expect((await sql.query('SELECT rev FROM filius_doc WHERE project_uuid = $1', [project])).rows[0].rev).toBe(1);
+    person.close();
+    await server.close();
+  });
+
+  it('compacts nothing after a visit that changed nothing', async () => {
+    const project = await newProject();
+    const rev = async () => (await sql.query('SELECT rev FROM filius_doc WHERE project_uuid = $1', [project])).rows[0].rev as number;
+    const server = await start({ idle: 100 });
+    const person = await join(server, project);
+    await new Promise((resolve) => setTimeout(resolve, 300));
+    person.close();
+    await until(() => server.sync.getDocumentsCount() === 0);
+    expect(await rev()).toBe(0);
+    expect(server.log.lines.some((line) => line.event === 'doc.compacted')).toBe(false);
+    await server.close();
+  });
+
+  it('stores everything before it closes, with people still there who come back at once', async () => {
+    const project = await newProject();
+    const server = await start();
+    const person = await join(server, project, true);
+    place(person.doc, 'a', 'pc', 0, 'person');
+    await until(async () => (await store.pending(project)) === 1);
+    await server.close();
+    expect(await store.pending(project)).toBe(0);
+    const stored = new Y.Doc();
+    Y.applyUpdate(stored, (await store.load(project))!.state);
+    expect(deviceCount(stored)).toBe(1);
+    person.close();
+  });
+
+  it('says so in the log when a project went while it was open, and stores nothing in its place', async () => {
+    const project = await newProject();
+    const server = await start();
+    const person = await join(server, project);
+    place(person.doc, 'a', 'pc', 0, 'person');
+    await until(async () => (await store.pending(project)) === 1);
+    await store.remove(project);
+    person.close();
+    await until(() => server.sync.getDocumentsCount() === 0);
+    expect(server.log.lines.filter((line) => line.event === 'store.failed').map((line) => line.project)).toEqual([project]);
+    expect(await store.load(project)).toBeUndefined();
+    await server.close();
+  });
+
+  it('makes a project once: a second one of the same id changes nothing', async () => {
+    const project = await newProject();
+    const first = (await store.load(project))!.state;
+    expect(await store.create(project, new Uint8Array([0, 0]))).toBe(false);
+    expect((await store.load(project))!.state).toEqual(first);
+  });
+
+  it("shows each person the other's changes at once", async () => {
+    const project = await newProject();
+    const server = await start();
+    const [mia, ben] = await Promise.all([join(server, project), join(server, project)]);
+    place(mia.doc, 'pc', 'pc', 0, 'mia');
+    place(ben.doc, 'switch', 'networkSwitch', 100, 'ben');
+    await until(() => deviceCount(mia.doc) === 2 && deviceCount(ben.doc) === 2);
+    connect(mia.doc, { node: 'pc' }, { node: 'switch' }, 'cable', 'mia');
+    await until(() => ben.doc.getMap(ROOTS.links).size === 1);
+    expect(readProject(ben.doc)).toEqual(readProject(mia.doc));
+    mia.close();
+    ben.close();
+    await server.close();
+  });
+
+  it(`compacts after ${COMPACT_AFTER_UPDATES} updates, and a minute after the last change`, async () => {
+    const project = await newProject();
+    const server = await start({ idle: 300 });
+    const person = await join(server, project);
+    const notes = person.doc.getMap<number>('test');
+    for (let index = 0; index < COMPACT_AFTER_UPDATES + 20; index++) notes.set('n', index);
+    await until(async () => (await sql.query('SELECT rev FROM filius_doc WHERE project_uuid = $1', [project])).rows[0].rev >= 1, 15_000);
+    // The compaction took the first 500; what came after waits — until the document has been quiet for a while.
+    await until(async () => (await store.pending(project)) === 0, 10_000);
+    expect(server.log.lines.filter((line) => line.event === 'doc.compacted').map((line) => line.updates)).toEqual([COMPACT_AFTER_UPDATES, 20]);
+    const stored = await store.load(project);
+    const reloaded = new Y.Doc();
+    Y.applyUpdate(reloaded, stored!.state);
+    expect(reloaded.getMap('test').get('n')).toBe(COMPACT_AFTER_UPDATES + 19);
+    person.close();
+    await server.close();
+  });
+
+  it('repairs what a merge leaves, once, as a client of its own', async () => {
+    const project = await newProject();
+    const server = await start();
+    const [mia, ben] = await Promise.all([join(server, project), join(server, project)]);
+    place(mia.doc, 'pc', 'pc', 0, 'mia');
+    place(mia.doc, 'switch', 'networkSwitch', 100, 'mia');
+    await until(() => deviceCount(ben.doc) === 2);
+    // Both plug a cable into the computer's one card at once; Mia's goes to a port that is gone too.
+    const port = topologyView(mia.doc).nodes.find((node) => node.id === 'pc')!.ports[0]!.id;
+    const switchPorts = topologyView(mia.doc)
+      .nodes.find((node) => node.id === 'switch')!
+      .ports.map((each) => each.id);
+    const cable = (doc: Y.Doc, id: string, targetPort: string) =>
+      doc.transact(() => {
+        const link = new Y.Map<unknown>();
+        for (const [key, value] of Object.entries({ sourceNode: 'pc', sourcePort: port, targetNode: 'switch', targetPort, order: 0 })) link.set(key, value);
+        doc.getMap(ROOTS.links).set(id, link);
+      }, 'person');
+    cable(mia.doc, 'cable-b', switchPorts[0]!);
+    cable(ben.doc, 'cable-a', switchPorts[1]!);
+    cable(mia.doc, 'cable-c', 'no-such-port');
+    await until(() => [...ben.doc.getMap(ROOTS.links).keys()].join() === 'cable-a' && [...mia.doc.getMap(ROOTS.links).keys()].join() === 'cable-a');
+    await new Promise((resolve) => setTimeout(resolve, 600));
+    const repairs = server.log.lines.filter((line) => line.event === 'janitor.repaired');
+    expect(repairs.reduce((sum, line) => sum + (line.danglingLinks as number) + (line.sharedPortLinks as number), 0)).toBe(2);
+    mia.close();
+    ben.close();
+    await server.close();
+  });
+
+  it('repairs a stored document when it is loaded, before anyone changes it', async () => {
+    const project = randomUUID();
+    const doc = new Y.Doc();
+    writeProject(doc, emptyProject());
+    doc.transact(() => {
+      const link = new Y.Map<unknown>();
+      for (const [key, value] of Object.entries({ sourceNode: 'gone', sourcePort: 'p', targetNode: 'gone too', targetPort: 'q', order: 0 })) link.set(key, value);
+      doc.getMap(ROOTS.links).set('stray', link);
+    });
+    await store.create(project, Y.encodeStateAsUpdate(doc));
+    const server = await start();
+    const person = await join(server, project);
+    await until(() => person.doc.getMap(ROOTS.links).size === 0);
+    expect(server.log.lines.find((line) => line.event === 'janitor.repaired')).toMatchObject({ project, danglingLinks: 1 });
+    person.close();
+    await server.close();
+  });
+
+  it("keeps the plan's measures: compacted after 500 updates or a minute of quiet, repaired 250 ms after a change", () => {
+    expect([COMPACT_AFTER_UPDATES, COMPACT_AFTER_IDLE_MILLISECONDS, JANITOR_DELAY_MILLISECONDS]).toEqual([500, 60_000, 250]);
+  });
+
+  it('refuses a project it does not know, and makes none', async () => {
+    const server = await start();
+    const project = randomUUID();
+    await expect(join(server, project)).rejects.toThrow('refused: unknownProject');
+    expect(await store.load(project)).toBeUndefined();
+    await expect(join(server, 'not-a-project')).rejects.toThrow('refused: notAProject');
+    expect(server.log.lines.some((line) => line.event === 'doc.unknown' && line.project === project)).toBe(true);
+    await server.close();
+  });
+
+  it('lets a reader see the document and change nothing', async () => {
+    const project = await newProject();
+    const writerServer = await start();
+    const writer = await join(writerServer, project);
+    place(writer.doc, 'pc', 'pc', 0, 'person');
+    await until(async () => (await store.pending(project)) >= 1);
+    writer.close();
+    await writerServer.close();
+    const server = await start({ access: { readOnly: true } });
+    const reader = await join(server, project);
+    expect(deviceCount(reader.doc)).toBe(1);
+    place(reader.doc, 'switch', 'networkSwitch', 0, 'reader');
+    await new Promise((resolve) => setTimeout(resolve, 400));
+    expect(await store.pending(project)).toBe(0);
+    reader.close();
+    await server.close();
+    const stored = new Y.Doc();
+    Y.applyUpdate(stored, (await store.load(project))!.state);
+    expect(deviceCount(stored)).toBe(1);
+  });
+
+  it('answers /healthz, refuses other ways in, and logs JSON', async () => {
+    const server = await start();
+    const health = await fetch(`http://127.0.0.1:${server.port}/healthz`);
+    expect([health.status, await health.json()]).toEqual([200, { status: 'ok', documents: 0, connections: 0 }]);
+    expect((await fetch(`http://127.0.0.1:${server.port}/nothing`)).status).toBe(404);
+    const refused = await new Promise<number>((resolve) => {
+      const socket = new WebSocket(`ws://127.0.0.1:${server.port}/elsewhere`);
+      socket.on('unexpected-response', (_request, response) => resolve(response.statusCode ?? 0));
+      socket.on('error', () => resolve(-1));
+    });
+    expect(refused).toBe(404);
+    await server.close();
+    expect(server.log.lines.map((line) => line.event)).toEqual(['server.listening', 'server.closed']);
+  });
+});

packages/server/test/tsconfig.json

@@ -0,0 +1,11 @@
+{
+  "extends": "../../../tsconfig.base.json",
+  "compilerOptions": {
+    "noEmit": true,
+    "composite": false,
+    "declaration": false,
+    "types": ["node"],
+    "resolveJsonModule": true
+  },
+  "include": ["."]
+}