feat: F5.1 server F keeps the shared documents

A new package, packages/server: Hocuspocus 3.4.4 on /sync, a project a
document, refused with a reason when the store has not got it; every update
stored in Postgres as it comes and compacted after 500 updates, a minute of
quiet and when the last person leaves (FOR UPDATE on the row written when
the project is made); the janitor runs only here, at loading and 250 ms
after a change that is not its own; readers change nothing; /healthz; JSON
log. On closing it takes no one new and closes the sockets too (Hocuspocus'
close is a message after which a client signs in again on the same socket).
The tests use real clients against a throwaway Postgres in Docker.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

packages/server/package.json

@@ -0,0 +1,30 @@
+{
+  "name": "@mdg-filius/server",
+  "version": "0.0.0",
+  "private": true,
+  "license": "GPL-3.0-only",
+  "type": "module",
+  "exports": {
+    ".": "./src/index.ts"
+  },
+  "scripts": {
+    "build": "esbuild src/main.ts --bundle --platform=node --format=esm --target=node20 --outfile=dist/main.mjs --external:pg-native --banner:js=\"import { createRequire } from 'node:module'; const require = createRequire(import.meta.url);\"",
+    "start": "node dist/main.mjs"
+  },
+  "dependencies": {
+    "@hocuspocus/server": "3.4.4",
+    "@mdg-filius/core": "0.0.0",
+    "@mdg-filius/doc": "0.0.0",
+    "@mdg-filius/protocol": "0.0.0",
+    "@mdg-filius/sim": "0.0.0",
+    "pg": "8.23.1",
+    "ws": "8.22.0",
+    "y-protocols": "1.0.6",
+    "yjs": "^13.6.33"
+  },
+  "devDependencies": {
+    "@hocuspocus/provider": "3.4.4",
+    "@types/pg": "8.23.1",
+    "@types/ws": "8.18.2"
+  }
+}

packages/server/src/index.ts

@@ -0,0 +1,7 @@
+// Server F (plan §8): the shared documents, and later the simulation rooms,
+// the sign-in and the dashboard's service.
+export * from './log.js';
+export * from './persistence.js';
+export * from './server.js';
+export * from './store.js';
+export * from './sync.js';

packages/server/src/log.ts

@@ -0,0 +1,25 @@
+// The service's log: one JSON object a line, in English (it is for running
+// the service, not for its users; plan §8.2): the time, the level, what
+// happened, and the project, person and room it happened in.
+
+export type LogFields = Readonly<Record<string, unknown>>;
+
+export interface Log {
+  info(event: string, fields?: LogFields): void;
+  warn(event: string, fields?: LogFields): void;
+  error(event: string, fields?: LogFields): void;
+}
+
+export function jsonLog(write: (line: string) => void = (line) => process.stdout.write(`${line}\n`), now: () => Date = () => new Date()): Log {
+  const entry =
+    (level: string) =>
+    (event: string, fields: LogFields = {}) => {
+      write(JSON.stringify({ time: now().toISOString(), level, event, ...fields }));
+    };
+  return { info: entry('info'), warn: entry('warn'), error: entry('error') };
+}
+
+/** An error as the log writes it. */
+export function errorText(error: unknown): string {
+  return error instanceof Error ? error.message : String(error);
+}

packages/server/src/main.ts

@@ -0,0 +1,35 @@
+// Server F started from its environment: FILIUS_PORT (8080), DATABASE_URL.
+// Until the sign-in comes (F5.2) every connection is refused, unless
+// FILIUS_INSECURE_DEV=1 lets everyone write — on a developer's machine only.
+
+import { jsonLog } from './log.js';
+import { startServer } from './server.js';
+import { PostgresStore } from './store.js';
+
+const log = jsonLog();
+const url = process.env.DATABASE_URL;
+if (url === undefined) {
+  log.error('config.missing', { variable: 'DATABASE_URL' });
+  process.exit(1);
+}
+const insecure = process.env.FILIUS_INSECURE_DEV === '1';
+if (insecure) log.warn('auth.insecureDev', { note: 'every connection may write' });
+const store = PostgresStore.connect(url);
+await store.migrate();
+const server = await startServer({
+  port: Number(process.env.FILIUS_PORT ?? 8080),
+  store,
+  log,
+  authenticate: async () => {
+    if (insecure) return { readOnly: false };
+    throw new Error('no sign-in yet');
+  },
+});
+for (const signal of ['SIGTERM', 'SIGINT'] as const) {
+  process.once(signal, () => {
+    void server
+      .close()
+      .then(() => store.close())
+      .then(() => process.exit(0));
+  });
+}

packages/server/src/persistence.ts

@@ -0,0 +1,80 @@
+// One project's document kept while it is open: every update stored as it
+// comes, in order (a queue, so that a compaction never overtakes an update
+// before it), and compacted after COMPACT_AFTER_UPDATES updates or when the
+// service asks (a minute after the last change, and when the last person
+// leaves; plan §5.4). A compaction stores the document as it is and drops
+// the updates stored before it; one stored after it stays, and applying it
+// again on loading changes nothing (Yjs).
+
+import * as Y from 'yjs';
+import { errorText, type Log } from './log.js';
+import type { DocStore } from './store.js';
+
+export const COMPACT_AFTER_UPDATES = 500;
+/** The origin of what is loaded from the store: not stored again. */
+export const STORE_ORIGIN = 'store';
+
+export class DocumentPersistence {
+  private queue: Promise<void> = Promise.resolve();
+  private lastSeq = '0';
+  /** Updates stored since the last compaction (in the queue's order). */
+  private sinceCompaction = 0;
+  /** Updates queued since the last compaction was queued: what decides the next one. */
+  private queued = 0;
+
+  constructor(
+    private readonly store: DocStore,
+    private readonly project: string,
+    private readonly log: Log,
+  ) {}
+
+  /** The stored document into a Y.Doc; false without the project. */
+  async load(doc: Y.Doc): Promise<boolean> {
+    const stored = await this.store.load(this.project);
+    if (stored === undefined) return false;
+    Y.transact(
+      doc,
+      () => {
+        Y.applyUpdate(doc, stored.state);
+        for (const update of stored.updates) Y.applyUpdate(doc, update);
+      },
+      STORE_ORIGIN,
+    );
+    this.lastSeq = stored.lastSeq;
+    this.sinceCompaction = stored.updates.length;
+    this.queued = stored.updates.length;
+    return true;
+  }
+
+  private enqueue(work: () => Promise<void>): Promise<void> {
+    this.queue = this.queue.then(work).catch((error: unknown) => this.log.error('store.failed', { project: this.project, error: errorText(error) }));
+    return this.queue;
+  }
+
+  /** An update stored; the document compacted once enough have come. */
+  append(update: Uint8Array, doc: Y.Doc): Promise<void> {
+    const stored = this.enqueue(async () => {
+      this.lastSeq = await this.store.append(this.project, update);
+      this.sinceCompaction += 1;
+    });
+    this.queued += 1;
+    return this.queued >= COMPACT_AFTER_UPDATES ? this.compact(doc) : stored;
+  }
+
+  /** The document as it is now stored as the project's state, the updates before dropped. */
+  compact(doc: Y.Doc): Promise<void> {
+    this.queued = 0;
+    return this.enqueue(async () => {
+      if (this.sinceCompaction === 0) return;
+      const updates = this.sinceCompaction;
+      await this.store.compact(this.project, Y.encodeStateAsUpdate(doc), this.lastSeq);
+      this.sinceCompaction = 0;
+      this.log.info('doc.compacted', { project: this.project, updates });
+    });
+  }
+
+  /** Everything stored that was asked for. */
+  flush(): Promise<void> {
+    return this.enqueue(async () => {});
+  }
+}

packages/server/src/server.ts

@@ -0,0 +1,74 @@
+// Server F's front door (plan §8.2): one HTTP server — /healthz for the
+// proxy, the WebSocket /sync for the shared documents; anything else is not
+// found. Closed, it first takes no one new (a client lets go and comes back
+// at once, which would keep its document open), then lets everyone go,
+// waits until every document is stored and unloaded, and stops.
+
+import { createServer, type Server } from 'node:http';
+import type { AddressInfo, Socket } from 'node:net';
+import type { Hocuspocus } from '@hocuspocus/server';
+import { WebSocketServer } from 'ws';
+import type { Log } from './log.js';
+import { createSync, type SyncOptions } from './sync.js';
+
+export interface FiliusServer {
+  readonly port: number;
+  readonly sync: Hocuspocus;
+  close(): Promise<void>;
+}
+
+export interface ServerOptions extends SyncOptions {
+  readonly port?: number;
+  readonly host?: string;
+  readonly log: Log;
+}
+
+function refuse(socket: Socket, status: string): void {
+  socket.end(`HTTP/1.1 ${status}\r\nConnection: close\r\n\r\n`);
+}
+
+export async function startServer(options: ServerOptions): Promise<FiliusServer> {
+  const sync = createSync(options);
+  const sockets = new WebSocketServer({ noServer: true });
+  let closing = false;
+  const http: Server = createServer((request, response) => {
+    const path = new URL(request.url ?? '/', 'http://filius').pathname;
+    if (request.method === 'GET' && path === '/healthz') {
+      response.writeHead(200, { 'content-type': 'application/json' });
+      response.end(JSON.stringify({ status: 'ok', documents: sync.getDocumentsCount(), connections: sync.getConnectionsCount() }));
+      return;
+    }
+    response.writeHead(404, { 'content-type': 'application/json' });
+    response.end(JSON.stringify({ error: 'notFound' }));
+  });
+  http.on('upgrade', (request, socket, head) => {
+    const path = new URL(request.url ?? '/', 'http://filius').pathname;
+    if (path !== '/sync') {
+      refuse(socket as Socket, '404 Not Found');
+      return;
+    }
+    if (closing) {
+      refuse(socket as Socket, '503 Service Unavailable');
+      return;
+    }
+    sockets.handleUpgrade(request, socket, head, (ws) => sync.handleConnection(ws, request));
+  });
+  await new Promise<void>((resolve) => http.listen(options.port ?? 0, options.host ?? '0.0.0.0', resolve));
+  const port = (http.address() as AddressInfo).port;
+  options.log.info('server.listening', { port });
+  return {
+    port,
+    sync,
+    async close() {
+      closing = true;
+      sync.closeConnections();
+      // Hocuspocus' close is a message on the socket, after which a client signs in again on it: the sockets go too.
+      for (const socket of sockets.clients) socket.close(1001, 'service stopping');
+      // Each document is stored and unloaded once its last person has gone.
+      for (let waited = 0; sync.getDocumentsCount() > 0 && waited < 10_000; waited += 50) await new Promise((resolve) => setTimeout(resolve, 50));
+      sockets.close();
+      await new Promise<void>((resolve) => http.close(() => resolve()));
+      options.log.info('server.closed', { documents: sync.getDocumentsCount() });
+    },
+  };
+}

packages/server/src/store.ts

@@ -0,0 +1,109 @@
+// Where F keeps each project's document (plan §5.4): its compacted state in
+// filius_doc, the updates since then in filius_doc_update, in the order they
+// came. A project's row is written when the project is made, so that the
+// compaction's FOR UPDATE has a row to lock (a lock on a missing row locks
+// nothing; Scratch's lesson 6.13).
+
+import pg from 'pg';
+
+export interface StoredDocument {
+  readonly state: Uint8Array;
+  readonly updates: readonly Uint8Array[];
+  /** The last update's sequence number, "0" without one. */
+  readonly lastSeq: string;
+  readonly rev: number;
+}
+
+export interface DocStore {
+  /** Makes the tables, if they are not there. */
+  migrate(): Promise<void>;
+  /** A new project with its first state; false if it is there already. */
+  create(project: string, state: Uint8Array): Promise<boolean>;
+  /** A project's state and the updates since, undefined without the project. */
+  load(project: string): Promise<StoredDocument | undefined>;
+  /** An update kept; its sequence number. */
+  append(project: string, update: Uint8Array): Promise<string>;
+  /** A new state, which holds every update up to `upToSeq`: those go. */
+  compact(project: string, state: Uint8Array, upToSeq: string): Promise<void>;
+  /** How many updates wait for the next compaction. */
+  pending(project: string): Promise<number>;
+  remove(project: string): Promise<void>;
+  close(): Promise<void>;
+}
+
+const SCHEMA = `
+CREATE TABLE IF NOT EXISTS filius_doc (
+  project_uuid uuid PRIMARY KEY,
+  state bytea NOT NULL,
+  rev integer NOT NULL DEFAULT 0,
+  updated_at timestamptz NOT NULL DEFAULT now()
+);
+CREATE TABLE IF NOT EXISTS filius_doc_update (
+  project_uuid uuid NOT NULL REFERENCES filius_doc ON DELETE CASCADE,
+  seq bigserial NOT NULL,
+  update bytea NOT NULL,
+  PRIMARY KEY (project_uuid, seq)
+);
+`;
+
+const bytes = (buffer: Buffer) => new Uint8Array(buffer.buffer, buffer.byteOffset, buffer.byteLength);
+
+export class PostgresStore implements DocStore {
+  constructor(private readonly pool: pg.Pool) {}
+
+  static connect(url: string): PostgresStore {
+    return new PostgresStore(new pg.Pool({ connectionString: url }));
+  }
+
+  async migrate(): Promise<void> {
+    await this.pool.query(SCHEMA);
+  }
+
+  async create(project: string, state: Uint8Array): Promise<boolean> {
+    const result = await this.pool.query('INSERT INTO filius_doc (project_uuid, state) VALUES ($1, $2) ON CONFLICT DO NOTHING', [project, Buffer.from(state)]);
+    return result.rowCount === 1;
+  }
+
+  async load(project: string): Promise<StoredDocument | undefined> {
+    const doc = await this.pool.query<{ state: Buffer; rev: number }>('SELECT state, rev FROM filius_doc WHERE project_uuid = $1', [project]);
+    const row = doc.rows[0];
+    if (row === undefined) return undefined;
+    const updates = await this.pool.query<{ seq: string; update: Buffer }>('SELECT seq, update FROM filius_doc_update WHERE project_uuid = $1 ORDER BY seq', [project]);
+    return { state: bytes(row.state), updates: updates.rows.map((each) => bytes(each.update)), lastSeq: updates.rows.at(-1)?.seq ?? '0', rev: row.rev };
+  }
+
+  async append(project: string, update: Uint8Array): Promise<string> {
+    const result = await this.pool.query<{ seq: string }>('INSERT INTO filius_doc_update (project_uuid, update) VALUES ($1, $2) RETURNING seq', [project, Buffer.from(update)]);
+    return result.rows[0]!.seq;
+  }
+
+  async compact(project: string, state: Uint8Array, upToSeq: string): Promise<void> {
+    const client = await this.pool.connect();
+    try {
+      await client.query('BEGIN');
+      const locked = await client.query('SELECT 1 FROM filius_doc WHERE project_uuid = $1 FOR UPDATE', [project]);
+      if (locked.rowCount !== 1) throw new Error(`no document for project ${project}`);
+      await client.query('UPDATE filius_doc SET state = $2, rev = rev + 1, updated_at = now() WHERE project_uuid = $1', [project, Buffer.from(state)]);
+      await client.query('DELETE FROM filius_doc_update WHERE project_uuid = $1 AND seq <= $2', [project, upToSeq]);
+      await client.query('COMMIT');
+    } catch (error) {
+      await client.query('ROLLBACK');
+      throw error;
+    } finally {
+      client.release();
+    }
+  }
+
+  async pending(project: string): Promise<number> {
+    const result = await this.pool.query<{ count: string }>('SELECT count(*) FROM filius_doc_update WHERE project_uuid = $1', [project]);
+    return Number(result.rows[0]!.count);
+  }
+
+  async remove(project: string): Promise<void> {
+    await this.pool.query('DELETE FROM filius_doc WHERE project_uuid = $1', [project]);
+  }
+
+  async close(): Promise<void> {
+    await this.pool.end();
+  }
+}

packages/server/src/sync.ts

@@ -0,0 +1,113 @@
+// The shared documents (/sync, plan §5): Hocuspocus, a project a document.
+// A project is loaded from the store when its first person comes (a project
+// the store has not got is refused, with its reason: projects are made
+// through the service, F5.2), every
+// update stored as it comes, compacted a minute after the last change, when
+// the last person leaves and after COMPACT_AFTER_UPDATES updates. The
+// janitor (plan §5.3) runs only here, as a client of its own: once when the
+// document is loaded, then 250 ms after the last update that was not its
+// own; its repair goes to everyone like any update.
+
+import { Hocuspocus, type onAuthenticatePayload } from '@hocuspocus/server';
+import type * as Y from 'yjs';
+import { JANITOR_ORIGIN, repair } from '@mdg-filius/doc';
+import { errorText, type Log } from './log.js';
+import { DocumentPersistence } from './persistence.js';
+import type { DocStore } from './store.js';
+
+/** How long after the last change a document is compacted (Hocuspocus' debounce of onStoreDocument). */
+export const COMPACT_AFTER_IDLE_MILLISECONDS = 60_000;
+/** Compacted at the latest so long after the first change, even while changes keep coming. */
+export const COMPACT_AT_LATEST_MILLISECONDS = 600_000;
+/** The janitor's wait after an update (entprellt). */
+export const JANITOR_DELAY_MILLISECONDS = 250;
+
+/** What a connection may do; thrown, it is refused. */
+export interface Access {
+  readonly readOnly: boolean;
+  /** Who, for the log. */
+  readonly user?: string;
+}
+
+export interface SyncOptions {
+  readonly store: DocStore;
+  readonly log: Log;
+  readonly authenticate: (payload: onAuthenticatePayload) => Promise<Access>;
+  readonly compactAfterIdleMilliseconds?: number;
+}
+
+/** A refusal of a connection, with the reason the person's editor is told (Hocuspocus sends an error's `reason`). */
+export function refusal(reason: string): Error & { readonly reason: string } {
+  return Object.assign(new Error(reason), { reason });
+}
+
+/** A project's id: a UUID, as the dashboard makes them. */
+export const isProjectId = (name: string) => /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/.test(name);
+
+/** The janitor on a document; the function that ends it. */
+function startJanitor(doc: Y.Doc, project: string, log: Log): () => void {
+  let timer: ReturnType<typeof setTimeout> | undefined;
+  const run = () => {
+    timer = undefined;
+    const repairs = repair(doc);
+    const count = repairs.danglingLinks.length + repairs.sharedPortLinks.length + repairs.madeFolders.length;
+    if (count > 0) log.info('janitor.repaired', { project, danglingLinks: repairs.danglingLinks.length, sharedPortLinks: repairs.sharedPortLinks.length, madeFolders: repairs.madeFolders.length });
+  };
+  const onUpdate = (_update: Uint8Array, origin: unknown) => {
+    if (origin === JANITOR_ORIGIN) return;
+    if (timer !== undefined) clearTimeout(timer);
+    timer = setTimeout(run, JANITOR_DELAY_MILLISECONDS);
+  };
+  run();
+  doc.on('update', onUpdate);
+  return () => {
+    if (timer !== undefined) clearTimeout(timer);
+    doc.off('update', onUpdate);
+  };
+}
+
+export function createSync({ store, log, authenticate, compactAfterIdleMilliseconds = COMPACT_AFTER_IDLE_MILLISECONDS }: SyncOptions): Hocuspocus {
+  const open = new Map<string, { readonly persistence: DocumentPersistence; stopJanitor?: () => void }>();
+  return new Hocuspocus({
+    quiet: true,
+    debounce: compactAfterIdleMilliseconds,
+    maxDebounce: COMPACT_AT_LATEST_MILLISECONDS,
+    async onAuthenticate(payload) {
+      if (!isProjectId(payload.documentName)) throw refusal('notAProject');
+      const access = await authenticate(payload);
+      payload.connectionConfig.readOnly = access.readOnly;
+      return access;
+    },
+    async onLoadDocument({ documentName, document }) {
+      const persistence = new DocumentPersistence(store, documentName, log);
+      if (!(await persistence.load(document))) {
+        log.warn('doc.unknown', { project: documentName });
+        throw refusal('unknownProject');
+      }
+      open.set(documentName, { persistence });
+      log.info('doc.loaded', { project: documentName });
+    },
+    async afterLoadDocument({ documentName, document }) {
+      const entry = open.get(documentName);
+      if (entry !== undefined) entry.stopJanitor = startJanitor(document, documentName, log);
+    },
+    async onChange({ documentName, document, update }) {
+      await open.get(documentName)?.persistence.append(update, document);
+    },
+    async onStoreDocument({ documentName, document }) {
+      await open.get(documentName)?.persistence.compact(document);
+    },
+    async afterUnloadDocument({ documentName }) {
+      const entry = open.get(documentName);
+      if (entry === undefined) return;
+      open.delete(documentName);
+      entry.stopJanitor?.();
+      try {
+        await entry.persistence.flush();
+      } catch (error) {
+        log.error('doc.unloadFailed', { project: documentName, error: errorText(error) });
+      }
+      log.info('doc.unloaded', { project: documentName });
+    },
+  });
+}

packages/server/test/database.ts

@@ -0,0 +1,42 @@
+// A Postgres of its own for the server's tests: FILIUS_TEST_DATABASE_URL
+// when it is set, else a throwaway container of the postgres image (Docker),
+// stopped and gone after the tests. Never a database of the dashboard.
+
+import { execFileSync } from 'node:child_process';
+import pg from 'pg';
+
+export interface TestDatabase {
+  readonly url: string;
+  stop(): Promise<void>;
+}
+
+export async function testDatabase(): Promise<TestDatabase> {
+  const given = process.env.FILIUS_TEST_DATABASE_URL;
+  if (given !== undefined) return { url: given, stop: async () => {} };
+  const name = `filius-test-pg-${process.pid}-${Date.now()}`;
+  execFileSync('docker', ['run', '-d', '--rm', '--name', name, '-e', 'POSTGRES_PASSWORD=filius', '-e', 'POSTGRES_DB=filius', '-p', '127.0.0.1::5432', 'postgres:16-alpine'], { stdio: 'pipe' });
+  const stop = async () => {
+    execFileSync('docker', ['stop', name], { stdio: 'pipe' });
+  };
+  try {
+    const port = execFileSync('docker', ['port', name, '5432/tcp']).toString().trim().split('\n')[0]!.split(':').pop();
+    const url = `postgres://postgres:filius@127.0.0.1:${port}/filius`;
+    // The image's first start runs a server on its socket only; the TCP port answers once the real one is up.
+    for (let tries = 0; ; tries++) {
+      const client = new pg.Client({ connectionString: url });
+      try {
+        await client.connect();
+        await client.query('SELECT 1');
+        await client.end();
+        return { url, stop };
+      } catch (error) {
+        await client.end().catch(() => {});
+        if (tries > 120) throw error;
+        await new Promise((resolve) => setTimeout(resolve, 250));
+      }
+    }
+  } catch (error) {
+    await stop();
+    throw error;
+  }
+}

packages/server/test/sync.test.ts

@@ -0,0 +1,360 @@
+// Server F's shared documents (F5.1) with real clients (Hocuspocus' own
+// provider over WebSocket) and a real Postgres: a project comes back as it
+// was after the service restarts, two people see each other's changes at
+// once, the updates are compacted (after 500, a minute after the last
+// change, when the last person leaves), the janitor repairs what a merge
+// leaves — once, as a client of its own —, a project the service does not
+// know is refused and not made, read-only people cannot write, /healthz
+// answers, and the log is JSON.
+
+import { randomUUID } from 'node:crypto';
+import { HocuspocusProvider, HocuspocusProviderWebsocket } from '@hocuspocus/provider';
+import pg from 'pg';
+import { afterAll, beforeAll, describe, expect, it } from 'vitest';
+import WebSocket from 'ws';
+import * as Y from 'yjs';
+import { addDevice, connect, emptyProject, readProject, ROOTS, topologyView, writeProject } from '@mdg-filius/doc';
+import { COMPACT_AFTER_IDLE_MILLISECONDS, COMPACT_AFTER_UPDATES, JANITOR_DELAY_MILLISECONDS, PostgresStore, startServer, type Access, type FiliusServer, type Log } from '../src/index.js';
+import { testDatabase, type TestDatabase } from './database.js';
+
+let database: TestDatabase;
+let store: PostgresStore;
+let sql: pg.Pool;
+
+beforeAll(async () => {
+  database = await testDatabase();
+  store = PostgresStore.connect(database.url);
+  await store.migrate();
+  sql = new pg.Pool({ connectionString: database.url });
+}, 60_000);
+
+afterAll(async () => {
+  await sql?.end();
+  await store?.close();
+  await database?.stop();
+});
+
+/** The log's lines, kept. */
+function memoryLog(): Log & { readonly lines: Record<string, unknown>[] } {
+  const lines: Record<string, unknown>[] = [];
+  const entry =
+    (level: string) =>
+    (event: string, fields: Record<string, unknown> = {}) => {
+      lines.push(JSON.parse(JSON.stringify({ level, event, ...fields })) as Record<string, unknown>);
+    };
+  return { lines, info: entry('info'), warn: entry('warn'), error: entry('error') };
+}
+
+async function start(options: { readonly access?: Access; readonly idle?: number } = {}): Promise<FiliusServer & { readonly log: ReturnType<typeof memoryLog> }> {
+  const log = memoryLog();
+  const server = await startServer({
+    port: 0,
+    host: '127.0.0.1',
+    store,
+    log,
+    authenticate: async () => options.access ?? { readOnly: false },
+    ...(options.idle === undefined ? {} : { compactAfterIdleMilliseconds: options.idle }),
+  });
+  return Object.assign(server, { log });
+}
+
+/** A new project as the service makes it: an empty FILIUS project. */
+async function newProject(): Promise<string> {
+  const project = randomUUID();
+  const doc = new Y.Doc();
+  writeProject(doc, emptyProject());
+  expect(await store.create(project, Y.encodeStateAsUpdate(doc))).toBe(true);
+  return project;
+}
+
+interface Client {
+  readonly doc: Y.Doc;
+  readonly provider: HocuspocusProvider;
+  close(): void;
+}
+
+/** A person in a project: synced, or refused. `eager` comes back at once when let go (no delay, as a client may). */
+function join(server: FiliusServer, project: string, eager = false): Promise<Client> {
+  const doc = new Y.Doc();
+  const socket = new HocuspocusProviderWebsocket({ url: `ws://127.0.0.1:${server.port}/sync`, WebSocketPolyfill: WebSocket, ...(eager ? { delay: 0, minDelay: 0, jitter: false } : {}) });
+  return new Promise((resolve, reject) => {
+    const timer = setTimeout(() => {
+      provider.destroy();
+      socket.destroy();
+      reject(new Error('not synced within 3 s'));
+    }, 3_000);
+    const provider = new HocuspocusProvider({
+      websocketProvider: socket,
+      name: project,
+      document: doc,
+      token: 'test',
+      onSynced: () => {
+        clearTimeout(timer);
+        resolve({ doc, provider, close: () => (provider.destroy(), socket.destroy()) });
+      },
+      onAuthenticationFailed: ({ reason }) => {
+        clearTimeout(timer);
+        provider.destroy();
+        socket.destroy();
+        reject(new Error(`refused: ${reason}`));
+      },
+      onClose: () => reject(new Error('closed')),
+    });
+    provider.attach();
+  });
+}
+
+const until = async (condition: () => boolean | Promise<boolean>, milliseconds = 5_000) => {
+  for (let waited = 0; !(await condition()); waited += 25) {
+    if (waited > milliseconds) throw new Error('timed out');
+    await new Promise((resolve) => setTimeout(resolve, 25));
+  }
+};
+const deviceCount = (doc: Y.Doc) => doc.getMap(ROOTS.nodes).size;
+const NAMES = { pc: 'Rechner', networkSwitch: 'Switch' } as const;
+/** A device as the palette puts one down. */
+const place = (doc: Y.Doc, id: string, kind: keyof typeof NAMES, x: number, origin: string) => addDevice(doc, { id, kind, name: NAMES[kind], x, y: 0, newId: randomUUID }, origin);
+
+describe('server F keeps the shared documents', () => {
+  it('gives a project back as it was after the service restarts, from its compacted state', async () => {
+    const project = await newProject();
+    let server = await start();
+    const first = await join(server, project);
+    place(first.doc, 'a', 'pc', 10, 'person');
+    place(first.doc, 'b', 'networkSwitch', 200, 'person');
+    await until(async () => (await store.pending(project)) >= 2);
+    first.close();
+    // The last person left: compacted and unloaded.
+    await until(() => server.sync.getDocumentsCount() === 0);
+    expect(await store.pending(project)).toBe(0);
+    await server.close();
+
+    server = await start();
+    const again = await join(server, project);
+    expect(topologyView(again.doc).nodes.map((node) => [node.id, node.kind, node.x])).toEqual([
+      ['a', 'pc', 10],
+      ['b', 'networkSwitch', 200],
+    ]);
+    // Loading stores nothing anew.
+    expect(await store.pending(project)).toBe(0);
+    again.close();
+    await server.close();
+  });
+
+  it('takes the updates a hard stop left uncompacted, and counts them towards the next compaction', async () => {
+    const project = await newProject();
+    // The service stopped hard: three updates stored, never compacted.
+    const before = new Y.Doc();
+    Y.applyUpdate(before, (await store.load(project))!.state);
+    before.on('update', (update: Uint8Array) => void store.append(project, update));
+    for (const [id, x] of [
+      ['a', 0],
+      ['b', 100],
+      ['c', 200],
+    ] as const)
+      place(before, id, 'pc', x, 'person');
+    await until(async () => (await store.pending(project)) === 3);
+    const server = await start();
+    const person = await join(server, project);
+    expect(topologyView(person.doc).nodes.map((node) => node.id)).toEqual(['a', 'b', 'c']);
+    const notes = person.doc.getMap<number>('test');
+    for (let index = 0; index < COMPACT_AFTER_UPDATES - 3; index++) notes.set('n', index);
+    await until(() => server.log.lines.some((line) => line.event === 'doc.compacted'));
+    expect(server.log.lines.find((line) => line.event === 'doc.compacted')!.updates).toBe(COMPACT_AFTER_UPDATES);
+    person.close();
+    await server.close();
+  });
+
+  it('compacts once after exactly 500 updates, and not again when the document has been quiet since', async () => {
+    const project = await newProject();
+    const server = await start({ idle: 200 });
+    const person = await join(server, project);
+    const notes = person.doc.getMap<number>('test');
+    for (let index = 0; index < COMPACT_AFTER_UPDATES; index++) notes.set('n', index);
+    await until(() => server.log.lines.some((line) => line.event === 'doc.compacted'), 15_000);
+    await new Promise((resolve) => setTimeout(resolve, 600));
+    expect(server.log.lines.filter((line) => line.event === 'doc.compacted').map((line) => line.updates)).toEqual([COMPACT_AFTER_UPDATES]);
+    expect((await sql.query('SELECT rev FROM filius_doc WHERE project_uuid = $1', [project])).rows[0].rev).toBe(1);
+    person.close();
+    await server.close();
+  });
+
+  it('compacts nothing after a visit that changed nothing', async () => {
+    const project = await newProject();
+    const rev = async () => (await sql.query('SELECT rev FROM filius_doc WHERE project_uuid = $1', [project])).rows[0].rev as number;
+    const server = await start({ idle: 100 });
+    const person = await join(server, project);
+    await new Promise((resolve) => setTimeout(resolve, 300));
+    person.close();
+    await until(() => server.sync.getDocumentsCount() === 0);
+    expect(await rev()).toBe(0);
+    expect(server.log.lines.some((line) => line.event === 'doc.compacted')).toBe(false);
+    await server.close();
+  });
+
+  it('stores everything before it closes, with people still there who come back at once', async () => {
+    const project = await newProject();
+    const server = await start();
+    const person = await join(server, project, true);
+    place(person.doc, 'a', 'pc', 0, 'person');
+    await until(async () => (await store.pending(project)) === 1);
+    await server.close();
+    expect(await store.pending(project)).toBe(0);
+    const stored = new Y.Doc();
+    Y.applyUpdate(stored, (await store.load(project))!.state);
+    expect(deviceCount(stored)).toBe(1);
+    person.close();
+  });
+
+  it('says so in the log when a project went while it was open, and stores nothing in its place', async () => {
+    const project = await newProject();
+    const server = await start();
+    const person = await join(server, project);
+    place(person.doc, 'a', 'pc', 0, 'person');
+    await until(async () => (await store.pending(project)) === 1);
+    await store.remove(project);
+    person.close();
+    await until(() => server.sync.getDocumentsCount() === 0);
+    expect(server.log.lines.filter((line) => line.event === 'store.failed').map((line) => line.project)).toEqual([project]);
+    expect(await store.load(project)).toBeUndefined();
+    await server.close();
+  });
+
+  it('makes a project once: a second one of the same id changes nothing', async () => {
+    const project = await newProject();
+    const first = (await store.load(project))!.state;
+    expect(await store.create(project, new Uint8Array([0, 0]))).toBe(false);
+    expect((await store.load(project))!.state).toEqual(first);
+  });
+
+  it("shows each person the other's changes at once", async () => {
+    const project = await newProject();
+    const server = await start();
+    const [mia, ben] = await Promise.all([join(server, project), join(server, project)]);
+    place(mia.doc, 'pc', 'pc', 0, 'mia');
+    place(ben.doc, 'switch', 'networkSwitch', 100, 'ben');
+    await until(() => deviceCount(mia.doc) === 2 && deviceCount(ben.doc) === 2);
+    connect(mia.doc, { node: 'pc' }, { node: 'switch' }, 'cable', 'mia');
+    await until(() => ben.doc.getMap(ROOTS.links).size === 1);
+    expect(readProject(ben.doc)).toEqual(readProject(mia.doc));
+    mia.close();
+    ben.close();
+    await server.close();
+  });
+
+  it(`compacts after ${COMPACT_AFTER_UPDATES} updates, and a minute after the last change`, async () => {
+    const project = await newProject();
+    const server = await start({ idle: 300 });
+    const person = await join(server, project);
+    const notes = person.doc.getMap<number>('test');
+    for (let index = 0; index < COMPACT_AFTER_UPDATES + 20; index++) notes.set('n', index);
+    await until(async () => (await sql.query('SELECT rev FROM filius_doc WHERE project_uuid = $1', [project])).rows[0].rev >= 1, 15_000);
+    // The compaction took the first 500; what came after waits — until the document has been quiet for a while.
+    await until(async () => (await store.pending(project)) === 0, 10_000);
+    expect(server.log.lines.filter((line) => line.event === 'doc.compacted').map((line) => line.updates)).toEqual([COMPACT_AFTER_UPDATES, 20]);
+    const stored = await store.load(project);
+    const reloaded = new Y.Doc();
+    Y.applyUpdate(reloaded, stored!.state);
+    expect(reloaded.getMap('test').get('n')).toBe(COMPACT_AFTER_UPDATES + 19);
+    person.close();
+    await server.close();
+  });
+
+  it('repairs what a merge leaves, once, as a client of its own', async () => {
+    const project = await newProject();
+    const server = await start();
+    const [mia, ben] = await Promise.all([join(server, project), join(server, project)]);
+    place(mia.doc, 'pc', 'pc', 0, 'mia');
+    place(mia.doc, 'switch', 'networkSwitch', 100, 'mia');
+    await until(() => deviceCount(ben.doc) === 2);
+    // Both plug a cable into the computer's one card at once; Mia's goes to a port that is gone too.
+    const port = topologyView(mia.doc).nodes.find((node) => node.id === 'pc')!.ports[0]!.id;
+    const switchPorts = topologyView(mia.doc)
+      .nodes.find((node) => node.id === 'switch')!
+      .ports.map((each) => each.id);
+    const cable = (doc: Y.Doc, id: string, targetPort: string) =>
+      doc.transact(() => {
+        const link = new Y.Map<unknown>();
+        for (const [key, value] of Object.entries({ sourceNode: 'pc', sourcePort: port, targetNode: 'switch', targetPort, order: 0 })) link.set(key, value);
+        doc.getMap(ROOTS.links).set(id, link);
+      }, 'person');
+    cable(mia.doc, 'cable-b', switchPorts[0]!);
+    cable(ben.doc, 'cable-a', switchPorts[1]!);
+    cable(mia.doc, 'cable-c', 'no-such-port');
+    await until(() => [...ben.doc.getMap(ROOTS.links).keys()].join() === 'cable-a' && [...mia.doc.getMap(ROOTS.links).keys()].join() === 'cable-a');
+    await new Promise((resolve) => setTimeout(resolve, 600));
+    const repairs = server.log.lines.filter((line) => line.event === 'janitor.repaired');
+    expect(repairs.reduce((sum, line) => sum + (line.danglingLinks as number) + (line.sharedPortLinks as number), 0)).toBe(2);
+    mia.close();
+    ben.close();
+    await server.close();
+  });
+
+  it('repairs a stored document when it is loaded, before anyone changes it', async () => {
+    const project = randomUUID();
+    const doc = new Y.Doc();
+    writeProject(doc, emptyProject());
+    doc.transact(() => {
+      const link = new Y.Map<unknown>();
+      for (const [key, value] of Object.entries({ sourceNode: 'gone', sourcePort: 'p', targetNode: 'gone too', targetPort: 'q', order: 0 })) link.set(key, value);
+      doc.getMap(ROOTS.links).set('stray', link);
+    });
+    await store.create(project, Y.encodeStateAsUpdate(doc));
+    const server = await start();
+    const person = await join(server, project);
+    await until(() => person.doc.getMap(ROOTS.links).size === 0);
+    expect(server.log.lines.find((line) => line.event === 'janitor.repaired')).toMatchObject({ project, danglingLinks: 1 });
+    person.close();
+    await server.close();
+  });
+
+  it("keeps the plan's measures: compacted after 500 updates or a minute of quiet, repaired 250 ms after a change", () => {
+    expect([COMPACT_AFTER_UPDATES, COMPACT_AFTER_IDLE_MILLISECONDS, JANITOR_DELAY_MILLISECONDS]).toEqual([500, 60_000, 250]);
+  });
+
+  it('refuses a project it does not know, and makes none', async () => {
+    const server = await start();
+    const project = randomUUID();
+    await expect(join(server, project)).rejects.toThrow('refused: unknownProject');
+    expect(await store.load(project)).toBeUndefined();
+    await expect(join(server, 'not-a-project')).rejects.toThrow('refused: notAProject');
+    expect(server.log.lines.some((line) => line.event === 'doc.unknown' && line.project === project)).toBe(true);
+    await server.close();
+  });
+
+  it('lets a reader see the document and change nothing', async () => {
+    const project = await newProject();
+    const writerServer = await start();
+    const writer = await join(writerServer, project);
+    place(writer.doc, 'pc', 'pc', 0, 'person');
+    await until(async () => (await store.pending(project)) >= 1);
+    writer.close();
+    await writerServer.close();
+    const server = await start({ access: { readOnly: true } });
+    const reader = await join(server, project);
+    expect(deviceCount(reader.doc)).toBe(1);
+    place(reader.doc, 'switch', 'networkSwitch', 0, 'reader');
+    await new Promise((resolve) => setTimeout(resolve, 400));
+    expect(await store.pending(project)).toBe(0);
+    reader.close();
+    await server.close();
+    const stored = new Y.Doc();
+    Y.applyUpdate(stored, (await store.load(project))!.state);
+    expect(deviceCount(stored)).toBe(1);
+  });
+
+  it('answers /healthz, refuses other ways in, and logs JSON', async () => {
+    const server = await start();
+    const health = await fetch(`http://127.0.0.1:${server.port}/healthz`);
+    expect([health.status, await health.json()]).toEqual([200, { status: 'ok', documents: 0, connections: 0 }]);
+    expect((await fetch(`http://127.0.0.1:${server.port}/nothing`)).status).toBe(404);
+    const refused = await new Promise<number>((resolve) => {
+      const socket = new WebSocket(`ws://127.0.0.1:${server.port}/elsewhere`);
+      socket.on('unexpected-response', (_request, response) => resolve(response.statusCode ?? 0));
+      socket.on('error', () => resolve(-1));
+    });
+    expect(refused).toBe(404);
+    await server.close();
+    expect(server.log.lines.map((line) => line.event)).toEqual(['server.listening', 'server.closed']);
+  });
+});

packages/server/test/tsconfig.json

@@ -0,0 +1,11 @@
+{
+  "extends": "../../../tsconfig.base.json",
+  "compilerOptions": {
+    "noEmit": true,
+    "composite": false,
+    "declaration": false,
+    "types": ["node"],
+    "resolveJsonModule": true
+  },
+  "include": ["."]
+}

packages/server/tsconfig.json

@@ -0,0 +1,10 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "compilerOptions": {
+    "rootDir": "src",
+    "outDir": "dist",
+    "types": ["node"]
+  },
+  "include": ["src"],
+  "references": [{ "path": "../core" }, { "path": "../doc" }, { "path": "../protocol" }, { "path": "../sim" }]
+}