feat(doc): F4.1 the shared document — a project in a Y.Doc and back, the janitor

packages/doc holds the editor's model (ImportedProject) losslessly in a Yjs
document (schema 1): ports with their settings, each node's settings as
maps per field, programs, files as a flat path map in creation order (text
as Y.Text), own Java applications, links, documentation items, archive
entries. The core's snapshot and the .fls export go through the project.
The janitor repairs dangling links, two links at one port and files
without their folder, deterministically and idempotently. All 23 corpus
files come back through a Yjs update with the same snapshot and export.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

packages/doc/test/collaboration.test.ts

@@ -0,0 +1,201 @@
+// Two people on one document: what Yjs merges, and what the janitor repairs
+// after a merge (plan §5.3). Each case starts from one example in two
+// replicas, changes them apart, exchanges their updates both ways and looks
+// at the project they hold.
+
+import { readFileSync } from 'node:fs';
+import { dirname, join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { describe, expect, it } from 'vitest';
+import * as Y from 'yjs';
+import { openProjectArchive, type ImportedProject } from '@mdg-filius/fls';
+import { readProject, repair, ROOTS, writeProject } from '../src/index.js';
+
+const corpus = join(dirname(fileURLToPath(import.meta.url)), '../../../corpus');
+
+function opened(file: string): ImportedProject {
+  let counter = 0;
+  return openProjectArchive(new Uint8Array(readFileSync(join(corpus, file))), () => `00000000-0000-0000-0000-${(++counter).toString(16).padStart(12, '0')}`);
+}
+
+/** Two replicas of the example, alike. */
+function replicas(file = 'java-examples/zwei_rechnernetze_komplett.fls'): [Y.Doc, Y.Doc] {
+  const first = new Y.Doc();
+  writeProject(first, opened(file));
+  const second = new Y.Doc();
+  Y.applyUpdate(second, Y.encodeStateAsUpdate(first));
+  return [first, second];
+}
+
+/** Each replica gets what the other has. */
+function exchange(first: Y.Doc, second: Y.Doc): void {
+  const toSecond = Y.encodeStateAsUpdate(first, Y.encodeStateVector(second));
+  const toFirst = Y.encodeStateAsUpdate(second, Y.encodeStateVector(first));
+  Y.applyUpdate(second, toSecond);
+  Y.applyUpdate(first, toFirst);
+}
+
+const nodes = (doc: Y.Doc) => doc.getMap<Y.Map<unknown>>(ROOTS.nodes);
+const links = (doc: Y.Doc) => doc.getMap<Y.Map<unknown>>(ROOTS.links);
+const nodeNamed = (doc: Y.Doc, name: string) => [...nodes(doc).entries()].find(([, node]) => node.get('name') === name)!;
+const settings = (doc: Y.Doc, name: string) => nodeNamed(doc, name)[1].get('settings') as Y.Map<Y.Map<unknown>>;
+const files = (doc: Y.Doc, name: string) => nodeNamed(doc, name)[1].get('files') as Y.Map<Y.Map<unknown>>;
+
+/** A switch's port no cable uses. */
+function freePort(doc: Y.Doc, switchName: string): [string, string] {
+  const [id, node] = nodeNamed(doc, switchName);
+  const used = new Set([...links(doc).values()].flatMap((link) => [`${link.get('sourceNode')}/${link.get('sourcePort')}`, `${link.get('targetNode')}/${link.get('targetPort')}`]));
+  const port = (node.get('ports') as Y.Array<Y.Map<unknown>>).toArray().find((each) => !used.has(`${id}/${each.get('id')}`))!;
+  return [id, port.get('id') as string];
+}
+
+function plug(doc: Y.Doc, id: string, [sourceNode, sourcePort]: [string, string], [targetNode, targetPort]: [string, string]): void {
+  const link = new Y.Map<unknown>();
+  link.set('order', links(doc).size);
+  link.set('sourceNode', sourceNode);
+  link.set('sourcePort', sourcePort);
+  link.set('targetNode', targetNode);
+  link.set('targetPort', targetPort);
+  links(doc).set(id, link);
+}
+
+describe('two people on one document', () => {
+  it('keeps both changes to two fields of one device', () => {
+    const [first, second] = replicas();
+    settings(first, 'Notebook 1.10').get('device')!.set('ipAddress', '141.99.1.20');
+    settings(second, 'Notebook 1.10').get('device')!.set('defaultGateway', '141.99.1.2');
+    exchange(first, second);
+    for (const doc of [first, second]) {
+      const project = readProject(doc);
+      const id = project.graph.nodes.find((node) => node.displayName === 'Notebook 1.10')!.id;
+      expect(project.deviceConfigurations.get(id)).toMatchObject({ ipAddress: '141.99.1.20', defaultGateway: '141.99.1.2' });
+    }
+  });
+
+  it('keeps both rows two people add to a list at once', () => {
+    const [first, second] = replicas();
+    const row = (name: string) => {
+      const map = new Y.Map<unknown>();
+      map.set('name', name);
+      map.set('type', 'A');
+      map.set('ttlSeconds', 3600);
+      map.set('target', '141.99.1.10');
+      return map;
+    };
+    const records = (doc: Y.Doc) => settings(doc, 'DNS-Server 2.10').get('dnsServer')!.get('records') as Y.Array<Y.Map<unknown>>;
+    records(first).push([row('eins.filius.de')]);
+    records(second).push([row('zwei.filius.de')]);
+    exchange(first, second);
+    const names = (doc: Y.Doc) => {
+      const project = readProject(doc);
+      const id = project.graph.nodes.find((node) => node.displayName === 'DNS-Server 2.10')!.id;
+      return project.dnsServerConfigurations.get(id)!.records.map((record) => record.name);
+    };
+    expect(names(first)).toEqual(names(second));
+    expect(names(first)).toEqual(expect.arrayContaining(['www.filius.de', 'eins.filius.de', 'zwei.filius.de']));
+  });
+
+  it('writes text two people type into one file together', () => {
+    const [first, second] = replicas();
+    const text = (doc: Y.Doc) => [...files(doc, 'Notebook 1.11').values()].find((entry) => entry.get('kind') === 'text')!.get('text') as Y.Text;
+    const before = text(first).toString();
+    text(first).insert(0, 'A');
+    text(second).insert(before.length, 'B');
+    exchange(first, second);
+    expect(text(first).toString()).toBe(`A${before}B`);
+    expect(text(second).toString()).toBe(`A${before}B`);
+  });
+});
+
+describe('the janitor', () => {
+  it('removes a cable to a device removed at the same time (I1)', () => {
+    const [first, second] = replicas();
+    const [notebook, notebookNode] = nodeNamed(second, 'Notebook 1.11');
+    const notebookPort = ((notebookNode.get('ports') as Y.Array<Y.Map<unknown>>).get(0).get('id') as string);
+    // One person removes the notebook with its cable, the other plugs a second cable into it.
+    first.transact(() => {
+      for (const [id, link] of links(first)) if (link.get('sourceNode') === notebook || link.get('targetNode') === notebook) links(first).delete(id);
+      nodes(first).delete(notebook);
+    });
+    plug(second, 'ffffffff-0000-0000-0000-000000000001', freePort(second, 'Switch 1'), [notebook, notebookPort]);
+    exchange(first, second);
+    expect(links(first).has('ffffffff-0000-0000-0000-000000000001')).toBe(true);
+
+    expect(repair(first)).toEqual({ danglingLinks: ['ffffffff-0000-0000-0000-000000000001'], sharedPortLinks: [], madeFolders: [] });
+    expect(links(first).has('ffffffff-0000-0000-0000-000000000001')).toBe(false);
+    // Idempotent: nothing left to do, and nothing written.
+    let updates = 0;
+    first.on('update', () => (updates += 1));
+    expect(repair(first)).toEqual({ danglingLinks: [], sharedPortLinks: [], madeFolders: [] });
+    expect(updates).toBe(0);
+  });
+
+  it('keeps the cable with the smaller id where two were plugged into one port at once (I2)', () => {
+    const [first, second] = replicas();
+    const port = freePort(first, 'Switch 1');
+    plug(first, 'eeeeeeee-0000-0000-0000-000000000002', port, freePort(first, 'Switch 2'));
+    plug(second, 'eeeeeeee-0000-0000-0000-000000000001', port, freePort(second, 'Switch 2'));
+    exchange(first, second);
+    // Both replicas repair, as two janitors would: they end alike.
+    expect(repair(first).sharedPortLinks).toEqual(['eeeeeeee-0000-0000-0000-000000000002']);
+    expect(repair(second).sharedPortLinks).toEqual(['eeeeeeee-0000-0000-0000-000000000002']);
+    exchange(first, second);
+    expect(readProject(first)).toEqual(readProject(second));
+    expect(links(first).has('eeeeeeee-0000-0000-0000-000000000001')).toBe(true);
+    expect(links(first).has('eeeeeeee-0000-0000-0000-000000000002')).toBe(false);
+  });
+
+  it('makes the folder of a file written into a folder removed at the same time (I4)', () => {
+    const [first, second] = replicas();
+    const folder = [...files(first, 'Notebook 1.11').entries()].find(([path, entry]) => entry.get('kind') === 'directory' && path.split('/').length === 2)![0];
+    first.transact(() => {
+      for (const path of [...files(first, 'Notebook 1.11').keys()]) if (path === folder || path.startsWith(`${folder}/`)) files(first, 'Notebook 1.11').delete(path);
+    });
+    const note = new Y.Map<unknown>();
+    note.set('order', 1000);
+    note.set('kind', 'text');
+    note.set('text', new Y.Text('neu'));
+    files(second, 'Notebook 1.11').set(`${folder}/sub/notiz.txt`, note);
+    exchange(first, second);
+
+    // Read tolerantly before the janitor: the file is left out, the project is whole.
+    const id = nodeNamed(first, 'Notebook 1.11')[0];
+    expect(readProject(first).virtualFileSystems.get(id)!.contains(`${folder}/sub/notiz.txt`)).toBe(false);
+    expect(repair(first).madeFolders).toEqual([`${id}:${folder}`, `${id}:${folder}/sub`]);
+    expect(readProject(first).virtualFileSystems.get(id)!.textFile(`${folder}/sub/notiz.txt`)).toBe('neu');
+    expect(repair(first).madeFolders).toEqual([]);
+  });
+
+  it('takes a folder in another case as the folder (I4)', () => {
+    const [doc] = replicas();
+    const [folder] = [...files(doc, 'Notebook 1.11').entries()].find(([path, entry]) => entry.get('kind') === 'directory' && path.split('/').length === 2)!;
+    const note = new Y.Map<unknown>();
+    note.set('order', 1000);
+    note.set('kind', 'text');
+    note.set('text', new Y.Text('neu'));
+    files(doc, 'Notebook 1.11').set(`${folder.toUpperCase()}/notiz.txt`, note);
+    expect(repair(doc).madeFolders).toEqual([]);
+  });
+});
+
+describe('a project in its document, beyond the corpus', () => {
+  it('keeps a Java long of an own application, binary files and an optional field left out', () => {
+    const project = opened('java-examples/anwendung_komplett.fls');
+    const [nodeId, applications] = [...project.ownApplicationsByNodeId][0]!;
+    project.ownApplicationsByNodeId.set(nodeId, [
+      ...applications,
+      { key: 'eigene', javaClass: 'demo.Eigene', bean: { kind: 'object', className: 'demo.Eigene', statements: [{ set: 'zahl', value: { kind: 'long', value: 12_345_678_901_234_567_890n } }] } },
+    ]);
+    const fileSystem = project.virtualFileSystems.get(nodeId)!;
+    fileSystem.writeBinaryFile('/daten.bin', new Uint8Array([0, 1, 2, 255]));
+    fileSystem.writeBinaryFile('/typ.bin', new Uint8Array([7]), 'application/octet-stream');
+    const doc = new Y.Doc();
+    writeProject(doc, project);
+    const other = new Y.Doc();
+    Y.applyUpdate(other, Y.encodeStateAsUpdate(doc));
+    const back = readProject(other);
+    expect(back.ownApplicationsByNodeId).toEqual(project.ownApplicationsByNodeId);
+    expect(back.virtualFileSystems.get(nodeId)!.entry('/daten.bin').content).toEqual({ kind: 'binary', data: new Uint8Array([0, 1, 2, 255]) });
+    expect(back.virtualFileSystems.get(nodeId)!.entry('/typ.bin').content).toEqual({ kind: 'binary', data: new Uint8Array([7]), mediaType: 'application/octet-stream' });
+  });
+});

packages/doc/test/projectDoc.test.ts

@@ -0,0 +1,70 @@
+// A project in its document and back, for every file of the corpus: written
+// into a document, sent as Yjs sends it (an update into a second document)
+// and read back, it is the same project — the same snapshot for the core,
+// the same .fls on export, each folder's files in the same order.
+
+import { readdirSync, readFileSync } from 'node:fs';
+import { dirname, join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { describe, expect, it } from 'vitest';
+import * as Y from 'yjs';
+import type { VirtualFileSystem } from '@mdg-filius/core';
+import { openProjectArchive, projectSnapshot, saveProjectArchive, type ImportedProject } from '@mdg-filius/fls';
+import { docArchive, docSnapshot, readProject, ROOTS, SCHEMA_VERSION, writeProject } from '../src/index.js';
+
+const corpus = join(dirname(fileURLToPath(import.meta.url)), '../../../corpus');
+const files = ['java-examples', 'java-tests', 'ipad-fixtures'].flatMap((folder) =>
+  readdirSync(join(corpus, folder))
+    .filter((name) => name.endsWith('.fls'))
+    .map((name) => join(folder, name)),
+);
+
+/** Ids that fall in the file's reverse order: the document must keep the order itself, not by its ids. */
+function opened(file: string): ImportedProject {
+  let counter = 0;
+  return openProjectArchive(new Uint8Array(readFileSync(join(corpus, file))), () => `00000000-0000-0000-0000-${(0xffff_ffff - ++counter).toString(16).padStart(12, '0')}`);
+}
+
+/** The document as another client gets it: its whole state as one update. */
+function sent(doc: Y.Doc): Y.Doc {
+  const other = new Y.Doc();
+  Y.applyUpdate(other, Y.encodeStateAsUpdate(doc));
+  return other;
+}
+
+/** Every folder's entries in the order they were made. */
+function creationOrder(files: VirtualFileSystem): string[] {
+  const paths: string[] = [];
+  const walk = (folder: string): void => {
+    for (const entry of files.entriesInCreationOrder(folder)) {
+      paths.push(entry.path);
+      if (entry.content.kind === 'directory') walk(entry.path);
+    }
+  };
+  walk('/');
+  return paths;
+}
+
+describe('a project in its document and back', () => {
+  it('reads the whole corpus', () => {
+    expect(files.length).toBe(23);
+  });
+
+  for (const file of files) {
+    it(file, () => {
+      const project = opened(file);
+      const doc = new Y.Doc();
+      writeProject(doc, project);
+      expect(doc.getMap(ROOTS.meta).get('schema')).toBe(SCHEMA_VERSION);
+      const received = sent(doc);
+      const back = readProject(received);
+
+      expect({ ...back, warnings: [] }).toEqual({ ...project, warnings: [] });
+      for (const [nodeId, fileSystem] of project.virtualFileSystems) {
+        expect(creationOrder(back.virtualFileSystems.get(nodeId)!), nodeId).toEqual(creationOrder(fileSystem));
+      }
+      expect(docSnapshot(received)).toEqual(projectSnapshot(project));
+      expect(docArchive(received).archive).toEqual(saveProjectArchive(project).archive);
+    });
+  }
+});

packages/doc/test/tsconfig.json

@@ -0,0 +1,15 @@
+{
+  "extends": "../../../tsconfig.base.json",
+  "compilerOptions": {
+    "noEmit": true,
+    "composite": false,
+    "declaration": false,
+    "types": [
+      "node"
+    ],
+    "resolveJsonModule": true
+  },
+  "include": [
+    "."
+  ]
+}

packages/doc/test/values.test.ts

@@ -0,0 +1,42 @@
+// The small parts of the document: plain values in Yjs and back, and the
+// order of entries two people placed at once.
+
+import { describe, expect, it } from 'vitest';
+import * as Y from 'yjs';
+import { fromY, inOrder, toY } from '../src/index.js';
+
+/** A value as the document holds it, inside a document (a Yjs type outside one cannot be read). */
+function held(value: Parameters<typeof toY>[0]): unknown {
+  const doc = new Y.Doc();
+  doc.getMap('root').set('value', toY(value));
+  return doc.getMap('root').get('value');
+}
+
+describe('plain values in the document', () => {
+  it('leaves out a field that is undefined', () => {
+    const map = held({ a: 1, b: undefined, c: null }) as Y.Map<unknown>;
+    expect([...map.keys()].sort()).toEqual(['a', 'c']);
+    expect(fromY(map)).toStrictEqual({ a: 1, c: null });
+  });
+
+  it('keeps lists, nested objects, bytes and the values of JSON', () => {
+    const value = { rows: [{ name: 'x', ttl: 3600 }, { name: 'y', ttl: 0 }], on: true, off: false, text: 'ä', none: null, bytes: new Uint8Array([1, 2]) };
+    const map = held(value) as Y.Map<unknown>;
+    expect(map.get('rows')).toBeInstanceOf(Y.Array);
+    expect((map.get('rows') as Y.Array<unknown>).get(0)).toBeInstanceOf(Y.Map);
+    expect(fromY(map)).toStrictEqual(value);
+  });
+});
+
+describe('the order of entries', () => {
+  it('orders by place, and two entries given one place by their keys', () => {
+    const doc = new Y.Doc();
+    const map = doc.getMap<Y.Map<unknown>>('entries');
+    for (const [key, order] of [['c', 1], ['b', 0], ['a', 1], ['d', -1]] as const) {
+      const entry = new Y.Map<unknown>();
+      entry.set('order', order);
+      map.set(key, entry);
+    }
+    expect(inOrder(map).map(([key]) => key)).toEqual(['d', 'b', 'a', 'c']);
+  });
+});