diff --git a/CHANGES.md b/CHANGES.md index 020d549..9fe9137 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -109,6 +109,17 @@ Swift-Pfade relativ zu `filius-ipad/ios/FiliusPad/TopologyEditor/` (Stand | `test/corpus.test.ts` | — | neu: 23 Korpus-Dateien, gegengeprüft mit `fflate.unzipSync` | | `test/pathFolding.test.ts`, `test/path-folding.*.json` | — | neu: welche Pfade dieselbe Datei meinen, Ergebnis aus dem Swift-Kern | +## packages/doc + +| Datei | Quelle | Art der Übertragung | +| --- | --- | --- | +| `src/projectDoc.ts` | Plan §5.2 (Schema 1); das Modell ist `ImportedProject` aus `packages/fls` | neu (F4.1): ein Projekt verlustfrei in ein `Y.Doc` und zurück — Geräte, Ports mit ihren Einstellungen, Einstellungen je Gerät, installierte und eingeschaltete Programme, Dateien (Text als `Y.Text`, je Ordner in der Reihenfolge des Anlegens), eigene Java-Anwendungen (als Text, `long` erhalten), Kabel, Doku-Elemente, Archiveinträge; Reihenfolge über `order`, Gleichstand nach Schlüssel; Dateien tolerant gelesen | +| `src/values.ts` | — | neu (F4.1): Einstellungen als `Y.Map` je Feld und Listen als `Y.Array`, damit gleichzeitige Änderungen verschiedener Felder und Zeilen beide bleiben | +| `src/janitor.ts` | Plan §5.3 | neu (F4.1): der Hausmeister, I1 (Kabel ins Leere), I2 (zwei Kabel an einem Port: kleinere Kennung bleibt), I4 (fehlender Ordner, ohne Rücksicht auf Groß-/Kleinschreibung); deterministisch, idempotent, Ursprung `janitor` | +| `src/snapshot.ts` | — | neu (F4.1): Schnappschuss des Kerns und `.fls`-Export eines Dokuments, beide über das Projekt (`projectSnapshot`, `saveProjectArchive`) | +| `test/projectDoc.test.ts` | — | neu (F4.1): alle 23 Korpusdateien → Dokument → Yjs-Update → Projekt: dasselbe Projekt, derselbe Schnappschuss, dieselbe `.fls`, dieselbe Reihenfolge je Ordner; Kennungen absichtlich gegen die Dateireihenfolge | +| `test/collaboration.test.ts`, `test/values.test.ts` | — | neu (F4.1): zwei Replikate — zwei Felder eines Geräts, zwei neue Zeilen, gemeinsames Tippen; der Hausmeister nach einem Zusammenführen (I1, I2 auf beiden Replikaten gleich, I4 auch in anderer Schreibweise), idempotent ohne Aktualisierung; `long` einer eigenen Anwendung, Binärdateien | + ## tools/swift-oracle Baut den unveränderten Swift-Kern (Stand `3b5ded0`) unter Linux, nur zwei diff --git a/README.md b/README.md index cbdffaa..95349c9 100644 --- a/README.md +++ b/README.md @@ -26,8 +26,8 @@ die Herkunft je Paket. Lizenz dieses Repos: **GPL-3.0** (`LICENSE`). | Pfad | Inhalt | | --- | --- | | `packages/core` | Simulationskern: Modell, Ereignis-Warteschlange, Protokolle, Anwendungen — ohne DOM, ohne Node | -| `packages/fls` | `.fls` lesen, schreiben, Unbekanntes bewahren *(folgt)* | -| `packages/doc` | gemeinsames Dokument (Yjs) *(folgt)* | +| `packages/fls` | `.fls` lesen, schreiben, Unbekanntes bewahren | +| `packages/doc` | gemeinsames Dokument (Yjs): Projekt ⇄ Dokument, Schnappschuss des Kerns, Hausmeister | | `packages/editor` | Oberfläche im Iframe des Dashboards *(folgt)* | | `server` | Filius-Dienst auf Server F *(folgt)* | diff --git a/package-lock.json b/package-lock.json index 8ada56e..ccdd336 100644 --- a/package-lock.json +++ b/package-lock.json @@ -471,6 +471,10 @@ "resolved": "packages/core", "link": true }, + "node_modules/@mdg-filius/doc": { + "resolved": "packages/doc", + "link": true + }, "node_modules/@mdg-filius/fls": { "resolved": "packages/fls", "link": true @@ -1175,6 +1179,16 @@ "node": "^8.16.0 || ^10.6.0 || >=11.0.0" } }, + "node_modules/isomorphic.js": { + "version": "0.2.5", + "resolved": "https://registry.npmjs.org/isomorphic.js/-/isomorphic.js-0.2.5.tgz", + "integrity": "sha512-PIeMbHqMt4DnUP3MA/Flc0HElYjMXArsw1qwJZcm9sqR8mq3l8NYizFMty0pWwE/tzIGH3EKK5+jes5mAr85yw==", + "license": "MIT", + "funding": { + "type": "GitHub Sponsors ❤", + "url": "https://github.com/sponsors/dmonad" + } + }, "node_modules/js-tokens": { "version": "9.0.1", "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-9.0.1.tgz", @@ -1182,6 +1196,27 @@ "dev": true, "license": "MIT" }, + "node_modules/lib0": { + "version": "0.2.119", + "resolved": "https://registry.npmjs.org/lib0/-/lib0-0.2.119.tgz", + "integrity": "sha512-vh+TiejVy8Xm+hCdnRfn9BthGNvNUZdEXltNjVZL3TzRIKP3s1lnQh8pWd/Q0wCgY0FPAGVVjFF4ZiHUV3LFgA==", + "license": "MIT", + "dependencies": { + "isomorphic.js": "^0.2.4" + }, + "bin": { + "0ecdsa-generate-keypair": "bin/0ecdsa-generate-keypair.js", + "0gentesthtml": "bin/gentesthtml.js", + "0serve": "bin/0serve.js" + }, + "engines": { + "node": ">=16" + }, + "funding": { + "type": "GitHub Sponsors ❤", + "url": "https://github.com/sponsors/dmonad" + } + }, "node_modules/loupe": { "version": "3.2.1", "resolved": "https://registry.npmjs.org/loupe/-/loupe-3.2.1.tgz", @@ -1669,11 +1704,38 @@ "integrity": "sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==", "license": "MIT" }, + "node_modules/yjs": { + "version": "13.6.33", + "resolved": "https://registry.npmjs.org/yjs/-/yjs-13.6.33.tgz", + "integrity": "sha512-q/UYvr1gOk4XII+Cu1CW2dINjYqJJFifTzogEaIARznrvX7uOlHsdTEXcnOBMuIsrCGEEKdNxi74kyMLZlKVfw==", + "license": "MIT", + "dependencies": { + "lib0": "^0.2.99" + }, + "engines": { + "node": ">=16.0.0", + "npm": ">=8.0.0" + }, + "funding": { + "type": "GitHub Sponsors ❤", + "url": "https://github.com/sponsors/dmonad" + } + }, "packages/core": { "name": "@mdg-filius/core", "version": "0.0.0", "license": "GPL-3.0-only" }, + "packages/doc": { + "name": "@mdg-filius/doc", + "version": "0.0.0", + "license": "GPL-3.0-only", + "dependencies": { + "@mdg-filius/core": "0.0.0", + "@mdg-filius/fls": "0.0.0", + "yjs": "^13.6.33" + } + }, "packages/fls": { "name": "@mdg-filius/fls", "version": "0.0.0", diff --git a/package.json b/package.json index 7a741d3..920a197 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,7 @@ }, "scripts": { "test": "vitest run", - "typecheck": "tsc -b && tsc -p packages/core/test && tsc -p packages/fls/test", + "typecheck": "tsc -b && tsc -p packages/core/test && tsc -p packages/fls/test && tsc -p packages/doc/test", "check": "npm run typecheck && npm run test", "hooks": "git config core.hooksPath scripts/hooks", "oracle": "tools/swift-oracle/run.sh", diff --git a/packages/doc/package.json b/packages/doc/package.json new file mode 100644 index 0000000..9691bce --- /dev/null +++ b/packages/doc/package.json @@ -0,0 +1,15 @@ +{ + "name": "@mdg-filius/doc", + "version": "0.0.0", + "private": true, + "license": "GPL-3.0-only", + "type": "module", + "exports": { + ".": "./src/index.ts" + }, + "dependencies": { + "@mdg-filius/core": "0.0.0", + "@mdg-filius/fls": "0.0.0", + "yjs": "^13.6.33" + } +} diff --git a/packages/doc/src/index.ts b/packages/doc/src/index.ts new file mode 100644 index 0000000..b6f63fe --- /dev/null +++ b/packages/doc/src/index.ts @@ -0,0 +1,5 @@ +// Public surface of the shared document. +export * from './janitor.js'; +export * from './projectDoc.js'; +export * from './snapshot.js'; +export * from './values.js'; diff --git a/packages/doc/src/janitor.ts b/packages/doc/src/janitor.ts new file mode 100644 index 0000000..0859f70 --- /dev/null +++ b/packages/doc/src/janitor.ts @@ -0,0 +1,100 @@ +// The janitor (plan §5.3): what a merge of two documents can leave that the +// model cannot hold, repaired deterministically. It depends only on the +// document, never on the clock, so that a repair run twice changes nothing +// the second time, and two replicas that repair the same document end +// alike. Only F runs it, as a client of its own (origin 'janitor'); clients +// draw tolerantly instead (a cable without an end is not drawn). +// +// I1 a cable to a node or port that is gone: removed +// I2 two cables at one port (plugged at once): the one with the smaller id stays +// I4 a file or folder without its folder: the folder made (paths compared +// without case, as the file system does) +// +// I3 (a switch made smaller than its ports in use) cannot happen here: a +// port is an entry of its node, and a cable at a removed port falls under +// I1. I5 (a client writing `runtime`) comes with the simulation room (F5); +// I6 (values out of range) is marked by the editor, never repaired. + +import * as Y from 'yjs'; +import { parentPath, swiftLowercased } from '@mdg-filius/core'; +import { ROOTS } from './projectDoc.js'; + +export const JANITOR_ORIGIN = 'janitor'; + +/** What one run repaired, by invariant. */ +export interface Repairs { + readonly danglingLinks: string[]; + readonly sharedPortLinks: string[]; + readonly madeFolders: string[]; +} + +/** The ids of a node's ports. */ +function portIds(node: Y.Map | undefined): Set { + const ports = node?.get('ports') as Y.Array> | undefined; + return new Set((ports?.toArray() ?? []).map((port) => port.get('id') as string)); +} + +/** Repairs the document in one transaction (Yjs sends nothing for one that changed nothing). */ +export function repair(doc: Y.Doc): Repairs { + const nodes = doc.getMap>(ROOTS.nodes); + const links = doc.getMap>(ROOTS.links); + const danglingLinks: string[] = []; + const sharedPortLinks: string[] = []; + const madeFolders: string[] = []; + + // I1, then I2 among the cables left, both in the order of their ids. + const ids = [...links.keys()].sort(); + const plugged = new Set(); + for (const id of ids) { + const link = links.get(id)!; + const ends = [ + [link.get('sourceNode') as string, link.get('sourcePort') as string], + [link.get('targetNode') as string, link.get('targetPort') as string], + ] as const; + if (ends.some(([node, port]) => !portIds(nodes.get(node)).has(port))) { + danglingLinks.push(id); + continue; + } + const keys = ends.map(([node, port]) => `${node}/${port}`); + if (keys.some((key) => plugged.has(key))) { + sharedPortLinks.push(id); + continue; + } + for (const key of keys) plugged.add(key); + } + + // I4: each node's files, in the order of the nodes' ids. + const folders: [Y.Map>, string[]][] = []; + for (const id of [...nodes.keys()].sort()) { + const files = nodes.get(id)!.get('files') as Y.Map> | undefined; + if (files === undefined) continue; + const known = (path: string) => swiftLowercased(path.normalize('NFC')); + const present = new Set([...files.keys()].map(known)); + const missing = new Map(); + for (const path of files.keys()) { + for (let folder = parentPath(path); folder !== '/' && !present.has(known(folder)); folder = parentPath(folder)) { + if (!missing.has(known(folder))) missing.set(known(folder), folder); + } + } + if (missing.size === 0) continue; + // Shallow first, so that a folder's folder comes before it. + const made = [...missing.values()].sort((left, right) => left.split('/').length - right.split('/').length || (left < right ? -1 : left > right ? 1 : 0)); + folders.push([files, made]); + madeFolders.push(...made.map((folder) => `${id}:${folder}`)); + } + + doc.transact(() => { + for (const id of [...danglingLinks, ...sharedPortLinks]) links.delete(id); + for (const [files, made] of folders) { + // A made folder comes before every entry it holds: below the smallest order. + let order = Math.min(...[...files.values()].map((entry) => entry.get('order') as number)) - made.length; + for (const folder of made) { + const entry = new Y.Map(); + entry.set('order', order++); + entry.set('kind', 'directory'); + files.set(folder, entry); + } + } + }, JANITOR_ORIGIN); + return { danglingLinks, sharedPortLinks, madeFolders }; +} diff --git a/packages/doc/src/projectDoc.ts b/packages/doc/src/projectDoc.ts new file mode 100644 index 0000000..378457c --- /dev/null +++ b/packages/doc/src/projectDoc.ts @@ -0,0 +1,333 @@ +// A project in its shared document and back (plan §5.2, schema 1). The +// document holds the editor's model — what the .fls reader makes of a file, +// `ImportedProject` — so that its snapshot for the core (projectSnapshot) +// and its export (saveProjectArchive) stay the one translation each, and a +// project read back from its document is the project written into it. +// +// meta schema, versionLine +// nodes id → { order, kind, name, labelPolicy, x, y, +// ports: [{ id, label, occupied, mac?, config? }], +// settings: { device?, routes?, firewall?, dnsServer?, … }, +// programs: program → { active }, +// files?: path → { order, kind, text | data, mediaType? }, +// own? } +// links id → { order, sourceNode, sourcePort, targetNode, targetPort } +// docs id → { kind, x, y, width, height, text, red, green, blue, alpha, +// font, size, bold, order } +// archive name → bytes (the project's own Java applications) +// +// Departures from §5.2, to keep the model whole: a port's settings are the +// model's InterfaceConfiguration under `config`, a node's settings its +// configurations by kind under `settings`; a node's and a link's place in +// the file is `order`; bytes stay in the document until F holds blobs (F5). + +import * as Y from 'yjs'; +import { + inDeterministicRenderOrder, + interfaceKey, + VirtualFileSystem, + type DocumentationItem, + type Id, + type InstallableProgram, + type TopologyLink, + type TopologyNode, + type VirtualFileContent, +} from '@mdg-filius/core'; +import type { FlsOwnApplication, ImportedProject } from '@mdg-filius/fls'; +import { fromY, toY, type PlainValue } from './values.js'; + +export const SCHEMA_VERSION = 1; + +/** The document's root types. */ +export const ROOTS = { meta: 'meta', nodes: 'nodes', links: 'links', docs: 'docs', archive: 'archive' } as const; + +type NodeSettingsField = + | 'deviceConfigurations' + | 'switchConfigurations' + | 'hostWirelessConfigurations' + | 'manualRoutesByNodeId' + | 'ripEnabledByNodeId' + | 'ipForwardingByNodeId' + | 'firewallConfigurationsByNodeId' + | 'dhcpClientConfigurations' + | 'dhcpServerConfigurations' + | 'portForwardingRowsByNodeId' + | 'remoteLinkConfigurations' + | 'webServerConfigurations' + | 'gnutellaConfigurations' + | 'dnsServerConfigurations' + | 'emailClientConfigurations' + | 'emailServerConfigurations' + | 'echoServerConfigurations' + | 'simpleClientConfigurations'; + +/** A node's settings in the document, by the model's map they come from. */ +export const NODE_SETTINGS: Readonly> = { + deviceConfigurations: 'device', + switchConfigurations: 'switch', + hostWirelessConfigurations: 'wireless', + manualRoutesByNodeId: 'routes', + ripEnabledByNodeId: 'rip', + ipForwardingByNodeId: 'ipForwarding', + firewallConfigurationsByNodeId: 'firewall', + dhcpClientConfigurations: 'dhcpClient', + dhcpServerConfigurations: 'dhcpServer', + portForwardingRowsByNodeId: 'portForwarding', + remoteLinkConfigurations: 'remoteLink', + webServerConfigurations: 'webServer', + gnutellaConfigurations: 'gnutella', + dnsServerConfigurations: 'dnsServer', + emailClientConfigurations: 'emailClient', + emailServerConfigurations: 'emailServer', + echoServerConfigurations: 'echoServer', + simpleClientConfigurations: 'simpleClient', +}; + +const SETTINGS_FIELDS = Object.keys(NODE_SETTINGS) as NodeSettingsField[]; + +/** The project's own applications as text: their beans are never edited, and hold Java longs (bigint). */ +function ownText(applications: readonly FlsOwnApplication[]): string { + return JSON.stringify(applications, (_, value: unknown) => (typeof value === 'bigint' ? { bigint: value.toString() } : value)); +} + +function ownApplications(text: string): FlsOwnApplication[] { + return JSON.parse(text, (_, value: unknown) => + value !== null && typeof value === 'object' && Object.keys(value).length === 1 && typeof (value as { bigint?: unknown }).bigint === 'string' + ? BigInt((value as { bigint: string }).bigint) + : value, + ) as FlsOwnApplication[]; +} + +function fileEntry(order: number, content: VirtualFileContent): Y.Map { + const entry = new Y.Map(); + entry.set('order', order); + entry.set('kind', content.kind); + if (content.kind === 'text') entry.set('text', new Y.Text(content.text)); + if (content.kind === 'binary' || content.kind === 'image') entry.set('data', content.data); + if ((content.kind === 'binary' || content.kind === 'image') && content.mediaType !== undefined) entry.set('mediaType', content.mediaType); + return entry; +} + +/** A device's files, each folder's entries in the order they were made (parents before their entries). */ +function filesMap(files: VirtualFileSystem): Y.Map> { + const map = new Y.Map>(); + let order = 0; + const walk = (folder: string): void => { + for (const entry of files.entriesInCreationOrder(folder)) { + map.set(entry.path, fileEntry(order++, entry.content)); + if (entry.content.kind === 'directory') walk(entry.path); + } + }; + walk('/'); + return map; +} + +function nodeMap(node: TopologyNode, order: number, project: ImportedProject): Y.Map { + const map = new Y.Map(); + map.set('order', order); + map.set('kind', node.kind); + map.set('name', node.displayName); + map.set('labelPolicy', node.hostLabelPolicy); + map.set('x', node.position.x); + map.set('y', node.position.y); + const ports = new Y.Array>(); + ports.push( + node.ports.map((port) => { + const entry = new Y.Map(); + entry.set('id', port.id); + entry.set('label', port.label); + entry.set('occupied', port.isOccupied); + if (port.importedMacAddress !== undefined) entry.set('mac', port.importedMacAddress); + const config = project.interfaceConfigurations.get(interfaceKey(node.id, port.id)); + if (config !== undefined) entry.set('config', toY(config as unknown as PlainValue)); + return entry; + }), + ); + map.set('ports', ports); + const settings = new Y.Map(); + for (const field of SETTINGS_FIELDS) { + const value = (project[field] as ReadonlyMap).get(node.id); + if (value !== undefined) settings.set(NODE_SETTINGS[field], toY(value as PlainValue)); + } + map.set('settings', settings); + const programs = new Y.Map>(); + for (const program of project.installedPrograms.get(node.id) ?? []) { + const entry = new Y.Map(); + entry.set('active', project.activePrograms.get(node.id)?.has(program) === true); + programs.set(program, entry); + } + map.set('programs', programs); + const files = project.virtualFileSystems.get(node.id); + if (files !== undefined) map.set('files', filesMap(files)); + const own = project.ownApplicationsByNodeId.get(node.id); + if (own !== undefined) map.set('own', ownText(own)); + return map; +} + +function linkMap(link: TopologyLink, order: number): Y.Map { + const map = new Y.Map(); + map.set('order', order); + map.set('sourceNode', link.sourceNodeId); + map.set('sourcePort', link.sourcePortId); + map.set('targetNode', link.targetNodeId); + map.set('targetPort', link.targetPortId); + return map; +} + +function documentationMap(item: DocumentationItem): Y.Map { + const map = new Y.Map(); + map.set('kind', item.kind); + map.set('x', item.frame.x); + map.set('y', item.frame.y); + map.set('width', item.frame.width); + map.set('height', item.frame.height); + map.set('text', new Y.Text(item.text)); + map.set('red', item.color.red); + map.set('green', item.color.green); + map.set('blue', item.color.blue); + map.set('alpha', item.color.alpha); + map.set('font', item.fontName); + map.set('size', item.fontSize); + map.set('bold', item.isBold); + map.set('order', item.order); + return map; +} + +/** Writes the project into an empty document, in one transaction. */ +export function writeProject(doc: Y.Doc, project: ImportedProject, origin: unknown = null): void { + doc.transact(() => { + const meta = doc.getMap(ROOTS.meta); + meta.set('schema', SCHEMA_VERSION); + meta.set('versionLine', project.versionLine); + const nodes = doc.getMap>(ROOTS.nodes); + project.graph.nodes.forEach((node, index) => nodes.set(node.id, nodeMap(node, index, project))); + const links = doc.getMap>(ROOTS.links); + project.graph.links.forEach((link, index) => links.set(link.id, linkMap(link, index))); + const docs = doc.getMap>(ROOTS.docs); + for (const item of project.documentationItems) docs.set(item.id, documentationMap(item)); + const archive = doc.getMap(ROOTS.archive); + for (const [name, bytes] of project.archiveEntries) archive.set(name, bytes); + }, origin); +} + +/** The entries of a Y.Map in their `order`, then by key: a place two people gave at once is decided the same everywhere. */ +export function inOrder(map: Y.Map>): [string, Y.Map][] { + return [...map.entries()].sort(([leftKey, left], [rightKey, right]) => { + const byOrder = (left.get('order') as number) - (right.get('order') as number); + return byOrder !== 0 ? byOrder : leftKey < rightKey ? -1 : leftKey > rightKey ? 1 : 0; + }); +} + +/** + * A device's files. Read tolerantly, as the editor draws: an entry the file + * system refuses (its folder missing before the janitor ran, or a name two + * people gave at once in different case) is left out, not the project. + */ +function fileSystemOf(files: Y.Map>): VirtualFileSystem { + const fileSystem = new VirtualFileSystem(); + for (const [path, entry] of inOrder(files)) { + const kind = entry.get('kind') as VirtualFileContent['kind']; + const mediaType = entry.get('mediaType') as string | undefined; + try { + if (kind === 'directory') fileSystem.createDirectory(path); + else if (kind === 'text') fileSystem.writeTextFile(path, (entry.get('text') as Y.Text).toString()); + else if (kind === 'image') fileSystem.writeImageFile(path, entry.get('data') as Uint8Array, mediaType!); + else fileSystem.writeBinaryFile(path, entry.get('data') as Uint8Array, mediaType); + } catch { + // Left out (see above). + } + } + return fileSystem; +} + +/** The project the document holds; its warnings are the reader's and stay with the import. */ +export function readProject(doc: Y.Doc): ImportedProject { + const meta = doc.getMap(ROOTS.meta); + const project: ImportedProject = { + versionLine: (meta.get('versionLine') as string | null | undefined) ?? null, + graph: { nodes: [], links: [] }, + deviceConfigurations: new Map(), + interfaceConfigurations: new Map(), + switchConfigurations: new Map(), + hostWirelessConfigurations: new Map(), + manualRoutesByNodeId: new Map(), + ripEnabledByNodeId: new Map(), + ipForwardingByNodeId: new Map(), + firewallConfigurationsByNodeId: new Map(), + dhcpClientConfigurations: new Map(), + dhcpServerConfigurations: new Map(), + portForwardingRowsByNodeId: new Map(), + remoteLinkConfigurations: new Map(), + virtualFileSystems: new Map(), + installedPrograms: new Map(), + activePrograms: new Map(), + webServerConfigurations: new Map(), + gnutellaConfigurations: new Map(), + dnsServerConfigurations: new Map(), + emailClientConfigurations: new Map(), + emailServerConfigurations: new Map(), + echoServerConfigurations: new Map(), + simpleClientConfigurations: new Map(), + documentationItems: [], + archiveEntries: new Map(doc.getMap(ROOTS.archive).entries()), + ownApplicationsByNodeId: new Map(), + warnings: [], + }; + for (const [id, map] of inOrder(doc.getMap>(ROOTS.nodes))) { + const ports = (map.get('ports') as Y.Array>).toArray(); + project.graph.nodes.push({ + id, + kind: map.get('kind') as TopologyNode['kind'], + displayName: map.get('name') as string, + hostLabelPolicy: map.get('labelPolicy') as TopologyNode['hostLabelPolicy'], + position: { x: map.get('x') as number, y: map.get('y') as number }, + ports: ports.map((port) => { + const mac = port.get('mac') as string | undefined; + const fields = { id: port.get('id') as Id, label: port.get('label') as string, isOccupied: port.get('occupied') as boolean }; + return mac === undefined ? fields : { ...fields, importedMacAddress: mac }; + }), + }); + for (const port of ports) { + const config = port.get('config'); + if (config !== undefined) project.interfaceConfigurations.set(interfaceKey(id, port.get('id') as Id), fromY(config) as never); + } + const settings = map.get('settings') as Y.Map; + for (const field of SETTINGS_FIELDS) { + const value = settings.get(NODE_SETTINGS[field]); + if (value !== undefined) (project[field] as Map).set(id, fromY(value)); + } + const programs = map.get('programs') as Y.Map>; + if (programs.size > 0) { + project.installedPrograms.set(id, new Set([...programs.keys()] as InstallableProgram[])); + const active = [...programs.entries()].filter(([, entry]) => entry.get('active') === true).map(([program]) => program as InstallableProgram); + if (active.length > 0) project.activePrograms.set(id, new Set(active)); + } + const files = map.get('files') as Y.Map> | undefined; + if (files !== undefined) project.virtualFileSystems.set(id, fileSystemOf(files)); + const own = map.get('own') as string | undefined; + if (own !== undefined) project.ownApplicationsByNodeId.set(id, ownApplications(own)); + } + for (const [id, map] of inOrder(doc.getMap>(ROOTS.links))) { + project.graph.links.push({ + id, + sourceNodeId: map.get('sourceNode') as Id, + sourcePortId: map.get('sourcePort') as Id, + targetNodeId: map.get('targetNode') as Id, + targetPortId: map.get('targetPort') as Id, + }); + } + project.documentationItems = inDeterministicRenderOrder( + [...doc.getMap>(ROOTS.docs).entries()].map(([id, map]) => ({ + id, + kind: map.get('kind') as DocumentationItem['kind'], + frame: { x: map.get('x') as number, y: map.get('y') as number, width: map.get('width') as number, height: map.get('height') as number }, + text: (map.get('text') as Y.Text).toString(), + color: { red: map.get('red') as number, green: map.get('green') as number, blue: map.get('blue') as number, alpha: map.get('alpha') as number }, + fontName: map.get('font') as string, + fontSize: map.get('size') as number, + isBold: map.get('bold') as boolean, + order: map.get('order') as number, + })), + ); + return project; +} diff --git a/packages/doc/src/snapshot.ts b/packages/doc/src/snapshot.ts new file mode 100644 index 0000000..93ffd54 --- /dev/null +++ b/packages/doc/src/snapshot.ts @@ -0,0 +1,18 @@ +// Where the document meets the core and the file format: both through the +// project it holds, so that the core never sees Yjs and the document never +// the core's snapshot (plan §5.2). + +import type * as Y from 'yjs'; +import type { TopologySnapshot } from '@mdg-filius/core'; +import { projectSnapshot, saveProjectArchive } from '@mdg-filius/fls'; +import { readProject } from './projectDoc.js'; + +/** The core's snapshot of the document, to start a simulation with. */ +export function docSnapshot(doc: Y.Doc): TopologySnapshot { + return projectSnapshot(readProject(doc)); +} + +/** The document as a .fls file, and what the export had to leave out. */ +export function docArchive(doc: Y.Doc): { readonly archive: Uint8Array; readonly warnings: string[] } { + return saveProjectArchive(readProject(doc)); +} diff --git a/packages/doc/src/values.ts b/packages/doc/src/values.ts new file mode 100644 index 0000000..1f7bfea --- /dev/null +++ b/packages/doc/src/values.ts @@ -0,0 +1,30 @@ +// Plain values in the document and back. A setting's object becomes a Y.Map +// of its fields, so that two people changing two fields of one device both +// keep their change (the last one wins only on the same field); a list +// becomes a Y.Array, so that two rows added at once are both kept. Text, +// numbers, booleans, null and bytes stay as they are. + +import * as Y from 'yjs'; + +/** What a setting is made of: JSON's values and bytes. */ +export type PlainValue = null | string | number | boolean | Uint8Array | readonly PlainValue[] | { readonly [key: string]: PlainValue | undefined }; + +/** The value as the document holds it; a field that is undefined is left out (exactOptionalPropertyTypes). */ +export function toY(value: PlainValue): unknown { + if (value === null || typeof value !== 'object' || value instanceof Uint8Array) return value; + if (Array.isArray(value)) { + const list = new Y.Array(); + list.push((value as readonly PlainValue[]).map(toY)); + return list; + } + const map = new Y.Map(); + for (const [key, field] of Object.entries(value)) if (field !== undefined) map.set(key, toY(field)); + return map; +} + +/** The plain value of what the document holds. */ +export function fromY(value: unknown): PlainValue { + if (value instanceof Y.Map) return Object.fromEntries([...value.entries()].map(([key, field]) => [key, fromY(field)])); + if (value instanceof Y.Array) return value.toArray().map(fromY); + return value as PlainValue; +} diff --git a/packages/doc/test/collaboration.test.ts b/packages/doc/test/collaboration.test.ts new file mode 100644 index 0000000..a90e5d5 --- /dev/null +++ b/packages/doc/test/collaboration.test.ts @@ -0,0 +1,201 @@ +// Two people on one document: what Yjs merges, and what the janitor repairs +// after a merge (plan §5.3). Each case starts from one example in two +// replicas, changes them apart, exchanges their updates both ways and looks +// at the project they hold. + +import { readFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { describe, expect, it } from 'vitest'; +import * as Y from 'yjs'; +import { openProjectArchive, type ImportedProject } from '@mdg-filius/fls'; +import { readProject, repair, ROOTS, writeProject } from '../src/index.js'; + +const corpus = join(dirname(fileURLToPath(import.meta.url)), '../../../corpus'); + +function opened(file: string): ImportedProject { + let counter = 0; + return openProjectArchive(new Uint8Array(readFileSync(join(corpus, file))), () => `00000000-0000-0000-0000-${(++counter).toString(16).padStart(12, '0')}`); +} + +/** Two replicas of the example, alike. */ +function replicas(file = 'java-examples/zwei_rechnernetze_komplett.fls'): [Y.Doc, Y.Doc] { + const first = new Y.Doc(); + writeProject(first, opened(file)); + const second = new Y.Doc(); + Y.applyUpdate(second, Y.encodeStateAsUpdate(first)); + return [first, second]; +} + +/** Each replica gets what the other has. */ +function exchange(first: Y.Doc, second: Y.Doc): void { + const toSecond = Y.encodeStateAsUpdate(first, Y.encodeStateVector(second)); + const toFirst = Y.encodeStateAsUpdate(second, Y.encodeStateVector(first)); + Y.applyUpdate(second, toSecond); + Y.applyUpdate(first, toFirst); +} + +const nodes = (doc: Y.Doc) => doc.getMap>(ROOTS.nodes); +const links = (doc: Y.Doc) => doc.getMap>(ROOTS.links); +const nodeNamed = (doc: Y.Doc, name: string) => [...nodes(doc).entries()].find(([, node]) => node.get('name') === name)!; +const settings = (doc: Y.Doc, name: string) => nodeNamed(doc, name)[1].get('settings') as Y.Map>; +const files = (doc: Y.Doc, name: string) => nodeNamed(doc, name)[1].get('files') as Y.Map>; + +/** A switch's port no cable uses. */ +function freePort(doc: Y.Doc, switchName: string): [string, string] { + const [id, node] = nodeNamed(doc, switchName); + const used = new Set([...links(doc).values()].flatMap((link) => [`${link.get('sourceNode')}/${link.get('sourcePort')}`, `${link.get('targetNode')}/${link.get('targetPort')}`])); + const port = (node.get('ports') as Y.Array>).toArray().find((each) => !used.has(`${id}/${each.get('id')}`))!; + return [id, port.get('id') as string]; +} + +function plug(doc: Y.Doc, id: string, [sourceNode, sourcePort]: [string, string], [targetNode, targetPort]: [string, string]): void { + const link = new Y.Map(); + link.set('order', links(doc).size); + link.set('sourceNode', sourceNode); + link.set('sourcePort', sourcePort); + link.set('targetNode', targetNode); + link.set('targetPort', targetPort); + links(doc).set(id, link); +} + +describe('two people on one document', () => { + it('keeps both changes to two fields of one device', () => { + const [first, second] = replicas(); + settings(first, 'Notebook 1.10').get('device')!.set('ipAddress', '141.99.1.20'); + settings(second, 'Notebook 1.10').get('device')!.set('defaultGateway', '141.99.1.2'); + exchange(first, second); + for (const doc of [first, second]) { + const project = readProject(doc); + const id = project.graph.nodes.find((node) => node.displayName === 'Notebook 1.10')!.id; + expect(project.deviceConfigurations.get(id)).toMatchObject({ ipAddress: '141.99.1.20', defaultGateway: '141.99.1.2' }); + } + }); + + it('keeps both rows two people add to a list at once', () => { + const [first, second] = replicas(); + const row = (name: string) => { + const map = new Y.Map(); + map.set('name', name); + map.set('type', 'A'); + map.set('ttlSeconds', 3600); + map.set('target', '141.99.1.10'); + return map; + }; + const records = (doc: Y.Doc) => settings(doc, 'DNS-Server 2.10').get('dnsServer')!.get('records') as Y.Array>; + records(first).push([row('eins.filius.de')]); + records(second).push([row('zwei.filius.de')]); + exchange(first, second); + const names = (doc: Y.Doc) => { + const project = readProject(doc); + const id = project.graph.nodes.find((node) => node.displayName === 'DNS-Server 2.10')!.id; + return project.dnsServerConfigurations.get(id)!.records.map((record) => record.name); + }; + expect(names(first)).toEqual(names(second)); + expect(names(first)).toEqual(expect.arrayContaining(['www.filius.de', 'eins.filius.de', 'zwei.filius.de'])); + }); + + it('writes text two people type into one file together', () => { + const [first, second] = replicas(); + const text = (doc: Y.Doc) => [...files(doc, 'Notebook 1.11').values()].find((entry) => entry.get('kind') === 'text')!.get('text') as Y.Text; + const before = text(first).toString(); + text(first).insert(0, 'A'); + text(second).insert(before.length, 'B'); + exchange(first, second); + expect(text(first).toString()).toBe(`A${before}B`); + expect(text(second).toString()).toBe(`A${before}B`); + }); +}); + +describe('the janitor', () => { + it('removes a cable to a device removed at the same time (I1)', () => { + const [first, second] = replicas(); + const [notebook, notebookNode] = nodeNamed(second, 'Notebook 1.11'); + const notebookPort = ((notebookNode.get('ports') as Y.Array>).get(0).get('id') as string); + // One person removes the notebook with its cable, the other plugs a second cable into it. + first.transact(() => { + for (const [id, link] of links(first)) if (link.get('sourceNode') === notebook || link.get('targetNode') === notebook) links(first).delete(id); + nodes(first).delete(notebook); + }); + plug(second, 'ffffffff-0000-0000-0000-000000000001', freePort(second, 'Switch 1'), [notebook, notebookPort]); + exchange(first, second); + expect(links(first).has('ffffffff-0000-0000-0000-000000000001')).toBe(true); + + expect(repair(first)).toEqual({ danglingLinks: ['ffffffff-0000-0000-0000-000000000001'], sharedPortLinks: [], madeFolders: [] }); + expect(links(first).has('ffffffff-0000-0000-0000-000000000001')).toBe(false); + // Idempotent: nothing left to do, and nothing written. + let updates = 0; + first.on('update', () => (updates += 1)); + expect(repair(first)).toEqual({ danglingLinks: [], sharedPortLinks: [], madeFolders: [] }); + expect(updates).toBe(0); + }); + + it('keeps the cable with the smaller id where two were plugged into one port at once (I2)', () => { + const [first, second] = replicas(); + const port = freePort(first, 'Switch 1'); + plug(first, 'eeeeeeee-0000-0000-0000-000000000002', port, freePort(first, 'Switch 2')); + plug(second, 'eeeeeeee-0000-0000-0000-000000000001', port, freePort(second, 'Switch 2')); + exchange(first, second); + // Both replicas repair, as two janitors would: they end alike. + expect(repair(first).sharedPortLinks).toEqual(['eeeeeeee-0000-0000-0000-000000000002']); + expect(repair(second).sharedPortLinks).toEqual(['eeeeeeee-0000-0000-0000-000000000002']); + exchange(first, second); + expect(readProject(first)).toEqual(readProject(second)); + expect(links(first).has('eeeeeeee-0000-0000-0000-000000000001')).toBe(true); + expect(links(first).has('eeeeeeee-0000-0000-0000-000000000002')).toBe(false); + }); + + it('makes the folder of a file written into a folder removed at the same time (I4)', () => { + const [first, second] = replicas(); + const folder = [...files(first, 'Notebook 1.11').entries()].find(([path, entry]) => entry.get('kind') === 'directory' && path.split('/').length === 2)![0]; + first.transact(() => { + for (const path of [...files(first, 'Notebook 1.11').keys()]) if (path === folder || path.startsWith(`${folder}/`)) files(first, 'Notebook 1.11').delete(path); + }); + const note = new Y.Map(); + note.set('order', 1000); + note.set('kind', 'text'); + note.set('text', new Y.Text('neu')); + files(second, 'Notebook 1.11').set(`${folder}/sub/notiz.txt`, note); + exchange(first, second); + + // Read tolerantly before the janitor: the file is left out, the project is whole. + const id = nodeNamed(first, 'Notebook 1.11')[0]; + expect(readProject(first).virtualFileSystems.get(id)!.contains(`${folder}/sub/notiz.txt`)).toBe(false); + expect(repair(first).madeFolders).toEqual([`${id}:${folder}`, `${id}:${folder}/sub`]); + expect(readProject(first).virtualFileSystems.get(id)!.textFile(`${folder}/sub/notiz.txt`)).toBe('neu'); + expect(repair(first).madeFolders).toEqual([]); + }); + + it('takes a folder in another case as the folder (I4)', () => { + const [doc] = replicas(); + const [folder] = [...files(doc, 'Notebook 1.11').entries()].find(([path, entry]) => entry.get('kind') === 'directory' && path.split('/').length === 2)!; + const note = new Y.Map(); + note.set('order', 1000); + note.set('kind', 'text'); + note.set('text', new Y.Text('neu')); + files(doc, 'Notebook 1.11').set(`${folder.toUpperCase()}/notiz.txt`, note); + expect(repair(doc).madeFolders).toEqual([]); + }); +}); + +describe('a project in its document, beyond the corpus', () => { + it('keeps a Java long of an own application, binary files and an optional field left out', () => { + const project = opened('java-examples/anwendung_komplett.fls'); + const [nodeId, applications] = [...project.ownApplicationsByNodeId][0]!; + project.ownApplicationsByNodeId.set(nodeId, [ + ...applications, + { key: 'eigene', javaClass: 'demo.Eigene', bean: { kind: 'object', className: 'demo.Eigene', statements: [{ set: 'zahl', value: { kind: 'long', value: 12_345_678_901_234_567_890n } }] } }, + ]); + const fileSystem = project.virtualFileSystems.get(nodeId)!; + fileSystem.writeBinaryFile('/daten.bin', new Uint8Array([0, 1, 2, 255])); + fileSystem.writeBinaryFile('/typ.bin', new Uint8Array([7]), 'application/octet-stream'); + const doc = new Y.Doc(); + writeProject(doc, project); + const other = new Y.Doc(); + Y.applyUpdate(other, Y.encodeStateAsUpdate(doc)); + const back = readProject(other); + expect(back.ownApplicationsByNodeId).toEqual(project.ownApplicationsByNodeId); + expect(back.virtualFileSystems.get(nodeId)!.entry('/daten.bin').content).toEqual({ kind: 'binary', data: new Uint8Array([0, 1, 2, 255]) }); + expect(back.virtualFileSystems.get(nodeId)!.entry('/typ.bin').content).toEqual({ kind: 'binary', data: new Uint8Array([7]), mediaType: 'application/octet-stream' }); + }); +}); diff --git a/packages/doc/test/projectDoc.test.ts b/packages/doc/test/projectDoc.test.ts new file mode 100644 index 0000000..63f07e5 --- /dev/null +++ b/packages/doc/test/projectDoc.test.ts @@ -0,0 +1,70 @@ +// A project in its document and back, for every file of the corpus: written +// into a document, sent as Yjs sends it (an update into a second document) +// and read back, it is the same project — the same snapshot for the core, +// the same .fls on export, each folder's files in the same order. + +import { readdirSync, readFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { describe, expect, it } from 'vitest'; +import * as Y from 'yjs'; +import type { VirtualFileSystem } from '@mdg-filius/core'; +import { openProjectArchive, projectSnapshot, saveProjectArchive, type ImportedProject } from '@mdg-filius/fls'; +import { docArchive, docSnapshot, readProject, ROOTS, SCHEMA_VERSION, writeProject } from '../src/index.js'; + +const corpus = join(dirname(fileURLToPath(import.meta.url)), '../../../corpus'); +const files = ['java-examples', 'java-tests', 'ipad-fixtures'].flatMap((folder) => + readdirSync(join(corpus, folder)) + .filter((name) => name.endsWith('.fls')) + .map((name) => join(folder, name)), +); + +/** Ids that fall in the file's reverse order: the document must keep the order itself, not by its ids. */ +function opened(file: string): ImportedProject { + let counter = 0; + return openProjectArchive(new Uint8Array(readFileSync(join(corpus, file))), () => `00000000-0000-0000-0000-${(0xffff_ffff - ++counter).toString(16).padStart(12, '0')}`); +} + +/** The document as another client gets it: its whole state as one update. */ +function sent(doc: Y.Doc): Y.Doc { + const other = new Y.Doc(); + Y.applyUpdate(other, Y.encodeStateAsUpdate(doc)); + return other; +} + +/** Every folder's entries in the order they were made. */ +function creationOrder(files: VirtualFileSystem): string[] { + const paths: string[] = []; + const walk = (folder: string): void => { + for (const entry of files.entriesInCreationOrder(folder)) { + paths.push(entry.path); + if (entry.content.kind === 'directory') walk(entry.path); + } + }; + walk('/'); + return paths; +} + +describe('a project in its document and back', () => { + it('reads the whole corpus', () => { + expect(files.length).toBe(23); + }); + + for (const file of files) { + it(file, () => { + const project = opened(file); + const doc = new Y.Doc(); + writeProject(doc, project); + expect(doc.getMap(ROOTS.meta).get('schema')).toBe(SCHEMA_VERSION); + const received = sent(doc); + const back = readProject(received); + + expect({ ...back, warnings: [] }).toEqual({ ...project, warnings: [] }); + for (const [nodeId, fileSystem] of project.virtualFileSystems) { + expect(creationOrder(back.virtualFileSystems.get(nodeId)!), nodeId).toEqual(creationOrder(fileSystem)); + } + expect(docSnapshot(received)).toEqual(projectSnapshot(project)); + expect(docArchive(received).archive).toEqual(saveProjectArchive(project).archive); + }); + } +}); diff --git a/packages/doc/test/tsconfig.json b/packages/doc/test/tsconfig.json new file mode 100644 index 0000000..b302fb2 --- /dev/null +++ b/packages/doc/test/tsconfig.json @@ -0,0 +1,15 @@ +{ + "extends": "../../../tsconfig.base.json", + "compilerOptions": { + "noEmit": true, + "composite": false, + "declaration": false, + "types": [ + "node" + ], + "resolveJsonModule": true + }, + "include": [ + "." + ] +} diff --git a/packages/doc/test/values.test.ts b/packages/doc/test/values.test.ts new file mode 100644 index 0000000..122de69 --- /dev/null +++ b/packages/doc/test/values.test.ts @@ -0,0 +1,42 @@ +// The small parts of the document: plain values in Yjs and back, and the +// order of entries two people placed at once. + +import { describe, expect, it } from 'vitest'; +import * as Y from 'yjs'; +import { fromY, inOrder, toY } from '../src/index.js'; + +/** A value as the document holds it, inside a document (a Yjs type outside one cannot be read). */ +function held(value: Parameters[0]): unknown { + const doc = new Y.Doc(); + doc.getMap('root').set('value', toY(value)); + return doc.getMap('root').get('value'); +} + +describe('plain values in the document', () => { + it('leaves out a field that is undefined', () => { + const map = held({ a: 1, b: undefined, c: null }) as Y.Map; + expect([...map.keys()].sort()).toEqual(['a', 'c']); + expect(fromY(map)).toStrictEqual({ a: 1, c: null }); + }); + + it('keeps lists, nested objects, bytes and the values of JSON', () => { + const value = { rows: [{ name: 'x', ttl: 3600 }, { name: 'y', ttl: 0 }], on: true, off: false, text: 'ä', none: null, bytes: new Uint8Array([1, 2]) }; + const map = held(value) as Y.Map; + expect(map.get('rows')).toBeInstanceOf(Y.Array); + expect((map.get('rows') as Y.Array).get(0)).toBeInstanceOf(Y.Map); + expect(fromY(map)).toStrictEqual(value); + }); +}); + +describe('the order of entries', () => { + it('orders by place, and two entries given one place by their keys', () => { + const doc = new Y.Doc(); + const map = doc.getMap>('entries'); + for (const [key, order] of [['c', 1], ['b', 0], ['a', 1], ['d', -1]] as const) { + const entry = new Y.Map(); + entry.set('order', order); + map.set(key, entry); + } + expect(inOrder(map).map(([key]) => key)).toEqual(['d', 'b', 'a', 'c']); + }); +}); diff --git a/packages/doc/tsconfig.json b/packages/doc/tsconfig.json new file mode 100644 index 0000000..2d77692 --- /dev/null +++ b/packages/doc/tsconfig.json @@ -0,0 +1,19 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "rootDir": "src", + "outDir": "dist", + "types": [] + }, + "include": [ + "src" + ], + "references": [ + { + "path": "../core" + }, + { + "path": "../fls" + } + ] +} diff --git a/tsconfig.json b/tsconfig.json index 2ceead3..7b03fd6 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -6,6 +6,9 @@ }, { "path": "packages/fls" + }, + { + "path": "packages/doc" } ] }