feat: F5.5 large binary files as blobs on server F

From 16 KB a picture's or binary file's entry names its bytes by their
SHA-256 instead of holding them; the bytes are on F, content-addressed
(GET/POST /blobs with the session's grant, from the editor's origin, at
most 16 MiB). A document's blob side says where they are: the editor's
fetches and uploads, the room thread's asks F's main thread, and a start
waits for the ones asked for; without a side everything stays in the
document. Text stays Y.Text. A blob whose bytes are not there is never
written back empty. A blob F has not got yet is asked for again a few
times, and a running room reads late bytes.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-04 16:46:50 +00:00
co-authored by Claude Opus 5.5
parent 47beda6b85
commit 4b33e36580
29 changed files with 1185 additions and 29 deletions
+1
View File
@@ -10,6 +10,7 @@
"dependencies": {
"@mdg-filius/core": "0.0.0",
"@mdg-filius/fls": "0.0.0",
"@noble/hashes": "2.4.0",
"yjs": "^13.6.33"
}
}
+95
View File
@@ -0,0 +1,95 @@
// A computer's large binary files outside the document (plan §5.1: no
// binary in the CRDT): from BLOB_THRESHOLD_BYTES on, a picture's or other
// binary file's entry holds `blob` — the SHA-256 of its bytes, in hex — and
// `size` instead of `data`; the bytes are on F, content-addressed. Text
// stays Y.Text whatever its size: two type into it together. Below the
// threshold a binary stays in the entry, which spares a small icon a fetch.
//
// Where the bytes are is a document's blob side (attachBlobs): the editor's
// on F (it fetches and uploads), the room thread's (it asks F's main
// thread). A document without one — the development page, the tests, the
// room in a worker of the page — keeps every binary in its entry. Reading a
// blob whose bytes have not come yet gives no bytes; whoever needs them all
// asks for the missing ones first (missingBlobs) and hears them come
// (onBlobsArrive).
import { sha256 } from '@noble/hashes/sha2.js';
import type * as Y from 'yjs';
import { ROOTS } from './projectDoc.js';
export const BLOB_THRESHOLD_BYTES = 16 * 1024;
export interface BlobSide {
/** A blob's bytes, if this side has them. */
bytes(sha: string): Uint8Array | undefined;
/** Bytes become a blob: kept here and taken to F. */
keep(sha: string, bytes: Uint8Array): void;
/** Hears blobs come; returns how to stop. */
onArrive(listener: () => void): () => void;
}
const sides = new WeakMap<Y.Doc, BlobSide>();
/** Where a document's blobs are, from now on. */
export function attachBlobs(doc: Y.Doc, side: BlobSide): void {
sides.set(doc, side);
}
export function blobSideOf(doc: Y.Doc | null | undefined): BlobSide | undefined {
return doc === null || doc === undefined ? undefined : sides.get(doc);
}
const hex = (bytes: Uint8Array) => Array.from(bytes, (byte) => byte.toString(16).padStart(2, '0')).join('');
/** A blob's name: the SHA-256 of its bytes, in hex. */
export function blobSha(bytes: Uint8Array): string {
return hex(sha256(bytes));
}
export const isBlobSha = (value: unknown): value is string => typeof value === 'string' && /^[0-9a-f]{64}$/.test(value);
/** Every blob the document's files name. */
export function blobsOf(doc: Y.Doc): Set<string> {
const found = new Set<string>();
for (const node of doc.getMap<Y.Map<unknown>>(ROOTS.nodes).values()) {
const files = node.get('files') as Y.Map<Y.Map<unknown>> | undefined;
for (const entry of files?.values() ?? []) {
const sha = entry.get('blob');
if (isBlobSha(sha)) found.add(sha);
}
}
return found;
}
/** The blobs the document names whose bytes its side has not got (all, without a side). */
export function missingBlobs(doc: Y.Doc): string[] {
const side = blobSideOf(doc);
return [...blobsOf(doc)].filter((sha) => side?.bytes(sha) === undefined);
}
/** Hears the document's blobs come (nothing without a side); returns how to stop. */
export function onBlobsArrive(doc: Y.Doc, listener: () => void): () => void {
return blobSideOf(doc)?.onArrive(listener) ?? (() => {});
}
/** A side that keeps blobs in memory and tells when some come (`arrived`); `taken` hears each kept one. */
export function memoryBlobSide(taken: (sha: string, bytes: Uint8Array) => void = () => {}): BlobSide & { arrived(blobs: ReadonlyMap<string, Uint8Array>): void } {
const held = new Map<string, Uint8Array>();
const listeners = new Set<() => void>();
return {
bytes: (sha) => held.get(sha),
keep(sha, bytes) {
if (held.has(sha)) return;
held.set(sha, bytes);
taken(sha, bytes);
},
onArrive(listener) {
listeners.add(listener);
return () => listeners.delete(listener);
},
arrived(blobs) {
for (const [sha, bytes] of blobs) held.set(sha, bytes);
if (blobs.size > 0) for (const listener of listeners) listener();
},
};
}
+2 -1
View File
@@ -29,6 +29,7 @@ import {
} from '@mdg-filius/core';
import { DEFAULT_MODEM_PORT, modemPair, type ImportedProject } from '@mdg-filius/fls';
import { emptyProject, inOrder, nodeEntry, ROOTS } from './projectDoc.js';
import { blobSideOf } from './blobs.js';
/** The kinds of the palette, in Java's order (GUIDesignSidebar.addComponentItemsToSidebar; the cable comes first, a tool). */
export const PALETTE: readonly Exclude<NodeKind, 'unsupported'>[] = ['pc', 'notebook', 'networkSwitch', 'gateway', 'router', 'remoteLink'];
@@ -120,7 +121,7 @@ export function addDevice(doc: Y.Doc, device: NewDevice, origin: unknown): void
const project = newDeviceProject(device);
doc.transact(() => {
const order = Math.max(-1, ...[...nodes(doc).values()].map((node) => node.get('order') as number)) + 1;
nodes(doc).set(device.id, nodeEntry(project.graph.nodes[0]!, order, project));
nodes(doc).set(device.id, nodeEntry(project.graph.nodes[0]!, order, project, blobSideOf(doc)));
}, origin);
}
+16 -4
View File
@@ -11,6 +11,7 @@
import * as Y from 'yjs';
import type { Id, VirtualFileContent, VirtualFileSystem } from '@mdg-filius/core';
import { VirtualFileSystem as FileSystem } from '@mdg-filius/core';
import { blobSha, blobSideOf } from './blobs.js';
import { fileEntry, fileSystemOf, inOrder, ROOTS } from './projectDoc.js';
export function sameBytes(left: Uint8Array, right: Uint8Array): boolean {
@@ -29,11 +30,21 @@ export function replaceText(text: Y.Text, next: string): void {
if (next.length - start - end > 0) text.insert(start, next.slice(start, next.length - end));
}
/** Whether a document entry holds the content. */
/**
* Whether a document entry holds the content (a blob by its bytes where they
* are here, else by its name).
*/
function holds(entry: Y.Map<unknown>, content: VirtualFileContent): boolean {
if (entry.get('kind') !== content.kind) return false;
if (content.kind === 'directory' || content.kind === 'text') return true;
return sameBytes(entry.get('data') as Uint8Array, content.data) && entry.get('mediaType') === content.mediaType;
if (entry.get('mediaType') !== content.mediaType) return false;
const blob = entry.get('blob');
if (typeof blob !== 'string') return sameBytes(entry.get('data') as Uint8Array, content.data);
const held = blobSideOf(entry.doc)?.bytes(blob);
// A blob whose bytes are not here was read as no bytes: written back so, that is no change (it would empty the file).
if (held === undefined && content.data.length === 0) return true;
if (entry.get('size') !== content.data.length) return false;
return held === undefined ? blobSha(content.data) === blob : sameBytes(held, content.data);
}
/**
@@ -42,6 +53,7 @@ function holds(entry: Y.Map<unknown>, content: VirtualFileContent): boolean {
* longer the file system's (a renamed entry), numbered anew.
*/
export function writeFiles(node: Y.Map<unknown>, fileSystem: VirtualFileSystem): void {
const blobs = blobSideOf(node.doc);
let files = node.get('files') as Y.Map<Y.Map<unknown>> | undefined;
const wanted: [string, VirtualFileContent][] = [];
const walk = (folder: string): void => {
@@ -62,9 +74,9 @@ export function writeFiles(node: Y.Map<unknown>, fileSystem: VirtualFileSystem):
for (const [path, content] of wanted) {
const entry = files.get(path);
if (entry === undefined) {
files.set(path, fileEntry(order++, content));
files.set(path, fileEntry(order++, content, blobs));
} else if (!holds(entry, content)) {
files.set(path, fileEntry(entry.get('order') as number, content));
files.set(path, fileEntry(entry.get('order') as number, content, blobs));
} else if (content.kind === 'text') {
replaceText(entry.get('text') as Y.Text, content.text);
}
+1
View File
@@ -1,4 +1,5 @@
// Public surface of the shared document.
export * from './blobs.js';
export * from './commands.js';
export * from './documentation.js';
export * from './fields.js';
+25 -9
View File
@@ -35,6 +35,7 @@ import {
} from '@mdg-filius/core';
import type { FlsOwnApplication, ImportedProject } from '@mdg-filius/fls';
import { fromY, toY, type PlainValue } from './values.js';
import { BLOB_THRESHOLD_BYTES, blobSha, blobSideOf, type BlobSide } from './blobs.js';
export const SCHEMA_VERSION = 1;
@@ -98,23 +99,29 @@ function ownApplications(text: string): FlsOwnApplication[] {
) as FlsOwnApplication[];
}
export function fileEntry(order: number, content: VirtualFileContent): Y.Map<unknown> {
/** A file's entry; a large binary as a blob when the document has a blob side (blobs.ts). */
export function fileEntry(order: number, content: VirtualFileContent, blobs?: BlobSide): Y.Map<unknown> {
const entry = new Y.Map<unknown>();
entry.set('order', order);
entry.set('kind', content.kind);
if (content.kind === 'text') entry.set('text', new Y.Text(content.text));
if (content.kind === 'binary' || content.kind === 'image') entry.set('data', content.data);
if ((content.kind === 'binary' || content.kind === 'image') && blobs !== undefined && content.data.length >= BLOB_THRESHOLD_BYTES) {
const sha = blobSha(content.data);
blobs.keep(sha, content.data);
entry.set('blob', sha);
entry.set('size', content.data.length);
} else if (content.kind === 'binary' || content.kind === 'image') entry.set('data', content.data);
if ((content.kind === 'binary' || content.kind === 'image') && content.mediaType !== undefined) entry.set('mediaType', content.mediaType);
return entry;
}
/** A device's files, each folder's entries in the order they were made (parents before their entries). */
function filesMap(files: VirtualFileSystem): Y.Map<Y.Map<unknown>> {
function filesMap(files: VirtualFileSystem, blobs: BlobSide | undefined): Y.Map<Y.Map<unknown>> {
const map = new Y.Map<Y.Map<unknown>>();
let order = 0;
const walk = (folder: string): void => {
for (const entry of files.entriesInCreationOrder(folder)) {
map.set(entry.path, fileEntry(order++, entry.content));
map.set(entry.path, fileEntry(order++, entry.content, blobs));
if (entry.content.kind === 'directory') walk(entry.path);
}
};
@@ -123,7 +130,7 @@ function filesMap(files: VirtualFileSystem): Y.Map<Y.Map<unknown>> {
}
/** A device's entry in the document, with what the project holds for it. */
export function nodeEntry(node: TopologyNode, order: number, project: ImportedProject): Y.Map<unknown> {
export function nodeEntry(node: TopologyNode, order: number, project: ImportedProject, blobs?: BlobSide): Y.Map<unknown> {
const map = new Y.Map<unknown>();
map.set('order', order);
map.set('kind', node.kind);
@@ -159,7 +166,7 @@ export function nodeEntry(node: TopologyNode, order: number, project: ImportedPr
}
map.set('programs', programs);
const files = project.virtualFileSystems.get(node.id);
if (files !== undefined) map.set('files', filesMap(files));
if (files !== undefined) map.set('files', filesMap(files, blobs));
const own = project.ownApplicationsByNodeId.get(node.id);
if (own !== undefined) map.set('own', ownText(own));
return map;
@@ -202,7 +209,7 @@ export function writeProject(doc: Y.Doc, project: ImportedProject, origin: unkno
meta.set('schema', SCHEMA_VERSION);
meta.set('versionLine', project.versionLine);
const nodes = doc.getMap<Y.Map<unknown>>(ROOTS.nodes);
project.graph.nodes.forEach((node, index) => nodes.set(node.id, nodeEntry(node, index, project)));
project.graph.nodes.forEach((node, index) => nodes.set(node.id, nodeEntry(node, index, project, blobSideOf(doc))));
const links = doc.getMap<Y.Map<unknown>>(ROOTS.links);
project.graph.links.forEach((link, index) => links.set(link.id, linkMap(link, index)));
const docs = doc.getMap<Y.Map<unknown>>(ROOTS.docs);
@@ -225,6 +232,15 @@ export function inOrder(map: Y.Map<Y.Map<unknown>>): [string, Y.Map<unknown>][]
* system refuses (its folder missing before the janitor ran, or a name two
* people gave at once in different case) is left out, not the project.
*/
const NO_BYTES = new Uint8Array(0);
/** A binary entry's bytes: its own, or its blob's (none while they have not come). */
export function bytesOf(entry: Y.Map<unknown>): Uint8Array {
const blob = entry.get('blob');
if (typeof blob === 'string') return blobSideOf(entry.doc)?.bytes(blob) ?? NO_BYTES;
return entry.get('data') as Uint8Array;
}
export function fileSystemOf(files: Y.Map<Y.Map<unknown>>): VirtualFileSystem {
const fileSystem = new VirtualFileSystem();
for (const [path, entry] of inOrder(files)) {
@@ -233,8 +249,8 @@ export function fileSystemOf(files: Y.Map<Y.Map<unknown>>): VirtualFileSystem {
try {
if (kind === 'directory') fileSystem.createDirectory(path);
else if (kind === 'text') fileSystem.writeTextFile(path, (entry.get('text') as Y.Text).toString());
else if (kind === 'image') fileSystem.writeImageFile(path, entry.get('data') as Uint8Array, mediaType!);
else fileSystem.writeBinaryFile(path, entry.get('data') as Uint8Array, mediaType);
else if (kind === 'image') fileSystem.writeImageFile(path, bytesOf(entry), mediaType!);
else fileSystem.writeBinaryFile(path, bytesOf(entry), mediaType);
} catch {
// Left out (see above).
}
+158
View File
@@ -0,0 +1,158 @@
// A computer's large binary files as blobs (plan §5.1): with a blob side, a
// picture or binary from 16 KB on is named by its SHA-256 in its entry and
// kept by the side, a smaller one stays in its entry, a text stays Y.Text
// whatever its size; without a side everything stays in the document as
// before. Read back with the side's bytes; a copy of the document whose
// side has not got them yet reads no bytes, names them as missing, and
// hears them come. Written again unchanged, nothing changes.
import { readFileSync } from 'node:fs';
import { describe, expect, it } from 'vitest';
import * as Y from 'yjs';
import { VirtualFileSystem } from '@mdg-filius/core';
import { openProjectArchive } from '@mdg-filius/fls';
import {
attachBlobs,
BLOB_THRESHOLD_BYTES,
blobsOf,
blobSha,
changeFiles,
emptyProject,
isBlobSha,
memoryBlobSide,
missingBlobs,
nodeFiles,
onBlobsArrive,
writeFiles,
writeProject,
ROOTS,
} from '../src/index.js';
const bytes = (length: number, seed = 1) => Uint8Array.from({ length }, (_, index) => (index * 31 + seed) % 251);
/** A document with one computer, its files as given, and a side (when given) attached before it is written. */
function docWith(files: (fs: VirtualFileSystem) => void, side?: ReturnType<typeof memoryBlobSide>): { doc: Y.Doc; node: string } {
const doc = new Y.Doc();
if (side !== undefined) attachBlobs(doc, side);
writeProject(doc, emptyProject());
const node = 'pc';
doc.transact(() => {
const entry = new Y.Map<unknown>();
doc.getMap<Y.Map<unknown>>(ROOTS.nodes).set(node, entry);
const fs = new VirtualFileSystem();
files(fs);
writeFiles(entry, fs);
});
return { doc, node };
}
const entryOf = (doc: Y.Doc, node: string, path: string) => (doc.getMap<Y.Map<unknown>>(ROOTS.nodes).get(node)!.get('files') as Y.Map<Y.Map<unknown>>).get(path)!;
describe('blobs', () => {
it('names a blob by the SHA-256 of its bytes, in hex', () => {
expect(blobSha(new TextEncoder().encode('abc'))).toBe('ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad');
expect([isBlobSha(blobSha(bytes(3))), isBlobSha('ABC'), isBlobSha(7)]).toEqual([true, false, false]);
expect(BLOB_THRESHOLD_BYTES).toBe(16 * 1024);
});
it('keeps a large picture and a large binary as blobs, a small one and every text in the document', () => {
const taken: string[] = [];
const side = memoryBlobSide((sha) => taken.push(sha));
const big = bytes(BLOB_THRESHOLD_BYTES);
const { doc, node } = docWith((fs) => {
fs.writeImageFile('/big.png', big, 'image/png');
fs.writeBinaryFile('/big.bin', bytes(BLOB_THRESHOLD_BYTES + 5, 2), 'application/zip');
fs.writeBinaryFile('/small.bin', bytes(BLOB_THRESHOLD_BYTES - 1, 3));
fs.writeTextFile('/long.txt', 'x'.repeat(BLOB_THRESHOLD_BYTES * 2));
}, side);
const picture = entryOf(doc, node, '/big.png');
expect([picture.get('blob'), picture.get('size'), picture.has('data'), picture.get('mediaType')]).toEqual([blobSha(big), big.length, false, 'image/png']);
expect(entryOf(doc, node, '/big.bin').get('blob')).toBe(blobSha(bytes(BLOB_THRESHOLD_BYTES + 5, 2)));
expect(entryOf(doc, node, '/small.bin').get('data')).toEqual(bytes(BLOB_THRESHOLD_BYTES - 1, 3));
expect(entryOf(doc, node, '/long.txt').get('text')).toBeInstanceOf(Y.Text);
expect(taken.sort()).toEqual([blobSha(big), blobSha(bytes(BLOB_THRESHOLD_BYTES + 5, 2))].sort());
// Read back with the side: the bytes.
const read = nodeFiles(doc, node)!;
expect(read.entry('/big.png')!.content).toEqual({ kind: 'image', data: big, mediaType: 'image/png' });
expect(blobsOf(doc)).toEqual(new Set(taken));
expect(missingBlobs(doc)).toEqual([]);
});
it('keeps the large pictures of a whole project as blobs when it is written with a side', () => {
const big = bytes(BLOB_THRESHOLD_BYTES * 2, 4);
const project = openProjectArchive(new Uint8Array(readFileSync(new URL('../../../corpus/java-examples/zwei_rechnernetze_komplett.fls', import.meta.url))), () => crypto.randomUUID());
const [node, files] = [...project.virtualFileSystems][0]!;
files.writeImageFile('/big.png', big, 'image/png');
const taken: string[] = [];
const doc = new Y.Doc();
attachBlobs(
doc,
memoryBlobSide((sha) => taken.push(sha)),
);
writeProject(doc, project);
expect(entryOf(doc, node, '/big.png').get('blob')).toBe(blobSha(big));
expect(taken).toEqual([blobSha(big)]);
});
it('keeps everything in the document without a side', () => {
const big = bytes(BLOB_THRESHOLD_BYTES * 2);
const { doc, node } = docWith((fs) => fs.writeImageFile('/big.png', big, 'image/png'));
expect(entryOf(doc, node, '/big.png').get('data')).toEqual(big);
expect(entryOf(doc, node, '/big.png').has('blob')).toBe(false);
expect(nodeFiles(doc, node)!.entry('/big.png')!.content).toEqual({ kind: 'image', data: big, mediaType: 'image/png' });
expect(missingBlobs(doc)).toEqual([]);
expect(onBlobsArrive(doc, () => undefined)).toBeTypeOf('function');
});
it('reads no bytes in a copy whose side has not got them, names them missing, and hears them come', () => {
const big = bytes(BLOB_THRESHOLD_BYTES * 2);
const { doc, node } = docWith((fs) => fs.writeImageFile('/big.png', big, 'image/png'), memoryBlobSide());
const copy = new Y.Doc();
const side = memoryBlobSide();
attachBlobs(copy, side);
Y.applyUpdate(copy, Y.encodeStateAsUpdate(doc));
expect(nodeFiles(copy, node)!.entry('/big.png')!.content).toEqual({ kind: 'image', data: new Uint8Array(0), mediaType: 'image/png' });
expect(missingBlobs(copy)).toEqual([blobSha(big)]);
let heard = 0;
const stop = onBlobsArrive(copy, () => (heard += 1));
side.arrived(new Map());
side.arrived(new Map([[blobSha(big), big]]));
stop();
side.arrived(new Map([[blobSha(big), big]]));
expect(heard).toBe(1);
expect(missingBlobs(copy)).toEqual([]);
expect(nodeFiles(copy, node)!.entry('/big.png')!.content).toMatchObject({ data: big });
// A copy without a side names every blob missing.
const bare = new Y.Doc();
Y.applyUpdate(bare, Y.encodeStateAsUpdate(doc));
expect(missingBlobs(bare)).toEqual([blobSha(big)]);
});
it('changes nothing when the same files are written again, with the bytes here or only their name', () => {
const big = bytes(BLOB_THRESHOLD_BYTES * 2);
const side = memoryBlobSide();
const { doc, node } = docWith((fs) => fs.writeImageFile('/big.png', big, 'image/png'), side);
const copy = new Y.Doc();
attachBlobs(copy, memoryBlobSide());
Y.applyUpdate(copy, Y.encodeStateAsUpdate(doc));
for (const each of [doc, copy]) {
let updates = 0;
each.on('update', () => (updates += 1));
changeFiles(each, node, 'person', (fs) => fs.writeImageFile('/big.png', big, 'image/png'));
expect(updates).toBe(0);
}
// Other bytes of the same size, another type, another size: a new entry each.
let updates = 0;
doc.on('update', () => (updates += 1));
changeFiles(doc, node, 'person', (fs) => fs.writeImageFile('/big.png', bytes(big.length, 9), 'image/png'));
expect(entryOf(doc, node, '/big.png').get('blob')).toBe(blobSha(bytes(big.length, 9)));
changeFiles(doc, node, 'person', (fs) => fs.writeImageFile('/big.png', bytes(big.length, 9), 'image/gif'));
expect(entryOf(doc, node, '/big.png').get('mediaType')).toBe('image/gif');
changeFiles(doc, node, 'person', (fs) => fs.writeImageFile('/big.png', bytes(big.length + 1, 9), 'image/gif'));
expect(entryOf(doc, node, '/big.png').get('size')).toBe(big.length + 1);
expect(updates).toBe(3);
// A copy without the bytes compares by the name.
changeFiles(copy, node, 'person', (fs) => fs.writeImageFile('/big.png', bytes(big.length, 5), 'image/png'));
expect(entryOf(copy, node, '/big.png').get('blob')).toBe(blobSha(bytes(big.length, 5)));
});
});