// A computer's large binary files as blobs (plan ยง5.1): with a blob side, a // picture or binary from 16 KB on is named by its SHA-256 in its entry and // kept by the side, a smaller one stays in its entry, a text stays Y.Text // whatever its size; without a side everything stays in the document as // before. Read back with the side's bytes; a copy of the document whose // side has not got them yet reads no bytes, names them as missing, and // hears them come. Written again unchanged, nothing changes. import { readFileSync } from 'node:fs'; import { describe, expect, it } from 'vitest'; import * as Y from 'yjs'; import { VirtualFileSystem } from '@mdg-filius/core'; import { openProjectArchive } from '@mdg-filius/fls'; import { attachBlobs, BLOB_THRESHOLD_BYTES, blobsOf, blobSha, changeFiles, emptyProject, isBlobSha, memoryBlobSide, missingBlobs, nodeFiles, onBlobsArrive, writeFiles, writeProject, ROOTS, } from '../src/index.js'; const bytes = (length: number, seed = 1) => Uint8Array.from({ length }, (_, index) => (index * 31 + seed) % 251); /** A document with one computer, its files as given, and a side (when given) attached before it is written. */ function docWith(files: (fs: VirtualFileSystem) => void, side?: ReturnType): { doc: Y.Doc; node: string } { const doc = new Y.Doc(); if (side !== undefined) attachBlobs(doc, side); writeProject(doc, emptyProject()); const node = 'pc'; doc.transact(() => { const entry = new Y.Map(); doc.getMap>(ROOTS.nodes).set(node, entry); const fs = new VirtualFileSystem(); files(fs); writeFiles(entry, fs); }); return { doc, node }; } const entryOf = (doc: Y.Doc, node: string, path: string) => (doc.getMap>(ROOTS.nodes).get(node)!.get('files') as Y.Map>).get(path)!; describe('blobs', () => { it('names a blob by the SHA-256 of its bytes, in hex', () => { expect(blobSha(new TextEncoder().encode('abc'))).toBe('ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad'); expect([isBlobSha(blobSha(bytes(3))), isBlobSha('ABC'), isBlobSha(7)]).toEqual([true, false, false]); expect(BLOB_THRESHOLD_BYTES).toBe(16 * 1024); }); it('keeps a large picture and a large binary as blobs, a small one and every text in the document', () => { const taken: string[] = []; const side = memoryBlobSide((sha) => taken.push(sha)); const big = bytes(BLOB_THRESHOLD_BYTES); const { doc, node } = docWith((fs) => { fs.writeImageFile('/big.png', big, 'image/png'); fs.writeBinaryFile('/big.bin', bytes(BLOB_THRESHOLD_BYTES + 5, 2), 'application/zip'); fs.writeBinaryFile('/small.bin', bytes(BLOB_THRESHOLD_BYTES - 1, 3)); fs.writeTextFile('/long.txt', 'x'.repeat(BLOB_THRESHOLD_BYTES * 2)); }, side); const picture = entryOf(doc, node, '/big.png'); expect([picture.get('blob'), picture.get('size'), picture.has('data'), picture.get('mediaType')]).toEqual([blobSha(big), big.length, false, 'image/png']); expect(entryOf(doc, node, '/big.bin').get('blob')).toBe(blobSha(bytes(BLOB_THRESHOLD_BYTES + 5, 2))); expect(entryOf(doc, node, '/small.bin').get('data')).toEqual(bytes(BLOB_THRESHOLD_BYTES - 1, 3)); expect(entryOf(doc, node, '/long.txt').get('text')).toBeInstanceOf(Y.Text); expect(taken.sort()).toEqual([blobSha(big), blobSha(bytes(BLOB_THRESHOLD_BYTES + 5, 2))].sort()); // Read back with the side: the bytes. const read = nodeFiles(doc, node)!; expect(read.entry('/big.png')!.content).toEqual({ kind: 'image', data: big, mediaType: 'image/png' }); expect(blobsOf(doc)).toEqual(new Set(taken)); expect(missingBlobs(doc)).toEqual([]); }); it('keeps the large pictures of a whole project as blobs when it is written with a side', () => { const big = bytes(BLOB_THRESHOLD_BYTES * 2, 4); const project = openProjectArchive(new Uint8Array(readFileSync(new URL('../../../corpus/java-examples/zwei_rechnernetze_komplett.fls', import.meta.url))), () => crypto.randomUUID()); const [node, files] = [...project.virtualFileSystems][0]!; files.writeImageFile('/big.png', big, 'image/png'); const taken: string[] = []; const doc = new Y.Doc(); attachBlobs( doc, memoryBlobSide((sha) => taken.push(sha)), ); writeProject(doc, project); expect(entryOf(doc, node, '/big.png').get('blob')).toBe(blobSha(big)); expect(taken).toEqual([blobSha(big)]); }); it('keeps everything in the document without a side', () => { const big = bytes(BLOB_THRESHOLD_BYTES * 2); const { doc, node } = docWith((fs) => fs.writeImageFile('/big.png', big, 'image/png')); expect(entryOf(doc, node, '/big.png').get('data')).toEqual(big); expect(entryOf(doc, node, '/big.png').has('blob')).toBe(false); expect(nodeFiles(doc, node)!.entry('/big.png')!.content).toEqual({ kind: 'image', data: big, mediaType: 'image/png' }); expect(missingBlobs(doc)).toEqual([]); expect(onBlobsArrive(doc, () => undefined)).toBeTypeOf('function'); }); it('reads no bytes in a copy whose side has not got them, names them missing, and hears them come', () => { const big = bytes(BLOB_THRESHOLD_BYTES * 2); const { doc, node } = docWith((fs) => fs.writeImageFile('/big.png', big, 'image/png'), memoryBlobSide()); const copy = new Y.Doc(); const side = memoryBlobSide(); attachBlobs(copy, side); Y.applyUpdate(copy, Y.encodeStateAsUpdate(doc)); expect(nodeFiles(copy, node)!.entry('/big.png')!.content).toEqual({ kind: 'image', data: new Uint8Array(0), mediaType: 'image/png' }); expect(missingBlobs(copy)).toEqual([blobSha(big)]); let heard = 0; const stop = onBlobsArrive(copy, () => (heard += 1)); side.arrived(new Map()); side.arrived(new Map([[blobSha(big), big]])); stop(); side.arrived(new Map([[blobSha(big), big]])); expect(heard).toBe(1); expect(missingBlobs(copy)).toEqual([]); expect(nodeFiles(copy, node)!.entry('/big.png')!.content).toMatchObject({ data: big }); // A copy without a side names every blob missing. const bare = new Y.Doc(); Y.applyUpdate(bare, Y.encodeStateAsUpdate(doc)); expect(missingBlobs(bare)).toEqual([blobSha(big)]); }); it('changes nothing when the same files are written again, with the bytes here or only their name', () => { const big = bytes(BLOB_THRESHOLD_BYTES * 2); const side = memoryBlobSide(); const { doc, node } = docWith((fs) => fs.writeImageFile('/big.png', big, 'image/png'), side); const copy = new Y.Doc(); attachBlobs(copy, memoryBlobSide()); Y.applyUpdate(copy, Y.encodeStateAsUpdate(doc)); for (const each of [doc, copy]) { let updates = 0; each.on('update', () => (updates += 1)); changeFiles(each, node, 'person', (fs) => fs.writeImageFile('/big.png', big, 'image/png')); expect(updates).toBe(0); } // Other bytes of the same size, another type, another size: a new entry each. let updates = 0; doc.on('update', () => (updates += 1)); changeFiles(doc, node, 'person', (fs) => fs.writeImageFile('/big.png', bytes(big.length, 9), 'image/png')); expect(entryOf(doc, node, '/big.png').get('blob')).toBe(blobSha(bytes(big.length, 9))); changeFiles(doc, node, 'person', (fs) => fs.writeImageFile('/big.png', bytes(big.length, 9), 'image/gif')); expect(entryOf(doc, node, '/big.png').get('mediaType')).toBe('image/gif'); changeFiles(doc, node, 'person', (fs) => fs.writeImageFile('/big.png', bytes(big.length + 1, 9), 'image/gif')); expect(entryOf(doc, node, '/big.png').get('size')).toBe(big.length + 1); expect(updates).toBe(3); // A copy without the bytes compares by the name. changeFiles(copy, node, 'person', (fs) => fs.writeImageFile('/big.png', bytes(big.length, 5), 'image/png')); expect(entryOf(copy, node, '/big.png').get('blob')).toBe(blobSha(bytes(big.length, 5))); }); });