mdg-filius/packages/doc/test/blobs.test.ts

// A computer's large binary files as blobs (plan §5.1): with a blob side, a
// picture or binary from 16 KB on is named by its SHA-256 in its entry and
// kept by the side, a smaller one stays in its entry, a text stays Y.Text
// whatever its size; without a side everything stays in the document as
// before. Read back with the side's bytes; a copy of the document whose
// side has not got them yet reads no bytes, names them as missing, and
// hears them come. Written again unchanged, nothing changes.

import { readFileSync } from 'node:fs';
import { describe, expect, it } from 'vitest';
import * as Y from 'yjs';
import { VirtualFileSystem } from '@mdg-filius/core';
import { openProjectArchive } from '@mdg-filius/fls';
import {
  attachBlobs,
  BLOB_THRESHOLD_BYTES,
  blobsOf,
  blobSha,
  changeFiles,
  emptyProject,
  isBlobSha,
  memoryBlobSide,
  missingBlobs,
  nodeFiles,
  onBlobsArrive,
  writeFiles,
  writeProject,
  ROOTS,
} from '../src/index.js';

const bytes = (length: number, seed = 1) => Uint8Array.from({ length }, (_, index) => (index * 31 + seed) % 251);

/** A document with one computer, its files as given, and a side (when given) attached before it is written. */
function docWith(files: (fs: VirtualFileSystem) => void, side?: ReturnType<typeof memoryBlobSide>): { doc: Y.Doc; node: string } {
  const doc = new Y.Doc();
  if (side !== undefined) attachBlobs(doc, side);
  writeProject(doc, emptyProject());
  const node = 'pc';
  doc.transact(() => {
    const entry = new Y.Map<unknown>();
    doc.getMap<Y.Map<unknown>>(ROOTS.nodes).set(node, entry);
    const fs = new VirtualFileSystem();
    files(fs);
    writeFiles(entry, fs);
  });
  return { doc, node };
}

const entryOf = (doc: Y.Doc, node: string, path: string) => (doc.getMap<Y.Map<unknown>>(ROOTS.nodes).get(node)!.get('files') as Y.Map<Y.Map<unknown>>).get(path)!;

describe('blobs', () => {
  it('names a blob by the SHA-256 of its bytes, in hex', () => {
    expect(blobSha(new TextEncoder().encode('abc'))).toBe('ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad');
    expect([isBlobSha(blobSha(bytes(3))), isBlobSha('ABC'), isBlobSha(7)]).toEqual([true, false, false]);
    expect(BLOB_THRESHOLD_BYTES).toBe(16 * 1024);
  });

  it('keeps a large picture and a large binary as blobs, a small one and every text in the document', () => {
    const taken: string[] = [];
    const side = memoryBlobSide((sha) => taken.push(sha));
    const big = bytes(BLOB_THRESHOLD_BYTES);
    const { doc, node } = docWith((fs) => {
      fs.writeImageFile('/big.png', big, 'image/png');
      fs.writeBinaryFile('/big.bin', bytes(BLOB_THRESHOLD_BYTES + 5, 2), 'application/zip');
      fs.writeBinaryFile('/small.bin', bytes(BLOB_THRESHOLD_BYTES - 1, 3));
      fs.writeTextFile('/long.txt', 'x'.repeat(BLOB_THRESHOLD_BYTES * 2));
    }, side);
    const picture = entryOf(doc, node, '/big.png');
    expect([picture.get('blob'), picture.get('size'), picture.has('data'), picture.get('mediaType')]).toEqual([blobSha(big), big.length, false, 'image/png']);
    expect(entryOf(doc, node, '/big.bin').get('blob')).toBe(blobSha(bytes(BLOB_THRESHOLD_BYTES + 5, 2)));
    expect(entryOf(doc, node, '/small.bin').get('data')).toEqual(bytes(BLOB_THRESHOLD_BYTES - 1, 3));
    expect(entryOf(doc, node, '/long.txt').get('text')).toBeInstanceOf(Y.Text);
    expect(taken.sort()).toEqual([blobSha(big), blobSha(bytes(BLOB_THRESHOLD_BYTES + 5, 2))].sort());
    // Read back with the side: the bytes.
    const read = nodeFiles(doc, node)!;
    expect(read.entry('/big.png')!.content).toEqual({ kind: 'image', data: big, mediaType: 'image/png' });
    expect(blobsOf(doc)).toEqual(new Set(taken));
    expect(missingBlobs(doc)).toEqual([]);
  });

  it('keeps the large pictures of a whole project as blobs when it is written with a side', () => {
    const big = bytes(BLOB_THRESHOLD_BYTES * 2, 4);
    const project = openProjectArchive(new Uint8Array(readFileSync(new URL('../../../corpus/java-examples/zwei_rechnernetze_komplett.fls', import.meta.url))), () => crypto.randomUUID());
    const [node, files] = [...project.virtualFileSystems][0]!;
    files.writeImageFile('/big.png', big, 'image/png');
    const taken: string[] = [];
    const doc = new Y.Doc();
    attachBlobs(
      doc,
      memoryBlobSide((sha) => taken.push(sha)),
    );
    writeProject(doc, project);
    expect(entryOf(doc, node, '/big.png').get('blob')).toBe(blobSha(big));
    expect(taken).toEqual([blobSha(big)]);
  });

  it('keeps everything in the document without a side', () => {
    const big = bytes(BLOB_THRESHOLD_BYTES * 2);
    const { doc, node } = docWith((fs) => fs.writeImageFile('/big.png', big, 'image/png'));
    expect(entryOf(doc, node, '/big.png').get('data')).toEqual(big);
    expect(entryOf(doc, node, '/big.png').has('blob')).toBe(false);
    expect(nodeFiles(doc, node)!.entry('/big.png')!.content).toEqual({ kind: 'image', data: big, mediaType: 'image/png' });
    expect(missingBlobs(doc)).toEqual([]);
    expect(onBlobsArrive(doc, () => undefined)).toBeTypeOf('function');
  });

  it('reads no bytes in a copy whose side has not got them, names them missing, and hears them come', () => {
    const big = bytes(BLOB_THRESHOLD_BYTES * 2);
    const { doc, node } = docWith((fs) => fs.writeImageFile('/big.png', big, 'image/png'), memoryBlobSide());
    const copy = new Y.Doc();
    const side = memoryBlobSide();
    attachBlobs(copy, side);
    Y.applyUpdate(copy, Y.encodeStateAsUpdate(doc));
    expect(nodeFiles(copy, node)!.entry('/big.png')!.content).toEqual({ kind: 'image', data: new Uint8Array(0), mediaType: 'image/png' });
    expect(missingBlobs(copy)).toEqual([blobSha(big)]);
    let heard = 0;
    const stop = onBlobsArrive(copy, () => (heard += 1));
    side.arrived(new Map());
    side.arrived(new Map([[blobSha(big), big]]));
    stop();
    side.arrived(new Map([[blobSha(big), big]]));
    expect(heard).toBe(1);
    expect(missingBlobs(copy)).toEqual([]);
    expect(nodeFiles(copy, node)!.entry('/big.png')!.content).toMatchObject({ data: big });
    // A copy without a side names every blob missing.
    const bare = new Y.Doc();
    Y.applyUpdate(bare, Y.encodeStateAsUpdate(doc));
    expect(missingBlobs(bare)).toEqual([blobSha(big)]);
  });

  it('changes nothing when the same files are written again, with the bytes here or only their name', () => {
    const big = bytes(BLOB_THRESHOLD_BYTES * 2);
    const side = memoryBlobSide();
    const { doc, node } = docWith((fs) => fs.writeImageFile('/big.png', big, 'image/png'), side);
    const copy = new Y.Doc();
    attachBlobs(copy, memoryBlobSide());
    Y.applyUpdate(copy, Y.encodeStateAsUpdate(doc));
    for (const each of [doc, copy]) {
      let updates = 0;
      each.on('update', () => (updates += 1));
      changeFiles(each, node, 'person', (fs) => fs.writeImageFile('/big.png', big, 'image/png'));
      expect(updates).toBe(0);
    }
    // Other bytes of the same size, another type, another size: a new entry each.
    let updates = 0;
    doc.on('update', () => (updates += 1));
    changeFiles(doc, node, 'person', (fs) => fs.writeImageFile('/big.png', bytes(big.length, 9), 'image/png'));
    expect(entryOf(doc, node, '/big.png').get('blob')).toBe(blobSha(bytes(big.length, 9)));
    changeFiles(doc, node, 'person', (fs) => fs.writeImageFile('/big.png', bytes(big.length, 9), 'image/gif'));
    expect(entryOf(doc, node, '/big.png').get('mediaType')).toBe('image/gif');
    changeFiles(doc, node, 'person', (fs) => fs.writeImageFile('/big.png', bytes(big.length + 1, 9), 'image/gif'));
    expect(entryOf(doc, node, '/big.png').get('size')).toBe(big.length + 1);
    expect(updates).toBe(3);
    // A copy without the bytes compares by the name.
    changeFiles(copy, node, 'person', (fs) => fs.writeImageFile('/big.png', bytes(big.length, 5), 'image/png'));
    expect(entryOf(copy, node, '/big.png').get('blob')).toBe(blobSha(bytes(big.length, 5)));
  });
});