feat: F5.5 large binary files as blobs on server F
From 16 KB a picture's or binary file's entry names its bytes by their SHA-256 instead of holding them; the bytes are on F, content-addressed (GET/POST /blobs with the session's grant, from the editor's origin, at most 16 MiB). A document's blob side says where they are: the editor's fetches and uploads, the room thread's asks F's main thread, and a start waits for the ones asked for; without a side everything stays in the document. Text stays Y.Text. A blob whose bytes are not there is never written back empty. A blob F has not got yet is asked for again a few times, and a running room reads late bytes. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -10,6 +10,7 @@
|
||||
"dependencies": {
|
||||
"@mdg-filius/core": "0.0.0",
|
||||
"@mdg-filius/fls": "0.0.0",
|
||||
"@noble/hashes": "2.4.0",
|
||||
"yjs": "^13.6.33"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,95 @@
|
||||
// A computer's large binary files outside the document (plan §5.1: no
|
||||
// binary in the CRDT): from BLOB_THRESHOLD_BYTES on, a picture's or other
|
||||
// binary file's entry holds `blob` — the SHA-256 of its bytes, in hex — and
|
||||
// `size` instead of `data`; the bytes are on F, content-addressed. Text
|
||||
// stays Y.Text whatever its size: two type into it together. Below the
|
||||
// threshold a binary stays in the entry, which spares a small icon a fetch.
|
||||
//
|
||||
// Where the bytes are is a document's blob side (attachBlobs): the editor's
|
||||
// on F (it fetches and uploads), the room thread's (it asks F's main
|
||||
// thread). A document without one — the development page, the tests, the
|
||||
// room in a worker of the page — keeps every binary in its entry. Reading a
|
||||
// blob whose bytes have not come yet gives no bytes; whoever needs them all
|
||||
// asks for the missing ones first (missingBlobs) and hears them come
|
||||
// (onBlobsArrive).
|
||||
|
||||
import { sha256 } from '@noble/hashes/sha2.js';
|
||||
import type * as Y from 'yjs';
|
||||
import { ROOTS } from './projectDoc.js';
|
||||
|
||||
export const BLOB_THRESHOLD_BYTES = 16 * 1024;
|
||||
|
||||
export interface BlobSide {
|
||||
/** A blob's bytes, if this side has them. */
|
||||
bytes(sha: string): Uint8Array | undefined;
|
||||
/** Bytes become a blob: kept here and taken to F. */
|
||||
keep(sha: string, bytes: Uint8Array): void;
|
||||
/** Hears blobs come; returns how to stop. */
|
||||
onArrive(listener: () => void): () => void;
|
||||
}
|
||||
|
||||
const sides = new WeakMap<Y.Doc, BlobSide>();
|
||||
|
||||
/** Where a document's blobs are, from now on. */
|
||||
export function attachBlobs(doc: Y.Doc, side: BlobSide): void {
|
||||
sides.set(doc, side);
|
||||
}
|
||||
|
||||
export function blobSideOf(doc: Y.Doc | null | undefined): BlobSide | undefined {
|
||||
return doc === null || doc === undefined ? undefined : sides.get(doc);
|
||||
}
|
||||
|
||||
const hex = (bytes: Uint8Array) => Array.from(bytes, (byte) => byte.toString(16).padStart(2, '0')).join('');
|
||||
|
||||
/** A blob's name: the SHA-256 of its bytes, in hex. */
|
||||
export function blobSha(bytes: Uint8Array): string {
|
||||
return hex(sha256(bytes));
|
||||
}
|
||||
|
||||
export const isBlobSha = (value: unknown): value is string => typeof value === 'string' && /^[0-9a-f]{64}$/.test(value);
|
||||
|
||||
/** Every blob the document's files name. */
|
||||
export function blobsOf(doc: Y.Doc): Set<string> {
|
||||
const found = new Set<string>();
|
||||
for (const node of doc.getMap<Y.Map<unknown>>(ROOTS.nodes).values()) {
|
||||
const files = node.get('files') as Y.Map<Y.Map<unknown>> | undefined;
|
||||
for (const entry of files?.values() ?? []) {
|
||||
const sha = entry.get('blob');
|
||||
if (isBlobSha(sha)) found.add(sha);
|
||||
}
|
||||
}
|
||||
return found;
|
||||
}
|
||||
|
||||
/** The blobs the document names whose bytes its side has not got (all, without a side). */
|
||||
export function missingBlobs(doc: Y.Doc): string[] {
|
||||
const side = blobSideOf(doc);
|
||||
return [...blobsOf(doc)].filter((sha) => side?.bytes(sha) === undefined);
|
||||
}
|
||||
|
||||
/** Hears the document's blobs come (nothing without a side); returns how to stop. */
|
||||
export function onBlobsArrive(doc: Y.Doc, listener: () => void): () => void {
|
||||
return blobSideOf(doc)?.onArrive(listener) ?? (() => {});
|
||||
}
|
||||
|
||||
/** A side that keeps blobs in memory and tells when some come (`arrived`); `taken` hears each kept one. */
|
||||
export function memoryBlobSide(taken: (sha: string, bytes: Uint8Array) => void = () => {}): BlobSide & { arrived(blobs: ReadonlyMap<string, Uint8Array>): void } {
|
||||
const held = new Map<string, Uint8Array>();
|
||||
const listeners = new Set<() => void>();
|
||||
return {
|
||||
bytes: (sha) => held.get(sha),
|
||||
keep(sha, bytes) {
|
||||
if (held.has(sha)) return;
|
||||
held.set(sha, bytes);
|
||||
taken(sha, bytes);
|
||||
},
|
||||
onArrive(listener) {
|
||||
listeners.add(listener);
|
||||
return () => listeners.delete(listener);
|
||||
},
|
||||
arrived(blobs) {
|
||||
for (const [sha, bytes] of blobs) held.set(sha, bytes);
|
||||
if (blobs.size > 0) for (const listener of listeners) listener();
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -29,6 +29,7 @@ import {
|
||||
} from '@mdg-filius/core';
|
||||
import { DEFAULT_MODEM_PORT, modemPair, type ImportedProject } from '@mdg-filius/fls';
|
||||
import { emptyProject, inOrder, nodeEntry, ROOTS } from './projectDoc.js';
|
||||
import { blobSideOf } from './blobs.js';
|
||||
|
||||
/** The kinds of the palette, in Java's order (GUIDesignSidebar.addComponentItemsToSidebar; the cable comes first, a tool). */
|
||||
export const PALETTE: readonly Exclude<NodeKind, 'unsupported'>[] = ['pc', 'notebook', 'networkSwitch', 'gateway', 'router', 'remoteLink'];
|
||||
@@ -120,7 +121,7 @@ export function addDevice(doc: Y.Doc, device: NewDevice, origin: unknown): void
|
||||
const project = newDeviceProject(device);
|
||||
doc.transact(() => {
|
||||
const order = Math.max(-1, ...[...nodes(doc).values()].map((node) => node.get('order') as number)) + 1;
|
||||
nodes(doc).set(device.id, nodeEntry(project.graph.nodes[0]!, order, project));
|
||||
nodes(doc).set(device.id, nodeEntry(project.graph.nodes[0]!, order, project, blobSideOf(doc)));
|
||||
}, origin);
|
||||
}
|
||||
|
||||
|
||||
@@ -11,6 +11,7 @@
|
||||
import * as Y from 'yjs';
|
||||
import type { Id, VirtualFileContent, VirtualFileSystem } from '@mdg-filius/core';
|
||||
import { VirtualFileSystem as FileSystem } from '@mdg-filius/core';
|
||||
import { blobSha, blobSideOf } from './blobs.js';
|
||||
import { fileEntry, fileSystemOf, inOrder, ROOTS } from './projectDoc.js';
|
||||
|
||||
export function sameBytes(left: Uint8Array, right: Uint8Array): boolean {
|
||||
@@ -29,11 +30,21 @@ export function replaceText(text: Y.Text, next: string): void {
|
||||
if (next.length - start - end > 0) text.insert(start, next.slice(start, next.length - end));
|
||||
}
|
||||
|
||||
/** Whether a document entry holds the content. */
|
||||
/**
|
||||
* Whether a document entry holds the content (a blob by its bytes where they
|
||||
* are here, else by its name).
|
||||
*/
|
||||
function holds(entry: Y.Map<unknown>, content: VirtualFileContent): boolean {
|
||||
if (entry.get('kind') !== content.kind) return false;
|
||||
if (content.kind === 'directory' || content.kind === 'text') return true;
|
||||
return sameBytes(entry.get('data') as Uint8Array, content.data) && entry.get('mediaType') === content.mediaType;
|
||||
if (entry.get('mediaType') !== content.mediaType) return false;
|
||||
const blob = entry.get('blob');
|
||||
if (typeof blob !== 'string') return sameBytes(entry.get('data') as Uint8Array, content.data);
|
||||
const held = blobSideOf(entry.doc)?.bytes(blob);
|
||||
// A blob whose bytes are not here was read as no bytes: written back so, that is no change (it would empty the file).
|
||||
if (held === undefined && content.data.length === 0) return true;
|
||||
if (entry.get('size') !== content.data.length) return false;
|
||||
return held === undefined ? blobSha(content.data) === blob : sameBytes(held, content.data);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -42,6 +53,7 @@ function holds(entry: Y.Map<unknown>, content: VirtualFileContent): boolean {
|
||||
* longer the file system's (a renamed entry), numbered anew.
|
||||
*/
|
||||
export function writeFiles(node: Y.Map<unknown>, fileSystem: VirtualFileSystem): void {
|
||||
const blobs = blobSideOf(node.doc);
|
||||
let files = node.get('files') as Y.Map<Y.Map<unknown>> | undefined;
|
||||
const wanted: [string, VirtualFileContent][] = [];
|
||||
const walk = (folder: string): void => {
|
||||
@@ -62,9 +74,9 @@ export function writeFiles(node: Y.Map<unknown>, fileSystem: VirtualFileSystem):
|
||||
for (const [path, content] of wanted) {
|
||||
const entry = files.get(path);
|
||||
if (entry === undefined) {
|
||||
files.set(path, fileEntry(order++, content));
|
||||
files.set(path, fileEntry(order++, content, blobs));
|
||||
} else if (!holds(entry, content)) {
|
||||
files.set(path, fileEntry(entry.get('order') as number, content));
|
||||
files.set(path, fileEntry(entry.get('order') as number, content, blobs));
|
||||
} else if (content.kind === 'text') {
|
||||
replaceText(entry.get('text') as Y.Text, content.text);
|
||||
}
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
// Public surface of the shared document.
|
||||
export * from './blobs.js';
|
||||
export * from './commands.js';
|
||||
export * from './documentation.js';
|
||||
export * from './fields.js';
|
||||
|
||||
@@ -35,6 +35,7 @@ import {
|
||||
} from '@mdg-filius/core';
|
||||
import type { FlsOwnApplication, ImportedProject } from '@mdg-filius/fls';
|
||||
import { fromY, toY, type PlainValue } from './values.js';
|
||||
import { BLOB_THRESHOLD_BYTES, blobSha, blobSideOf, type BlobSide } from './blobs.js';
|
||||
|
||||
export const SCHEMA_VERSION = 1;
|
||||
|
||||
@@ -98,23 +99,29 @@ function ownApplications(text: string): FlsOwnApplication[] {
|
||||
) as FlsOwnApplication[];
|
||||
}
|
||||
|
||||
export function fileEntry(order: number, content: VirtualFileContent): Y.Map<unknown> {
|
||||
/** A file's entry; a large binary as a blob when the document has a blob side (blobs.ts). */
|
||||
export function fileEntry(order: number, content: VirtualFileContent, blobs?: BlobSide): Y.Map<unknown> {
|
||||
const entry = new Y.Map<unknown>();
|
||||
entry.set('order', order);
|
||||
entry.set('kind', content.kind);
|
||||
if (content.kind === 'text') entry.set('text', new Y.Text(content.text));
|
||||
if (content.kind === 'binary' || content.kind === 'image') entry.set('data', content.data);
|
||||
if ((content.kind === 'binary' || content.kind === 'image') && blobs !== undefined && content.data.length >= BLOB_THRESHOLD_BYTES) {
|
||||
const sha = blobSha(content.data);
|
||||
blobs.keep(sha, content.data);
|
||||
entry.set('blob', sha);
|
||||
entry.set('size', content.data.length);
|
||||
} else if (content.kind === 'binary' || content.kind === 'image') entry.set('data', content.data);
|
||||
if ((content.kind === 'binary' || content.kind === 'image') && content.mediaType !== undefined) entry.set('mediaType', content.mediaType);
|
||||
return entry;
|
||||
}
|
||||
|
||||
/** A device's files, each folder's entries in the order they were made (parents before their entries). */
|
||||
function filesMap(files: VirtualFileSystem): Y.Map<Y.Map<unknown>> {
|
||||
function filesMap(files: VirtualFileSystem, blobs: BlobSide | undefined): Y.Map<Y.Map<unknown>> {
|
||||
const map = new Y.Map<Y.Map<unknown>>();
|
||||
let order = 0;
|
||||
const walk = (folder: string): void => {
|
||||
for (const entry of files.entriesInCreationOrder(folder)) {
|
||||
map.set(entry.path, fileEntry(order++, entry.content));
|
||||
map.set(entry.path, fileEntry(order++, entry.content, blobs));
|
||||
if (entry.content.kind === 'directory') walk(entry.path);
|
||||
}
|
||||
};
|
||||
@@ -123,7 +130,7 @@ function filesMap(files: VirtualFileSystem): Y.Map<Y.Map<unknown>> {
|
||||
}
|
||||
|
||||
/** A device's entry in the document, with what the project holds for it. */
|
||||
export function nodeEntry(node: TopologyNode, order: number, project: ImportedProject): Y.Map<unknown> {
|
||||
export function nodeEntry(node: TopologyNode, order: number, project: ImportedProject, blobs?: BlobSide): Y.Map<unknown> {
|
||||
const map = new Y.Map<unknown>();
|
||||
map.set('order', order);
|
||||
map.set('kind', node.kind);
|
||||
@@ -159,7 +166,7 @@ export function nodeEntry(node: TopologyNode, order: number, project: ImportedPr
|
||||
}
|
||||
map.set('programs', programs);
|
||||
const files = project.virtualFileSystems.get(node.id);
|
||||
if (files !== undefined) map.set('files', filesMap(files));
|
||||
if (files !== undefined) map.set('files', filesMap(files, blobs));
|
||||
const own = project.ownApplicationsByNodeId.get(node.id);
|
||||
if (own !== undefined) map.set('own', ownText(own));
|
||||
return map;
|
||||
@@ -202,7 +209,7 @@ export function writeProject(doc: Y.Doc, project: ImportedProject, origin: unkno
|
||||
meta.set('schema', SCHEMA_VERSION);
|
||||
meta.set('versionLine', project.versionLine);
|
||||
const nodes = doc.getMap<Y.Map<unknown>>(ROOTS.nodes);
|
||||
project.graph.nodes.forEach((node, index) => nodes.set(node.id, nodeEntry(node, index, project)));
|
||||
project.graph.nodes.forEach((node, index) => nodes.set(node.id, nodeEntry(node, index, project, blobSideOf(doc))));
|
||||
const links = doc.getMap<Y.Map<unknown>>(ROOTS.links);
|
||||
project.graph.links.forEach((link, index) => links.set(link.id, linkMap(link, index)));
|
||||
const docs = doc.getMap<Y.Map<unknown>>(ROOTS.docs);
|
||||
@@ -225,6 +232,15 @@ export function inOrder(map: Y.Map<Y.Map<unknown>>): [string, Y.Map<unknown>][]
|
||||
* system refuses (its folder missing before the janitor ran, or a name two
|
||||
* people gave at once in different case) is left out, not the project.
|
||||
*/
|
||||
const NO_BYTES = new Uint8Array(0);
|
||||
|
||||
/** A binary entry's bytes: its own, or its blob's (none while they have not come). */
|
||||
export function bytesOf(entry: Y.Map<unknown>): Uint8Array {
|
||||
const blob = entry.get('blob');
|
||||
if (typeof blob === 'string') return blobSideOf(entry.doc)?.bytes(blob) ?? NO_BYTES;
|
||||
return entry.get('data') as Uint8Array;
|
||||
}
|
||||
|
||||
export function fileSystemOf(files: Y.Map<Y.Map<unknown>>): VirtualFileSystem {
|
||||
const fileSystem = new VirtualFileSystem();
|
||||
for (const [path, entry] of inOrder(files)) {
|
||||
@@ -233,8 +249,8 @@ export function fileSystemOf(files: Y.Map<Y.Map<unknown>>): VirtualFileSystem {
|
||||
try {
|
||||
if (kind === 'directory') fileSystem.createDirectory(path);
|
||||
else if (kind === 'text') fileSystem.writeTextFile(path, (entry.get('text') as Y.Text).toString());
|
||||
else if (kind === 'image') fileSystem.writeImageFile(path, entry.get('data') as Uint8Array, mediaType!);
|
||||
else fileSystem.writeBinaryFile(path, entry.get('data') as Uint8Array, mediaType);
|
||||
else if (kind === 'image') fileSystem.writeImageFile(path, bytesOf(entry), mediaType!);
|
||||
else fileSystem.writeBinaryFile(path, bytesOf(entry), mediaType);
|
||||
} catch {
|
||||
// Left out (see above).
|
||||
}
|
||||
|
||||
@@ -0,0 +1,158 @@
|
||||
// A computer's large binary files as blobs (plan §5.1): with a blob side, a
|
||||
// picture or binary from 16 KB on is named by its SHA-256 in its entry and
|
||||
// kept by the side, a smaller one stays in its entry, a text stays Y.Text
|
||||
// whatever its size; without a side everything stays in the document as
|
||||
// before. Read back with the side's bytes; a copy of the document whose
|
||||
// side has not got them yet reads no bytes, names them as missing, and
|
||||
// hears them come. Written again unchanged, nothing changes.
|
||||
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import * as Y from 'yjs';
|
||||
import { VirtualFileSystem } from '@mdg-filius/core';
|
||||
import { openProjectArchive } from '@mdg-filius/fls';
|
||||
import {
|
||||
attachBlobs,
|
||||
BLOB_THRESHOLD_BYTES,
|
||||
blobsOf,
|
||||
blobSha,
|
||||
changeFiles,
|
||||
emptyProject,
|
||||
isBlobSha,
|
||||
memoryBlobSide,
|
||||
missingBlobs,
|
||||
nodeFiles,
|
||||
onBlobsArrive,
|
||||
writeFiles,
|
||||
writeProject,
|
||||
ROOTS,
|
||||
} from '../src/index.js';
|
||||
|
||||
const bytes = (length: number, seed = 1) => Uint8Array.from({ length }, (_, index) => (index * 31 + seed) % 251);
|
||||
|
||||
/** A document with one computer, its files as given, and a side (when given) attached before it is written. */
|
||||
function docWith(files: (fs: VirtualFileSystem) => void, side?: ReturnType<typeof memoryBlobSide>): { doc: Y.Doc; node: string } {
|
||||
const doc = new Y.Doc();
|
||||
if (side !== undefined) attachBlobs(doc, side);
|
||||
writeProject(doc, emptyProject());
|
||||
const node = 'pc';
|
||||
doc.transact(() => {
|
||||
const entry = new Y.Map<unknown>();
|
||||
doc.getMap<Y.Map<unknown>>(ROOTS.nodes).set(node, entry);
|
||||
const fs = new VirtualFileSystem();
|
||||
files(fs);
|
||||
writeFiles(entry, fs);
|
||||
});
|
||||
return { doc, node };
|
||||
}
|
||||
|
||||
const entryOf = (doc: Y.Doc, node: string, path: string) => (doc.getMap<Y.Map<unknown>>(ROOTS.nodes).get(node)!.get('files') as Y.Map<Y.Map<unknown>>).get(path)!;
|
||||
|
||||
describe('blobs', () => {
|
||||
it('names a blob by the SHA-256 of its bytes, in hex', () => {
|
||||
expect(blobSha(new TextEncoder().encode('abc'))).toBe('ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad');
|
||||
expect([isBlobSha(blobSha(bytes(3))), isBlobSha('ABC'), isBlobSha(7)]).toEqual([true, false, false]);
|
||||
expect(BLOB_THRESHOLD_BYTES).toBe(16 * 1024);
|
||||
});
|
||||
|
||||
it('keeps a large picture and a large binary as blobs, a small one and every text in the document', () => {
|
||||
const taken: string[] = [];
|
||||
const side = memoryBlobSide((sha) => taken.push(sha));
|
||||
const big = bytes(BLOB_THRESHOLD_BYTES);
|
||||
const { doc, node } = docWith((fs) => {
|
||||
fs.writeImageFile('/big.png', big, 'image/png');
|
||||
fs.writeBinaryFile('/big.bin', bytes(BLOB_THRESHOLD_BYTES + 5, 2), 'application/zip');
|
||||
fs.writeBinaryFile('/small.bin', bytes(BLOB_THRESHOLD_BYTES - 1, 3));
|
||||
fs.writeTextFile('/long.txt', 'x'.repeat(BLOB_THRESHOLD_BYTES * 2));
|
||||
}, side);
|
||||
const picture = entryOf(doc, node, '/big.png');
|
||||
expect([picture.get('blob'), picture.get('size'), picture.has('data'), picture.get('mediaType')]).toEqual([blobSha(big), big.length, false, 'image/png']);
|
||||
expect(entryOf(doc, node, '/big.bin').get('blob')).toBe(blobSha(bytes(BLOB_THRESHOLD_BYTES + 5, 2)));
|
||||
expect(entryOf(doc, node, '/small.bin').get('data')).toEqual(bytes(BLOB_THRESHOLD_BYTES - 1, 3));
|
||||
expect(entryOf(doc, node, '/long.txt').get('text')).toBeInstanceOf(Y.Text);
|
||||
expect(taken.sort()).toEqual([blobSha(big), blobSha(bytes(BLOB_THRESHOLD_BYTES + 5, 2))].sort());
|
||||
// Read back with the side: the bytes.
|
||||
const read = nodeFiles(doc, node)!;
|
||||
expect(read.entry('/big.png')!.content).toEqual({ kind: 'image', data: big, mediaType: 'image/png' });
|
||||
expect(blobsOf(doc)).toEqual(new Set(taken));
|
||||
expect(missingBlobs(doc)).toEqual([]);
|
||||
});
|
||||
|
||||
it('keeps the large pictures of a whole project as blobs when it is written with a side', () => {
|
||||
const big = bytes(BLOB_THRESHOLD_BYTES * 2, 4);
|
||||
const project = openProjectArchive(new Uint8Array(readFileSync(new URL('../../../corpus/java-examples/zwei_rechnernetze_komplett.fls', import.meta.url))), () => crypto.randomUUID());
|
||||
const [node, files] = [...project.virtualFileSystems][0]!;
|
||||
files.writeImageFile('/big.png', big, 'image/png');
|
||||
const taken: string[] = [];
|
||||
const doc = new Y.Doc();
|
||||
attachBlobs(
|
||||
doc,
|
||||
memoryBlobSide((sha) => taken.push(sha)),
|
||||
);
|
||||
writeProject(doc, project);
|
||||
expect(entryOf(doc, node, '/big.png').get('blob')).toBe(blobSha(big));
|
||||
expect(taken).toEqual([blobSha(big)]);
|
||||
});
|
||||
|
||||
it('keeps everything in the document without a side', () => {
|
||||
const big = bytes(BLOB_THRESHOLD_BYTES * 2);
|
||||
const { doc, node } = docWith((fs) => fs.writeImageFile('/big.png', big, 'image/png'));
|
||||
expect(entryOf(doc, node, '/big.png').get('data')).toEqual(big);
|
||||
expect(entryOf(doc, node, '/big.png').has('blob')).toBe(false);
|
||||
expect(nodeFiles(doc, node)!.entry('/big.png')!.content).toEqual({ kind: 'image', data: big, mediaType: 'image/png' });
|
||||
expect(missingBlobs(doc)).toEqual([]);
|
||||
expect(onBlobsArrive(doc, () => undefined)).toBeTypeOf('function');
|
||||
});
|
||||
|
||||
it('reads no bytes in a copy whose side has not got them, names them missing, and hears them come', () => {
|
||||
const big = bytes(BLOB_THRESHOLD_BYTES * 2);
|
||||
const { doc, node } = docWith((fs) => fs.writeImageFile('/big.png', big, 'image/png'), memoryBlobSide());
|
||||
const copy = new Y.Doc();
|
||||
const side = memoryBlobSide();
|
||||
attachBlobs(copy, side);
|
||||
Y.applyUpdate(copy, Y.encodeStateAsUpdate(doc));
|
||||
expect(nodeFiles(copy, node)!.entry('/big.png')!.content).toEqual({ kind: 'image', data: new Uint8Array(0), mediaType: 'image/png' });
|
||||
expect(missingBlobs(copy)).toEqual([blobSha(big)]);
|
||||
let heard = 0;
|
||||
const stop = onBlobsArrive(copy, () => (heard += 1));
|
||||
side.arrived(new Map());
|
||||
side.arrived(new Map([[blobSha(big), big]]));
|
||||
stop();
|
||||
side.arrived(new Map([[blobSha(big), big]]));
|
||||
expect(heard).toBe(1);
|
||||
expect(missingBlobs(copy)).toEqual([]);
|
||||
expect(nodeFiles(copy, node)!.entry('/big.png')!.content).toMatchObject({ data: big });
|
||||
// A copy without a side names every blob missing.
|
||||
const bare = new Y.Doc();
|
||||
Y.applyUpdate(bare, Y.encodeStateAsUpdate(doc));
|
||||
expect(missingBlobs(bare)).toEqual([blobSha(big)]);
|
||||
});
|
||||
|
||||
it('changes nothing when the same files are written again, with the bytes here or only their name', () => {
|
||||
const big = bytes(BLOB_THRESHOLD_BYTES * 2);
|
||||
const side = memoryBlobSide();
|
||||
const { doc, node } = docWith((fs) => fs.writeImageFile('/big.png', big, 'image/png'), side);
|
||||
const copy = new Y.Doc();
|
||||
attachBlobs(copy, memoryBlobSide());
|
||||
Y.applyUpdate(copy, Y.encodeStateAsUpdate(doc));
|
||||
for (const each of [doc, copy]) {
|
||||
let updates = 0;
|
||||
each.on('update', () => (updates += 1));
|
||||
changeFiles(each, node, 'person', (fs) => fs.writeImageFile('/big.png', big, 'image/png'));
|
||||
expect(updates).toBe(0);
|
||||
}
|
||||
// Other bytes of the same size, another type, another size: a new entry each.
|
||||
let updates = 0;
|
||||
doc.on('update', () => (updates += 1));
|
||||
changeFiles(doc, node, 'person', (fs) => fs.writeImageFile('/big.png', bytes(big.length, 9), 'image/png'));
|
||||
expect(entryOf(doc, node, '/big.png').get('blob')).toBe(blobSha(bytes(big.length, 9)));
|
||||
changeFiles(doc, node, 'person', (fs) => fs.writeImageFile('/big.png', bytes(big.length, 9), 'image/gif'));
|
||||
expect(entryOf(doc, node, '/big.png').get('mediaType')).toBe('image/gif');
|
||||
changeFiles(doc, node, 'person', (fs) => fs.writeImageFile('/big.png', bytes(big.length + 1, 9), 'image/gif'));
|
||||
expect(entryOf(doc, node, '/big.png').get('size')).toBe(big.length + 1);
|
||||
expect(updates).toBe(3);
|
||||
// A copy without the bytes compares by the name.
|
||||
changeFiles(copy, node, 'person', (fs) => fs.writeImageFile('/big.png', bytes(big.length, 5), 'image/png'));
|
||||
expect(entryOf(copy, node, '/big.png').get('blob')).toBe(blobSha(bytes(big.length, 5)));
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user