feat: F5.5 large binary files as blobs on server F

From 16 KB a picture's or binary file's entry names its bytes by their
SHA-256 instead of holding them; the bytes are on F, content-addressed
(GET/POST /blobs with the session's grant, from the editor's origin, at
most 16 MiB). A document's blob side says where they are: the editor's
fetches and uploads, the room thread's asks F's main thread, and a start
waits for the ones asked for; without a side everything stays in the
document. Text stays Y.Text. A blob whose bytes are not there is never
written back empty. A blob F has not got yet is asked for again a few
times, and a running room reads late bytes.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

packages/server/src/blobRoutes.ts

@@ -0,0 +1,72 @@
+// The editor's way to the blobs (blobs.ts): GET /blobs/<sha>?project=… for
+// whoever has a grant to the project, POST /blobs?project=… (the bytes as
+// the body) for whoever may write in it — not in a frozen project, and only
+// from the editor's origin (a page elsewhere under the same domain would
+// bring the person's cookie along). The answer to a POST is the name F gave
+// the bytes. For the editor's origins F answers CORS, so that a page served
+// from elsewhere (the development page) can reach it with its cookie.
+
+import type { IncomingMessage, ServerResponse } from 'node:http';
+import { BLOB_MAX_BYTES, type BlobStore } from './blobs.js';
+import type { Log } from './log.js';
+import type { SessionStore } from './sessions.js';
+import { cookieValue, SESSION_COOKIE } from './sync.js';
+import { isUuid } from './ticket.js';
+
+export interface BlobRouteOptions {
+  readonly blobs: BlobStore;
+  readonly sessions: SessionStore;
+  readonly log: Log;
+  readonly origins: readonly string[];
+}
+
+export async function handleBlobs(request: IncomingMessage, response: ServerResponse, url: URL, { blobs, sessions, log, origins }: BlobRouteOptions): Promise<void> {
+  const origin = request.headers.origin;
+  const allowed = origin !== undefined && origins.includes(origin);
+  const headers: Record<string, string> = allowed ? { 'access-control-allow-origin': origin, 'access-control-allow-credentials': 'true', vary: 'Origin' } : {};
+  const send = (status: number, value: object) => {
+    response.writeHead(status, { ...headers, 'content-type': 'application/json', 'cache-control': 'no-store' });
+    response.end(JSON.stringify(value));
+  };
+  if (request.method === 'OPTIONS') {
+    response.writeHead(allowed ? 204 : 403, { ...headers, 'access-control-allow-methods': 'GET, POST', 'access-control-allow-headers': 'content-type', 'access-control-max-age': '600' });
+    response.end();
+    return;
+  }
+  const project = url.searchParams.get('project');
+  if (!isUuid(project)) return send(400, { error: 'notAProject' });
+  const key = cookieValue(request.headers.cookie, SESSION_COOKIE);
+  const access = key === undefined ? ({ kind: 'notSignedIn' } as const) : await sessions.access(key, project);
+  if (access.kind !== 'granted') return send(403, { error: access.kind });
+  const name = /^\/blobs\/([^/]+)$/.exec(url.pathname)?.[1];
+  if (request.method === 'GET' && name !== undefined) {
+    const bytes = (await blobs.get([name])).get(name);
+    if (bytes === undefined) return send(404, { error: 'unknownBlob' });
+    // A blob's bytes never change under its name.
+    response.writeHead(200, { ...headers, 'content-type': 'application/octet-stream', 'content-length': String(bytes.length), 'cache-control': 'private, max-age=31536000, immutable' });
+    response.end(bytes);
+    return;
+  }
+  if (request.method === 'POST' && url.pathname === '/blobs') {
+    if (!allowed) {
+      log.warn('blob.foreignOrigin', { project, origin: origin ?? null });
+      return send(403, { error: 'foreignOrigin' });
+    }
+    if (access.level !== 'write') return send(403, { error: 'readOnly' });
+    if (access.frozen) return send(403, { error: 'frozen' });
+    const parts: Buffer[] = [];
+    let size = 0;
+    for await (const part of request as AsyncIterable<Buffer>) {
+      size += part.length;
+      if (size > BLOB_MAX_BYTES) {
+        log.warn('blob.tooLarge', { project, user: access.person.login });
+        return send(413, { error: 'tooLarge' });
+      }
+      parts.push(part);
+    }
+    const sha = await blobs.put(Buffer.concat(parts));
+    log.info('blob.kept', { project, user: access.person.login, sha, size });
+    return send(201, { sha });
+  }
+  send(404, { error: 'notFound' });
+}

packages/server/src/blobs.ts

@@ -0,0 +1,52 @@
+// The large binary files of the projects' computers (plan §5.1,
+// packages/doc's blobs.ts), content-addressed: named by the SHA-256 of
+// their bytes, kept once however many projects hold them. F computes the
+// name itself; what a client says a blob is called is not believed.
+//
+// Not yet: forgetting a blob no project names any more (they stay; the
+// commits keep theirs in the dashboard, plan §10.1).
+
+import { createHash } from 'node:crypto';
+import type pg from 'pg';
+
+/** A blob at most (a picture or a file put into a simulated computer). */
+export const BLOB_MAX_BYTES = 16 * 1024 * 1024;
+
+export interface BlobStore {
+  migrate(): Promise<void>;
+  /** Bytes kept; their name. */
+  put(bytes: Uint8Array): Promise<string>;
+  /** The blobs of these names that F has. */
+  get(shas: readonly string[]): Promise<Map<string, Uint8Array>>;
+}
+
+export const shaOf = (bytes: Uint8Array) => createHash('sha256').update(bytes).digest('hex');
+
+const SCHEMA = `
+CREATE TABLE IF NOT EXISTS filius_blob (
+  sha text PRIMARY KEY,
+  bytes bytea NOT NULL,
+  size integer NOT NULL,
+  created_at timestamptz NOT NULL DEFAULT now()
+);
+`;
+
+export class PostgresBlobs implements BlobStore {
+  constructor(private readonly pool: pg.Pool) {}
+
+  async migrate(): Promise<void> {
+    await this.pool.query(SCHEMA);
+  }
+
+  async put(bytes: Uint8Array): Promise<string> {
+    const sha = shaOf(bytes);
+    await this.pool.query('INSERT INTO filius_blob (sha, bytes, size) VALUES ($1, $2, $3) ON CONFLICT DO NOTHING', [sha, Buffer.from(bytes), bytes.length]);
+    return sha;
+  }
+
+  async get(shas: readonly string[]): Promise<Map<string, Uint8Array>> {
+    if (shas.length === 0) return new Map();
+    const result = await this.pool.query<{ sha: string; bytes: Buffer }>('SELECT sha, bytes FROM filius_blob WHERE sha = ANY($1)', [shas]);
+    return new Map(result.rows.map((row) => [row.sha, new Uint8Array(row.bytes.buffer, row.bytes.byteOffset, row.bytes.byteLength)]));
+  }
+}

packages/server/src/index.ts

@@ -1,6 +1,8 @@
 // Server F (plan §8): the shared documents, the simulation rooms, the
 // sign-in and the dashboard's service.
 export * from './auth.js';
+export * from './blobRoutes.js';
+export * from './blobs.js';
 export * from './log.js';
 export * from './persistence.js';
 export * from './server.js';

packages/server/src/main.ts

@@ -7,6 +7,7 @@
 // machine without HTTPS). The rooms' thread is the bundle beside this one
 // (dist/simWorker.mjs).
 
+import { PostgresBlobs } from './blobs.js';
 import { jsonLog } from './log.js';
 import { startServer } from './server.js';
 import { PostgresSessions } from './sessions.js';
@@ -23,12 +24,15 @@ const secureCookie = process.env.FILIUS_INSECURE_COOKIE !== '1';
 if (!secureCookie) log.warn('auth.insecureCookie', { note: 'the session cookie goes over plain HTTP' });
 const store = PostgresStore.connect(config.DATABASE_URL!);
 const sessions = new PostgresSessions(store.pool);
+const blobs = new PostgresBlobs(store.pool);
 await store.migrate();
 await sessions.migrate();
+await blobs.migrate();
 const server = await startServer({
   port: Number(process.env.FILIUS_PORT ?? 8080),
   store,
   sessions,
+  blobs,
   log,
   ticketSecret: config.FILIUS_TICKET_SECRET!,
   serviceKey: config.FILIUS_SERVICE_KEY!,

packages/server/src/server.ts

@@ -1,6 +1,7 @@
 // Server F's front door (plan §8.2): one HTTP server — /healthz for the
 // proxy, /__auth for the editor's frame (ticket in, cookie out), /svc/* for
-// the dashboard (with the service key), the WebSockets /sync for the shared
+// the dashboard (with the service key), /blobs for the large binary files
+// of the projects' computers, the WebSockets /sync for the shared
 // documents and /sim?project=… for the simulation rooms (the session's grant
 // decides there as at /sync; a refusal closes the socket with its reason);
 // anything else is not found. A WebSocket from a page of another
@@ -18,6 +19,8 @@ import type { AddressInfo, Socket } from 'node:net';
 import { SIM_REFUSED_CODE, type SyncRefusal } from '@mdg-filius/protocol';
 import { WebSocketServer, type WebSocket } from 'ws';
 import { exchangeTicket } from './auth.js';
+import { handleBlobs } from './blobRoutes.js';
+import type { BlobStore } from './blobs.js';
 import { errorText, type Log } from './log.js';
 import { handleService } from './service.js';
 import { holdMessages, SimRooms } from './simRooms.js';
@@ -46,6 +49,7 @@ export interface ServerOptions extends SyncOptions {
   readonly secureCookie: boolean;
   /** The bundled room thread (dist/simWorker.mjs). */
   readonly workerUrl: URL;
+  readonly blobs: BlobStore;
   readonly simThreads?: number;
   readonly lingerMilliseconds?: number;
   readonly pingMilliseconds?: number;
@@ -65,6 +69,7 @@ export async function startServer(options: ServerOptions): Promise<FiliusServer>
   const { hocuspocus } = sync;
   const sims = new SimRooms({
     hocuspocus,
+    blobs: options.blobs,
     log: options.log,
     workerUrl: options.workerUrl,
     ...(options.simThreads === undefined ? {} : { threads: options.simThreads }),
@@ -105,6 +110,7 @@ export async function startServer(options: ServerOptions): Promise<FiliusServer>
     if (request.method === 'GET' && path === '/healthz') json(response, 200, { status: 'ok', documents: hocuspocus.getDocumentsCount(), connections: hocuspocus.getConnectionsCount(), rooms: sims.stats().rooms });
     else if (request.method === 'GET' && path === '/__auth') await exchangeTicket(request, response, options);
     else if (path.startsWith('/svc/')) await handleService(request, response, path, { ...options, sync, sims });
+    else if (path === '/blobs' || path.startsWith('/blobs/')) await handleBlobs(request, response, new URL(request.url ?? '/', 'http://filius'), options);
     else json(response, 404, { error: 'notFound' });
   };
   const http: Server = createServer((request, response) => {

packages/server/src/simChannel.ts

@@ -4,10 +4,16 @@
 // rooms, each with a replica of its project's document: the document's
 // updates go both ways, a person's messages go in, the room's come out,
 // each for one person (a room tells its people apart; the main thread
-// numbers them).
+// numbers them). A room's large binary files (blobs) are F's: the thread
+// asks for the ones it has not got, and gives F the ones the simulation
+// makes before the document's change that names them.
 
 import type { HostMessage } from '@mdg-filius/protocol';
 
+/** How often, and how long apart, a thread asks again for a blob F has not got (yet). */
+export const BLOB_ASKS = 5;
+export const BLOB_ASK_AGAIN_MILLISECONDS = 1_000;
+
 export type ToRoomThread =
   /** A room for a project, its document as it is now. */
   | { readonly type: 'open'; readonly room: string; readonly state: Uint8Array }
@@ -19,10 +25,16 @@ export type ToRoomThread =
   /** The simulation stopped (what it left written back first); answered by `stopped`. */
   | { readonly type: 'stop'; readonly room: string }
   /** The room ended — stopped first with `writeBack`, else as it is (its project is gone); answered by `closed`. */
-  | { readonly type: 'close'; readonly room: string; readonly writeBack: boolean };
+  | { readonly type: 'close'; readonly room: string; readonly writeBack: boolean }
+  /** The blobs asked for: those F has, and the names of those it has not. */
+  | { readonly type: 'blobs'; readonly room: string; readonly blobs: readonly { readonly sha: string; readonly bytes: Uint8Array }[]; readonly missing: readonly string[] };
 
 export type FromRoomThread =
   | { readonly type: 'doc'; readonly room: string; readonly update: Uint8Array }
   | { readonly type: 'host'; readonly room: string; readonly client: number; readonly message: HostMessage }
   | { readonly type: 'stopped'; readonly room: string }
+  /** Blobs the document names that the thread has not got. */
+  | { readonly type: 'needBlobs'; readonly room: string; readonly shas: readonly string[] }
+  /** A blob the simulation made (before the change that names it). */
+  | { readonly type: 'blob'; readonly room: string; readonly bytes: Uint8Array }
   | { readonly type: 'closed'; readonly room: string };

packages/server/src/simRooms.ts

@@ -3,7 +3,11 @@
 // with the fewest rooms takes a new one). A room keeps its project's
 // document open on F (a direct connection: what the simulation leaves goes
 // into the document even when no one has it open) and gets every change of
-// it; what the room writes comes back as the room's own change.
+// it; what the room writes comes back as the room's own change. The blobs
+// its document names come from F's store when the thread asks; a blob the
+// simulation makes is kept before the change that names it is applied (the
+// room's messages are taken in order), so that no one is told of a file
+// whose bytes are not there.
 //
 // Who may do what: everyone in the project watches; only with a write grant
 // does one start, stop, set the speed, hold "Pakete verwerfen", clear the
@@ -23,6 +27,7 @@ import type { DirectConnection, Hocuspocus } from '@hocuspocus/server';
 import * as Y from 'yjs';
 import { SIM_CLOSED_CODE, SIM_ROOM_FAILED_CODE, WRITING_KINDS, type ClientMessageKind, type HostMessage, type SyncClosing } from '@mdg-filius/protocol';
 import type WebSocket from 'ws';
+import type { BlobStore } from './blobs.js';
 import { errorText, type Log } from './log.js';
 import type { FromRoomThread, ToRoomThread } from './simChannel.js';
 
@@ -39,6 +44,7 @@ export interface SimPerson {
 
 export interface SimRoomsOptions {
   readonly hocuspocus: Hocuspocus;
+  readonly blobs: BlobStore;
   readonly log: Log;
   /** The bundled room thread (dist/simWorker.mjs). */
   readonly workerUrl: URL;
@@ -64,6 +70,8 @@ interface Room {
   frozen: boolean;
   linger: ReturnType<typeof setTimeout> | undefined;
   ended: boolean;
+  /** The thread's changes and blobs, taken in order. */
+  taken: Promise<void>;
 }
 
 type RawData = Buffer | ArrayBuffer | Buffer[];
@@ -124,9 +132,30 @@ export class SimRooms {
   private fromThread(thread: Thread, message: FromRoomThread): void {
     const room = [...thread.rooms].find((each) => each.project === message.room);
     if (room === undefined) return;
+    const inOrder = (work: () => Promise<void> | void) => {
+      room.taken = room.taken.then(work).catch((error: unknown) => this.options.log.error('sim.blobFailed', { project: room.project, error: errorText(error) }));
+    };
     switch (message.type) {
       case 'doc':
-        Y.applyUpdate(room.doc, message.update, room);
+        inOrder(() => Y.applyUpdate(room.doc, message.update, room));
+        return;
+      case 'blob':
+        inOrder(async () => void (await this.options.blobs.put(message.bytes)));
+        return;
+      case 'needBlobs':
+        void this.options.blobs.get(message.shas).then(
+          (found) =>
+            this.post(room, {
+              type: 'blobs',
+              room: room.project,
+              blobs: [...found].map(([sha, bytes]) => ({ sha, bytes })),
+              missing: message.shas.filter((sha) => !found.has(sha)),
+            }),
+          (error: unknown) => {
+            this.options.log.error('sim.blobFailed', { project: room.project, error: errorText(error) });
+            this.post(room, { type: 'blobs', room: room.project, blobs: [], missing: message.shas });
+          },
+        );
         return;
       case 'host':
         room.clients.get(message.client)?.socket.send(JSON.stringify(message.message));
@@ -168,6 +197,7 @@ export class SimRooms {
       frozen,
       linger: undefined,
       ended: false,
+      taken: Promise.resolve(),
     };
     thread.rooms.add(room);
     this.post(room, { type: 'open', room: project, state: Y.encodeStateAsUpdate(doc) });
@@ -267,6 +297,8 @@ export class SimRooms {
     const done = (async () => {
       this.rooms.delete(room.project);
       await this.ask(room, { type: 'close', room: room.project, writeBack });
+      // What it wrote back is in the document before the document is let go.
+      await room.taken;
       this.forget(room);
       await room.direct.disconnect();
       this.options.log.info('sim.roomClosed', { project: room.project, writeBack });

packages/server/src/simWorker.ts

@@ -1,12 +1,18 @@
 // A thread of server F that holds simulation rooms (simChannel.ts): each
 // room a replica of its project's document and the room that reads it, all
 // ticking together every TICK_MILLISECONDS. Bundled on its own
-// (dist/simWorker.mjs), started by the main thread.
+// (dist/simWorker.mjs), started by the main thread. The blobs a room's
+// document names are asked of the main thread as they come into it; a start
+// waits until the ones asked for have come (or F said it has not got them),
+// so that the simulation starts with the files' bytes. One F has not got —
+// yet: the editor that made it may still be taking it there — is asked for
+// again a few times; when it comes, the running simulation reads it.
 
 import { parentPort } from 'node:worker_threads';
 import * as Y from 'yjs';
+import { attachBlobs, memoryBlobSide, missingBlobs } from '@mdg-filius/doc';
 import { SimulationRoom, TICK_MILLISECONDS } from '@mdg-filius/sim';
-import type { FromRoomThread, ToRoomThread } from './simChannel.js';
+import { BLOB_ASK_AGAIN_MILLISECONDS, BLOB_ASKS, type FromRoomThread, type ToRoomThread } from './simChannel.js';
 
 /** The origin of what came from the main thread: not sent back. */
 const MAIN_ORIGIN = 'main';
@@ -16,21 +22,43 @@ interface HeldRoom {
   readonly room: SimulationRoom;
   /** The main thread's number of a person → the room's. */
   readonly clients: Map<number, number>;
+  readonly blobs: ReturnType<typeof memoryBlobSide>;
+  /** Blobs asked for and not answered yet. */
+  readonly asked: Set<string>;
+  /** Blobs F said it has not got: asked again a while later, a few times (`misses` counts). */
+  readonly unavailable: Set<string>;
+  readonly misses: Map<string, number>;
+  /** Starts waiting for the blobs asked for. */
+  readonly waiting: { readonly client: number; readonly message: unknown }[];
 }
 
 const port = parentPort!;
 const rooms = new Map<string, HeldRoom>();
 const post = (message: FromRoomThread) => port.postMessage(message);
 
+/** Asks the main thread for the blobs the room's document names and the room has not got. */
+function askForBlobs(name: string, held: HeldRoom): void {
+  const shas = missingBlobs(held.doc).filter((sha) => !held.asked.has(sha) && !held.unavailable.has(sha));
+  if (shas.length === 0) return;
+  for (const sha of shas) held.asked.add(sha);
+  post({ type: 'needBlobs', room: name, shas });
+}
+
+const isStart = (message: unknown) => typeof message === 'object' && message !== null && (message as { kind?: unknown }).kind === 'start';
+
 port.on('message', (message: ToRoomThread) => {
   if (message.type === 'open') {
     const doc = new Y.Doc();
+    const blobs = memoryBlobSide((_sha, bytes) => post({ type: 'blob', room: message.room, bytes }));
+    attachBlobs(doc, blobs);
     Y.applyUpdate(doc, message.state, MAIN_ORIGIN);
     doc.on('update', (update: Uint8Array, origin: unknown) => {
       if (origin !== MAIN_ORIGIN && rooms.get(message.room)?.doc === doc) post({ type: 'doc', room: message.room, update });
     });
     const room = new SimulationRoom(doc, { now: () => performance.now(), seed: () => Math.floor(Math.random() * 2 ** 31) });
-    rooms.set(message.room, { doc, room, clients: new Map() });
+    const held: HeldRoom = { doc, room, clients: new Map(), blobs, asked: new Set(), unavailable: new Set(), misses: new Map(), waiting: [] };
+    rooms.set(message.room, held);
+    askForBlobs(message.room, held);
     return;
   }
   const held = rooms.get(message.room);
@@ -38,7 +66,32 @@ port.on('message', (message: ToRoomThread) => {
   switch (message.type) {
     case 'doc':
       Y.applyUpdate(held.doc, message.update, MAIN_ORIGIN);
+      askForBlobs(message.room, held);
       return;
+    case 'blobs': {
+      held.blobs.arrived(new Map(message.blobs.map(({ sha, bytes }) => [sha, bytes])));
+      for (const { sha } of message.blobs) held.asked.delete(sha);
+      if (message.blobs.length > 0) held.room.blobsArrived();
+      for (const sha of message.missing) {
+        held.asked.delete(sha);
+        const asks = (held.misses.get(sha) ?? 0) + 1;
+        held.misses.set(sha, asks);
+        held.unavailable.add(sha);
+        if (asks < BLOB_ASKS)
+          setTimeout(() => {
+            if (rooms.get(message.room) !== held) return;
+            held.unavailable.delete(sha);
+            askForBlobs(message.room, held);
+          }, BLOB_ASK_AGAIN_MILLISECONDS);
+      }
+      if (held.asked.size === 0) {
+        for (const { client, message: waiting } of held.waiting.splice(0)) {
+          const roomClient = held.clients.get(client);
+          if (roomClient !== undefined) held.room.receive(roomClient, waiting);
+        }
+      }
+      return;
+    }
     case 'connect':
       held.clients.set(
         message.client,
@@ -47,7 +100,9 @@ port.on('message', (message: ToRoomThread) => {
       return;
     case 'client': {
       const client = held.clients.get(message.client);
-      if (client !== undefined) held.room.receive(client, message.message);
+      if (client === undefined) return;
+      if (isStart(message.message) && held.asked.size > 0) held.waiting.push({ client: message.client, message: message.message });
+      else held.room.receive(client, message.message);
       return;
     }
     case 'disconnect': {

packages/server/test/blobs.test.ts

@@ -0,0 +1,268 @@
+// The large binary files on F (F5.5): kept by their SHA-256 through POST
+// /blobs from the editor's origin with a write grant, given back through GET
+// /blobs/<sha> to whoever has a grant, with CORS for the editor's origins;
+// refused from another origin, to a reader, in a frozen project, without a
+// session, beyond the size. A room gets the blobs its document names before
+// its simulation starts: a picture copied in a command line is the same
+// blob, and a file it did not touch stays one.
+
+import { randomUUID } from 'node:crypto';
+import { readFileSync } from 'node:fs';
+import { afterAll, beforeAll, describe, expect, it } from 'vitest';
+import WebSocket from 'ws';
+import * as Y from 'yjs';
+import { attachBlobs, BLOB_THRESHOLD_BYTES, blobSha, changeFiles, memoryBlobSide, nodeFiles, ROOTS, writeProject } from '@mdg-filius/doc';
+import { openProjectArchive } from '@mdg-filius/fls';
+import type { HostMessage } from '@mdg-filius/protocol';
+import { BLOB_MAX_BYTES, PostgresBlobs, PostgresSessions, PostgresStore, shaOf } from '../src/index.js';
+import { BLOB_ASK_AGAIN_MILLISECONDS, BLOB_ASKS } from '../src/simChannel.js';
+import { testDatabase, type TestDatabase } from './database.js';
+import { EDITOR_ORIGIN, newProject, SERVICE_KEY, signIn, startTestServer, until, type TestServer } from './harness.js';
+
+let database: TestDatabase;
+let store: PostgresStore;
+let sessions: PostgresSessions;
+let blobs: PostgresBlobs;
+
+beforeAll(async () => {
+  database = await testDatabase();
+  store = PostgresStore.connect(database.url);
+  sessions = new PostgresSessions(store.pool);
+  blobs = new PostgresBlobs(store.pool);
+  await store.migrate();
+  await sessions.migrate();
+  await blobs.migrate();
+}, 60_000);
+
+afterAll(async () => {
+  await store?.close();
+  await database?.stop();
+});
+
+const picture = (length = BLOB_THRESHOLD_BYTES * 2, seed = 7) => Uint8Array.from({ length }, (_, index) => (index * 13 + seed) % 256);
+
+function post(server: TestServer, project: string, body: Uint8Array, headers: Record<string, string>): Promise<Response> {
+  return fetch(`${server.base}/blobs?project=${project}`, { method: 'POST', headers: { 'content-type': 'application/octet-stream', ...headers }, body });
+}
+const get = (server: TestServer, project: string, sha: string, headers: Record<string, string>) => fetch(`${server.base}/blobs/${sha}?project=${project}`, { headers });
+
+/** F's store, slower by so many milliseconds in reading. */
+function slowly(inner: PostgresBlobs, { get }: { readonly get: number }): PostgresBlobs {
+  return Object.assign(Object.create(inner) as PostgresBlobs, {
+    get: async (shas: readonly string[]) => (await new Promise((resolve) => setTimeout(resolve, get)), inner.get(shas)),
+  });
+}
+
+/** FILIUS' example with a picture as a blob on the notebook (its bytes not in F's store). */
+async function projectWithPicture(seed: number) {
+  const project = randomUUID();
+  const imported = openProjectArchive(new Uint8Array(readFileSync(new URL('../../../corpus/java-examples/internetwork_komplett.fls', import.meta.url))), () => randomUUID());
+  const notebook = imported.graph.nodes.find((each) => each.displayName === 'Notebook 1.10')!.id;
+  const doc = new Y.Doc();
+  attachBlobs(doc, memoryBlobSide());
+  writeProject(doc, imported);
+  const bytes = picture(BLOB_THRESHOLD_BYTES * 2, seed);
+  changeFiles(doc, notebook, 'person', (files) => files.writeImageFile('/spaet.png', bytes, 'image/png'));
+  await store.create(project, Y.encodeStateAsUpdate(doc));
+  return { project, notebook, bytes };
+}
+
+/** In a project's room by WebSocket, signed in with a write grant. */
+async function enterRoom(server: TestServer, project: string) {
+  const cookie = await signIn(server, project);
+  const socket = new WebSocket(`ws://127.0.0.1:${server.port}/sim?project=${project}`, { headers: { cookie } });
+  const got: HostMessage[] = [];
+  socket.on('message', (data) => got.push(JSON.parse(data.toString()) as HostMessage));
+  socket.on('error', () => {});
+  await new Promise((resolve) => socket.on('open', resolve));
+  return { got, send: (message: object) => socket.send(JSON.stringify({ v: 1, ...message })), close: () => socket.close() };
+}
+
+/** A computer's files as F's store holds them. */
+async function storedFiles(project: string, node: string): Promise<Y.Map<Y.Map<unknown>>> {
+  const stored = await store.load(project);
+  const doc = new Y.Doc();
+  Y.applyUpdate(doc, stored!.state);
+  for (const update of stored!.updates) Y.applyUpdate(doc, update);
+  return doc.getMap<Y.Map<unknown>>(ROOTS.nodes).get(node)!.get('files') as Y.Map<Y.Map<unknown>>;
+}
+
+describe('the blobs on F', () => {
+  it('keeps bytes under the name F gives them, and gives them back to whoever has a grant', async () => {
+    const server = await startTestServer(store, sessions);
+    const project = await newProject(store);
+    const cookie = await signIn(server, project, { login: 'ab12' });
+    const bytes = picture();
+    const kept = await post(server, project, bytes, { cookie, origin: EDITOR_ORIGIN });
+    expect([kept.status, await kept.json()]).toEqual([201, { sha: blobSha(bytes) }]);
+    expect(kept.headers.get('access-control-allow-origin')).toBe(EDITOR_ORIGIN);
+    expect(kept.headers.get('access-control-allow-credentials')).toBe('true');
+    expect(shaOf(bytes)).toBe(blobSha(bytes));
+    // Again: the same name, kept once.
+    expect(await (await post(server, project, bytes, { cookie, origin: EDITOR_ORIGIN })).json()).toEqual({ sha: blobSha(bytes) });
+    expect((await store.pool.query('SELECT count(*)::int AS n FROM filius_blob WHERE sha = $1', [blobSha(bytes)])).rows[0].n).toBe(1);
+    const reader = await signIn(server, project, { level: 'read' });
+    const back = await get(server, project, blobSha(bytes), { cookie: reader, origin: EDITOR_ORIGIN });
+    expect(back.status).toBe(200);
+    expect(new Uint8Array(await back.arrayBuffer())).toEqual(bytes);
+    expect([back.headers.get('content-type'), back.headers.get('cache-control'), back.headers.get('access-control-allow-origin')]).toEqual([
+      'application/octet-stream',
+      'private, max-age=31536000, immutable',
+      EDITOR_ORIGIN,
+    ]);
+    // Without an origin (the same origin), no CORS.
+    expect((await get(server, project, blobSha(bytes), { cookie: reader })).headers.get('access-control-allow-origin')).toBeNull();
+    expect(server.log.lines.find((line) => line.event === 'blob.kept')).toMatchObject({ project, user: 'ab12', sha: blobSha(bytes), size: bytes.length });
+    await server.close();
+  });
+
+  it('answers the preflight of the editor’s origin, and no other', async () => {
+    const server = await startTestServer(store, sessions);
+    const ask = (origin: string) => fetch(`${server.base}/blobs?project=x`, { method: 'OPTIONS', headers: { origin, 'access-control-request-method': 'POST' } });
+    const ours = await ask(EDITOR_ORIGIN);
+    expect([ours.status, ours.headers.get('access-control-allow-methods'), ours.headers.get('access-control-allow-headers')]).toEqual([204, 'GET, POST', 'content-type']);
+    expect((await ask('https://elsewhere.test')).status).toBe(403);
+    await server.close();
+  });
+
+  it('refuses another origin, a reader, a frozen project, no session, no project and too many bytes', async () => {
+    const server = await startTestServer(store, sessions);
+    const project = await newProject(store);
+    const cookie = await signIn(server, project);
+    const reader = await signIn(server, project, { level: 'read' });
+    const bytes = picture(100);
+    const answer = async (response: Response) => [response.status, await response.json()];
+    expect(await answer(await post(server, project, bytes, { cookie }))).toEqual([403, { error: 'foreignOrigin' }]);
+    expect(await answer(await post(server, project, bytes, { cookie, origin: 'https://elsewhere.test' }))).toEqual([403, { error: 'foreignOrigin' }]);
+    expect(await answer(await post(server, project, bytes, { cookie: reader, origin: EDITOR_ORIGIN }))).toEqual([403, { error: 'readOnly' }]);
+    expect(await answer(await post(server, project, bytes, { origin: EDITOR_ORIGIN }))).toEqual([403, { error: 'notSignedIn' }]);
+    expect(await answer(await post(server, 'nothing', bytes, { cookie, origin: EDITOR_ORIGIN }))).toEqual([400, { error: 'notAProject' }]);
+    expect(await answer(await post(server, randomUUID(), bytes, { cookie, origin: EDITOR_ORIGIN }))).toEqual([403, { error: 'unknownProject' }]);
+    expect(await answer(await post(server, project, new Uint8Array(BLOB_MAX_BYTES + 1), { cookie, origin: EDITOR_ORIGIN }))).toEqual([413, { error: 'tooLarge' }]);
+    expect(server.log.lines.filter((line) => line.event === 'blob.foreignOrigin')).toHaveLength(2);
+    await fetch(`${server.base}/svc/projects/${project}/freeze`, { method: 'POST', headers: { 'x-filius-service-key': SERVICE_KEY }, body: JSON.stringify({ frozen: true }) });
+    expect(await answer(await post(server, project, bytes, { cookie, origin: EDITOR_ORIGIN }))).toEqual([403, { error: 'frozen' }]);
+    expect(await answer(await get(server, project, 'f'.repeat(64), { cookie }))).toEqual([404, { error: 'unknownBlob' }]);
+    expect(await answer(await get(server, project, 'not-a-sha', { cookie }))).toEqual([404, { error: 'unknownBlob' }]);
+    expect(await answer(await get(server, randomUUID(), blobSha(bytes), { cookie }))).toEqual([403, { error: 'unknownProject' }]);
+    expect(await answer(await fetch(`${server.base}/blobs/x/y?project=${project}`, { headers: { cookie } }))).toEqual([404, { error: 'notFound' }]);
+    expect(BLOB_MAX_BYTES).toBe(16 * 1024 * 1024);
+    await server.close();
+  });
+
+  it('gives a room the blobs of its document before the simulation starts', async () => {
+    const server = await startTestServer(store, sessions);
+    // FILIUS' example, with a picture as a blob on the notebook.
+    const project = randomUUID();
+    const imported = openProjectArchive(new Uint8Array(readFileSync(new URL('../../../corpus/java-examples/internetwork_komplett.fls', import.meta.url))), () => randomUUID());
+    const notebook = imported.graph.nodes.find((each) => each.displayName === 'Notebook 1.10')!.id;
+    const doc = new Y.Doc();
+    const side = memoryBlobSide();
+    attachBlobs(doc, side);
+    writeProject(doc, imported);
+    const bytes = picture();
+    changeFiles(doc, notebook, 'person', (files) => files.writeImageFile('/bild.png', bytes, 'image/png'));
+    await blobs.put(bytes);
+    await store.create(project, Y.encodeStateAsUpdate(doc));
+    const cookie = await signIn(server, project);
+    const socket = new WebSocket(`ws://127.0.0.1:${server.port}/sim?project=${project}`, { headers: { cookie } });
+    const got: HostMessage[] = [];
+    socket.on('message', (data) => got.push(JSON.parse(data.toString()) as HostMessage));
+    await new Promise((resolve) => socket.on('open', resolve));
+    const send = (message: object) => socket.send(JSON.stringify({ v: 1, ...message }));
+    send({ kind: 'start' });
+    await until(() => got.some((message) => message.kind === 'state' && message.mode === 'simulation'));
+    send({ kind: 'input', node: notebook, app: 'terminal', action: 'enter', args: { line: 'cp bild.png kopie.png' }, reqId: 'r' });
+    await until(() => got.some((message) => message.kind === 'ack'));
+    send({ kind: 'stop' });
+    await until(() => got.filter((message) => message.kind === 'state' && message.mode === 'design').length === 2);
+    socket.close();
+    await server.close();
+    const stored = await store.load(project);
+    const after = new Y.Doc();
+    Y.applyUpdate(after, stored!.state);
+    for (const update of stored!.updates) Y.applyUpdate(after, update);
+    const files = after.getMap<Y.Map<unknown>>(ROOTS.nodes).get(notebook)!.get('files') as Y.Map<Y.Map<unknown>>;
+    // The copy is the same blob: the room had the bytes; the original is untouched.
+    expect([files.get('/kopie.png')?.get('blob'), files.get('/bild.png')?.get('blob')]).toEqual([blobSha(bytes), blobSha(bytes)]);
+    expect(nodeFiles(after, notebook)!.entry('/kopie.png')!.content).toMatchObject({ kind: 'image', mediaType: 'image/png' });
+  }, 30_000);
+
+  it('asks again for a blob F gets only after the start, and the running simulation reads it', async () => {
+    const server = await startTestServer(store, sessions);
+    const project = randomUUID();
+    const imported = openProjectArchive(new Uint8Array(readFileSync(new URL('../../../corpus/java-examples/internetwork_komplett.fls', import.meta.url))), () => randomUUID());
+    const notebook = imported.graph.nodes.find((each) => each.displayName === 'Notebook 1.10')!.id;
+    const doc = new Y.Doc();
+    attachBlobs(doc, memoryBlobSide());
+    writeProject(doc, imported);
+    const bytes = picture(BLOB_THRESHOLD_BYTES * 2, 42);
+    changeFiles(doc, notebook, 'person', (files) => files.writeImageFile('/spaet.png', bytes, 'image/png'));
+    await store.create(project, Y.encodeStateAsUpdate(doc));
+    const cookie = await signIn(server, project);
+    const socket = new WebSocket(`ws://127.0.0.1:${server.port}/sim?project=${project}`, { headers: { cookie } });
+    const got: HostMessage[] = [];
+    socket.on('message', (data) => got.push(JSON.parse(data.toString()) as HostMessage));
+    await new Promise((resolve) => socket.on('open', resolve));
+    socket.send(JSON.stringify({ v: 1, kind: 'start' }));
+    await until(() => got.some((message) => message.kind === 'state' && message.mode === 'simulation'));
+    // Taken to F only now (an editor still uploading it); the thread asks again a second later.
+    await blobs.put(bytes);
+    await new Promise((resolve) => setTimeout(resolve, BLOB_ASK_AGAIN_MILLISECONDS + 500));
+    socket.send(JSON.stringify({ v: 1, kind: 'input', node: notebook, app: 'terminal', action: 'enter', args: { line: 'cp spaet.png kopie.png' }, reqId: 'r' }));
+    await until(() => got.some((message) => message.kind === 'ack'));
+    socket.close();
+    await server.close();
+    const stored = await store.load(project);
+    const after = new Y.Doc();
+    Y.applyUpdate(after, stored!.state);
+    for (const update of stored!.updates) Y.applyUpdate(after, update);
+    const files = after.getMap<Y.Map<unknown>>(ROOTS.nodes).get(notebook)!.get('files') as Y.Map<Y.Map<unknown>>;
+    expect(files.get('/kopie.png')?.get('blob')).toBe(blobSha(bytes));
+    expect(BLOB_ASKS).toBe(5);
+  }, 30_000);
+
+  it("starts a simulation only once the blobs asked for have come, however slow F's store is", async () => {
+    const { project, notebook, bytes } = await projectWithPicture(51);
+    await blobs.put(bytes);
+    const server = await startTestServer(store, sessions, { blobs: slowly(blobs, { get: 800 }) });
+    const room = await enterRoom(server, project);
+    room.send({ kind: 'start' });
+    await until(() => room.got.some((message) => message.kind === 'state' && message.mode === 'simulation'), 5_000);
+    room.send({ kind: 'input', node: notebook, app: 'terminal', action: 'enter', args: { line: 'cp spaet.png kopie.png' }, reqId: 'r' });
+    await until(() => room.got.some((message) => message.kind === 'ack'));
+    room.close();
+    await server.close();
+    expect((await storedFiles(project, notebook)).get('/kopie.png')?.get('blob')).toBe(blobSha(bytes));
+  }, 30_000);
+
+  it('starts a room whose blob F has not got, and empties no file by it', async () => {
+    const server = await startTestServer(store, sessions);
+    const project = randomUUID();
+    const imported = openProjectArchive(new Uint8Array(readFileSync(new URL('../../../corpus/java-examples/internetwork_komplett.fls', import.meta.url))), () => randomUUID());
+    const notebook = imported.graph.nodes.find((each) => each.displayName === 'Notebook 1.10')!.id;
+    const doc = new Y.Doc();
+    attachBlobs(doc, memoryBlobSide());
+    writeProject(doc, imported);
+    const bytes = picture(BLOB_THRESHOLD_BYTES * 3, 99);
+    changeFiles(doc, notebook, 'person', (files) => files.writeImageFile('/fehlt.png', bytes, 'image/png'));
+    await store.create(project, Y.encodeStateAsUpdate(doc));
+    const cookie = await signIn(server, project);
+    const socket = new WebSocket(`ws://127.0.0.1:${server.port}/sim?project=${project}`, { headers: { cookie } });
+    const got: HostMessage[] = [];
+    socket.on('message', (data) => got.push(JSON.parse(data.toString()) as HostMessage));
+    await new Promise((resolve) => socket.on('open', resolve));
+    socket.send(JSON.stringify({ v: 1, kind: 'start' }));
+    await until(() => got.some((message) => message.kind === 'state' && message.mode === 'simulation'));
+    socket.send(JSON.stringify({ v: 1, kind: 'input', node: notebook, app: 'terminal', action: 'enter', args: { line: 'touch neu.txt' }, reqId: 'r' }));
+    await until(() => got.some((message) => message.kind === 'ack'));
+    socket.close();
+    await server.close();
+    const stored = await store.load(project);
+    const after = new Y.Doc();
+    Y.applyUpdate(after, stored!.state);
+    for (const update of stored!.updates) Y.applyUpdate(after, update);
+    const files = after.getMap<Y.Map<unknown>>(ROOTS.nodes).get(notebook)!.get('files') as Y.Map<Y.Map<unknown>>;
+    expect([files.get('/fehlt.png')?.get('blob'), files.has('/neu.txt')]).toEqual([blobSha(bytes), true]);
+  }, 30_000);
+});

packages/server/test/harness.ts

@@ -16,7 +16,7 @@ import { expect } from 'vitest';
 import WebSocket from 'ws';
 import * as Y from 'yjs';
 import { emptyProject, writeProject } from '@mdg-filius/doc';
-import { startServer, type DocStore, type FiliusServer, type Log, type ServerOptions, type SessionStore } from '../src/index.js';
+import { PostgresBlobs, startServer, type DocStore, type PostgresStore, type FiliusServer, type Log, type ServerOptions, type SessionStore } from '../src/index.js';
 import { signTicket } from './tickets.js';
 
 export const TICKET_SECRET = 'test-ticket-secret';
@@ -38,6 +38,13 @@ export function memoryLog(): MemoryLog {
 
 export type TestServer = FiliusServer & { readonly log: MemoryLog; readonly base: string };
 
+/** F's blob store in the test database (the table made, once a file). */
+async function testBlobs(store: DocStore): Promise<PostgresBlobs> {
+  const blobs = new PostgresBlobs((store as PostgresStore).pool);
+  await blobs.migrate();
+  return blobs;
+}
+
 let roomThread: Promise<URL> | undefined;
 
 /** src/simWorker.ts bundled into a directory of its own, once a test file. */
@@ -63,6 +70,7 @@ export async function startTestServer(store: DocStore, sessions: SessionStore, o
     origins: [EDITOR_ORIGIN],
     secureCookie: false,
     workerUrl: await bundledRoomThread(),
+    blobs: await testBlobs(store),
     simThreads: 1,
     ...options,
   });