feat: F5.5 large binary files as blobs on server F

From 16 KB a picture's or binary file's entry names its bytes by their
SHA-256 instead of holding them; the bytes are on F, content-addressed
(GET/POST /blobs with the session's grant, from the editor's origin, at
most 16 MiB). A document's blob side says where they are: the editor's
fetches and uploads, the room thread's asks F's main thread, and a start
waits for the ones asked for; without a side everything stays in the
document. Text stays Y.Text. A blob whose bytes are not there is never
written back empty. A blob F has not got yet is asked for again a few
times, and a running room reads late bytes.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

packages/editor/src/desktop/fileExplorer.ts

@@ -41,7 +41,7 @@ import {
   type FolderEntry,
   type Id,
 } from '@mdg-filius/core';
-import { changeFiles, nodeFiles } from '@mdg-filius/doc';
+import { changeFiles, nodeFiles, onBlobsArrive } from '@mdg-filius/doc';
 import { WindowState, windowOf } from './windowState.js';
 
 /** A row of the explorer's tree: every folder and file, each folder's entries after it in the order they were made. */
@@ -214,6 +214,8 @@ export class ImageViewerWindow extends WindowState {
   ) {
     super();
     doc.on('update', () => this.pathValue !== null && this.changed());
+    // A picture that is a blob shows once its bytes have come.
+    onBlobsArrive(doc, () => this.pathValue !== null && this.changed());
   }
 
   private files(): VirtualFileSystem {

packages/editor/src/i18n/de.json

@@ -367,6 +367,7 @@
   "remote.refused": "Der Filius-Dienst hat die Verbindung abgelehnt ({reason}).",
   "remote.revoked": "Ihr Zugang zu diesem Projekt wurde beendet.",
   "remote.unknownProject": "Dieses Projekt gibt es nicht mehr.",
+  "remote.uploadFailed": "Eine Datei konnte nicht zum Filius-Dienst gebracht werden; andere sehen sie nicht. Laden Sie die Seite neu und versuchen Sie es noch einmal.",
   "report.apps": "Anwendungen",
   "report.baseConfig": "Basiskonfiguration",
   "report.components": "Komponenten",

packages/editor/src/i18n/en.json

@@ -367,6 +367,7 @@
   "remote.refused": "The Filius service refused the connection ({reason}).",
   "remote.revoked": "Your access to this project has ended.",
   "remote.unknownProject": "This project no longer exists.",
+  "remote.uploadFailed": "A file could not be taken to the Filius service; others do not see it. Reload the page and try again.",
   "report.apps": "Applications",
   "report.baseConfig": "Base Configuration",
   "report.components": "Components",

packages/editor/src/remote/SharedProject.tsx

@@ -4,9 +4,14 @@
 // refuses or lets go — of the document or the room — a word why, instead of
 // the editor. Who only reads gets the editor locked (Editor's readOnly).
 // The others in the project are shown in the editor and told to whoever
-// embeds it (`onPeople`: the dashboard's header, over the bridge).
+// embeds it (`onPeople`: the dashboard's header, over the bridge). The
+// document's large binary files come from F and go to it (remoteBlobs):
+// fetched as the document names them; one that could not be taken to F is
+// said.
 
 import { useEffect, useMemo, useState } from 'react';
+import { attachBlobs } from '@mdg-filius/doc';
+import { remoteBlobs, type RemoteBlobs } from './remoteBlobs.js';
 import { peopleOf, type PresenceUser } from './presence.js';
 import { usePresence } from './usePresence.js';
 import { Editor } from '../Editor.js';
@@ -18,7 +23,25 @@ import { connectSharedDoc, type SharedDocState } from './sharedDoc.js';
 type Ready = Extract<SharedDocState, { kind: 'ready' }>;
 
 /** The editor on the ready document, with the others in the project. */
-function ReadyEditor({ state, createSimHost, onPeople }: { readonly state: Ready; readonly createSimHost: SimHostFactory; readonly onPeople: ((people: readonly PresenceUser[]) => void) | undefined }) {
+function ReadyEditor({
+  state,
+  createSimHost,
+  onPeople,
+  blobs,
+}: {
+  readonly state: Ready;
+  readonly createSimHost: SimHostFactory;
+  readonly onPeople: ((people: readonly PresenceUser[]) => void) | undefined;
+  readonly blobs: RemoteBlobs;
+}) {
+  // The blob side before anything reads or writes a file of the document.
+  useMemo(() => attachBlobs(state.doc, blobs), [state.doc, blobs]);
+  useEffect(() => {
+    const fetchMissing = () => void blobs.fetchMissing(state.doc);
+    fetchMissing();
+    state.doc.on('update', fetchMissing);
+    return () => state.doc.off('update', fetchMissing);
+  }, [state.doc, blobs]);
   const presence = usePresence(state.awareness, state.you);
   const key = JSON.stringify(peopleOf(presence.peers, state.you.id));
   useEffect(() => onPeople?.(JSON.parse(key) as PresenceUser[]), [key, onPeople]);
@@ -39,6 +62,8 @@ export function SharedProject({ server, project, onPeople }: SharedProjectProps)
   const [state, setState] = useState<SharedDocState>({ kind: 'connecting' });
   /** F ended the room for this page: the editor goes as when the document is let go. */
   const [roomEnd, setRoomEnd] = useState<string | null>(null);
+  const [uploadFailed, setUploadFailed] = useState(false);
+  const blobs = useMemo(() => remoteBlobs({ server, project, onFailure: () => setUploadFailed(true) }), [server, project]);
   useEffect(() => connectSharedDoc({ server, project, onState: setState }), [server, project]);
   const createSimHost = useMemo<SimHostFactory>(
     () => () => {
@@ -63,5 +88,14 @@ export function SharedProject({ server, project, onPeople }: SharedProjectProps)
         {t('remote.connecting')}
       </p>
     );
-  return <ReadyEditor key={state.doc.guid} state={state} createSimHost={createSimHost} onPeople={onPeople} />;
+  return (
+    <>
+      {uploadFailed ? (
+        <p role="alert" className="remote-notice">
+          {t('remote.uploadFailed')}
+        </p>
+      ) : null}
+      <ReadyEditor key={state.doc.guid} state={state} createSimHost={createSimHost} onPeople={onPeople} blobs={blobs} />
+    </>
+  );
 }

packages/editor/src/remote/remoteBlobs.ts

@@ -0,0 +1,89 @@
+// The editor's blob side on F (packages/doc's blobs.ts): the blobs the
+// document names are fetched from F as they come into it (each once, with
+// the cookie; one F has not got — yet: the editor that made it may still be
+// taking it there — is asked for again a few times, a while apart; one the
+// network lost is asked for with the next change). A blob made here (a file
+// put into a computer) is kept at once and taken to F — tried again once
+// after a while, then told as failed.
+// What arrives is told to the document's listeners, so that a picture
+// shows once its bytes are here.
+
+import type * as Y from 'yjs';
+import { memoryBlobSide, missingBlobs, type BlobSide } from '@mdg-filius/doc';
+
+/** The wait before an upload is tried again. */
+export const UPLOAD_RETRY_MILLISECONDS = 2_000;
+/** How often, and how long apart, a blob F has not got is asked for again. */
+export const BLOB_ASKS = 5;
+export const BLOB_ASK_AGAIN_MILLISECONDS = 2_000;
+
+export interface RemoteBlobsOptions {
+  /** F's address. */
+  readonly server: string;
+  readonly project: string;
+  /** A blob that could not be taken to F. */
+  readonly onFailure?: (sha: string) => void;
+  readonly fetch?: typeof fetch;
+}
+
+export interface RemoteBlobs extends BlobSide {
+  /** Fetches what the document names and is not here; done when every answer has come. */
+  fetchMissing(doc: Y.Doc): Promise<void>;
+}
+
+export function remoteBlobs({ server, project, onFailure, fetch: request = (input, init) => fetch(input, init) }: RemoteBlobsOptions): RemoteBlobs {
+  const address = (path: string) => `${new URL(path, server).href}?project=${encodeURIComponent(project)}`;
+  const upload = async (sha: string, bytes: Uint8Array, tries: number): Promise<void> => {
+    try {
+      const response = await request(address('/blobs'), { method: 'POST', credentials: 'include', headers: { 'content-type': 'application/octet-stream' }, body: bytes as BodyInit });
+      const answer = response.ok ? ((await response.json()) as { sha?: unknown }) : undefined;
+      if (answer?.sha === sha) return;
+    } catch {
+      // Tried again below.
+    }
+    if (tries > 1) {
+      await new Promise((resolve) => setTimeout(resolve, UPLOAD_RETRY_MILLISECONDS));
+      return upload(sha, bytes, tries - 1);
+    }
+    onFailure?.(sha);
+  };
+  const side = memoryBlobSide((sha, bytes) => void upload(sha, bytes, 2));
+  const asked = new Set<string>();
+  /** Blobs F has not got (not asked for until their wait is over), and how often F said so. */
+  const unavailable = new Set<string>();
+  const misses = new Map<string, number>();
+  const blobs: RemoteBlobs = {
+    bytes: side.bytes,
+    keep: side.keep,
+    onArrive: side.onArrive,
+    async fetchMissing(doc) {
+      const wanted = missingBlobs(doc).filter((sha) => !asked.has(sha) && !unavailable.has(sha));
+      for (const sha of wanted) asked.add(sha);
+      const found = await Promise.all(
+        wanted.map(async (sha): Promise<[string, Uint8Array] | undefined> => {
+          try {
+            const response = await request(address(`/blobs/${sha}`), { credentials: 'include' });
+            if (response.ok) return [sha, new Uint8Array(await response.arrayBuffer())];
+            if (response.status === 404) {
+              const asks = (misses.get(sha) ?? 0) + 1;
+              misses.set(sha, asks);
+              unavailable.add(sha);
+              if (asks < BLOB_ASKS)
+                setTimeout(() => {
+                  unavailable.delete(sha);
+                  void blobs.fetchMissing(doc);
+                }, BLOB_ASK_AGAIN_MILLISECONDS);
+            }
+            return undefined;
+          } catch {
+            return undefined;
+          } finally {
+            asked.delete(sha);
+          }
+        }),
+      );
+      side.arrived(new Map(found.filter((each) => each !== undefined)));
+    },
+  };
+  return blobs;
+}

packages/editor/src/sheet/sheet.css

@@ -396,3 +396,19 @@
   height: 10px;
   border-radius: 5px;
 }
+
+/* Over the editor on a project of F: something went wrong that the person should know. */
+.remote-notice {
+  position: fixed;
+  left: 50%;
+  bottom: 16px;
+  z-index: 100;
+  transform: translateX(-50%);
+  max-width: min(90vw, 560px);
+  margin: 0;
+  padding: 10px 14px;
+  border: 1px solid var(--danger);
+  border-radius: 8px;
+  background: var(--danger-soft);
+  color: var(--danger-on-soft);
+}