feat: F6.1 FILIUS files on F: projects from examples and .fls, copies, export

/svc/examples lists FILIUS' examples, which lie beside F in the image (GPL
like this repository, kept out of the dashboard); /svc/projects makes a
project from one; /svc/projects/<id>/import takes a .fls (up to 64 MiB),
/copy copies a project as it is now, /export gives it back as a .fls with its
warnings. Large files go into the blob store on import and come back from it
on export, which is refused when one is missing.

Found on the way: Node's request body is a Buffer, and Yjs takes no Buffer
into a document, so every .fls with binary files would have failed; and
fflate, now in the bundle, imports createRequire itself, which the build's
banner declared again, so F did not start. The banner names it otherwise and
a test builds and starts the bundle. The proxy's /svc/ takes 70 MB.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

packages/server/src/index.ts

@@ -15,3 +15,4 @@ export * from './simRooms.js';
 export * from './store.js';
 export * from './sync.js';
 export * from './ticket.js';
+export * from './transfer.js';

packages/server/src/main.ts

@@ -7,7 +7,8 @@
 // machine without HTTPS). The rooms' thread is the bundle beside this one
 // (dist/simWorker.mjs), the editor's page and bundle the folder editor/
 // beside it (FILIUS_EDITOR_DIR elsewhere); without that folder F answers
-// only its own routes and says so in the log.
+// only its own routes and says so in the log. FILIUS' examples lie in the
+// folder examples/ beside it (FILIUS_EXAMPLES_DIR elsewhere).
 
 import { existsSync } from 'node:fs';
 import { join } from 'node:path';
@@ -37,6 +38,8 @@ await blobs.migrate();
 const editorFolder = process.env.FILIUS_EDITOR_DIR ?? fileURLToPath(new URL('./editor/', import.meta.url));
 const editor = existsSync(join(editorFolder, 'index.html')) ? editorFiles(editorFolder) : undefined;
 if (editor === undefined) log.warn('editor.missing', { folder: editorFolder });
+const examplesFolder = process.env.FILIUS_EXAMPLES_DIR ?? fileURLToPath(new URL('./examples/', import.meta.url));
+if (!existsSync(examplesFolder)) log.warn('examples.missing', { folder: examplesFolder });
 const server = await startServer({
   port: Number(process.env.FILIUS_PORT ?? 8080),
   store,
@@ -49,6 +52,7 @@ const server = await startServer({
   secureCookie,
   workerUrl: new URL('./simWorker.mjs', import.meta.url),
   ...(editor === undefined ? {} : { editor }),
+  examplesFolder,
 });
 for (const signal of ['SIGTERM', 'SIGINT'] as const) {
   process.once(signal, () => {

packages/server/src/server.ts

@@ -54,6 +54,8 @@ export interface ServerOptions extends SyncOptions {
   readonly blobs: BlobStore;
   /** The editor's page and bundle; without them only F's own routes answer (development, tests). */
   readonly editor?: EditorFiles;
+  /** The folder of FILIUS' examples (/svc/examples); none: there are none. */
+  readonly examplesFolder?: string;
   readonly simThreads?: number;
   readonly lingerMilliseconds?: number;
   readonly pingMilliseconds?: number;

packages/server/src/service.ts

@@ -6,24 +6,33 @@
 // a running simulation stops, what it left written back), end a person's
 // grant to a project or all their sessions (whoever is connected, to the
 // document or the room, is let go), and list who is signed in to what (the
-// dashboard checks it against its shares). Snapshots, restoring,
-// copying and the .fls come with the commits (F6).
+// dashboard checks it against its shares). FILIUS files (F6.1,
+// transfer.ts): make a project from one of FILIUS' examples or from a .fls
+// the dashboard sends, copy a project (as it is now, also while people work
+// in it), and give a project as a .fls (with what it could not hold, as
+// warnings in X-Filius-Warnings, base64url JSON). Snapshots and restoring come
+// with the commits (F7).
 
 import { createHash, timingSafeEqual } from 'node:crypto';
 import type { IncomingMessage, ServerResponse } from 'node:http';
 import * as Y from 'yjs';
 import { emptyProject, writeProject } from '@mdg-filius/doc';
 import type { SyncClosing } from '@mdg-filius/protocol';
+import type { BlobStore } from './blobs.js';
 import type { Log } from './log.js';
 import type { SessionStore } from './sessions.js';
 import type { SimRooms } from './simRooms.js';
 import type { DocStore } from './store.js';
 import { accessMessage, type ConnectionContext, type Sync } from './sync.js';
 import { isUuid } from './ticket.js';
+import { archiveOf, exampleArchive, exampleNames, stateOfArchive } from './transfer.js';
 
 export const SERVICE_KEY_HEADER = 'x-filius-service-key';
 /** A request's body at most (the service sends small JSON). */
 export const SERVICE_BODY_BYTES = 64 * 1024;
+/** A FILIUS file to import at most: pictures of up to 16 MiB each (BLOB_MAX_BYTES) may be in it. */
+export const IMPORT_BYTES = 64 * 1024 * 1024;
+export const WARNINGS_HEADER = 'x-filius-warnings';
 /** Hocuspocus' code for a connection ended for want of rights. */
 const FORBIDDEN = 4403;
 
@@ -34,8 +43,14 @@ export interface ServiceOptions {
   readonly sims: SimRooms;
   readonly log: Log;
   readonly serviceKey: string;
+  readonly blobs: BlobStore;
+  /** The folder of FILIUS' examples (beside main.mjs in the image); none: there are none. */
+  readonly examplesFolder?: string;
 }
 
+/** What a route answers: JSON, or a FILIUS file with its warnings. */
+type Answer = readonly [number, object] | { readonly file: Uint8Array; readonly warnings: readonly string[] };
+
 class HttpError extends Error {
   constructor(
     readonly status: number,
@@ -52,17 +67,22 @@ export function serviceKeyMatches(given: string | string[] | undefined, expected
   return expected !== '' && typeof given === 'string' && timingSafeEqual(digest(given), digest(expected));
 }
 
-async function body(request: IncomingMessage): Promise<Record<string, unknown>> {
+async function bytes(request: IncomingMessage, most: number): Promise<Buffer> {
   const parts: Buffer[] = [];
   let size = 0;
   for await (const part of request as AsyncIterable<Buffer>) {
     size += part.length;
-    if (size > SERVICE_BODY_BYTES) throw new HttpError(413, 'tooLarge');
+    if (size > most) throw new HttpError(413, 'tooLarge');
     parts.push(part);
   }
+  return Buffer.concat(parts);
+}
+
+async function body(request: IncomingMessage): Promise<Record<string, unknown>> {
+  const given = await bytes(request, SERVICE_BODY_BYTES);
   let value: unknown;
   try {
-    value = JSON.parse(Buffer.concat(parts).toString('utf8'));
+    value = JSON.parse(given.toString('utf8'));
   } catch {
     throw new HttpError(400, 'notJson');
   }
@@ -88,27 +108,92 @@ export async function handleService(request: IncomingMessage, response: ServerRe
     return;
   }
   try {
-    const [status, value] = await route(request, path, options);
-    send(status, value);
+    const answer = await route(request, path, options);
+    if (!('file' in answer)) send(...answer);
+    else {
+      response.writeHead(200, {
+        'content-type': 'application/octet-stream',
+        'content-length': String(answer.file.length),
+        'cache-control': 'no-store',
+        [WARNINGS_HEADER]: Buffer.from(JSON.stringify(answer.warnings)).toString('base64url'),
+      });
+      response.end(answer.file);
+    }
   } catch (error) {
     if (!(error instanceof HttpError)) throw error;
     send(error.status, { error: error.code });
   }
 }
 
-async function route(request: IncomingMessage, path: string, { store, sessions, sync, sims, log }: ServiceOptions): Promise<[number, object]> {
+/** A FILIUS file as a new project's first state; refused when it is none. */
+async function readArchive(archive: Uint8Array, blobs: BlobStore, log: Log, project: string) {
+  try {
+    return await stateOfArchive(archive, blobs);
+  } catch (error) {
+    log.warn('svc.notAFiliusFile', { project, bytes: archive.length, error: error instanceof Error ? error.message : String(error) });
+    throw new HttpError(422, 'notAFiliusFile');
+  }
+}
+
+async function route(request: IncomingMessage, path: string, { store, sessions, sync, sims, log, blobs, examplesFolder }: ServiceOptions): Promise<Answer> {
   const close = (connections: ReturnType<Sync['connections']>, reason: SyncClosing) => {
     for (const connection of connections) connection.close({ code: FORBIDDEN, reason });
     return connections.length;
   };
-  const project = /^\/svc\/projects\/([^/]+)(\/freeze)?$/.exec(path);
+  /** A project's document as it is now, held while `use` runs (a project no one has open is loaded for it). */
+  const withDocument = async <T>(id: string, use: (doc: Y.Doc) => Promise<T>): Promise<T> => {
+    const direct = await sync.hocuspocus.openDirectConnection(id, { service: true }).catch(() => undefined);
+    if (direct === undefined) throw new HttpError(404, 'unknownProject');
+    try {
+      return await use(direct.document!);
+    } finally {
+      await direct.disconnect();
+    }
+  };
+  const project = /^\/svc\/projects\/([^/]+)(\/freeze|\/import|\/copy|\/export)?$/.exec(path);
+  if (request.method === 'GET' && path === '/svc/examples') return [200, { examples: await exampleNames(examplesFolder) }];
   if (request.method === 'POST' && path === '/svc/projects') {
-    const id = uuid((await body(request)).project, 'notAProject');
-    const doc = new Y.Doc();
-    writeProject(doc, emptyProject());
-    if (!(await store.create(id, Y.encodeStateAsUpdate(doc)))) throw new HttpError(409, 'exists');
-    log.info('svc.created', { project: id });
-    return [201, { project: id }];
+    const given = await body(request);
+    const id = uuid(given.project, 'notAProject');
+    let made: { readonly state: Uint8Array; readonly warnings: readonly string[] };
+    if (given.example === undefined) {
+      const doc = new Y.Doc();
+      writeProject(doc, emptyProject());
+      made = { state: Y.encodeStateAsUpdate(doc), warnings: [] };
+    } else {
+      const archive = await exampleArchive(examplesFolder, given.example);
+      if (archive === undefined) throw new HttpError(404, 'unknownExample');
+      made = await readArchive(archive, blobs, log, id);
+    }
+    if (!(await store.create(id, made.state))) throw new HttpError(409, 'exists');
+    log.info('svc.created', { project: id, ...(given.example === undefined ? {} : { example: given.example }) });
+    return [201, { project: id, warnings: made.warnings }];
+  }
+  if (request.method === 'POST' && project !== null && project[2] === '/import') {
+    const id = uuid(project[1], 'notAProject');
+    const archive = await bytes(request, IMPORT_BYTES);
+    const made = await readArchive(archive, blobs, log, id);
+    if (!(await store.create(id, made.state))) throw new HttpError(409, 'exists');
+    log.info('svc.imported', { project: id, bytes: archive.length, warnings: made.warnings.length });
+    return [201, { project: id, warnings: made.warnings }];
+  }
+  if (request.method === 'POST' && project !== null && project[2] === '/copy') {
+    const id = uuid(project[1], 'notAProject');
+    const to = uuid((await body(request)).to, 'notAProject');
+    const state = await withDocument(id, async (doc) => Y.encodeStateAsUpdate(doc));
+    if (!(await store.create(to, state))) throw new HttpError(409, 'exists');
+    log.info('svc.copied', { project: id, to });
+    return [201, { project: to }];
+  }
+  if (request.method === 'GET' && project !== null && project[2] === '/export') {
+    const id = uuid(project[1], 'notAProject');
+    const made = await withDocument(id, (doc) => archiveOf(doc, blobs));
+    if ('missing' in made) {
+      log.error('svc.exportMissingBlobs', { project: id, missing: made.missing });
+      throw new HttpError(409, 'missingBlobs');
+    }
+    log.info('svc.exported', { project: id, bytes: made.archive.length, warnings: made.warnings.length });
+    return { file: made.archive, warnings: made.warnings };
   }
   if (request.method === 'DELETE' && project !== null && project[2] === undefined) {
     const id = uuid(project[1], 'notAProject');

packages/server/src/transfer.ts

@@ -0,0 +1,61 @@
+// Projects into and out of F as FILIUS files (plan §11, F6.1): a project's
+// first state from a .fls — one of FILIUS' examples, which lie beside F (they
+// are GPL like this repository and stay out of the dashboard), or a file the
+// dashboard sends —, and a project's document as a .fls. A large file of a
+// computer goes into F's blob store before the state that names it (as the
+// editor does with an upload); a .fls takes the blobs' bytes back from the
+// store, and is not made when one of them is missing (it would hold the file
+// empty, without a word).
+
+import { randomUUID } from 'node:crypto';
+import { readdir, readFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import * as Y from 'yjs';
+import { attachBlobs, blobsOf, docArchive, memoryBlobSide, writeProject } from '@mdg-filius/doc';
+import { openProjectArchive } from '@mdg-filius/fls';
+import type { BlobStore } from './blobs.js';
+
+/** A FILIUS file as a project's first state, with what Java's file held that the project cannot (warnings); throws on a file that is none. */
+export async function stateOfArchive(archive: Uint8Array, blobs: BlobStore): Promise<{ readonly state: Uint8Array; readonly warnings: readonly string[] }> {
+  // A plain Uint8Array, not the request's Buffer: the files' bytes are pieces of it, and Yjs takes no
+  // Buffer into a document ("Unexpected content type").
+  const project = openProjectArchive(new Uint8Array(archive), randomUUID);
+  const kept: Uint8Array[] = [];
+  const doc = new Y.Doc();
+  attachBlobs(
+    doc,
+    memoryBlobSide((_sha, bytes) => kept.push(bytes)),
+  );
+  writeProject(doc, project);
+  for (const bytes of kept) await blobs.put(bytes);
+  return { state: Y.encodeStateAsUpdate(doc), warnings: project.warnings };
+}
+
+/** A project's document as a FILIUS file, the blobs' bytes from F's store; the blobs the store does not have, if any. */
+export async function archiveOf(doc: Y.Doc, blobs: BlobStore): Promise<{ readonly archive: Uint8Array; readonly warnings: readonly string[] } | { readonly missing: readonly string[] }> {
+  const wanted = [...blobsOf(doc)];
+  const found = await blobs.get(wanted);
+  const missing = wanted.filter((sha) => !found.has(sha));
+  if (missing.length > 0) return { missing };
+  // A copy with the bytes beside it: the live document has no blob side on F's main thread.
+  const copy = new Y.Doc();
+  const side = memoryBlobSide();
+  side.arrived(found);
+  attachBlobs(copy, side);
+  Y.applyUpdate(copy, Y.encodeStateAsUpdate(doc));
+  return docArchive(copy);
+}
+
+const EXAMPLE_SUFFIX = '.fls';
+
+/** The examples in F's folder, by name (the file's name without .fls; Node lists a folder sorted, libuv sorts it); none without the folder. */
+export async function exampleNames(folder: string | undefined): Promise<string[]> {
+  const files = folder === undefined ? [] : await readdir(folder).catch(() => []);
+  return files.filter((file) => file.endsWith(EXAMPLE_SUFFIX)).map((file) => file.slice(0, -EXAMPLE_SUFFIX.length));
+}
+
+/** An example's file; undefined for anything that is not the name of one of the folder's (never a path). */
+export async function exampleArchive(folder: string | undefined, name: unknown): Promise<Uint8Array | undefined> {
+  if (folder === undefined || !(await exampleNames(folder)).includes(name as string)) return undefined;
+  return new Uint8Array(await readFile(join(folder, `${name as string}${EXAMPLE_SUFFIX}`)));
+}