feat: F6.1 FILIUS files on F: projects from examples and .fls, copies, export
/svc/examples lists FILIUS' examples, which lie beside F in the image (GPL like this repository, kept out of the dashboard); /svc/projects makes a project from one; /svc/projects/<id>/import takes a .fls (up to 64 MiB), /copy copies a project as it is now, /export gives it back as a .fls with its warnings. Large files go into the blob store on import and come back from it on export, which is refused when one is missing. Found on the way: Node's request body is a Buffer, and Yjs takes no Buffer into a document, so every .fls with binary files would have failed; and fflate, now in the bundle, imports createRequire itself, which the build's banner declared again, so F did not start. The banner names it otherwise and a test builds and starts the bundle. The proxy's /svc/ takes 70 MB. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5.5
@@ -261,7 +261,7 @@ describe("the dashboard's service", () => {
|
||||
it('makes an empty project once', async () => {
|
||||
const server = await start();
|
||||
const project = randomUUID();
|
||||
expect(await answer(await service(server, 'POST', '/svc/projects', { project }))).toEqual([201, { project }]);
|
||||
expect(await answer(await service(server, 'POST', '/svc/projects', { project }))).toEqual([201, { project, warnings: [] }]);
|
||||
expect(await answer(await service(server, 'POST', '/svc/projects', { project }))).toEqual([409, { error: 'exists' }]);
|
||||
const doc = new Y.Doc();
|
||||
Y.applyUpdate(doc, (await store.load(project))!.state);
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
// Server F as the image runs it: built by the package's own build script
|
||||
// and started with node. Without its environment it stops at once and says
|
||||
// what is missing — which it can only say when the bundle loads at all (a
|
||||
// bundled library that declared createRequire again kept it from loading).
|
||||
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { describe, expect, it } from 'vitest';
|
||||
|
||||
const server = fileURLToPath(new URL('..', import.meta.url));
|
||||
|
||||
describe('the built server', () => {
|
||||
it('loads, and stops for want of its environment', () => {
|
||||
const built = spawnSync('npm', ['run', 'build'], { cwd: server, encoding: 'utf8' });
|
||||
expect(built.status, built.stderr).toBe(0);
|
||||
const started = spawnSync(process.execPath, ['dist/main.mjs'], { cwd: server, encoding: 'utf8', env: { PATH: process.env.PATH }, timeout: 20_000 });
|
||||
expect(started.status, started.stderr).toBe(1);
|
||||
expect(JSON.parse(started.stdout.trim().split('\n').at(-1)!)).toMatchObject({
|
||||
event: 'config.missing',
|
||||
variables: ['DATABASE_URL', 'FILIUS_TICKET_SECRET', 'FILIUS_SERVICE_KEY', 'FILIUS_ORIGINS'],
|
||||
});
|
||||
}, 60_000);
|
||||
});
|
||||
@@ -0,0 +1,275 @@
|
||||
// FILIUS files through the dashboard's service (F6.1): the examples beside
|
||||
// F, a project made from one and from a file the dashboard sends — its
|
||||
// large pictures in F's blob store —, a project copied while people work in
|
||||
// it, a project given back as a FILIUS file (with its warnings, and never
|
||||
// with a picture it no longer has), and what is refused.
|
||||
//
|
||||
// A .fls is compared in a canonical form: read with ids counted from one and
|
||||
// written again (a file's bytes depend on the ids a reading gave its parts).
|
||||
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { copyFileSync, mkdtempSync, readFileSync, writeFileSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
|
||||
import * as Y from 'yjs';
|
||||
import { attachBlobs, memoryBlobSide, nodeFiles, readProject, writeProject } from '@mdg-filius/doc';
|
||||
import { openProjectArchive, readFlsArchive, saveProjectArchive, writeFlsArchive, type ImportedProject } from '@mdg-filius/fls';
|
||||
import { IMPORT_BYTES, PostgresBlobs, PostgresSessions, PostgresStore, SERVICE_KEY_HEADER, shaOf, WARNINGS_HEADER } from '../src/index.js';
|
||||
import { testDatabase, type TestDatabase } from './database.js';
|
||||
import { enter, SERVICE_KEY, startTestServer, until, type TestServer } from './harness.js';
|
||||
|
||||
const corpus = new URL('../../../corpus/', import.meta.url);
|
||||
const EXAMPLES = new URL('java-examples/', corpus).pathname;
|
||||
const example = (name: string) => new Uint8Array(readFileSync(new URL(`java-examples/${name}.fls`, corpus)));
|
||||
|
||||
/**
|
||||
* A FILIUS file as it reads: the same for the same project, whatever ids a
|
||||
* reading gave it. A switch's SSID too, which a file without one gets from
|
||||
* its id when it is read (and keeps from then on).
|
||||
*/
|
||||
function canonical(archive: Uint8Array): string {
|
||||
let counter = 0;
|
||||
const project = openProjectArchive(archive, () => `00000000-0000-4000-8000-${(++counter).toString(16).padStart(12, '0')}`);
|
||||
const xml = new TextDecoder().decode(readFlsArchive(saveProjectArchive(project).archive).configurationXml);
|
||||
return xml.replace(/(<void property="SSID">\s*<string>)[0-9a-f]{6}(<\/string>)/g, '$1ssid$2');
|
||||
}
|
||||
|
||||
/** An example whose switch is of a kind FILIUS does not know: read, it warns. */
|
||||
function withForeignDevice(): Uint8Array {
|
||||
const contents = readFlsArchive(example('internetwork_komplett'));
|
||||
const xml = new TextDecoder().decode(contents.configurationXml).replace('filius.hardware.knoten.Switch', 'filius.hardware.knoten.Unbekannt');
|
||||
return writeFlsArchive(new TextEncoder().encode(xml), contents.supplementalEntries);
|
||||
}
|
||||
|
||||
const documents = async () => ((await (await fetch(`${server.base}/healthz`)).json()) as { documents: number }).documents;
|
||||
|
||||
let database: TestDatabase;
|
||||
let store: PostgresStore;
|
||||
let sessions: PostgresSessions;
|
||||
let blobs: PostgresBlobs;
|
||||
let server: TestServer;
|
||||
|
||||
beforeAll(async () => {
|
||||
database = await testDatabase();
|
||||
store = PostgresStore.connect(database.url);
|
||||
sessions = new PostgresSessions(store.pool);
|
||||
blobs = new PostgresBlobs(store.pool);
|
||||
await store.migrate();
|
||||
await sessions.migrate();
|
||||
await blobs.migrate();
|
||||
server = await startTestServer(store, sessions, { examplesFolder: EXAMPLES });
|
||||
}, 60_000);
|
||||
|
||||
afterAll(async () => {
|
||||
await server?.close();
|
||||
await store?.close();
|
||||
await database?.stop();
|
||||
});
|
||||
|
||||
const service = (method: string, path: string, body?: string | Uint8Array, json = true) =>
|
||||
fetch(`${server.base}${path}`, { method, headers: { [SERVICE_KEY_HEADER]: SERVICE_KEY, ...(json ? { 'content-type': 'application/json' } : {}) }, ...(body === undefined ? {} : { body }) });
|
||||
const exported = async (project: string) => {
|
||||
const response = await service('GET', `/svc/projects/${project}/export`);
|
||||
return { response, archive: new Uint8Array(await response.arrayBuffer()) };
|
||||
};
|
||||
const warningsOf = (response: Response) => JSON.parse(Buffer.from(response.headers.get(WARNINGS_HEADER) ?? '', 'base64url').toString('utf8')) as string[];
|
||||
|
||||
describe("FILIUS' examples", () => {
|
||||
it('lists the files of the folder beside F, by name', async () => {
|
||||
const names = ((await (await service('GET', '/svc/examples')).json()) as { examples: string[] }).examples;
|
||||
expect(names).toHaveLength(18);
|
||||
expect(names).toContain('internetwork_komplett');
|
||||
expect(names).toEqual([...names].sort());
|
||||
});
|
||||
|
||||
it('lists none without a folder, nor when the folder is not there', async () => {
|
||||
const gone = await startTestServer(store, sessions, { examplesFolder: join(tmpdir(), `filius-no-examples-${randomUUID()}`) });
|
||||
try {
|
||||
expect(await (await fetch(`${gone.base}/svc/examples`, { headers: { [SERVICE_KEY_HEADER]: SERVICE_KEY } })).json()).toEqual({ examples: [] });
|
||||
} finally {
|
||||
await gone.close();
|
||||
}
|
||||
const bare = await startTestServer(store, sessions);
|
||||
try {
|
||||
const answer = await fetch(`${bare.base}/svc/examples`, { headers: { [SERVICE_KEY_HEADER]: SERVICE_KEY } });
|
||||
expect(await answer.json()).toEqual({ examples: [] });
|
||||
const made = await fetch(`${bare.base}/svc/projects`, {
|
||||
method: 'POST',
|
||||
headers: { [SERVICE_KEY_HEADER]: SERVICE_KEY },
|
||||
body: JSON.stringify({ project: randomUUID(), example: 'internetwork_komplett' }),
|
||||
});
|
||||
expect([made.status, await made.json()]).toEqual([404, { error: 'unknownExample' }]);
|
||||
} finally {
|
||||
await bare.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("lists only the folder's FILIUS files, and tells what one held that a project cannot", async () => {
|
||||
const folder = mkdtempSync(join(tmpdir(), 'filius-examples-'));
|
||||
// Made out of order: the list comes sorted all the same.
|
||||
writeFileSync(join(folder, 'fremd.fls'), withForeignDevice());
|
||||
writeFileSync(join(folder, 'LIESMICH.txt'), 'keine FILIUS-Datei');
|
||||
copyFileSync(new URL('java-examples/webserver.fls', corpus), join(folder, 'webserver.fls'));
|
||||
const own = await startTestServer(store, sessions, { examplesFolder: folder });
|
||||
try {
|
||||
const ask = (method: string, path: string, body?: string) => fetch(`${own.base}${path}`, { method, headers: { [SERVICE_KEY_HEADER]: SERVICE_KEY }, ...(body === undefined ? {} : { body }) });
|
||||
expect(await (await ask('GET', '/svc/examples')).json()).toEqual({ examples: ['fremd', 'webserver'] });
|
||||
const project = randomUUID();
|
||||
const made = (await (await ask('POST', '/svc/projects', JSON.stringify({ project, example: 'fremd' }))).json()) as { warnings: string[] };
|
||||
expect(made.warnings).toEqual(openProjectArchive(withForeignDevice(), randomUUID).warnings);
|
||||
expect(made.warnings.length).toBeGreaterThan(0);
|
||||
} finally {
|
||||
await own.close();
|
||||
}
|
||||
});
|
||||
|
||||
it('makes a project from one, the same as its file', async () => {
|
||||
const project = randomUUID();
|
||||
const made = await service('POST', '/svc/projects', JSON.stringify({ project, example: 'internetwork_komplett' }));
|
||||
expect([made.status, await made.json()]).toEqual([201, { project, warnings: [] }]);
|
||||
const { response, archive } = await exported(project);
|
||||
expect(response.status).toBe(200);
|
||||
expect(canonical(archive)).toBe(canonical(example('internetwork_komplett')));
|
||||
const person = await enter(server, project);
|
||||
expect(readProject(person.doc).graph.nodes.map((node) => node.displayName)).toContain('Notebook 1.10');
|
||||
person.close();
|
||||
});
|
||||
|
||||
it.each([['nichtda'], ['../../package'], ['java-examples/internetwork_komplett'], [7]])('knows no example %j', async (name) => {
|
||||
const made = await service('POST', '/svc/projects', JSON.stringify({ project: randomUUID(), example: name }));
|
||||
expect([made.status, await made.json()]).toEqual([404, { error: 'unknownExample' }]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('a FILIUS file the dashboard sends', () => {
|
||||
it('becomes a project, the same as the file', async () => {
|
||||
const project = randomUUID();
|
||||
const made = await service('POST', `/svc/projects/${project}/import`, example('nat_with_mapped_port'), false);
|
||||
expect([made.status, await made.json()]).toEqual([201, { project, warnings: [] }]);
|
||||
expect(canonical((await exported(project)).archive)).toBe(canonical(example('nat_with_mapped_port')));
|
||||
expect(server.log.lines).toContainEqual(expect.objectContaining({ event: 'svc.imported', project }));
|
||||
});
|
||||
|
||||
it("takes a large picture into F's blob store, and gives it back in the file", async () => {
|
||||
const picture = Uint8Array.from({ length: 40_000 }, (_, index) => (index * 7) % 251);
|
||||
const original = openProjectArchive(example('webserver'), randomUUID);
|
||||
const computer = original.graph.nodes.find((node) => original.virtualFileSystems.has(node.id))!;
|
||||
original.virtualFileSystems.get(computer.id)!.writeBinaryFile('/bild.png', picture, 'image/png');
|
||||
const file = saveProjectArchive(original).archive;
|
||||
const project = randomUUID();
|
||||
expect((await service('POST', `/svc/projects/${project}/import`, file, false)).status).toBe(201);
|
||||
expect(Buffer.from((await blobs.get([shaOf(picture)])).get(shaOf(picture)) ?? [])).toEqual(Buffer.from(picture));
|
||||
const person = await enter(server, project);
|
||||
// F gave the parts ids of its own: the computer is found by its name.
|
||||
const node = [...person.doc.getMap<Y.Map<unknown>>('nodes').values()].find((each) => each.get('name') === computer.displayName)!;
|
||||
const entry = (node.get('files') as Y.Map<Y.Map<unknown>>).get('/bild.png')!;
|
||||
expect([entry.get('blob'), entry.get('data')]).toEqual([shaOf(picture), undefined]);
|
||||
person.close();
|
||||
const back = openProjectArchive((await exported(project)).archive, randomUUID);
|
||||
const again = back.graph.nodes.find((each) => each.displayName === computer.displayName)!;
|
||||
const content = back.virtualFileSystems.get(again.id)!.entry('/bild.png').content;
|
||||
expect(content.kind === 'binary' || content.kind === 'image' ? [...content.data] : content).toEqual([...picture]);
|
||||
});
|
||||
|
||||
it('tells what the file held that a project cannot', async () => {
|
||||
const project = randomUUID();
|
||||
const made = (await (await service('POST', `/svc/projects/${project}/import`, withForeignDevice(), false)).json()) as { warnings: string[] };
|
||||
expect(made.warnings).toEqual(openProjectArchive(withForeignDevice(), randomUUID).warnings);
|
||||
expect(made.warnings.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it('is refused when it is no FILIUS file, too large, or for a project there is', async () => {
|
||||
const garbage = await service('POST', `/svc/projects/${randomUUID()}/import`, new Uint8Array([1, 2, 3]), false);
|
||||
expect([garbage.status, await garbage.json()]).toEqual([422, { error: 'notAFiliusFile' }]);
|
||||
const large = await service('POST', `/svc/projects/${randomUUID()}/import`, new Uint8Array(IMPORT_BYTES + 1), false);
|
||||
expect([large.status, await large.json()]).toEqual([413, { error: 'tooLarge' }]);
|
||||
const project = randomUUID();
|
||||
await service('POST', '/svc/projects', JSON.stringify({ project }));
|
||||
const twice = await service('POST', `/svc/projects/${project}/import`, example('webserver'), false);
|
||||
expect([twice.status, await twice.json()]).toEqual([409, { error: 'exists' }]);
|
||||
const notAProject = await service('POST', '/svc/projects/nope/import', example('webserver'), false);
|
||||
expect([notAProject.status, await notAProject.json()]).toEqual([400, { error: 'notAProject' }]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('a copy of a project', () => {
|
||||
it('lets go of a project no one has open, once it is copied or given as a file', async () => {
|
||||
const project = randomUUID();
|
||||
await service('POST', '/svc/projects', JSON.stringify({ project, example: 'webserver' }));
|
||||
const before = await documents();
|
||||
expect((await service('POST', `/svc/projects/${project}/copy`, JSON.stringify({ to: randomUUID() }))).status).toBe(201);
|
||||
expect((await exported(project)).response.status).toBe(200);
|
||||
await until(async () => (await documents()) === before);
|
||||
});
|
||||
|
||||
it('is the project as it is now, while someone works in it, and goes its own way after', async () => {
|
||||
const project = randomUUID();
|
||||
await service('POST', '/svc/projects', JSON.stringify({ project, example: 'internetwork_komplett' }));
|
||||
const ben = await enter(server, project);
|
||||
const nodes = ben.doc.getMap<Y.Map<unknown>>('nodes');
|
||||
const [first] = [...nodes.keys()];
|
||||
nodes.get(first!)!.set('name', 'Bens Rechner');
|
||||
await until(() => ben.provider.unsyncedChanges === 0);
|
||||
const copy = randomUUID();
|
||||
const made = await service('POST', `/svc/projects/${project}/copy`, JSON.stringify({ to: copy }));
|
||||
expect([made.status, await made.json()]).toEqual([201, { project: copy }]);
|
||||
const mia = await enter(server, copy);
|
||||
expect(mia.doc.getMap<Y.Map<unknown>>('nodes').get(first!)!.get('name')).toBe('Bens Rechner');
|
||||
mia.doc.getMap<Y.Map<unknown>>('nodes').get(first!)!.set('name', 'Mias Rechner');
|
||||
await until(() => mia.provider.unsyncedChanges === 0);
|
||||
expect(nodes.get(first!)!.get('name')).toBe('Bens Rechner');
|
||||
ben.close();
|
||||
mia.close();
|
||||
});
|
||||
|
||||
it('is refused onto a project there is, and of one there is not — which then can be made', async () => {
|
||||
const project = randomUUID();
|
||||
await service('POST', '/svc/projects', JSON.stringify({ project }));
|
||||
const onto = await service('POST', `/svc/projects/${project}/copy`, JSON.stringify({ to: project }));
|
||||
expect([onto.status, await onto.json()]).toEqual([409, { error: 'exists' }]);
|
||||
const missing = randomUUID();
|
||||
const of = await service('POST', `/svc/projects/${missing}/copy`, JSON.stringify({ to: randomUUID() }));
|
||||
expect([of.status, await of.json()]).toEqual([404, { error: 'unknownProject' }]);
|
||||
expect((await service('POST', '/svc/projects', JSON.stringify({ project: missing }))).status).toBe(201);
|
||||
(await enter(server, missing)).close();
|
||||
const bad = await service('POST', `/svc/projects/${project}/copy`, JSON.stringify({ to: 'nope' }));
|
||||
expect([bad.status, await bad.json()]).toEqual([400, { error: 'notAProject' }]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('a project as a FILIUS file', () => {
|
||||
it("names what Java's file cannot hold", async () => {
|
||||
const original: ImportedProject = openProjectArchive(example('webserver'), randomUUID);
|
||||
const [model] = original.graph.nodes;
|
||||
original.graph.nodes.push({ ...model!, id: randomUUID(), kind: 'unsupported', displayName: 'Fremdgerät', ports: model!.ports.map((port) => ({ ...port, id: randomUUID() })) });
|
||||
const doc = new Y.Doc();
|
||||
writeProject(doc, original);
|
||||
const project = randomUUID();
|
||||
await store.create(project, Y.encodeStateAsUpdate(doc));
|
||||
const { response } = await exported(project);
|
||||
expect(response.status).toBe(200);
|
||||
expect(warningsOf(response)).toEqual(["node 'Fremdgerät': a kind Java FILIUS does not have, left out"]);
|
||||
});
|
||||
|
||||
it('is not made without a picture it names, and not of a project there is not', async () => {
|
||||
const doc = new Y.Doc();
|
||||
attachBlobs(doc, memoryBlobSide());
|
||||
const original = openProjectArchive(example('webserver'), randomUUID);
|
||||
const computer = original.graph.nodes.find((node) => original.virtualFileSystems.has(node.id))!;
|
||||
original.virtualFileSystems.get(computer.id)!.writeBinaryFile(
|
||||
'/fehlt.png',
|
||||
Uint8Array.from({ length: 30_000 }, (_, index) => index % 13),
|
||||
'image/png',
|
||||
);
|
||||
writeProject(doc, original);
|
||||
expect(nodeFiles(doc, computer.id)?.contains('/fehlt.png')).toBe(true);
|
||||
const project = randomUUID();
|
||||
await store.create(project, Y.encodeStateAsUpdate(doc));
|
||||
const refused = await service('GET', `/svc/projects/${project}/export`);
|
||||
expect([refused.status, await refused.json()]).toEqual([409, { error: 'missingBlobs' }]);
|
||||
expect(server.log.lines).toContainEqual(expect.objectContaining({ event: 'svc.exportMissingBlobs', project }));
|
||||
const none = await service('GET', `/svc/projects/${randomUUID()}/export`);
|
||||
expect([none.status, await none.json()]).toEqual([404, { error: 'unknownProject' }]);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user