mdg-filius/tools/live/m9.ts

// F9's load test on a live server F: two classes of 30 at once, every
// project in the action mode (plan F9: M3 under load). 30 people work
// alone, 30 in 15 group projects of two — 45 projects, each the largest
// example. Everybody has the document (/sync) and the room (/sim) open, as
// an editor has, moves a pointer (presence) every 2 s and pings across the
// network every 6 s; in a group both move a device every 10 s.
//
// Measured, a line a minute:
// - the room's lag: once a second per person, an action for a device that
//   does not exist — answered at once between two ticks of the room's
//   thread (M3's probe), so the time to the answer is how far behind it is;
// - the way of a change: a device one member moves, until the other's
//   document has it (through F, both in this process: one clock);
// - what got lost: probes without an answer, sockets that closed;
// - what F says of itself (/healthz): documents, connections, rooms.
// Every project made is deleted at the end.
//
// Against an F a dashboard reconciles (test-filius with dev), sessions of
// people and projects the dashboard does not know are ended at its next run
// — measured on 11.10.2026: all 120 sockets at once in the second minute.
// With M9_DASHBOARD (its address), M9_SESSION (a session cookie
// "name=value") and M9_USER (that person's uuid) the projects are made
// through the dashboard as hers, and everybody signs in as her: the
// reconciliation keeps them, and F carries the same load.
//
//   npx vite-node tools/live/m9.ts [minutes] [alone] [groups]

import { randomUUID } from 'node:crypto';
import * as Y from 'yjs';
import { moveDevice, ROOTS } from '@mdg-filius/doc';
import { BASE, cleanUp, joinDoc, openRoom, seededProject, signIn, type Room, type Shared } from './lib.js';

const EXAMPLE = 'java-examples/Internet_Komplett_mit_eMail_Webserver_Intranet_Portforwarding_Firewall_DHCP_DE.fls';
const minutes = Number(process.argv[2] ?? 10);
const ALONE = Number(process.argv[3] ?? 30);
const GROUPS = Number(process.argv[4] ?? 15);

interface Person {
  readonly name: string;
  readonly shared: Shared;
  readonly room: Room;
  readonly timers: NodeJS.Timeout[];
}

const DASHBOARD = process.env.M9_DASHBOARD;
const SESSION = process.env.M9_SESSION;
const USER = process.env.M9_USER;
const EXAMPLE_NAME = 'Internet_Komplett_mit_eMail_Webserver_Intranet_Portforwarding_Firewall_DHCP_DE';
const dashboardProjects: string[] = [];

/** A project made through the dashboard, as M9_USER's, from the example. */
async function dashboardProject(index: number): Promise<string> {
  const r = await fetch(`${DASHBOARD}/api/filius/projects`, {
    method: 'POST',
    headers: { cookie: SESSION!, 'content-type': 'application/json' },
    body: JSON.stringify({ name: `Lastlauf ${index + 1}`, example: EXAMPLE_NAME }),
  });
  if (r.status !== 201) throw new Error(`dashboard made no project: ${r.status} ${await r.text()}`);
  const project = ((await r.json()) as { uuid: string }).uuid;
  dashboardProjects.push(project);
  return project;
}

async function dashboardCleanUp(): Promise<void> {
  for (const project of dashboardProjects.splice(0)) {
    for (const [method, path] of [
      ['DELETE', ''],
      ['DELETE', '/purge'],
    ] as const)
      await fetch(`${DASHBOARD}/api/filius/projects/${project}${path}`, { method, headers: { cookie: SESSION! } });
  }
}

/** A device's id in the shared document, by its name. */
function deviceNamed(doc: Y.Doc, name: string): string {
  const found = [...doc.getMap<Y.Map<unknown>>(ROOTS.nodes).entries()].find(([, node]) => node.get('name') === name);
  if (found === undefined) throw new Error(`no device ${name}`);
  return found[0];
}

const people: Person[] = [];
let probes: number[] = [];
let ways: number[] = [];
let lostProbes = 0;
let roomErrors = 0;
let closes: string[] = [];
const pending = new Map<string, number>();
let counter = 0;

/** The closings, counted by way and reason: a reconnecting provider closes again and again. */
const closedByReason = () =>
  Object.entries(
    closes.reduce<Record<string, number>>((counts, line) => {
      const key = line.replace(/ [^:]+:/, ':');
      counts[key] = (counts[key] ?? 0) + 1;
      return counts;
    }, {}),
  )
    .map(([reason, count]) => `${count}× ${reason}`)
    .join('; ');

const quantile = (sorted: number[], q: number) => sorted[Math.min(sorted.length - 1, Math.floor(q * sorted.length))] ?? NaN;
const ms = (value: number) => (Number.isNaN(value) ? '–' : `${Math.round(value)} ms`);
const pause = (milliseconds: number) => new Promise((resolve) => setTimeout(resolve, milliseconds));

/** One person in a project: document and room, pointer, pings and probes. */
async function join(project: string, name: string, starts: boolean): Promise<Person> {
  const cookie = await signIn(project, { user: USER ?? randomUUID(), name });
  const shared = await joinDoc(project, cookie);
  const pingingDevice = deviceNamed(shared.doc, 'iPad 1');
  const room = await openRoom(project, cookie);
  room.onMessage = (message) => {
    if (message.kind !== 'error' || typeof message.reqId !== 'string') return;
    if (pending.has(message.reqId)) {
      probes.push(performance.now() - pending.get(message.reqId)!);
      pending.delete(message.reqId);
    } else if (message.code !== 'unknownNode') roomErrors += 1;
  };
  void room.closed.then((reason) => closes.push(`/sim ${name}: ${reason}`));
  shared.provider.on('close', ({ event }: { event: { code: number; reason: string } }) => closes.push(`/sync ${name}: ${event.code} ${event.reason}`));
  if (starts) {
    room.send({ kind: 'speed', value: 100 });
    room.send({ kind: 'start' });
  }
  await room.waitFor((message) => message.kind === 'state' && message.mode === 'simulation', 30_000);
  room.send({ kind: 'sub', topics: ['activity', `app:${pingingDevice}/terminal`] });
  const awareness = shared.provider.awareness;
  awareness.setLocalStateField('user', { name, color: '#3366cc' });
  const timers: NodeJS.Timeout[] = [];
  const point = () => awareness.setLocalStateField('pointer', { x: Math.random() * 1000, y: Math.random() * 700 });
  const ping = () => room.send({ kind: 'input', node: pingingDevice, app: 'terminal', action: 'enter', args: { line: 'ping 120.130.140.80' }, reqId: `ping${counter++}` });
  const probe = () => {
    const reqId = `p${counter++}`;
    pending.set(reqId, performance.now());
    room.send({ kind: 'input', node: 'probe', app: 'terminal', action: 'enter', args: { line: '' }, reqId });
  };
  // Spread over the period, as people are: nobody clicks in step.
  timers.push(setTimeout(() => timers.push(setInterval(point, 2000)), Math.random() * 2000));
  timers.push(setTimeout(() => timers.push(setInterval(ping, 6000)), Math.random() * 6000));
  timers.push(setTimeout(() => timers.push(setInterval(probe, 1000)), Math.random() * 1000));
  const person = { name, shared, room, timers };
  people.push(person);
  return person;
}

/** In a group, each moves a device every 10 s; the partner's document is watched for it. */
function moveTogether(mover: Person, watcher: Person, device: string): void {
  let sent = 0;
  let x = 100;
  watcher.shared.doc.on('update', () => {
    if (sent === 0) return;
    const node = watcher.shared.doc.getMap<Y.Map<unknown>>(ROOTS.nodes).get(device);
    if (node?.get('x') === x) {
      ways.push(performance.now() - sent);
      sent = 0;
    }
  });
  const move = () => {
    x = 100 + Math.round(Math.random() * 600);
    sent = performance.now();
    moveDevice(mover.shared.doc, device, x, 200 + Math.round(Math.random() * 300), 'm9');
  };
  mover.timers.push(setTimeout(() => mover.timers.push(setInterval(move, 10_000)), Math.random() * 10_000));
}

try {
  const started = performance.now();
  console.log(`setting up: ${ALONE} alone, ${GROUPS} groups of two — ${ALONE + GROUPS} projects, ${ALONE + 2 * GROUPS} people`);
  for (let index = 0; index < ALONE + GROUPS; index++) {
    const project = DASHBOARD === undefined ? (await seededProject(EXAMPLE)).project : await dashboardProject(index);
    if (index < ALONE) {
      await join(project, `Allein ${index + 1}`, true);
    } else {
      const group = index - ALONE + 1;
      const first = await join(project, `Gruppe ${group} A`, true);
      const second = await join(project, `Gruppe ${group} B`, false);
      moveTogether(first, second, deviceNamed(first.shared.doc, 'Heimrouter'));
      moveTogether(second, first, deviceNamed(first.shared.doc, 'webserver.de (120.130.140.80)'));
    }
    if ((index + 1) % 5 === 0) console.log(`  ${index + 1} projects, ${people.length} people (${((performance.now() - started) / 1000).toFixed(0)} s)`);
  }
  console.log(`all in, ${people.length} people, holding ${minutes} min`);
  probes = [];
  ways = [];
  const all = { probes: [] as number[], ways: [] as number[] };
  for (let minute = 1; minute <= minutes; minute++) {
    await pause(60_000);
    const sortedProbes = [...probes].sort((a, b) => a - b);
    const sortedWays = [...ways].sort((a, b) => a - b);
    const now = performance.now();
    const overdue = [...pending.values()].filter((at) => now - at > 10_000).length;
    lostProbes += overdue;
    for (const [reqId, at] of pending) if (now - at > 10_000) pending.delete(reqId);
    all.probes.push(...probes);
    all.ways.push(...ways);
    console.log(
      `min ${minute}: room lag p50 ${ms(quantile(sortedProbes, 0.5))}, p95 ${ms(quantile(sortedProbes, 0.95))}, max ${ms(sortedProbes.at(-1) ?? NaN)} (${probes.length} probes, ${overdue} lost) · a change to the partner p50 ${ms(quantile(sortedWays, 0.5))}, p95 ${ms(quantile(sortedWays, 0.95))}, max ${ms(sortedWays.at(-1) ?? NaN)} (${ways.length}) · room errors ${roomErrors} · closed ${closes.length}`,
    );
    probes = [];
    ways = [];
    const health = await fetch(`${BASE}/healthz`)
      .then((r) => r.json())
      .catch((error: Error) => ({ error: error.message }));
    console.log(`        F: ${JSON.stringify(health)}`);
  }
  const sortedProbes = all.probes.sort((a, b) => a - b);
  const sortedWays = all.ways.sort((a, b) => a - b);
  console.log(
    `whole run: room lag p50 ${ms(quantile(sortedProbes, 0.5))}, p95 ${ms(quantile(sortedProbes, 0.95))}, p99 ${ms(quantile(sortedProbes, 0.99))}, max ${ms(sortedProbes.at(-1) ?? NaN)} of ${sortedProbes.length}; lost ${lostProbes}; a change to the partner p50 ${ms(quantile(sortedWays, 0.5))}, p95 ${ms(quantile(sortedWays, 0.95))}, max ${ms(sortedWays.at(-1) ?? NaN)} of ${sortedWays.length}; room errors ${roomErrors}; closed: ${closes.length === 0 ? 'none' : closedByReason()}`,
  );
} finally {
  for (const person of people) {
    for (const timer of person.timers) clearInterval(timer);
    person.room.close();
    person.shared.close();
  }
  await cleanUp();
  await dashboardCleanUp();
}