mdg-filius/packages/core/test/oracle.test.ts

// Replays every scenario in test/scenarios with the TypeScript engine and
// compares the result with what the filius-ipad Swift engine produced for the
// same file (<name>.swift.json, written by tools/swift-oracle/run.sh).
// Every trace event must match: identity, time, node, headers, detail.

import { existsSync, readdirSync, readFileSync } from 'node:fs';
import { dirname, join } from 'node:path';
import { fileURLToPath } from 'node:url';
import { describe, expect, it } from 'vitest';
import {
  NetworkRuntimeEngine,
  type DhcpClientStatus,
  type IPv4DeliveryResult,
  type IcmpObservation,
  type TraceEvent,
} from '../src/index.js';
import { snapshotOf, type Scenario } from './scenario.js';

const scenarioDirectory = join(dirname(fileURLToPath(import.meta.url)), 'scenarios');

const upper = (id: string) => id.toUpperCase();

function dhcpStatusJson(status: DhcpClientStatus) {
  return {
    state: status.state,
    errors: status.errorCount,
    server: status.selectedServerIPAddress ?? null,
    offered: status.offeredIPAddress ?? null,
    succeeded: status.succeeded,
  };
}

function traceJson(event: TraceEvent): Record<string, unknown> {
  return {
    id: event.id,
    time: event.timeMilliseconds,
    node: upper(event.nodeId),
    direction: event.direction,
    layer: event.layer,
    operation: event.operation,
    before: event.beforeHeaders.map((field) => [field.name, field.value]),
    after: event.afterHeaders.map((field) => [field.name, field.value]),
    ...(event.frameIdentity === undefined ? {} : { frame: event.frameIdentity }),
    ...(event.packetIdentity === undefined ? {} : { packet: event.packetIdentity }),
    ...(event.interfaceId === undefined ? {} : { interface: upper(event.interfaceId) }),
    ...(event.detail === undefined ? {} : { detail: event.detail }),
  };
}

function deliveryJson(result: IPv4DeliveryResult): Record<string, unknown> {
  switch (result.type) {
    case 'delivered':
      return { type: 'delivered', packet: result.packetIdentity, node: upper(result.nodeId) };
    case 'icmpError':
      return { type: 'icmpError', packet: result.packetIdentity, node: upper(result.nodeId), kind: result.kind };
    case 'dropped':
      return {
        type: 'dropped',
        packet: result.packetIdentity,
        ...(result.nodeId === undefined ? {} : { node: upper(result.nodeId) }),
      };
  }
}

function observationJson(item: IcmpObservation): Record<string, unknown> {
  return {
    kind: item.message.kind,
    identifier: item.message.identifier,
    sequence: item.message.sequenceNumber,
    sender: item.packet.senderIPAddress,
  };
}

function run(scenario: Scenario): Record<string, unknown> {
  // Swift's cables carry all frames at once; everything else is compared as it is.
  const engine = new NetworkRuntimeEngine(scenario.seed, undefined, { swiftCables: true, swiftArp: true, swiftSends: true, swiftRouters: true });
  if (scenario.speedPercent !== undefined) engine.setSimulationSpeed(scenario.speedPercent);
  engine.handle({ type: 'start', snapshot: snapshotOf(scenario), seed: scenario.seed, initialTimeMilliseconds: 0 });

  const results: unknown[] = [];
  for (const step of scenario.steps) {
    switch (step.op) {
      case 'ping':
        results.push(
          deliveryJson(
            engine.sendIcmpEcho(step.from!, step.target!, {
              timeToLive: step.ttl ?? 64,
              identifier: step.identifier ?? 1,
              sequenceNumber: step.sequence ?? 1,
            }),
          ),
        );
        break;
      case 'traceroute':
        results.push(
          engine
            .traceIcmpEcho(step.from!, step.target!, {
              maximumTimeToLive: step.maxTtl ?? 64,
              identifier: step.identifier ?? 1,
            })
            .map(observationJson),
        );
        break;
      case 'resolveMac':
        results.push(engine.resolveMacAddress(step.node!, step.target!) ?? null);
        break;
      case 'advance':
        engine.handle({ type: 'advance', toMilliseconds: step.to! });
        results.push(engine.state.currentTimeMilliseconds);
        break;
      case 'packetLoss':
        engine.setGlobalPacketLossEnabled(step.enabled!);
        results.push(engine.state.globalPacketLossEnabled);
        break;
      case 'sat':
        results.push(
          engine.switchSatEntries(step.node!).map((entry) => ({
            mac: entry.macAddress,
            port: entry.portLabel,
            updated: entry.updatedAtMilliseconds,
          })),
        );
        break;
      case 'speed':
        engine.setSimulationSpeed(step.percent!);
        results.push(engine.simulationSpeed.percent);
        break;
      case 'bindUdp':
        results.push(
          engine.bindUdpSocket(step.node!, {
            ...(step.port === undefined ? {} : { localPort: step.port }),
            ...(step.localIp === undefined ? {} : { localIPAddress: step.localIp }),
            ...(step.remoteIp === undefined ? {} : { remoteIPAddress: step.remoteIp }),
            ...(step.remotePort === undefined ? {} : { remotePort: step.remotePort }),
          }) ?? null,
        );
        break;
      case 'sendUdp': {
        const result = engine.sendUdp(step.socket!, new TextEncoder().encode(step.payload ?? ''), {
          ...(step.target === undefined ? {} : { ipAddress: step.target }),
          ...(step.port === undefined ? {} : { port: step.port }),
        });
        results.push(result === undefined ? null : deliveryJson(result));
        break;
      }
      case 'receiveUdp': {
        const item = engine.receiveUdp(step.socket!, step.timeout ?? 0);
        results.push(
          item === undefined
            ? null
            : {
                sender: item.senderIPAddress,
                receiver: item.receiverIPAddress,
                sourcePort: item.datagram.sourcePort,
                destinationPort: item.datagram.destinationPort,
                payload: new TextDecoder().decode(item.datagram.payload),
                at: item.receivedAtMilliseconds,
              },
        );
        break;
      }
      case 'openTcpServer':
        results.push(engine.openTcpServerSocket(step.node!, step.port!, step.localIp) ?? null);
        break;
      case 'openTcpClient':
        results.push(
          engine.openTcpClientSocket(step.node!, step.target!, step.port!, {
            ...(step.localIp === undefined ? {} : { localIPAddress: step.localIp }),
            ...(step.localPort === undefined ? {} : { localPort: step.localPort }),
          }) ?? null,
        );
        break;
      case 'connectTcp':
        results.push(engine.connectTcpWithResult(step.socket!));
        break;
      case 'sendTcp':
        results.push(engine.sendTcp(step.socket!, new TextEncoder().encode((step.payload ?? '').repeat(step.repeatCount ?? 1))));
        break;
      case 'receiveTcp': {
        const message = engine.receiveTcp(step.socket!);
        results.push(message === undefined ? null : new TextDecoder().decode(message));
        break;
      }
      case 'acceptedTcp':
        results.push(engine.state.tcpAcceptedSocketIdsByListenerId.get(step.socket!) ?? []);
        break;
      case 'peerTcp':
        results.push(engine.peerTcpSocketId(step.socket!) ?? null);
        break;
      case 'cleanTcp':
        results.push(engine.cleanClosedTcpSocket(step.socket!));
        break;
      case 'runDhcp': {
        const status = engine.runDhcpClient(step.node!);
        results.push(status === undefined ? null : dhcpStatusJson(status));
        break;
      }
      case 'closeSocket':
        engine.closeSocket(step.socket!);
        results.push(!engine.state.socketsById.has(step.socket!));
        break;
      default:
        throw new Error(`unknown step ${step.op}`);
    }
  }

  const state = engine.state;
  const mapOf = <V>(map: Map<string, V>, convert: (value: V) => unknown) =>
    Object.fromEntries([...map].map(([key, value]) => [upper(key), convert(value)]));
  return {
    results,
    traces: state.packetTraces.map(traceJson),
    time: state.currentTimeMilliseconds,
    pendingEvents: state.pendingEvents.length,
    arp: mapOf(state.arpCachesByNodeId, (cache) =>
      Object.fromEntries([...cache].map(([ip, entry]) => [ip, entry.macAddress])),
    ),
    switchTables: mapOf(state.switchForwardingTablesByNodeId, (table) =>
      Object.fromEntries([...table].map(([mac, port]) => [mac, upper(port)])),
    ),
    icmpObservations: mapOf(state.icmpObservationsByNodeId, (items) => items.map(observationJson)),
    devices: Object.fromEntries(
      [...state.topologySnapshot.deviceConfigurations].map(([node, device]) => [
        upper(node),
        [device.ipAddress, device.subnetMask, device.defaultGateway, device.dnsServer],
      ]),
    ),
    dhcp: {
      clients: mapOf(state.dhcpClientStatusesByNodeId, dhcpStatusJson),
      leases: [...state.dhcpLeasesByIPAddress]
        .sort(([lhs], [rhs]) => (lhs < rhs ? -1 : 1))
        .map(([key, lease]) => [key, lease.clientMacAddress, lease.ipAddress, lease.expiresAtMilliseconds ?? null]),
      offers: [...state.dhcpOffersByIPAddress]
        .sort(([lhs], [rhs]) => (lhs < rhs ? -1 : 1))
        .map(([key, offer]) => [key, offer.clientMacAddress, offer.expiresAtMilliseconds]),
      blacklist: mapOf(state.dhcpBlacklistByServerNodeId, (entries) =>
        [...entries].sort(([lhs], [rhs]) => (lhs < rhs ? -1 : 1)).map(([address, expiry]) => [address, expiry]),
      ),
      servers: mapOf(state.dhcpServerSocketIdsByNodeId, upper),
    },
    sockets: [...state.socketsById.values()]
      .sort((lhs, rhs) => (upper(lhs.id) < upper(rhs.id) ? -1 : 1))
      .map((socket) => ({
        id: upper(socket.id),
        node: upper(socket.nodeId),
        protocol: socket.protocol,
        localIp: socket.localIPAddress ?? null,
        localPort: socket.localPort,
        remoteIp: socket.remoteIPAddress ?? null,
        remotePort: socket.remotePort ?? null,
        queued: state.udpReceiveQueuesBySocketId.get(socket.id)?.length ?? 0,
        tcpState: socket.tcpState ?? null,
        parent: socket.parentListenerSocketId === undefined ? null : upper(socket.parentListenerSocketId),
        session: ((session) =>
          session === undefined
            ? null
            : {
                attempts: session.sendAttempts,
                next: session.nextSendSequenceNumber,
                remote: session.remoteSequenceNumber,
                pending: session.lastSentSegment !== undefined,
                assembly: session.receiveAssembly.length,
                messages: session.receivedMessages.length,
              })(state.tcpSessionsById.get(socket.id)),
      })),
    firewallDecisions: state.firewallDecisions.map((decision) => ({
      packet: decision.packetIdentity,
      node: upper(decision.nodeId),
      accepted: decision.accepted,
      ...(decision.ruleIndex === undefined ? {} : { rule: decision.ruleIndex }),
    })),
  };
}

// Scenarios with a Swift result (the Java comparisons have scenarios of their own).
const scenarioNames = readdirSync(scenarioDirectory)
  .filter((name) => name.endsWith('.scenario.json') && existsSync(join(scenarioDirectory, name.replace('.scenario.json', '.swift.json'))))
  .map((name) => name.replace('.scenario.json', ''))
  .sort();

/**
 * Where the Swift engine is wrong, per scenario: its result corrected. None
 * at the moment. (Until F2.3b: dhcp-lan, the status a directly run DHCP
 * client reported — Swift wrote back a stale copy after a nested run.)
 */
const SWIFT_ERRORS: Record<string, (expected: Record<string, unknown>) => Record<string, unknown>> = {};

const ECHO_KINDS = new Set(['echoRequest', 'echoReply']);

/**
 * Java instead of Swift in every scenario: an ICMP error's identifier and
 * sequence number are 0 (RFC 792: unused; Java's sendeICMP), where Swift
 * copies them from the echo request it is about. Measured with Java FILIUS
 * (the capture tables of arpJava and natJava).
 */
function withJavaIcmpErrors(expected: Record<string, unknown>): Record<string, unknown> {
  type Header = [string, string];
  const headers = (fields: Header[]): Header[] => {
    const kind = fields.find(([name]) => name === 'kind')?.[1];
    if (kind === undefined || ECHO_KINDS.has(kind)) return fields;
    return fields.map(([name, value]) => [name, name === 'identifier' || name === 'sequence' ? '0' : value]);
  };
  const traces = (expected.traces as { before: Header[]; after: Header[] }[]).map((trace) => ({ ...trace, before: headers(trace.before), after: headers(trace.after) }));
  const observation = (item: { kind?: string }) => (item.kind === undefined || ECHO_KINDS.has(item.kind) ? item : { ...item, identifier: 0, sequence: 0 });
  const observations = Object.fromEntries(
    Object.entries(expected.icmpObservations as Record<string, { kind: string }[]>).map(([node, items]) => [node, items.map(observation)]),
  );
  // A traceroute's result is its observations (other results are lists too: a switch's table).
  const results = (expected.results as unknown[]).map((result) => (Array.isArray(result) ? (result as { kind?: string }[]).map(observation) : result));
  return { ...expected, traces, icmpObservations: observations, results };
}

/**
 * Scenarios where the engine follows Java instead of Swift; the Java
 * comparisons check that behaviour (dhcpJava.test.ts, tcpJava.test.ts,
 * measured with Java FILIUS). Here
 * they must differ from Swift, or the departure is not real.
 */
const JAVA_INSTEAD_OF_SWIFT: Record<string, string> = {
  'dhcp-lan': 'DHCP client: one message per attempt, the ARP check waits a round trip, which follows the speed',
  'dhcp-two-servers': 'DHCP client: the second OFFER is read and dropped, no REQUEST twice (Swift: answered with NAK)',
  'dhcp-conflict': 'DHCP client: the ARP check finds the address taken and declines (Swift: it does not wait, two cards share the address)',
  'dhcp-no-server': 'DHCP client: a link-local address after five timeouts (Swift: the old address after ten)',
  'tcp-lan': 'TCP: the other end closes by itself on FIN, TIME_WAIT lasts 3 round trips, the accepted socket has the listener\'s sequence number',
  'tcp-routed-loss': 'TCP: 3 round trips per attempt follow the speed (8250 ms at 90 %), failed data closes with FIN',
  'tcp-closing': 'TCP: TIME_WAIT lasts 3 round trips and ends with the next segment; the other end closes by itself',
};

/**
 * Scenarios that follow Java from a trace on: the traces before it must match
 * Swift, the rest must differ (the Java comparisons check that behaviour).
 */
const SWIFT_UNTIL_TRACE: Record<string, { readonly trace: number; readonly reason: string }> = {
  'firewall-and-loss': { trace: 36, reason: "a ping without an answer waits a round trip (Java's sendProbe), Swift returns at once" },
  'routed-two-networks': { trace: 156, reason: 'the router answers host unreachable when ARP has given up after two round trips, Swift at once' },
  'switch-lan': { trace: 155, reason: 'ARP for an address no card has asks again after a round trip, then host unreachable' },
};

describe('same result as the Swift engine', () => {
  it('corrects Swift only where it errs', () => {
    for (const [name, correct] of Object.entries(SWIFT_ERRORS)) {
      const expected = JSON.parse(readFileSync(join(scenarioDirectory, `${name}.swift.json`), 'utf8')) as Record<string, unknown>;
      expect(JSON.stringify(correct(expected)), name).not.toBe(JSON.stringify(expected));
    }
  });

  it('corrects ICMP errors where Swift has them', () => {
    const corrected = scenarioNames.filter((name) => {
      const expected = JSON.parse(readFileSync(join(scenarioDirectory, `${name}.swift.json`), 'utf8')) as Record<string, unknown>;
      return JSON.stringify(withJavaIcmpErrors(expected)) !== JSON.stringify(expected);
    });
    expect(corrected.length).toBeGreaterThan(0);
  });

  it('has scenarios with Swift results', () => {
    expect(scenarioNames.length).toBeGreaterThan(0);
  });

  for (const [name, reason] of Object.entries(JAVA_INSTEAD_OF_SWIFT)) {
    it(`${name} follows Java instead: ${reason}`, () => {
      const scenario = JSON.parse(readFileSync(join(scenarioDirectory, `${name}.scenario.json`), 'utf8')) as Scenario;
      const swift = JSON.parse(readFileSync(join(scenarioDirectory, `${name}.swift.json`), 'utf8')) as Record<string, unknown>;
      expect(run(scenario)).not.toEqual(swift);
    });
  }

  for (const [name, { trace, reason }] of Object.entries(SWIFT_UNTIL_TRACE)) {
    it(`${name} matches Swift until trace #${trace}, then follows Java: ${reason}`, () => {
      const scenario = JSON.parse(readFileSync(join(scenarioDirectory, `${name}.scenario.json`), 'utf8')) as Scenario;
      const swift = withJavaIcmpErrors(JSON.parse(readFileSync(join(scenarioDirectory, `${name}.swift.json`), 'utf8')) as Record<string, unknown>);
      const actual = run(scenario);
      const expectedTraces = swift.traces as unknown[];
      const actualTraces = actual.traces as unknown[];
      for (let index = 0; index < trace - 1; index += 1) {
        expect(actualTraces[index], `trace event #${index + 1}`).toEqual(expectedTraces[index]);
      }
      expect(actualTraces[trace - 1]).not.toEqual(expectedTraces[trace - 1]);
    });
  }

  for (const name of scenarioNames.filter((each) => !(each in JAVA_INSTEAD_OF_SWIFT) && !(each in SWIFT_UNTIL_TRACE))) {
    it(name, () => {
      const scenario = JSON.parse(readFileSync(join(scenarioDirectory, `${name}.scenario.json`), 'utf8')) as Scenario;
      const swift = JSON.parse(readFileSync(join(scenarioDirectory, `${name}.swift.json`), 'utf8')) as Record<string, unknown>;
      const expected = withJavaIcmpErrors(SWIFT_ERRORS[name]?.(swift) ?? swift);
      const actual = run(scenario);

      // Traces first and one by one: the first difference is the one that matters.
      const expectedTraces = expected.traces as unknown[];
      const actualTraces = actual.traces as unknown[];
      for (let index = 0; index < Math.max(expectedTraces.length, actualTraces.length); index += 1) {
        expect(actualTraces[index], `trace event #${index + 1}`).toEqual(expectedTraces[index]);
      }
      expect(actual).toEqual(expected);
    });
  }
});