julian.kiedaischandClaude Opus 5.5
24ebcea67f
A resolver's lookup on the new dns-lan network (A, MX, a referral by NS, an unknown name, a missing server): every frame and the call's result. ScenarioNetwork installs a scenario's DNS servers. The Swift comparison reads only scenarios with a Swift result. Measured: Java's text messages (ID=... QR=... header, question and record lines), no question in the answer, RCODE 0 always, a referral in the answer section, an iterative resolver waiting 5 RTT per query. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
341 lines
14 KiB
TypeScript
341 lines
14 KiB
TypeScript
// Replays every scenario in test/scenarios with the TypeScript engine and
|
|
// compares the result with what the filius-ipad Swift engine produced for the
|
|
// same file (<name>.swift.json, written by tools/swift-oracle/run.sh).
|
|
// Every trace event must match: identity, time, node, headers, detail.
|
|
|
|
import { existsSync, readdirSync, readFileSync } from 'node:fs';
|
|
import { dirname, join } from 'node:path';
|
|
import { fileURLToPath } from 'node:url';
|
|
import { describe, expect, it } from 'vitest';
|
|
import {
|
|
NetworkRuntimeEngine,
|
|
type DhcpClientStatus,
|
|
type IPv4DeliveryResult,
|
|
type IcmpObservation,
|
|
type TraceEvent,
|
|
} from '../src/index.js';
|
|
import { snapshotOf, type Scenario } from './scenario.js';
|
|
|
|
const scenarioDirectory = join(dirname(fileURLToPath(import.meta.url)), 'scenarios');
|
|
|
|
const upper = (id: string) => id.toUpperCase();
|
|
|
|
function dhcpStatusJson(status: DhcpClientStatus) {
|
|
return {
|
|
state: status.state,
|
|
errors: status.errorCount,
|
|
server: status.selectedServerIPAddress ?? null,
|
|
offered: status.offeredIPAddress ?? null,
|
|
succeeded: status.succeeded,
|
|
};
|
|
}
|
|
|
|
function traceJson(event: TraceEvent): Record<string, unknown> {
|
|
return {
|
|
id: event.id,
|
|
time: event.timeMilliseconds,
|
|
node: upper(event.nodeId),
|
|
direction: event.direction,
|
|
layer: event.layer,
|
|
operation: event.operation,
|
|
before: event.beforeHeaders.map((field) => [field.name, field.value]),
|
|
after: event.afterHeaders.map((field) => [field.name, field.value]),
|
|
...(event.frameIdentity === undefined ? {} : { frame: event.frameIdentity }),
|
|
...(event.packetIdentity === undefined ? {} : { packet: event.packetIdentity }),
|
|
...(event.interfaceId === undefined ? {} : { interface: upper(event.interfaceId) }),
|
|
...(event.detail === undefined ? {} : { detail: event.detail }),
|
|
};
|
|
}
|
|
|
|
function deliveryJson(result: IPv4DeliveryResult): Record<string, unknown> {
|
|
switch (result.type) {
|
|
case 'delivered':
|
|
return { type: 'delivered', packet: result.packetIdentity, node: upper(result.nodeId) };
|
|
case 'icmpError':
|
|
return { type: 'icmpError', packet: result.packetIdentity, node: upper(result.nodeId), kind: result.kind };
|
|
case 'dropped':
|
|
return {
|
|
type: 'dropped',
|
|
packet: result.packetIdentity,
|
|
...(result.nodeId === undefined ? {} : { node: upper(result.nodeId) }),
|
|
};
|
|
}
|
|
}
|
|
|
|
function observationJson(item: IcmpObservation): Record<string, unknown> {
|
|
return {
|
|
kind: item.message.kind,
|
|
identifier: item.message.identifier,
|
|
sequence: item.message.sequenceNumber,
|
|
sender: item.packet.senderIPAddress,
|
|
};
|
|
}
|
|
|
|
function run(scenario: Scenario): Record<string, unknown> {
|
|
// Swift's cables carry all frames at once; everything else is compared as it is.
|
|
const engine = new NetworkRuntimeEngine(scenario.seed, undefined, { swiftCables: true });
|
|
if (scenario.speedPercent !== undefined) engine.setSimulationSpeed(scenario.speedPercent);
|
|
engine.handle({ type: 'start', snapshot: snapshotOf(scenario), seed: scenario.seed, initialTimeMilliseconds: 0 });
|
|
|
|
const results: unknown[] = [];
|
|
for (const step of scenario.steps) {
|
|
switch (step.op) {
|
|
case 'ping':
|
|
results.push(
|
|
deliveryJson(
|
|
engine.sendIcmpEcho(step.from!, step.target!, {
|
|
timeToLive: step.ttl ?? 64,
|
|
identifier: step.identifier ?? 1,
|
|
sequenceNumber: step.sequence ?? 1,
|
|
}),
|
|
),
|
|
);
|
|
break;
|
|
case 'traceroute':
|
|
results.push(
|
|
engine
|
|
.traceIcmpEcho(step.from!, step.target!, {
|
|
maximumTimeToLive: step.maxTtl ?? 64,
|
|
identifier: step.identifier ?? 1,
|
|
})
|
|
.map(observationJson),
|
|
);
|
|
break;
|
|
case 'resolveMac':
|
|
results.push(engine.resolveMacAddress(step.node!, step.target!) ?? null);
|
|
break;
|
|
case 'advance':
|
|
engine.handle({ type: 'advance', toMilliseconds: step.to! });
|
|
results.push(engine.state.currentTimeMilliseconds);
|
|
break;
|
|
case 'packetLoss':
|
|
engine.setGlobalPacketLossEnabled(step.enabled!);
|
|
results.push(engine.state.globalPacketLossEnabled);
|
|
break;
|
|
case 'sat':
|
|
results.push(
|
|
engine.switchSatEntries(step.node!).map((entry) => ({
|
|
mac: entry.macAddress,
|
|
port: entry.portLabel,
|
|
updated: entry.updatedAtMilliseconds,
|
|
})),
|
|
);
|
|
break;
|
|
case 'speed':
|
|
engine.setSimulationSpeed(step.percent!);
|
|
results.push(engine.simulationSpeed.percent);
|
|
break;
|
|
case 'bindUdp':
|
|
results.push(
|
|
engine.bindUdpSocket(step.node!, {
|
|
...(step.port === undefined ? {} : { localPort: step.port }),
|
|
...(step.localIp === undefined ? {} : { localIPAddress: step.localIp }),
|
|
...(step.remoteIp === undefined ? {} : { remoteIPAddress: step.remoteIp }),
|
|
...(step.remotePort === undefined ? {} : { remotePort: step.remotePort }),
|
|
}) ?? null,
|
|
);
|
|
break;
|
|
case 'sendUdp': {
|
|
const result = engine.sendUdp(step.socket!, new TextEncoder().encode(step.payload ?? ''), {
|
|
...(step.target === undefined ? {} : { ipAddress: step.target }),
|
|
...(step.port === undefined ? {} : { port: step.port }),
|
|
});
|
|
results.push(result === undefined ? null : deliveryJson(result));
|
|
break;
|
|
}
|
|
case 'receiveUdp': {
|
|
const item = engine.receiveUdp(step.socket!, step.timeout ?? 0);
|
|
results.push(
|
|
item === undefined
|
|
? null
|
|
: {
|
|
sender: item.senderIPAddress,
|
|
receiver: item.receiverIPAddress,
|
|
sourcePort: item.datagram.sourcePort,
|
|
destinationPort: item.datagram.destinationPort,
|
|
payload: new TextDecoder().decode(item.datagram.payload),
|
|
at: item.receivedAtMilliseconds,
|
|
},
|
|
);
|
|
break;
|
|
}
|
|
case 'openTcpServer':
|
|
results.push(engine.openTcpServerSocket(step.node!, step.port!, step.localIp) ?? null);
|
|
break;
|
|
case 'openTcpClient':
|
|
results.push(
|
|
engine.openTcpClientSocket(step.node!, step.target!, step.port!, {
|
|
...(step.localIp === undefined ? {} : { localIPAddress: step.localIp }),
|
|
...(step.localPort === undefined ? {} : { localPort: step.localPort }),
|
|
}) ?? null,
|
|
);
|
|
break;
|
|
case 'connectTcp':
|
|
results.push(engine.connectTcpWithResult(step.socket!));
|
|
break;
|
|
case 'sendTcp':
|
|
results.push(engine.sendTcp(step.socket!, new TextEncoder().encode((step.payload ?? '').repeat(step.repeatCount ?? 1))));
|
|
break;
|
|
case 'receiveTcp': {
|
|
const message = engine.receiveTcp(step.socket!);
|
|
results.push(message === undefined ? null : new TextDecoder().decode(message));
|
|
break;
|
|
}
|
|
case 'acceptedTcp':
|
|
results.push(engine.state.tcpAcceptedSocketIdsByListenerId.get(step.socket!) ?? []);
|
|
break;
|
|
case 'peerTcp':
|
|
results.push(engine.peerTcpSocketId(step.socket!) ?? null);
|
|
break;
|
|
case 'cleanTcp':
|
|
results.push(engine.cleanClosedTcpSocket(step.socket!));
|
|
break;
|
|
case 'runDhcp': {
|
|
const status = engine.runDhcpClient(step.node!);
|
|
results.push(status === undefined ? null : dhcpStatusJson(status));
|
|
break;
|
|
}
|
|
case 'closeSocket':
|
|
engine.closeSocket(step.socket!);
|
|
results.push(!engine.state.socketsById.has(step.socket!));
|
|
break;
|
|
default:
|
|
throw new Error(`unknown step ${step.op}`);
|
|
}
|
|
}
|
|
|
|
const state = engine.state;
|
|
const mapOf = <V>(map: Map<string, V>, convert: (value: V) => unknown) =>
|
|
Object.fromEntries([...map].map(([key, value]) => [upper(key), convert(value)]));
|
|
return {
|
|
results,
|
|
traces: state.packetTraces.map(traceJson),
|
|
time: state.currentTimeMilliseconds,
|
|
pendingEvents: state.pendingEvents.length,
|
|
arp: mapOf(state.arpCachesByNodeId, (cache) =>
|
|
Object.fromEntries([...cache].map(([ip, entry]) => [ip, entry.macAddress])),
|
|
),
|
|
switchTables: mapOf(state.switchForwardingTablesByNodeId, (table) =>
|
|
Object.fromEntries([...table].map(([mac, port]) => [mac, upper(port)])),
|
|
),
|
|
icmpObservations: mapOf(state.icmpObservationsByNodeId, (items) => items.map(observationJson)),
|
|
devices: Object.fromEntries(
|
|
[...state.topologySnapshot.deviceConfigurations].map(([node, device]) => [
|
|
upper(node),
|
|
[device.ipAddress, device.subnetMask, device.defaultGateway, device.dnsServer],
|
|
]),
|
|
),
|
|
dhcp: {
|
|
clients: mapOf(state.dhcpClientStatusesByNodeId, dhcpStatusJson),
|
|
leases: [...state.dhcpLeasesByIPAddress]
|
|
.sort(([lhs], [rhs]) => (lhs < rhs ? -1 : 1))
|
|
.map(([key, lease]) => [key, lease.clientMacAddress, lease.ipAddress, lease.expiresAtMilliseconds ?? null]),
|
|
offers: [...state.dhcpOffersByIPAddress]
|
|
.sort(([lhs], [rhs]) => (lhs < rhs ? -1 : 1))
|
|
.map(([key, offer]) => [key, offer.clientMacAddress, offer.expiresAtMilliseconds]),
|
|
blacklist: mapOf(state.dhcpBlacklistByServerNodeId, (entries) =>
|
|
[...entries].sort(([lhs], [rhs]) => (lhs < rhs ? -1 : 1)).map(([address, expiry]) => [address, expiry]),
|
|
),
|
|
servers: mapOf(state.dhcpServerSocketIdsByNodeId, upper),
|
|
},
|
|
sockets: [...state.socketsById.values()]
|
|
.sort((lhs, rhs) => (upper(lhs.id) < upper(rhs.id) ? -1 : 1))
|
|
.map((socket) => ({
|
|
id: upper(socket.id),
|
|
node: upper(socket.nodeId),
|
|
protocol: socket.protocol,
|
|
localIp: socket.localIPAddress ?? null,
|
|
localPort: socket.localPort,
|
|
remoteIp: socket.remoteIPAddress ?? null,
|
|
remotePort: socket.remotePort ?? null,
|
|
queued: state.udpReceiveQueuesBySocketId.get(socket.id)?.length ?? 0,
|
|
tcpState: socket.tcpState ?? null,
|
|
parent: socket.parentListenerSocketId === undefined ? null : upper(socket.parentListenerSocketId),
|
|
session: ((session) =>
|
|
session === undefined
|
|
? null
|
|
: {
|
|
attempts: session.sendAttempts,
|
|
next: session.nextSendSequenceNumber,
|
|
remote: session.remoteSequenceNumber,
|
|
pending: session.lastSentSegment !== undefined,
|
|
assembly: session.receiveAssembly.length,
|
|
messages: session.receivedMessages.length,
|
|
})(state.tcpSessionsById.get(socket.id)),
|
|
})),
|
|
firewallDecisions: state.firewallDecisions.map((decision) => ({
|
|
packet: decision.packetIdentity,
|
|
node: upper(decision.nodeId),
|
|
accepted: decision.accepted,
|
|
...(decision.ruleIndex === undefined ? {} : { rule: decision.ruleIndex }),
|
|
})),
|
|
};
|
|
}
|
|
|
|
// Scenarios with a Swift result (the Java comparisons have scenarios of their own).
|
|
const scenarioNames = readdirSync(scenarioDirectory)
|
|
.filter((name) => name.endsWith('.scenario.json') && existsSync(join(scenarioDirectory, name.replace('.scenario.json', '.swift.json'))))
|
|
.map((name) => name.replace('.scenario.json', ''))
|
|
.sort();
|
|
|
|
/**
|
|
* Where the Swift engine is wrong, per scenario: its result corrected. None
|
|
* at the moment. (Until F2.3b: dhcp-lan, the status a directly run DHCP
|
|
* client reported — Swift wrote back a stale copy after a nested run.)
|
|
*/
|
|
const SWIFT_ERRORS: Record<string, (expected: Record<string, unknown>) => Record<string, unknown>> = {};
|
|
|
|
/**
|
|
* Scenarios where the engine follows Java instead of Swift; the Java
|
|
* comparisons check that behaviour (dhcpJava.test.ts, tcpJava.test.ts,
|
|
* measured with Java FILIUS). Here
|
|
* they must differ from Swift, or the departure is not real.
|
|
*/
|
|
const JAVA_INSTEAD_OF_SWIFT: Record<string, string> = {
|
|
'dhcp-lan': 'DHCP client: one message per attempt, the ARP check waits a round trip, which follows the speed',
|
|
'dhcp-two-servers': 'DHCP client: the second OFFER is read and dropped, no REQUEST twice (Swift: answered with NAK)',
|
|
'dhcp-conflict': 'DHCP client: the ARP check finds the address taken and declines (Swift: it does not wait, two cards share the address)',
|
|
'dhcp-no-server': 'DHCP client: a link-local address after five timeouts (Swift: the old address after ten)',
|
|
'tcp-lan': 'TCP: the other end closes by itself on FIN, TIME_WAIT lasts 3 round trips, the accepted socket has the listener\'s sequence number',
|
|
'tcp-routed-loss': 'TCP: 3 round trips per attempt follow the speed (8250 ms at 90 %), failed data closes with FIN',
|
|
'tcp-closing': 'TCP: TIME_WAIT lasts 3 round trips and ends with the next segment; the other end closes by itself',
|
|
};
|
|
|
|
describe('same result as the Swift engine', () => {
|
|
it('corrects Swift only where it errs', () => {
|
|
for (const [name, correct] of Object.entries(SWIFT_ERRORS)) {
|
|
const expected = JSON.parse(readFileSync(join(scenarioDirectory, `${name}.swift.json`), 'utf8')) as Record<string, unknown>;
|
|
expect(JSON.stringify(correct(expected)), name).not.toBe(JSON.stringify(expected));
|
|
}
|
|
});
|
|
|
|
it('has scenarios with Swift results', () => {
|
|
expect(scenarioNames.length).toBeGreaterThan(0);
|
|
});
|
|
|
|
for (const [name, reason] of Object.entries(JAVA_INSTEAD_OF_SWIFT)) {
|
|
it(`${name} follows Java instead: ${reason}`, () => {
|
|
const scenario = JSON.parse(readFileSync(join(scenarioDirectory, `${name}.scenario.json`), 'utf8')) as Scenario;
|
|
const swift = JSON.parse(readFileSync(join(scenarioDirectory, `${name}.swift.json`), 'utf8')) as Record<string, unknown>;
|
|
expect(run(scenario)).not.toEqual(swift);
|
|
});
|
|
}
|
|
|
|
for (const name of scenarioNames.filter((each) => !(each in JAVA_INSTEAD_OF_SWIFT))) {
|
|
it(name, () => {
|
|
const scenario = JSON.parse(readFileSync(join(scenarioDirectory, `${name}.scenario.json`), 'utf8')) as Scenario;
|
|
const swift = JSON.parse(readFileSync(join(scenarioDirectory, `${name}.swift.json`), 'utf8')) as Record<string, unknown>;
|
|
const expected = SWIFT_ERRORS[name]?.(swift) ?? swift;
|
|
const actual = run(scenario);
|
|
|
|
// Traces first and one by one: the first difference is the one that matters.
|
|
const expectedTraces = expected.traces as unknown[];
|
|
const actualTraces = actual.traces as unknown[];
|
|
for (let index = 0; index < Math.max(expectedTraces.length, actualTraces.length); index += 1) {
|
|
expect(actualTraces[index], `trace event #${index + 1}`).toEqual(expectedTraces[index]);
|
|
}
|
|
expect(actual).toEqual(expected);
|
|
});
|
|
}
|
|
});
|