julian.kiedaisch/mdg-filius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(core): Gnutella search and download after Java, measured against Java

Browse Source 
GnutellaRun adds a line of four peers: a search goes on as a query, each
peer answers with a hit for each matching file and the hits come back the
way the query came; a hit's file is fetched by HTTP on port 6346. The
file type names of the model move from the project export and import into
the core, so that the download uses them too.

Departures from Java: a peer that sent hits keeps its connection, so that
the hits of farther peers come back too (Java closes it and loses them),
and sizes are in characters.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
julian.kiedaisch
2026-10-02 14:59:26 +00:00
co-authored by Claude Opus 5.5
parent b4a95e6cac
commit ce5b10c1f3
13 changed files with 39934 additions and 130 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/core/test/gnutellaJava.test.ts
+193 -36
View File
@@ -11,7 +11,7 @@ import { readFileSync } from 'node:fs';
import { dirname, join } from 'node:path';
import { fileURLToPath } from 'node:url';
import { describe, expect, it } from 'vitest';
import { withSnapshot, type NetworkRuntimeEngine } from '../src/index.js';
import { GNUTELLA_FOLDER, javaBase64, javaFileTypeName, VirtualFileSystem, withSnapshot, type NetworkRuntimeEngine } from '../src/index.js';
import {
engineText,
expectSameCaptures,
@@ -39,7 +39,7 @@ interface JavaRun extends JavaTables {
acts: [number, string, string][];
/** time, host, kind (call or peers), text */
events: [number, string, string, string][];
programs: Record<string, { peers: string[]; results: string[] }>;
programs: Record<string, { peers: string[]; results: string[]; files?: [string, string, string][] }>;
}
/** The stories, and how many connections each makes one after the other (Java looks every 100 ms whether one is made). */
@@ -50,14 +50,17 @@ const STORIES: Record<string, { readonly looks: number }> = {
full: { looks: 3 },
reset: { looks: 1 },
errors: { looks: 1 },
search: { looks: 3 },
searchBoth: { looks: 3 },
};
/** The ids of Java's own pings, in the order they were made: the engine draws its own, these take their place. */
/** The ids of Java's own pings and queries, in the order they were made: the engine draws its own, these take their place. */
function javaPingIds(java: JavaRun): number[] {
const pings = Object.values(java.frames)
.flat()
.flatMap((frame) => {
const match = /^(\d+)\/\/0x00\/\/0\/\//.exec(frame.payload ?? '');
// A ping or a query with no hops yet: the program's own.
const match = /^(\d+)\/\/0x(?:00|80)\/\/0\/\//.exec(frame.payload ?? '');
return match === null ? [] : [[frame.time, Number(match[1])] as const];
})
.sort((lhs, rhs) => lhs[0] - rhs[0]);
@@ -65,19 +68,86 @@ function javaPingIds(java: JavaRun): number[] {
}
/** Each program's known peers as it told them, each change once (Java tells its window also when nothing changed, and an empty list at its start). */
function javaPeers(java: JavaRun): Record<string, [number, string][]> {
const peers: Record<string, [number, string][]> = Object.fromEntries(PEERS.map((host) => [host, []]));
/**
* Each program's known peers, or its results, as it told them, each change
* once (Java tells its window also when nothing changed, and an empty list
* at its start); results only when there are some (Java empties them without
* telling).
*/
function javaTold(java: JavaRun, wanted: 'peers' | 'results'): Record<string, [number, string][]> {
const told: Record<string, [number, string][]> = Object.fromEntries(PEERS.map((host) => [host, []]));
for (const [time, host, kind, text] of java.events) {
if (kind !== 'peers') continue;
const list = peers[host]!;
if (kind !== wanted || (wanted === 'results' && text === '')) continue;
const list = told[host]!;
if ((list.at(-1)?.[1] ?? '') === text) continue;
list.push([time, text]);
}
return peers;
return told;
}
function enginePeers(engine: NetworkRuntimeEngine): Record<string, [number, string][]> {
return Object.fromEntries(PEERS.map((host) => [host, engine.gnutella(host)!.log.all.map((line): [number, string] => [line.timeMilliseconds, line.peers.join(',')])]));
function engineTold(engine: NetworkRuntimeEngine, wanted: 'peers' | 'results'): Record<string, [number, string][]> {
return Object.fromEntries(
PEERS.map((host) => {
const list: [number, string][] = [];
for (const line of engine.gnutella(host)!.log.all) {
const text = line.kind === 'peers' ? (wanted === 'peers' ? line.peers.join(',') : undefined) : wanted === 'results' ? line.results.join('|') : undefined;
if (text === undefined || (wanted === 'results' && text === '') || list.at(-1)?.[1] === text) continue;
list.push([line.timeMilliseconds, text]);
}
return [host, list];
}),
);
}
/** Java's run with each result's text changed, where it tells them and at the end. */
function withResults(java: JavaRun, change: (results: string[], host: string) => string[]): JavaRun {
return {
...java,
events: java.events.map(([time, host, kind, text]) => [time, host, kind, kind === 'results' && text !== '' ? change(text.split('|'), host).join('|') : text]),
programs: Object.fromEntries(
Object.entries(java.programs).map(([host, program]) => [host, { ...program, results: program.results.length === 0 ? [] : change(program.results, host) }]),
),
};
}
/** A hit's size as Java's Datei.holeGroesse says it (the text decoded as Base64), and as the engine (its characters). */
const SIZES: Record<string, [number, number]> = { 'Lied der Woche.txt': [3, 8], 'notizen.txt': [3, 5], 'lied.txt': [6, 10] };
const withSizes = (results: string[]) =>
results.map((result) =>
result.replace(/\/(.+): (\d+) B$/, (whole, name: string, size: string) => (SIZES[name]?.[0] === Number(size) ? `/${name}: ${SIZES[name]![1]} B` : whole)),
);
/**
* Where Java errs, its run corrected: a hit's size in characters (search,
* searchBoth); in searchBoth also the hit of 10.0.0.2, which Java loses
* because .31 closes the connection after its own hit (gnutella.ts).
*/
const JAVA_BUGS: Record<string, (java: JavaRun) => JavaRun> = {
search: (java) => withResults(java, withSizes),
searchBoth: (java) => {
const sized = withResults(java, withSizes);
const farHit = '10.0.0.2/Lied der Woche.txt: 8 B';
const last = sized.events.findLast(([, host, kind, text]) => host === CLIENT && kind === 'results' && text !== '')!;
return {
...sized,
events: [...sized.events, [last[0] + 1, CLIENT, 'results', `${last[3]}|${farHit}`]],
programs: { ...sized.programs, [CLIENT]: { ...sized.programs[CLIENT]!, results: [...sized.programs[CLIENT]!.results, farHit] } },
};
},
};
/** Where a result's time is not compared: a result Java never had (JAVA_BUGS). */
const RESULTS_WITHOUT_TIMES = new Set(['searchBoth']);
/** Each program's files in /peer2peer as GnutellaRun writes Java's: name, type, content. */
function engineFiles(engine: NetworkRuntimeEngine, host: string): [string, string, string][] {
const files = engine.state.topologySnapshot.fileSystemsByNodeId.get(host)!;
return files.entriesInCreationOrder(GNUTELLA_FOLDER).flatMap((entry): [string, string, string][] => {
const content = entry.content;
if (content.kind === 'directory') return [];
const name = entry.path.slice(entry.path.lastIndexOf('/') + 1);
return [[name, javaFileTypeName(name, content), content.kind === 'text' ? content.text : javaBase64(content.data)]];
});
}
/**
@@ -94,7 +164,38 @@ function enginePeers(engine: NetworkRuntimeEngine): Record<string, [number, stri
* frames between the two hosts on the server's port, in the order the
* connections began — with their data; ARP on its own.
*/
const BY_CONNECTION = new Set(['twoPaths', 'full']);
const BY_CONNECTION = new Set(['twoPaths', 'full', 'search', 'searchBoth']);
/**
* search, searchBoth: a hit names a size, Java's another than the engine's
* (JAVA_BUGS), as a pong its files' bytes, so the length of the text and
* the numbers after it on its connection differ; the sizes, bytes,
* sequence numbers and lengths are left out of the frames compared. A connection Java's server closes after its
* hits (gnutella.ts) is compared up to that FIN; one the client closes (a
* download's) wholly.
*/
const SEARCHING = new Set(['search', 'searchBoth']);
const withoutSizes = (text: string) =>
text
.replace(/ seq=\d+ ack=\d+ length=\d+/, '')
.replace(/: \d+ B\/\//g, ': n B//')
// A pong's bytes, and its payload length that counts their bits.
.replace(/(\/\/0x01\/\/\d+\/\/\d+\/\/)\d+(\/\/.*\/\/6346\/\/\d+\/\/)\d+$/, '$1n$2n');
/** A card's frames without those of each connection from the FIN its server sent first. */
function beforeServerFins(frames: readonly JavaFrame[]): JavaFrame[] {
const closedAt = new Map<string, number>();
for (const frame of frames) {
const client = frame.sourcePort === 6346 ? `${frame.destinationIp}:${frame.destinationPort}` : `${frame.sourceIp}:${frame.sourcePort}`;
if (frame.tcpFlags?.includes('F') !== true || closedAt.has(client)) continue;
closedAt.set(client, frame.sourcePort === 6346 ? frame.time : Number.POSITIVE_INFINITY);
}
return frames.filter((frame) => {
if (frame.tcpFlags === undefined) return true;
const client = frame.sourcePort === 6346 ? `${frame.destinationIp}:${frame.destinationPort}` : `${frame.sourceIp}:${frame.sourcePort}`;
return frame.time < (closedAt.get(client) ?? Number.POSITIVE_INFINITY);
});
}
interface FlowFrame {
readonly time: number;
@@ -146,22 +247,54 @@ const UNTIL: Record<string, (java: JavaRun) => number> = {
twoPaths: secondPing,
};
/** searchBoth: each side's frames up to its first hit, after which Java loses the hit of 10.0.0.2 (JAVA_BUGS). */
const UP_TO_FIRST_HIT = new Set(['searchBoth']);
const FIRST_HIT = '//0x81//';
/** When the engine's first hit showed on a card. */
function firstEngineHit(engine: NetworkRuntimeEngine, scenario: Scenario): number {
const times = scenarioCards(scenario).flatMap((card) =>
engine.capture.frames(card.portId).flatMap(({ timeMilliseconds, frame }) => {
const packet = frame.payload.kind === 'ipv4' ? frame.payload.packet : undefined;
const text = packet?.payload.kind === 'tcp' ? new TextDecoder().decode(packet.payload.segment.payload) : '';
return text.includes(FIRST_HIT) ? [timeMilliseconds] : [];
}),
);
return Math.min(...times);
}
describe('the Gnutella program as Java FILIUS runs it', () => {
const scenario = JSON.parse(readFileSync(join(scenarioDirectory, 'terminal-net.scenario.json'), 'utf8')) as Scenario;
for (const [name, story] of Object.entries(STORIES)) {
it(name, () => {
const java = JSON.parse(readFileSync(join(scenarioDirectory, `gnutella-java-${name}.java.json`), 'utf8')) as JavaRun;
const measured = JSON.parse(readFileSync(join(scenarioDirectory, `gnutella-java-${name}.java.json`), 'utf8')) as JavaRun;
const java = JAVA_BUGS[name]?.(measured) ?? measured;
const maximumKnownPeers = (host: string) =>
name === 'full' ? (host === ECHO ? 2 : host === FAR ? 3 : 1) : SEARCHING.has(name) ? (host === CLIENT || host === FAR ? 1 : 2) : 3;
const files = new Map(PEERS.map((host) => [host, new VirtualFileSystem()]));
if (SEARCHING.has(name)) {
for (const host of [ECHO, FAR]) files.get(host)!.createDirectory(GNUTELLA_FOLDER);
if (name === 'searchBoth') files.get(ECHO)!.writeTextFile(`${GNUTELLA_FOLDER}/lied.txt`, 'Hallo Welt');
files.get(ECHO)!.writeTextFile(`${GNUTELLA_FOLDER}/notizen.txt`, 'Notiz');
files.get(FAR)!.writeTextFile(`${GNUTELLA_FOLDER}/Lied der Woche.txt`, 'La la la');
}
const snapshot = withSnapshot(snapshotOf(scenario), {
gnutellaByNodeId: new Map(PEERS.map((host) => [host, { maximumKnownPeers: name !== 'full' ? 3 : host === ECHO ? 2 : host === FAR ? 3 : 1 }])),
gnutellaByNodeId: new Map(PEERS.map((host) => [host, { maximumKnownPeers: maximumKnownPeers(host) }])),
fileSystemsByNodeId: files,
});
const { engine } = watchedEngine(scenario, snapshot);
const ids = javaPingIds(java);
engine.drawGnutellaGuid = () => ids.shift() ?? 1;
for (const [time, host, call] of java.acts) {
engine.handle({ type: 'advance', toMilliseconds: time });
if (call.startsWith('join ')) engine.gnutella(host)!.join(call.slice('join '.length));
else engine.gnutella(host)!.reset();
const program = engine.gnutella(host)!;
if (call.startsWith('join ')) program.join(call.slice('join '.length));
else if (call.startsWith('search ')) program.search(call.slice('search '.length));
else if (call.startsWith('download ')) program.download(Number(call.slice('download '.length)));
else if (call.startsWith('remove ')) files.get(host)!.deleteItem(`${GNUTELLA_FOLDER}/${call.slice('remove '.length)}`);
else if (call === 'clear') program.clearResults();
else program.reset();
}
engine.handle({ type: 'advance', toMilliseconds: Math.max(...java.events.map(([time]) => time)) + 10_000 });
@@ -169,21 +302,37 @@ describe('the Gnutella program as Java FILIUS runs it', () => {
const until = UNTIL[name]?.(java) ?? Number.POSITIVE_INFINITY;
if (BY_CONNECTION.has(name)) {
const portOf = new Map(scenarioCards(scenario).map((card) => [card.key, card.portId]));
const masked = (text: string) => (SEARCHING.has(name) ? withoutSizes(text) : text);
// search, searchBoth: a bare acknowledgement left out (Java's server closes after its hits, and the client's
// acknowledgement of them falls after that FIN).
const compared = (frame: FlowFrame) => !SEARCHING.has(name) || !/ TCP \S+ > \S+ A $/.test(frame.text);
const data = (text: string | undefined) => withClientPortText(text ?? '').replace(/^\d+\/\/0x/, 'x//0x');
for (const [card, list] of Object.entries(java.frames)) {
const javaUntil = UP_TO_FIRST_HIT.has(name)
? Math.min(
...Object.values(java.frames)
.flat()
.filter((frame) => frame.payload?.includes(FIRST_HIT) === true)
.map((frame) => frame.time),
)
: until;
const expected = connections(
list
.filter((frame) => frame.time < until)
.map((frame) => ({ time: frame.time, raw: javaText(frame), text: `${withClientPortText(javaText(frame))} ${data(frame.payload)}` })),
(SEARCHING.has(name) ? beforeServerFins(list) : list)
.filter((frame) => frame.time < javaUntil)
.map((frame) => ({ time: frame.time, raw: javaText(frame), text: masked(`${withClientPortText(javaText(frame))} ${data(frame.payload)}`) }))
.filter(compared),
);
const captured = engine.capture.frames(portOf.get(card)!).filter((each) => each.timeMilliseconds < until);
const engineUntil = UP_TO_FIRST_HIT.has(name) ? firstEngineHit(engine, scenario) : until;
const captured = engine.capture.frames(portOf.get(card)!).filter((each) => each.timeMilliseconds < engineUntil);
const actual = connections(
captured.map(({ timeMilliseconds, frame }) => {
const packet = frame.payload.kind === 'ipv4' ? frame.payload.packet : undefined;
const bytes = packet?.payload.kind === 'tcp' ? packet.payload.segment.payload : new Uint8Array();
const text = engineText(frame);
return { time: timeMilliseconds, raw: text, text: `${withClientPortText(text)} ${data(bytes.length === 0 ? undefined : new TextDecoder().decode(bytes))}` };
}),
captured
.map(({ timeMilliseconds, frame }) => {
const packet = frame.payload.kind === 'ipv4' ? frame.payload.packet : undefined;
const bytes = packet?.payload.kind === 'tcp' ? packet.payload.segment.payload : new Uint8Array();
const text = engineText(frame);
return { time: timeMilliseconds, raw: text, text: masked(`${withClientPortText(text)} ${data(bytes.length === 0 ? undefined : new TextDecoder().decode(bytes))}`) };
})
.filter(compared),
);
expect([...actual.keys()].sort(), `${card}'s connections`).toEqual([...expected.keys()].sort());
for (const [key, flows] of expected) {
@@ -200,17 +349,25 @@ describe('the Gnutella program as Java FILIUS runs it', () => {
expectSameCaptures(engine, scenario, java.frames, { text: withClientPortText, tables: java, until: UNTIL[name]?.(java), tolerance });
}
const expected = javaPeers(java);
const actual = enginePeers(engine);
for (const host of PEERS) {
expect(
actual[host]!.map(([, peers]) => peers),
`${host}'s peers`,
).toEqual(expected[host]!.map(([, peers]) => peers));
for (const [index, [javaTime]] of expected[host]!.entries()) {
expect(Math.abs(javaTime - actual[host]![index]![0]), `${host}'s peers #${index + 1}`).toBeLessThanOrEqual(tolerance);
for (const wanted of ['peers', 'results'] as const) {
const expected = javaTold(java, wanted);
const actual = engineTold(engine, wanted);
for (const host of PEERS) {
expect(
actual[host]!.map(([, told]) => told),
`${host}'s ${wanted}`,
).toEqual(expected[host]!.map(([, told]) => told));
if (wanted === 'results' && RESULTS_WITHOUT_TIMES.has(name)) continue;
for (const [index, [javaTime]] of expected[host]!.entries()) {
expect(Math.abs(javaTime - actual[host]![index]![0]), `${host}'s ${wanted} #${index + 1}`).toBeLessThanOrEqual(tolerance);
}
}
expect(engine.gnutella(host)!.peers, `${host}'s peers at the end`).toEqual(java.programs[host]!.peers);
}
for (const host of PEERS) {
const program = engine.gnutella(host)!;
expect(program.peers, `${host}'s peers at the end`).toEqual(java.programs[host]!.peers);
expect(program.results, `${host}'s results at the end`).toEqual(java.programs[host]!.results);
if (java.programs[host]!.files !== undefined) expect(engineFiles(engine, host), `${host}'s files`).toEqual(java.programs[host]!.files);
}
});
}