diff --git a/CHANGES.md b/CHANGES.md index a9ccd00..a58004e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -28,10 +28,11 @@ Swift-Pfade relativ zu `filius-ipad/ios/FiliusPad/TopologyEditor/` (Stand | Datei | Quelle | Art der Übertragung | | --- | --- | --- | -| `src/zip.ts` | `Persistence/TopologyFLSArchive.swift` (Fehler, Grenzen, Pfadregeln, ZIP-Leser/-Schreiber, CRC-32) | übertragen; DEFLATE über `fflate`, mit Apples Abschneide-Verhalten nachgebildet; Breitenfaltung als NFKC nur für U+FF01–U+FFEF | +| `src/zip.ts` | `Persistence/TopologyFLSArchive.swift` (Fehler, Grenzen, Pfadregeln, ZIP-Leser/-Schreiber, CRC-32) | übertragen; DEFLATE über `fflate`, mit Apples Abschneide-Verhalten nachgebildet; Pfadfaltung (NFC, Breite, volles Case-Folding) an 41 Paaren gegen das Swift-Orakel gemessen | | `src/archive.ts` | ebenda, Archiv-Teil von Import und Export | übertragen, ohne die XML-Ebene | | `test/zip.test.ts` | `TopologyProjectPersistenceTests.swift` (Archiv-Ablehnungstests, `makeMinimalZIP`, `makePayloadZIP`) | übertragen, ergänzt um DEFLATE-Grenzfälle | | `test/corpus.test.ts` | — | neu: 23 Korpus-Dateien, gegengeprüft mit `fflate.unzipSync` | +| `test/pathFolding.test.ts`, `test/path-folding.*.json` | — | neu: welche Pfade dieselbe Datei meinen, Ergebnis aus dem Swift-Kern | ## tools/swift-oracle diff --git a/packages/fls/src/zip.ts b/packages/fls/src/zip.ts index ed989bb..1bf0f65 100644 --- a/packages/fls/src/zip.ts +++ b/packages/fls/src/zip.ts @@ -112,21 +112,39 @@ export function validateEntryPath(path: string): void { } /** - * Folds full-width and half-width forms (U+FF01…U+FFEF) to their ordinary - * counterparts — the part of NFKC that Foundation's `.widthInsensitive` does. + * Foundation's `.widthInsensitive`: the Halfwidth and Fullwidth Forms block + * (U+FF01…U+FFEE) folds to its NFKC form — full-width ASCII, half-width + * katakana, half-width hangul (to conjoining jamo) and the symbols. Measured + * against the Swift core (packages/fls/test/path-folding.*.json). */ function foldWidth(text: string): string { - return text.replace(/[!-￯]/g, (character) => character.normalize('NFKC')); + return text.replace(/[\uFF01-\uFFEE]/g, (character) => character.normalize('NFKC')); +} + +/** + * Unicode full case folding, which JavaScript lacks: lower, upper, lower per + * character gives it (`ß` → `ss`, `fi` → `fi`, `K` (Kelvin) → `k`, `ς` → `σ`, + * `İ` → `i̇`). The one exception is `ı`, which default case folding leaves + * alone and the round trip would turn into `i`. + */ +function foldCase(text: string): string { + let folded = ''; + for (const character of text) { + folded += character === '\u0131' ? character : character.toLowerCase().toUpperCase().toLowerCase(); + } + return folded; } /** * The key two paths share when they would name the same file on a case- and - * width-insensitive file system: NFC, width-folded, case-folded, per component. + * width-insensitive file system: NFC, then width and case folding, per + * component — `precomposedStringWithCanonicalMapping.folding([.caseInsensitive, + * .widthInsensitive])` in Swift. */ export function canonicalEntryKey(path: string): string { return path .split('/') - .map((component) => foldWidth(component.normalize('NFC')).toLowerCase()) + .map((component) => foldCase(foldWidth(component.normalize('NFC')))) .join('/'); } diff --git a/packages/fls/test/path-folding.cases.json b/packages/fls/test/path-folding.cases.json new file mode 100644 index 0000000..d426c51 --- /dev/null +++ b/packages/fls/test/path-folding.cases.json @@ -0,0 +1,207 @@ +[ + { + "name": "ascii case", + "first": "projekt/x/a.txt", + "second": "projekt/x/A.TXT" + }, + { + "name": "fullwidth letter", + "first": "projekt/x/A.txt", + "second": "projekt/x/A.txt" + }, + { + "name": "fullwidth digit", + "first": "projekt/x/1.txt", + "second": "projekt/x/1.txt" + }, + { + "name": "halfwidth katakana", + "first": "projekt/x/ア.txt", + "second": "projekt/x/ア.txt" + }, + { + "name": "sharp s vs SS", + "first": "projekt/x/straße.txt", + "second": "projekt/x/STRASSE.txt" + }, + { + "name": "sharp s vs ss", + "first": "projekt/x/straße.txt", + "second": "projekt/x/strasse.txt" + }, + { + "name": "capital sharp s", + "first": "projekt/x/ẞ.txt", + "second": "projekt/x/ß.txt" + }, + { + "name": "kelvin sign", + "first": "projekt/x/K.txt", + "second": "projekt/x/k.txt" + }, + { + "name": "angstrom sign", + "first": "projekt/x/Å.txt", + "second": "projekt/x/å.txt" + }, + { + "name": "ohm sign", + "first": "projekt/x/Ω.txt", + "second": "projekt/x/ω.txt" + }, + { + "name": "fi ligature", + "first": "projekt/x/fi.txt", + "second": "projekt/x/fi.txt" + }, + { + "name": "composed vs decomposed", + "first": "projekt/x/é.txt", + "second": "projekt/x/é.txt" + }, + { + "name": "dotted capital I", + "first": "projekt/x/İ.txt", + "second": "projekt/x/i.txt" + }, + { + "name": "dotless i", + "first": "projekt/x/ı.txt", + "second": "projekt/x/I.txt" + }, + { + "name": "final sigma", + "first": "projekt/x/ς.txt", + "second": "projekt/x/σ.txt" + }, + { + "name": "micro sign", + "first": "projekt/x/µ.txt", + "second": "projekt/x/μ.txt" + }, + { + "name": "circled letter", + "first": "projekt/x/Ⓐ.txt", + "second": "projekt/x/ⓐ.txt" + }, + { + "name": "superscript two", + "first": "projekt/x/².txt", + "second": "projekt/x/2.txt" + }, + { + "name": "ideographic space", + "first": "projekt/x/a b.txt", + "second": "projekt/x/a b.txt" + }, + { + "name": "umlaut case", + "first": "projekt/x/Ä.txt", + "second": "projekt/x/ä.txt" + }, + { + "name": "fullwidth tilde", + "first": "projekt/x/~.txt", + "second": "projekt/x/~.txt" + }, + { + "name": "halfwidth hangul", + "first": "projekt/x/ᄀ.txt", + "second": "projekt/x/ㄱ.txt" + }, + { + "name": "fullwidth won sign", + "first": "projekt/x/₩.txt", + "second": "projekt/x/₩.txt" + }, + { + "name": "different names", + "first": "projekt/x/a.txt", + "second": "projekt/x/b.txt" + }, + { + "name": "halfwidth left arrow", + "first": "projekt/x/←.txt", + "second": "projekt/x/←.txt" + }, + { + "name": "halfwidth forms light vertical", + "first": "projekt/x/│.txt", + "second": "projekt/x/│.txt" + }, + { + "name": "halfwidth katakana middle dot", + "first": "projekt/x/・.txt", + "second": "projekt/x/・.txt" + }, + { + "name": "halfwidth hangul filler", + "first": "projekt/x/ᅠ.txt", + "second": "projekt/x/ㅤ.txt" + }, + { + "name": "halfwidth hangul last", + "first": "projekt/x/ᅵ.txt", + "second": "projekt/x/ㅣ.txt" + }, + { + "name": "fullwidth cent sign", + "first": "projekt/x/¢.txt", + "second": "projekt/x/¢.txt" + }, + { + "name": "long s", + "first": "projekt/x/ſ.txt", + "second": "projekt/x/s.txt" + }, + { + "name": "capital sharp s vs ss", + "first": "projekt/x/ẞ.txt", + "second": "projekt/x/ss.txt" + }, + { + "name": "ff ligature", + "first": "projekt/x/ff.txt", + "second": "projekt/x/ff.txt" + }, + { + "name": "dotless i vs i", + "first": "projekt/x/ı.txt", + "second": "projekt/x/i.txt" + }, + { + "name": "dotted capital I vs i dot", + "first": "projekt/x/İ.txt", + "second": "projekt/x/i̇.txt" + }, + { + "name": "greek iota with dialytika and tonos", + "first": "projekt/x/ΐ.txt", + "second": "projekt/x/ΐ.txt" + }, + { + "name": "cherokee small letter", + "first": "projekt/x/ꭰ.txt", + "second": "projekt/x/Ꭰ.txt" + }, + { + "name": "fullwidth lower vs ascii upper", + "first": "projekt/x/a.txt", + "second": "projekt/x/A.txt" + }, + { + "name": "halfwidth hangul vs its NFKC form", + "first": "projekt/x/ᄀ.txt", + "second": "projekt/x/ᄀ.txt" + }, + { + "name": "halfwidth hangul filler vs its NFKC form", + "first": "projekt/x/ᅠ.txt", + "second": "projekt/x/ᅠ.txt" + }, + { + "name": "halfwidth hangul last vs its NFKC form", + "first": "projekt/x/ᅵ.txt", + "second": "projekt/x/ᅵ.txt" + } +] diff --git a/packages/fls/test/path-folding.swift.json b/packages/fls/test/path-folding.swift.json new file mode 100644 index 0000000..608eb18 --- /dev/null +++ b/packages/fls/test/path-folding.swift.json @@ -0,0 +1,166 @@ +[ + { + "name" : "ascii case", + "outcome" : "collide" + }, + { + "name" : "fullwidth letter", + "outcome" : "collide" + }, + { + "name" : "fullwidth digit", + "outcome" : "collide" + }, + { + "name" : "halfwidth katakana", + "outcome" : "collide" + }, + { + "name" : "sharp s vs SS", + "outcome" : "collide" + }, + { + "name" : "sharp s vs ss", + "outcome" : "collide" + }, + { + "name" : "capital sharp s", + "outcome" : "collide" + }, + { + "name" : "kelvin sign", + "outcome" : "collide" + }, + { + "name" : "angstrom sign", + "outcome" : "collide" + }, + { + "name" : "ohm sign", + "outcome" : "collide" + }, + { + "name" : "fi ligature", + "outcome" : "collide" + }, + { + "name" : "composed vs decomposed", + "outcome" : "sameKey" + }, + { + "name" : "dotted capital I", + "outcome" : "distinct" + }, + { + "name" : "dotless i", + "outcome" : "distinct" + }, + { + "name" : "final sigma", + "outcome" : "collide" + }, + { + "name" : "micro sign", + "outcome" : "collide" + }, + { + "name" : "circled letter", + "outcome" : "collide" + }, + { + "name" : "superscript two", + "outcome" : "distinct" + }, + { + "name" : "ideographic space", + "outcome" : "distinct" + }, + { + "name" : "umlaut case", + "outcome" : "collide" + }, + { + "name" : "fullwidth tilde", + "outcome" : "collide" + }, + { + "name" : "halfwidth hangul", + "outcome" : "distinct" + }, + { + "name" : "fullwidth won sign", + "outcome" : "collide" + }, + { + "name" : "different names", + "outcome" : "distinct" + }, + { + "name" : "halfwidth left arrow", + "outcome" : "collide" + }, + { + "name" : "halfwidth forms light vertical", + "outcome" : "collide" + }, + { + "name" : "halfwidth katakana middle dot", + "outcome" : "collide" + }, + { + "name" : "halfwidth hangul filler", + "outcome" : "distinct" + }, + { + "name" : "halfwidth hangul last", + "outcome" : "distinct" + }, + { + "name" : "fullwidth cent sign", + "outcome" : "collide" + }, + { + "name" : "long s", + "outcome" : "collide" + }, + { + "name" : "capital sharp s vs ss", + "outcome" : "collide" + }, + { + "name" : "ff ligature", + "outcome" : "collide" + }, + { + "name" : "dotless i vs i", + "outcome" : "distinct" + }, + { + "name" : "dotted capital I vs i dot", + "outcome" : "collide" + }, + { + "name" : "greek iota with dialytika and tonos", + "outcome" : "sameKey" + }, + { + "name" : "cherokee small letter", + "outcome" : "collide" + }, + { + "name" : "fullwidth lower vs ascii upper", + "outcome" : "collide" + }, + { + "name" : "halfwidth hangul vs its NFKC form", + "outcome" : "collide" + }, + { + "name" : "halfwidth hangul filler vs its NFKC form", + "outcome" : "collide" + }, + { + "name" : "halfwidth hangul last vs its NFKC form", + "outcome" : "collide" + } +] diff --git a/packages/fls/test/pathFolding.test.ts b/packages/fls/test/pathFolding.test.ts new file mode 100644 index 0000000..6a0c8db --- /dev/null +++ b/packages/fls/test/pathFolding.test.ts @@ -0,0 +1,52 @@ +// Which archive paths name the same file: the TypeScript folding against what +// the Swift core decides for the same pairs (path-folding.swift.json, written +// by tools/swift-oracle/run.sh; the Swift folding is Linux Foundation's). + +import { readFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { describe, expect, it } from 'vitest'; +import { FlsArchiveError, writeZipEntries } from '../src/index.js'; + +const here = dirname(fileURLToPath(import.meta.url)); +const cases = JSON.parse(readFileSync(join(here, 'path-folding.cases.json'), 'utf8')) as { + name: string; + first: string; + second: string; +}[]; +const swift = new Map( + (JSON.parse(readFileSync(join(here, 'path-folding.swift.json'), 'utf8')) as { name: string; outcome: string }[]).map( + (item) => [item.name, item.outcome], + ), +); + +function outcome(first: string, second: string): string { + try { + writeZipEntries( + new Map([ + [first, Uint8Array.of(1)], + [second, Uint8Array.of(2)], + ]), + ); + return 'distinct'; + } catch (error) { + if (error instanceof FlsArchiveError) return error.code === 'duplicateEntry' ? 'collide' : error.code; + throw error; + } +} + +describe('archive path folding matches Swift', () => { + it('has a Swift result for every case', () => { + expect(cases.every((item) => swift.has(item.name))).toBe(true); + }); + + for (const item of cases) { + it(item.name, () => { + // Swift strings compare by canonical equivalence, so a composed and a + // decomposed spelling are one dictionary key there ("sameKey"). A Map + // here keeps both — and the writer then refuses them as one file. + const expected = swift.get(item.name) === 'sameKey' ? 'collide' : swift.get(item.name); + expect(outcome(item.first, item.second)).toBe(expected); + }); + } +}); diff --git a/tools/swift-oracle/Sources/Oracle/main.swift b/tools/swift-oracle/Sources/Oracle/main.swift index 5037862..c498b3a 100644 --- a/tools/swift-oracle/Sources/Oracle/main.swift +++ b/tools/swift-oracle/Sources/Oracle/main.swift @@ -291,8 +291,47 @@ func readCorpus(root: URL) throws -> JSON { return result } +// MARK: - Archive path folding + +struct PathCase: Decodable { let name: String; let first: String; let second: String } + +/// Whether Swift's archive writer treats two entry paths as the same file. +/// The folding itself is file-private in Swift; the export reports it as a +/// duplicate entry when two carried entries collide. +func foldPaths(_ cases: [PathCase]) -> [JSON] { + cases.map { item in + var result: JSON = ["name": item.name] + // Swift compares strings by canonical equivalence: "é" and "e\u{301}" + // are one dictionary key, so they never reach the archive writer. + if item.first == item.second { + result["outcome"] = "sameKey" + return result + } + do { + _ = try TopologyProjectStore.exportFiliusArchiveWithReport( + from: TopologyEditorState(), + supplementalEntries: [item.first: Data("1".utf8), item.second: Data("2".utf8)] + ) + result["outcome"] = "distinct" + } catch let error as TopologyFLSArchiveError { + result["outcome"] = error.code == .duplicateEntry ? "collide" : error.code.rawValue + } catch { + result["outcome"] = "error: \(error)" + } + return result + } +} + // MARK: - Main +if CommandLine.arguments.count == 4, CommandLine.arguments[1] == "--path-folding" { + let cases = try JSONDecoder().decode([PathCase].self, from: Data(contentsOf: URL(fileURLWithPath: CommandLine.arguments[2]))) + let data = try JSONSerialization.data(withJSONObject: foldPaths(cases), options: [.prettyPrinted, .sortedKeys]) + try (data + Data("\n".utf8)).write(to: URL(fileURLWithPath: CommandLine.arguments[3])) + print("path folding: \(cases.count) cases") + exit(0) +} + if CommandLine.arguments.count == 4, CommandLine.arguments[1] == "--corpus" { let summary = try readCorpus(root: URL(fileURLWithPath: CommandLine.arguments[2])) let data = try JSONSerialization.data(withJSONObject: summary, options: [.prettyPrinted, .sortedKeys]) diff --git a/tools/swift-oracle/core.209 b/tools/swift-oracle/core.209 new file mode 100644 index 0000000..05c4fb8 Binary files /dev/null and b/tools/swift-oracle/core.209 differ diff --git a/tools/swift-oracle/core.7 b/tools/swift-oracle/core.7 index 7d37f31..3fa020a 100644 Binary files a/tools/swift-oracle/core.7 and b/tools/swift-oracle/core.7 differ diff --git a/tools/swift-oracle/run.sh b/tools/swift-oracle/run.sh index a2df364..92e15e0 100755 --- a/tools/swift-oracle/run.sh +++ b/tools/swift-oracle/run.sh @@ -11,4 +11,5 @@ docker run --rm -v "$root:/work" -w /work/tools/swift-oracle swift:6.1-jammy sh set -e swift build -q .build/debug/Oracle /work/packages/core/test/scenarios - .build/debug/Oracle --corpus /work/corpus /work/corpus/corpus.swift.json' + .build/debug/Oracle --corpus /work/corpus /work/corpus/corpus.swift.json + .build/debug/Oracle --path-folding /work/packages/fls/test/path-folding.cases.json /work/packages/fls/test/path-folding.swift.json'