From 6a5423b7a56d3e2c442119ee9621cbbd769907d6 Mon Sep 17 00:00:00 2001 From: Mx-Iris Date: Thu, 3 Sep 2026 18:35:30 +0800 Subject: [PATCH 1/4] chore(deps): raise the swift-demangling floor to 0.6.3 (proposal large-stack-executor-and-cross-version-parallelism) 0.6.1 re-ranked the hop pool's QoS per call and slowed dump/interface 3-4x; 0.6.2 partitioned the pool by class and restored the speed; 0.6.3 ships the large-stack TaskExecutor the proposal adopts. The floor skips 0.6.1. The proposal moves to In Progress: the user asked for the implementation on top of the self-contained ABI branch. --- Documentations/Evolutions/README.md | 2 +- ...aft-large-stack-executor-and-cross-version-parallelism.md | 5 +++-- Package.swift | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/Documentations/Evolutions/README.md b/Documentations/Evolutions/README.md index 1cfc4d24..1b4a1eb1 100644 --- a/Documentations/Evolutions/README.md +++ b/Documentations/Evolutions/README.md @@ -26,4 +26,4 @@ | [0017](0017-macho-dependencies-module.md) | 依赖闭包下沉为 MachODependencies 模块：两套依赖加载合一 | Implemented | | [0016](0016-exported-only-interface.md) | Interface 只打印导出声明（`--exported-only`）：提案 0008 标注的过滤形态，打印期按描述符 / 派生符号 / 扩展目标裁决 | Implemented | | [draft](draft-self-contained-abi-layer.md) | ABI 层自包含：MachOSwiftSection 不再依赖符号索引——描述符只暴露实现地址，符号查询上移 SwiftInspection，值类型下沉 MachOResolving | In Progress | -| [draft](draft-large-stack-executor-and-cross-version-parallelism.md) | 大栈任务执行器接入与跨版本并行准备：打印路径零线程跳转（执行器本体在 swift-demangling 0014），diff / evolution 多版本并行 | Draft | +| [draft](draft-large-stack-executor-and-cross-version-parallelism.md) | 大栈任务执行器接入与跨版本并行准备：打印路径零线程跳转（执行器本体在 swift-demangling 0014），diff / evolution 多版本并行 | In Progress | diff --git a/Documentations/Evolutions/draft-large-stack-executor-and-cross-version-parallelism.md b/Documentations/Evolutions/draft-large-stack-executor-and-cross-version-parallelism.md index a3fc0b56..dc6a0b5c 100644 --- a/Documentations/Evolutions/draft-large-stack-executor-and-cross-version-parallelism.md +++ b/Documentations/Evolutions/draft-large-stack-executor-and-cross-version-parallelism.md @@ -1,12 +1,12 @@ # Draft - 大栈任务执行器接入与跨版本并行准备 -- **状态**: Draft +- **状态**: In Progress - **作者**: JH - **创建日期**: 2026-09-03 - **最后更新**: 2026-09-03 - **所属愿景**: 无 - **关联提案**: swift-demangling 提案 0014「大栈 TaskExecutor」（上游前置，执行器本体在那边；本提案只做接入）；[draft-self-contained-abi-layer](draft-self-contained-abi-layer.md)（同一轮调研产物，互不依赖） -- **实现分支 / PR**: 待定 +- **实现分支 / PR**: `feature/large-stack-executor-and-cross-version-parallelism`（基于 `feature/self-contained-abi-layer`，PR 待开） - **配套文档**: 待定 —— 落地时登记实现说明的链接 ## 摘要 @@ -188,3 +188,4 @@ AGENTS.md（`SwiftInterface` / `SwiftIndexing` 条目补执行器一句；测试 | 2026-09-03 | 库入口自装偏好；macOS 15 以下静默回退；默认并行上限取核数 | 第二轮 | | 2026-09-03 | 上游提案由本人在 sibling 仓库起草；上游发版号 0.6.3 | 第三轮与收尾确认；用户告知 0.6.2 已实测恢复 | | 2026-09-03 | 上游 0.6.3 已发版，接口与四处修订对齐 | swift-demangling 会话通知：tag `8f32e30`、实现 `eaf7e76`；`StackSafeExecutor.taskExecutor` / `LargeStackTaskExecutor`；优先级映射改用 `JobPriority` 原始值、只用稳态额度、双级回退、16 MB 实测深度。状态仍为 Draft，等用户置 Accepted 后再抬 pin 与开工 | +| 2026-09-03 | Accepted → In Progress | 用户指示「基于上一个 PR 实现 async 提案」，视为批准；分支自 ABI 提案的分支切出，第一步抬 swift-demangling pin 到 0.6.3 | diff --git a/Package.swift b/Package.swift index 16ed9c40..a00c778d 100644 --- a/Package.swift +++ b/Package.swift @@ -159,7 +159,7 @@ extension Package.Dependency { ), remote: .package( url: "https://github.com/MxIris-Reverse-Engineering/swift-demangling", - "0.6.0" ..< "0.7.0", + "0.6.3" ..< "0.7.0", ), ) From 4c6bdd4d8076c8cccaedeb4c154581f0078d448c Mon Sep 17 00:00:00 2001 From: Mx-Iris Date: Thu, 3 Sep 2026 22:30:30 +0800 Subject: [PATCH 2/4] feat(concurrency): run async entry points on the large-stack task executor and prepare versions in parallel (proposal draft-large-stack-executor-and-cross-version-parallelism) swift-demangling's StackSafeExecutor decides per demangle / print / remangle whether to hop to its 8 MB pool by probing the calling thread's remaining stack; cooperative threads carry 512 KB, so an async print loop paid one thread round trip and a semaphore wait per printed symbol. The new MachOSymbols.LargeStackTaskExecution.run sets swift-demangling 0.6.3's 16 MB LargeStackTaskExecutor as the task executor preference around every library entry point (indexer prepare, interface builder prepare/printRoot, diffable builder prepare, evolution builder prepare/render, diff renderer, the printer's per-definition entries, the six Dumpable.dump conformers), so the whole pipeline runs inline. Nesting is a no-op, no unstructured Task exists in the library, and below macOS 15 / iOS 18 the body runs unchanged; isEnabled (seeded by MACHO_SWIFT_SECTION_LARGE_STACK_EXECUTOR=0) turns it off. Multi-version preparation is parallel: AnySwiftEvolutionInterfaceBuilder .prepare(maximumConcurrentPreparations:) (default = processor count, 1 = the old serial order), swift-section diff / evolution --jobs N, all through a windowed Collection.concurrentMap(maximumConcurrency:_:) in Utilities. Verified: 1637 tests pass; rendering A/B 78 pairs byte-identical with the executor on and off; SwiftUICore/SwiftUI dump+interface 16-23% faster, a three-version SwiftUI evolution 2.0x faster. Bumps to 0.19.0. --- .github/workflows/macOS.yml | 2 +- AGENTS.md | 17 ++- Changelogs/0.19.0.md | 33 +++++ ...-executor-and-cross-version-parallelism.md | 8 +- Documentations/Glossary.md | 7 + .../LargeStackTaskExecutorAdoption.md | 107 ++++++++++++++ .../Internal/Modules/SwiftInterface.md | 4 +- .../Internal/ProjectEvolutionLog.md | 23 +++ .../2026-07-31-node-store-migration-review.md | 1 + ...-executor-and-cross-version-parallelism.md | 50 +++++++ Documentations/README.md | 1 + .../LargeStackTaskExecution.swift | 77 ++++++++++ .../Extensions/Node+.swift | 39 ++--- .../Dumpable/AssociatedType+Dumpable.swift | 5 +- .../SwiftDump/Dumpable/Class+Dumpable.swift | 5 +- .../SwiftDump/Dumpable/Enum+Dumpable.swift | 5 +- .../Dumpable/Protocol+Dumpable.swift | 5 +- .../ProtocolConformance+Dumpable.swift | 5 +- .../SwiftDump/Dumpable/Struct+Dumpable.swift | 5 +- .../SwiftDeclarationIndexer.swift | 13 +- .../AnySwiftEvolutionInterfaceBuilder.swift | 44 ++++-- .../SwiftDiffableInterfaceBuilder.swift | 12 ++ .../SwiftDiffableInterfaceRenderer.swift | 12 +- .../SwiftEvolutionInterfaceBuilder.swift | 6 +- .../SwiftInterfaceBuilder.swift | 17 ++- .../SwiftDeclarationPrinter.swift | 31 +++- Sources/Utilities/BoundedConcurrentMap.swift | 50 +++++++ .../swift-section/Commands/DiffCommand.swift | 31 +++- .../Commands/EvolutionCommand.swift | 25 +++- Sources/swift-section/Version.swift | 2 +- .../LargeStackTaskExecutionTests.swift | 113 +++++++++++++++ .../BoundedConcurrentMapTests.swift | 136 ++++++++++++++++++ .../SwiftEvolutionInterfaceBuilderTests.swift | 41 +++++- .../DiffCommandValidationTests.swift | 45 ++++++ .../EvolutionCommandValidationTests.swift | 16 +++ 35 files changed, 930 insertions(+), 63 deletions(-) create mode 100644 Changelogs/0.19.0.md create mode 100644 Documentations/Internal/LargeStackTaskExecutorAdoption.md create mode 100644 Documentations/Internal/TaskReports/2026-09-03-large-stack-executor-and-cross-version-parallelism.md create mode 100644 Sources/MachOSymbols/LargeStackTaskExecution.swift create mode 100644 Sources/Utilities/BoundedConcurrentMap.swift create mode 100644 Tests/MachOSymbolsTests/LargeStackTaskExecutionTests.swift create mode 100644 Tests/SwiftInterfaceTests/BoundedConcurrentMapTests.swift create mode 100644 Tests/SwiftSectionCommandTests/DiffCommandValidationTests.swift diff --git a/.github/workflows/macOS.yml b/.github/workflows/macOS.yml index 5f7ecfb4..8e905aa9 100644 --- a/.github/workflows/macOS.yml +++ b/.github/workflows/macOS.yml @@ -106,4 +106,4 @@ jobs: -c ${{ matrix.configuration }} \ --enable-experimental-prebuilts \ --build-path .build-test-${{ matrix.configuration }} \ - --filter '\.(SymbolTestsCoreDumpSnapshotTests|SymbolTestsCoreInterfaceSnapshotTests|SymbolTestsCoreCoverageInvariantTests|STCoreE2ETests|STCoreTests|GenericSpecializationTests|MultiPayloadEnumTests|MetadataReaderDemanglingTests|DependencyLoadNameTests|DependencyClosureTests|FileDependencyLocatorTests|SwiftInterfaceBuilderDependenciesTests|ManglingPrefixTests|MethodDescriptorTests|MethodOverrideDescriptorTests|ProtocolRequirementTests|ResilientWitnessTests)(/|$)' + --filter '\.(SymbolTestsCoreDumpSnapshotTests|SymbolTestsCoreInterfaceSnapshotTests|SymbolTestsCoreCoverageInvariantTests|STCoreE2ETests|STCoreTests|GenericSpecializationTests|MultiPayloadEnumTests|MetadataReaderDemanglingTests|DependencyLoadNameTests|DependencyClosureTests|FileDependencyLocatorTests|SwiftInterfaceBuilderDependenciesTests|ManglingPrefixTests|MethodDescriptorTests|MethodOverrideDescriptorTests|ProtocolRequirementTests|ResilientWitnessTests|LargeStackTaskExecutionTests|BoundedConcurrentMapTests|SwiftEvolutionInterfaceBuilderTests|EvolutionCommandValidationTests|DiffCommandValidationTests)(/|$)' diff --git a/AGENTS.md b/AGENTS.md index 130fbfa9..8a0d1ff9 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -134,6 +134,7 @@ The interface generation is split into layered peer modules over a shared `Swift - `SwiftIndexEvents` - event namespace (Payload/Dispatcher/Handler) emitted by both indexer and printer. **Library code never writes to a process stream** (evolution proposal 0005): every degradation — a dropped definition, a skipped descriptor, a dependency that would not load — is dispatched as an event, and the *host* decides where it lands (a GUI attaches `OSLogEventHandler`, the CLI attaches `ConsoleEventHandler`, which reports on **stderr**; stdout carries the generated Swift / JSON, so writing there corrupts the product output — issue #102). `Dispatcher.dispatch` has a **floor**: with no handler attached it reports failures through `#log` rather than dropping them, so forgetting to attach a sink degrades to "somewhere findable" instead of silence. What counts as a failure is `Payload.unhandledFailureDescription`, an exhaustive `switch` on purpose — a new failure case must opt *in* explicitly, since the alternative silently escapes the floor. Two modules sit below the event layer and cannot reach it (`SwiftDeclaration` depends on `SwiftDeclarationRendering`, so naming the event types there would cycle): `Node+OpaqueType` takes an injected `OpaqueTypeDegradationReporter` closure, `MultiPayloadEnumDescriptorCache` logs directly; both land on the same `#log` floor. Never use `FileHandle.standardError/Output.write(_:)` — that overload raises an uncatchable ObjC exception on a closed or broken stream and aborts the host; use `fputs` / `fwrite` (`write(contentsOf:)` needs macOS 10.15.4, above this package's 10.15 floor). Pinned by a source scan (`PrintFailureEventTests.libraryModulesWriteToNoProcessStream`) carrying an explicit, shrink-only list of baseline offenders. See [Documentations/Internal/EventBasedDegradationReporting.md](Documentations/Internal/EventBasedDegradationReporting.md) **SwiftIndexing** - Builds the `SwiftDeclaration` model from a Mach-O image +- `SwiftDeclarationIndexer.prepare()` runs its body on the demangler's large-stack task executor through `LargeStackTaskExecution.run` (evolution proposal `large-stack-executor-and-cross-version-parallelism`; see the `MachOSymbols` entry) — as do `SwiftInterfaceBuilder.prepare()` / `printRoot()`, `SwiftDiffableInterfaceBuilder.prepare()`, the evolution builder's `prepare` / render entries, the diff renderer's two entries, the printer's four per-definition entries and the six `Dumpable.dump(using:in:)` conformers. Output is independent of the executor; the wrap is about where the task runs, not what it produces - `SwiftDeclarationIndexer` - Indexes types, extensions, conformances. Its `deinit` cleans up the three per-image caches (symbol store, interned-name store, demangle memo) via `PerImageCacheEvictionRegistry`, under two rules: eviction is claimed **per CACHE** (each of the three sampled separately at `prepare()`, by whichever indexer's `prepare()` found it absent and therefore built it) and performed by the image's LAST live indexer, so an earlier-deinitializing indexer never wipes the caches out from under a live sibling. The per-cache split matters because only the symbol store is necessarily an indexer's: the interned-name store and the demangle memo are also populated by SwiftLayout, `SwiftDeclarationRendering` and `SwiftSpecialization` through `MetadataReader`, so a "dump, then build the interface" sequence fills those two with no symbol store at all — a single claim sampled from the symbol store alone read that state backwards and evicted live non-indexer work's caches. Registration is keyed on indexer identity (`ObjectIdentifier`), not counted, so a concurrent double `prepare()` (its `isPrepared` guard is a plain check-then-set on an async entry point) cannot strand the population above zero and leak all three caches for the process lifetime. Entries built by non-indexer callers are never claimed and never evicted (pinned by `PerImageCacheEvictionTests`, including `indexerDoesNotEvictCachesItDidNotBuild`) - The section-wrapper populations the index passes consume (`types` / `protocols` / `protocolConformances` / `associatedTypes` and the parsed-value keyed conformance maps) are **indexing transients** since proposal 0002 — released when `prepare()` finishes, with no public projection. The retained conformance facts are the name-level maps `conformingProtocolNamesByTypeName` / `conformingTypesByProtocolName` (+ their merged `all*` variants), which is all any post-indexing consumer (including `SwiftSpecialization`'s `ConformanceProvider`) reads. - `SwiftIndexEventReporter`, `OSLogEventHandler`, `ConsoleEventHandler` - event handlers @@ -158,7 +159,8 @@ The interface generation is split into layered peer modules over a shared `Swift - `TypeDefinition` specialization behavior (`specialize(...)`, `specializedChildren`) **SwiftInterface** - Thin orchestrator tying indexing + printing into a full interface dump (module reference: [Documentations/Internal/Modules/SwiftInterface.md](Documentations/Internal/Modules/SwiftInterface.md)) -- `SwiftInterfaceBuilder` - Main builder, call `prepare()` then `printRoot()` +- `SwiftInterfaceBuilder` - Main builder, call `prepare()` then `printRoot()`; both run on the large-stack task executor (`LargeStackTaskExecution.run`, see `MachOSymbols`) +- Cross-version preparation is **parallel** (evolution proposal `large-stack-executor-and-cross-version-parallelism`): `AnySwiftEvolutionInterfaceBuilder.prepare(maximumConcurrentPreparations:)` (default = processor count; `1` = the serial oldest-first order; below 1 clamps) indexes the versions through `Collection.concurrentMap(maximumConcurrency:_:)` (`Utilities`, a windowed task group: source-ordered results, first failure rethrown, pending elements never started), and the CLI's `diff` / `evolution` index their inputs the same way under `--jobs N`. Safe because versions are different files — caches key on UUID, descriptor reads go through a memory mapping — and the window is capped at the processor count because a preparation occupies its executor thread. The result is byte-identical to serial (pinned by `parallelPreparationMatchesSerialPreparation`); only event delivery interleaves. Intra-version (per-definition) parallelism is deliberately NOT done: MachOKit's `MachOFile` reads share one `FileHandle` (seek + read), and `index(in:)`'s `isIndexed` guard is a plain check-then-set - `InterfaceUnionWalker` - the SHARED structure walk behind both comparison renderers (evolution proposal 0014): N versions erased as `[any InterfaceVersionRendering]` (`InterfaceVersionUnit` = per-version `SwiftDiffableInterfaceBuilder` + a printer sharing its dispatcher) in, block-grouped lines out. The walker owns the STRUCTURE — matching and union ordering (newest version's order as spine, absent declarations appended from their last-carrying version; keys first-wins per side, EMISSION INCLUDED, mirroring `ABIDiffer.keyed` — a later same-keyed element is never emitted twice, which the header-failure tests' replace-not-append injections rely on), extension-container splitting + header construction (`ABIDiffer.extensionContainerKey`, one source of truth), member construction (`UnionRenderableMember`, identity/payload keys from the same `MemberRecord` projections the differ freezes), the `MemberCategory.allCases` schedule, and body composition order — while an `InterfaceUnionEmitting` strategy owns the PRESENTATION. `SwiftDiffableInterfaceRenderer` is a public shell whose generics erase at construction: `DiffUnionStrategy` keeps the genuinely two-sided semantics (`HeaderOutcome` pairing with failed-side stand-in, `-`/`+` pairs for modified members with the identical-rendering collapse, `DiffContainerAssembler` markers); `SwiftEvolutionInterfaceRenderer` is the N-way annotation strategy (latest-renderable header, `EvolutionAnnotationIndex` lookups, anchor rules). Members render at printer level 0 IN THE WALKER: the variable/subscript printers bake accessor-block interior indentation ABSOLUTELY from `level` while both format layers indent every line by its own indentLevel — real-level rendering double-indented `get`/`}` (the diff path carried exactly that artifact until the unification; pinned by `DiffMemberIndentationTests`). The format layers stay split on purpose (`DiffMarking`/`EvolutionMarking` + the two assemblers): markers apply per line, annotations anchor per unit — genuinely different semantics, not duplication - `SwiftEvolutionInterfaceBuilder` - N ≥ 2 versions rendered as ONE **union interface with lifecycle annotations** (evolution proposal 0013; the N-way analogue of the `SwiftDiffableInterfaceBuilder`+`Renderer` pair). Division of labor is the load-bearing rule: annotation facts come SOLELY from `ABIEvolutionBuilder` over the versions' frozen snapshots (`EvolutionAnnotationIndex` joins on the same `ABIKey`/`MemberRecord` constructions `ABIDiffer` freezes — a lookup miss IS the "present throughout, never changed" verdict and renders bare), while text renders from the live models: each declaration from the LAST version that has it, so a modified member shows one line (newest generation) with the old shape in the phrase (`modified in 26.0: old → new`; an identical arrow collapses to the bare phrase). Union order = newest version's order as spine, absent declarations appended in their last-carrying version's order (the shared `InterfaceUnionWalker`'s matching — see its entry above). Format layer (`EvolutionMarking`): trailing `// [●●○] removed in 26.0` comments (bitmap + phrases, legend header mapping positions to labels), per-block column alignment capped at 72 (overflow → own line one level deeper), annotation anchored on the unit's DECLARATION line (members: first line, attributes print inline; container headers: last line, the one with the brace — a computed property's annotation never sinks to its accessor block's closing brace), reporter-mirrored warnings tail. Members render at printer level 0 — enforced in the shared walker for BOTH comparison paths, see the `InterfaceUnionWalker` entry. The public surface is TWO types: `AnySwiftEvolutionInterfaceBuilder`, the type-ERASED runtime-N workhorse (each version erases at construction via `InterfaceVersionRendering`/`InterfaceVersionUnit`; homogeneous `[MachO]` init + heterogeneous pack init, both available everywhere — packs in *function* position need no availability gate), and `SwiftEvolutionInterfaceBuilder`, the pack-generic façade for compile-time-fixed axes (`@available(macOS 14…)` — packs in a TYPE's generic parameter list DO need the Swift 5.9 runtime; constructs-then-erases, behavior byte-identical by construction, pinned by `packGenericFacadeMatchesTheErasedBuilder`). A pack's arity is compile-time, so runtime-N (the CLI, RuntimeViewer's user-picked versions) always goes through the erased type; same-element requirements (`repeat each MachO == M`) are not yet supported by the toolchain, so the array init cannot live on the pack type. All inputs must be binaries (snapshots carry no renderable interface); protocols' stripped `pwtslot:` records are not rendered (no declaration — same as `diff --interface`). CLI: `swift-section evolution --interface` (mutually exclusive with `--json`/`--summary-only`; colorizes by event kind at the CLI layer). Structured stream for hosts: `@_spi(Support) annotatedBlocks()` → `[[EvolutionLine]]` - `SwiftInterfaceBuilderOpaqueTypeProvider` - Expands `some` return types from the opaque type descriptor's generic requirements. Primary-associated-type arguments are **attributed per protocol** (evolution proposal 0011), never distributed blindly across the composition: a same-type constraint attaches to the protocol its **anchor** names (the associated type's declaring protocol, kept by the demangler in `dependentAssociatedTypeRef` — identity comparison only, so it works offline via bind symbols), to a protocol whose refine closure contains the anchor, or — for compiler-collapsed equivalence classes like `TestCollection<[A]>` — by name fallback (only when no anchor matched, the protocol itself declares that associated-type name, the candidate is unique, and the anchor lies **outside** the composition; a collapsed pin and a never-pinned same-named member are byte-identical in the descriptor, so an in-composition anchor forbids the fallback). Protocol facts (refine signature + associated-type names) resolve through `ProtocolFactsResolver`: a reachable descriptor (same image on both readers; **any** image in-process — `MachOImage` dereferences the requirement's indirect pointer cross-image) merged with `BuiltinStandardLibraryProtocolFacts` (the only source of primary names/order — SE-0346 leaves no runtime trace — and the offline fallback for bind-only externals). Unknown facts attach nothing (a missed parameter beats a fabricated one), so `MachOFile` and `MachOImage` output depth may differ — accepted, in-process is strictly additive. Composition order is the descriptor's canonical order; source order is not recoverable. See [Documentations/Internal/OpaquePrimaryAssociatedTypeAttribution.md](Documentations/Internal/OpaquePrimaryAssociatedTypeAttribution.md) (implementation note) and [Documentations/Internal/OpaqueReturnTypeResolution.md](Documentations/Internal/OpaqueReturnTypeResolution.md) (domain deep-dive: descriptor encoding, anchor/collapse mechanics, byte-level debugging). @@ -225,7 +227,7 @@ Printing and indexing are peers — neither depends on the other. - **MachOFoundation** - `MachOBase` plus `MachOSymbols` and `MachODependencies` — the umbrella for everything above the ABI model - **MachOReading** - File reading abstractions - **MachOResolving** - Address/offset resolution; also home of the symbol **value** types `Symbol` (offset + name + `isExternal`), `Symbols` (every name at one offset — identical code folding leaves several) and `SymbolOrElement` (a bind-table symbol or a resolved element). They carry no lookup behavior: `Symbols` is deliberately not `Resolvable`, because "the symbols at this offset" is a query against the symbol index, not a read -- **MachOSymbols** - Symbol table parsing and demangling — the symbol *index*, one layer above the ABI model; the value types it vends live in `MachOResolving`. Index-backed lookups: `symbols(offset:)` on `MachORepresentableWithCache`, `Symbol.resolve(from:in:)` and the process-wide `Symbol.resolvesSymbolUsingIndexStore` switch. `SymbolIndexStore`'s offset and member indexes hold their row lists in `SymbolRowBucket` (evolution proposal 0003): the dominant single-row case stays inline in the dictionary slot, only a bucket that collects a second row allocates an array; iteration order is insertion order, so query output is byte-identical to the former `[UInt32]` buckets. The member/typeInfo/thunk-attribute indexes are keyed by printed type name FIRST and interned context node SECOND, and the name key is NOT injective — it is printed with `.interfaceTypeBuilderOnly`, which strips private discriminators, so same-named private types from different files share one name bucket (issue #115: the dump path's name-only lookups merged both types' members into each declaration). Any consumer resolving *one* type's members/info/attributes must use the node-taking overloads (`memberSymbols(of:for:node:in:)`, `methodDescriptorMemberSymbols(of:for:node:in:)`, `typeInfo(for:node:in:)`, `thunkAttributeMembers(of:for:node:in:)`); the name-only forms deliberately flatten every sub-bucket and are only for "all types printing as this name" aggregation. See [Documentations/Internal/PrivateTypeMemberAttribution.md](Documentations/Internal/PrivateTypeMemberAttribution.md). `Storage` also carries the image's **export facts** (evolution proposal 0008): both symtab collection legs filter on `!nlist.isExternal`, so exported symbols' rows come only from the export-trie leg, whose row minting is conditional — trie membership is therefore collected explicitly in the same pass (a per-row bitmap plus a fallback name set for offset-less re-exports), backing the tri-state `isExported(name:in:)` (`nil` = the image has no export information; never annotate then) and `isExportedIncludingDerivedSymbols(name:in:)` (extends over the `Tj`/`Tq`/`Tu`/`TjTu` appended-suffix forms) +- **MachOSymbols** - Symbol table parsing and demangling — the symbol *index*, one layer above the ABI model; the value types it vends live in `MachOResolving`. Also home of `LargeStackTaskExecution` (evolution proposal `large-stack-executor-and-cross-version-parallelism`): `run(_:)` sets swift-demangling's 16 MB `LargeStackTaskExecutor` (`StackSafeExecutor.taskExecutor`, `@_spi(Internals)`, 0.6.3+) as the task executor preference around a library entry point. The demangler decides per call whether to hop to its 8 MB pool by probing the CALLING thread's remaining stack against a 2 MB floor — cooperative and libdispatch threads carry 512 KB, so an async print loop paid one thread round trip + semaphore wait per printed symbol (8–21 µs, 1.14–2.28×); on an executor thread the probe passes at every entry, synchronous callees included, so the whole pipeline runs inline (the effect of `withLargeStack` extended to a whole task). Nesting is a no-op (already on the executor = no switch), an unstructured `Task {}` does NOT inherit the preference (SE-0417 — never start one inside a wrapped entry; child tasks and default actors do inherit), the main actor keeps its own executor (its 8 MB stack passes the probe anyway), and below macOS 15 / iOS 18 (or off Darwin) `run` executes the body unchanged. Process-wide off switch `isEnabled` (hosts with their own executor) seeded by `MACHO_SWIFT_SECTION_LARGE_STACK_EXECUTOR=0` (the A/B and timing runs compare one binary both ways). See [Documentations/Internal/LargeStackTaskExecutorAdoption.md](Documentations/Internal/LargeStackTaskExecutorAdoption.md). Index-backed lookups: `symbols(offset:)` on `MachORepresentableWithCache`, `Symbol.resolve(from:in:)` and the process-wide `Symbol.resolvesSymbolUsingIndexStore` switch. `SymbolIndexStore`'s offset and member indexes hold their row lists in `SymbolRowBucket` (evolution proposal 0003): the dominant single-row case stays inline in the dictionary slot, only a bucket that collects a second row allocates an array; iteration order is insertion order, so query output is byte-identical to the former `[UInt32]` buckets. The member/typeInfo/thunk-attribute indexes are keyed by printed type name FIRST and interned context node SECOND, and the name key is NOT injective — it is printed with `.interfaceTypeBuilderOnly`, which strips private discriminators, so same-named private types from different files share one name bucket (issue #115: the dump path's name-only lookups merged both types' members into each declaration). Any consumer resolving *one* type's members/info/attributes must use the node-taking overloads (`memberSymbols(of:for:node:in:)`, `methodDescriptorMemberSymbols(of:for:node:in:)`, `typeInfo(for:node:in:)`, `thunkAttributeMembers(of:for:node:in:)`); the name-only forms deliberately flatten every sub-bucket and are only for "all types printing as this name" aggregation. See [Documentations/Internal/PrivateTypeMemberAttribution.md](Documentations/Internal/PrivateTypeMemberAttribution.md). `Storage` also carries the image's **export facts** (evolution proposal 0008): both symtab collection legs filter on `!nlist.isExternal`, so exported symbols' rows come only from the export-trie leg, whose row minting is conditional — trie membership is therefore collected explicitly in the same pass (a per-row bitmap plus a fallback name set for offset-less re-exports), backing the tri-state `isExported(name:in:)` (`nil` = the image has no export information; never annotate then) and `isExportedIncludingDerivedSymbols(name:in:)` (extends over the `Tj`/`Tq`/`Tu`/`TjTu` appended-suffix forms) - **MachOPointers** - Pointer types (relative, indirect, etc.), including `SymbolOrElementPointer` (absorbed the former `MachOSymbolPointers` target): an indirectable target that resolves either to a bind-table symbol (`Symbol`, via `MachOBindRebaseResolving`) or to an element - **MachOCaches** - dyld shared cache support - **MachODependencies** - The one dependency-resolution implementation every feature shares (evolution proposal 0017; re-exported by `MachOFoundation`, so nothing above it needs an extra import). `DependencyClosure` walks a root's `LC_LOAD_DYLIB`-family load commands — `.direct` (the root's own list, load-command order) or `.transitive` (breadth-first, so a lazily indexing consumer meets the root's direct dependencies first) — deduplicated by **bare image name** (`DependencyLoadName.bareImageName(of:)`: last path component, FIRST extension stripped — `libobjc.A.dylib` → `libobjc`; the exact rule `MachOImage(name:)` matches on, which is why handing that lookup a raw load path resolves nothing — the bug `SwiftInterfaceBuilderDependencies`'s image initializer carried until this module), root excluded, misses recorded in `unresolvedLoadNames` rather than dropped. Locators: `InProcessDependencyLocator` (active dyld) and `FileDependencyLocator` (`DependencySearchPath`s: explicit files + dyld caches; **exact install path first, ranked bare name second** via `DyldCacheImageSearchMode.matchRank`, so a macOS cache's `/System/iOSSupport` Catalyst SwiftUI never shadows the native one; each cache indexed ONCE lazily, since a per-lookup `machOFile(by:)` scan is `O(dependencies × cache size)`; fat explicit files contribute the root's architecture; unopenable search paths land in `searchPathLoadFailures`, never thrown). Consumers: `SwiftLayout.ImageUniverse` (transitive), `SwiftInterface.SwiftInterfaceBuilderDependencies` (DIRECT on purpose — TypeIndexing generates one SourceKit interface per dependency module, so the transitive closure would bring back the whole-SDK generation that once disabled the target; it dispatches `searchPathLoadFailures` as `renderingDegraded(.dependencyLoad)` events and exposes `unresolvedLoadNames`), `swift-section interface --resolve-c-module-names`. This module sits below the event layer: failures are data, never logged here. See [Documentations/Internal/Modules/MachODependencies.md](Documentations/Internal/Modules/MachODependencies.md) @@ -288,6 +290,17 @@ final class Foo { Tests use `MACHO_SWIFT_SECTION_SILENT_TEST=1` to suppress verbose output. +**Swift Testing runs test bodies on 512 KB cooperative threads.** A test that +calls a library entry point gets the large-stack executor through the entry's +own `LargeStackTaskExecution.run` (macOS 15+; CI runs macOS 26), but a test +that drives the demangler or printer DIRECTLY in a deep recursion runs on the +cooperative thread and hops per call exactly as a host would; wrap such a body +in `LargeStackTaskExecution.run` if the hop cost or the 8 MB pool depth limit +(`KnownIssues.md` #4 upstream) is what is being measured. Do not read a "the +suite passed" as "the executor was used": `LargeStackTaskExecutionTests` pins +the executor behavior by thread identity, everything else is executor-agnostic +by design. + **On-the-fly-compiled fixture dylibs need a class.** A struct-only fixture module compiles to a dylib with NO `__DATA` segment, and the pinned MachOKit release mis-walks that layout's chained-fixup pages during `resolveBind` — diff --git a/Changelogs/0.19.0.md b/Changelogs/0.19.0.md new file mode 100644 index 00000000..a60b7e5a --- /dev/null +++ b/Changelogs/0.19.0.md @@ -0,0 +1,33 @@ +# 0.19.0 + +A minor release on top of `0.18.0`. Two related changes, both output-neutral: the library's async entry points now run on swift-demangling's large-stack task executor, and multi-version preparation (`diff` / `evolution`) is parallel. Evolution proposal `large-stack-executor-and-cross-version-parallelism`; implementation note `Documentations/Internal/LargeStackTaskExecutorAdoption.md`. + +## What changed + +1. **Async entry points run on a 16 MB task executor.** The demangler decides per call whether to hop to its 8 MB pool by probing the calling thread's remaining stack; Swift Concurrency's cooperative threads carry 512 KB, so an async print loop paid one thread round trip per printed symbol. `MachOSymbols.LargeStackTaskExecution.run` sets swift-demangling 0.6.3's `LargeStackTaskExecutor` as the task executor preference around every library entry — indexer `prepare()`, `SwiftInterfaceBuilder.prepare()` / `printRoot()`, `SwiftDiffableInterfaceBuilder.prepare()`, the evolution builder's `prepare` / render entries, the diff renderer, the printer's four per-definition entries and the six `Dumpable.dump(using:in:)` conformers — so the whole pipeline demangles, prints and remangles inline. Hosts change nothing. Requires macOS 15 / iOS 18 / tvOS 18 / watchOS 11 / visionOS 2 at runtime; below that (or off Darwin) the body runs exactly as before. `LargeStackTaskExecution.isEnabled` (seeded by `MACHO_SWIFT_SECTION_LARGE_STACK_EXECUTOR=0`) turns it off process-wide. +2. **Cross-version preparation is parallel.** `AnySwiftEvolutionInterfaceBuilder.prepare(maximumConcurrentPreparations:)` (and the pack-generic façade) indexes the versions concurrently, defaulting to the processor count; `1` is the former oldest-first serial order. `swift-section diff` and `swift-section evolution` index their inputs the same way and gained `--jobs N`. The result is byte-identical to serial preparation; only event delivery interleaves. Intra-version parallelism is out of scope (MachOKit's file reads share one `FileHandle`). +3. **`Collection.concurrentMap(maximumConcurrency:_:)`** (`Utilities`): a windowed task-group map — source-ordered results, first failure rethrown, pending elements never started. + +## Measured + +Release binaries, host dyld cache (macOS 26.5.2, 10-core Apple Silicon), two runs each; outputs byte-identical across every configuration. + +| Configuration | SwiftUICore dump | SwiftUICore interface | SwiftUI dump | SwiftUI interface | +|---|---|---|---|---| +| 0.18.0 (swift-demangling 0.6.0) | 48.8 s / 48.6 s | 56.4 s / 57.0 s | 79.4 s / 79.9 s | 87.5 s / 89.4 s | +| 0.19.0, executor off | 48.6 s / 48.7 s | 56.4 s / 55.7 s | 78.0 s / 79.7 s | 88.4 s / 89.7 s | +| 0.19.0, executor on (default) | **40.6 s / 40.4 s** | **47.0 s / 47.1 s** | **61.3 s / 60.2 s** | **71.2 s / 70.9 s** | + +`swift-section evolution` over three archived SwiftUI caches (macOS 15.5 / 26.5.2 / 27.0 beta 6): `--interface` 306.7 s → 151.9 s, lineage report 282.4 s → 139.4 s (executor off + `--jobs 1` → executor on + default parallelism), identical output. + +## Compatibility + +- Additive at the source level: no existing signature changed; `prepare()` keeps working through the new parameter's default. +- **Dependency floor**: swift-demangling `0.6.3 ..< 0.7.0` (was `0.6.0 ..< 0.7.0`). 0.6.1 re-ranked the demangler's pool QoS per hop and slowed `dump` / `interface` 3–4×; 0.6.2 fixed that; 0.6.3 adds the executor. Other pins unchanged. +- Rendered output is byte-identical (dump and interface, verified with the rendering A/B harness, executor on and off). +- Snapshot `formatVersion` unchanged. + +## Requirements + +- Swift 6.2+ +- Xcode 26.0+ diff --git a/Documentations/Evolutions/draft-large-stack-executor-and-cross-version-parallelism.md b/Documentations/Evolutions/draft-large-stack-executor-and-cross-version-parallelism.md index dc6a0b5c..44549a83 100644 --- a/Documentations/Evolutions/draft-large-stack-executor-and-cross-version-parallelism.md +++ b/Documentations/Evolutions/draft-large-stack-executor-and-cross-version-parallelism.md @@ -7,7 +7,7 @@ - **所属愿景**: 无 - **关联提案**: swift-demangling 提案 0014「大栈 TaskExecutor」（上游前置，执行器本体在那边；本提案只做接入）；[draft-self-contained-abi-layer](draft-self-contained-abi-layer.md)（同一轮调研产物，互不依赖） - **实现分支 / PR**: `feature/large-stack-executor-and-cross-version-parallelism`（基于 `feature/self-contained-abi-layer`，PR 待开） -- **配套文档**: 待定 —— 落地时登记实现说明的链接 +- **配套文档**: [LargeStackTaskExecutorAdoption.md](../Internal/LargeStackTaskExecutorAdoption.md)（实现说明）；任务报告 [2026-09-03-large-stack-executor-and-cross-version-parallelism.md](../Internal/TaskReports/2026-09-03-large-stack-executor-and-cross-version-parallelism.md) ## 摘要 @@ -189,3 +189,9 @@ AGENTS.md（`SwiftInterface` / `SwiftIndexing` 条目补执行器一句；测试 | 2026-09-03 | 上游提案由本人在 sibling 仓库起草；上游发版号 0.6.3 | 第三轮与收尾确认；用户告知 0.6.2 已实测恢复 | | 2026-09-03 | 上游 0.6.3 已发版，接口与四处修订对齐 | swift-demangling 会话通知：tag `8f32e30`、实现 `eaf7e76`；`StackSafeExecutor.taskExecutor` / `LargeStackTaskExecutor`；优先级映射改用 `JobPriority` 原始值、只用稳态额度、双级回退、16 MB 实测深度。状态仍为 Draft，等用户置 Accepted 后再抬 pin 与开工 | | 2026-09-03 | Accepted → In Progress | 用户指示「基于上一个 PR 实现 async 提案」，视为批准；分支自 ABI 提案的分支切出，第一步抬 swift-demangling pin 到 0.6.3 | +| 2026-09-03 | 落地偏差：`isEnabled` 初值读环境变量 | 提案只有静态开关；加 `MACHO_SWIFT_SECTION_LARGE_STACK_EXECUTOR=0` 是为了让渲染 A/B 与计时用同一个二进制比较开 / 关，宿主不必重编 | +| 2026-09-03 | 落地偏差：并行用通用 `concurrentMap(maximumConcurrency:)` 而非 `async let` | `DiffCommand` 与 evolution 的 lineage 输入同样需要窗口化，一个 `Utilities` 帮手三处复用，`--jobs 1` 与默认走同一条代码路径 | +| 2026-09-03 | 落地偏差：CLI 的 `dump` 循环不再额外包一层 | 六个 `Dumpable.dump` 已各自包裹，CLI 逐类型进出执行器约一万跳、零点几秒，远小于原来逐符号跳转 | +| 2026-09-03 | 核对：库内零处非结构化 `Task {}` | 唯一的 `withTaskGroup` 在 `TypeIndexing.TypeDatabase`，结构化、继承偏好；落地步骤 2 无需改动 | +| 2026-09-03 | 验证：全量 1637 测试通过；计时 −16% ~ −23%（单版本）、2.0×（三版本 evolution）；四种配置输出逐字节一致 | 数据见实现说明「实测数据」；0.6.0 → 0.6.3 仅抬 pin 持平，证明 0.6.1 的回归未带入 | +| 2026-09-03 | 收尾判断：写实现说明；「大栈执行器」入术语表 | 实现说明记录探测机制为何免改调用点、入口清单与嵌套免费、回退与开关、并行安全性与不做版本内并行的原因、计时表；术语在 AGENTS.md / 提案 / 实现说明 / 账本多处出现，登记 `Documentations/Glossary.md` | diff --git a/Documentations/Glossary.md b/Documentations/Glossary.md index cada335e..e55e4de5 100644 --- a/Documentations/Glossary.md +++ b/Documentations/Glossary.md @@ -100,6 +100,13 @@ diff / evolution 两条对比渲染路共享结构遍历核心（`InterfaceUnion - **主要出现在**：`Sources/SwiftInterface/InterfaceUnionWalker.swift`（协议与遍历器）、`SwiftDiffableInterfaceRenderer.swift`（`DiffUnionStrategy`）、`SwiftEvolutionInterfaceRenderer.swift`（evolution 策略） - **延伸阅读**：[提案 0014](Evolutions/0014-unify-interface-renderers.md) +### large-stack executor（大栈执行器，`LargeStackTaskExecution`） + +swift-demangling 0.6.3 起提供的 `TaskExecutor`（`StackSafeExecutor.taskExecutor`，线程栈 16 MB，`@_spi(Internals)`）。demangler 每次 demangle / print / remangle 都按**调用线程的剩余栈**决定要不要跳到它的 8 MB 线程池——协作线程只有 512 KB，探针永远不过，async 打印循环因此每打印一个符号付一次线程往返；task 跑在大栈执行器的线程上时探针每个入口都通过，全程原地执行、零跳转。本库通过 `MachOSymbols.LargeStackTaskExecution.run` 在库入口（索引器 prepare、interface builder、printer 逐定义入口、diff / evolution、dump）自装偏好，宿主零改动；macOS 15 / iOS 18 以下静默回退为原样执行。与「跳转池」（demangler 自己的 8 MB `LargeStackThreadPool`，同步 `withLargeStack` 批次用）是两个池：执行器的 job 是整段 task，会占线程上百秒，不能挤占同步跳转的额度。 + +- **主要出现在**：`Sources/MachOSymbols/LargeStackTaskExecution.swift`、各 async 入口的 `LargeStackTaskExecution.run { … }` +- **延伸阅读**：[LargeStackTaskExecutorAdoption.md](Internal/LargeStackTaskExecutorAdoption.md)、提案 [draft-large-stack-executor-and-cross-version-parallelism](Evolutions/draft-large-stack-executor-and-cross-version-parallelism.md)、上游 swift-demangling `Documentations/StackSafety.md` 第八节 + ### late-name 路径（`lateDemangledNode(forName:)`） sweep 覆盖范围之外的名字走的旁路：demangle 后 intern 进 `Storage` 自持的一个可追加 side store，名字 → 裁决字典保证一个名字只 demangle 一次（拒绝也缓存为 `nil` 裁决、不再重试）。与主表冻结不可变的性质相对。 diff --git a/Documentations/Internal/LargeStackTaskExecutorAdoption.md b/Documentations/Internal/LargeStackTaskExecutorAdoption.md new file mode 100644 index 00000000..8b4682e0 --- /dev/null +++ b/Documentations/Internal/LargeStackTaskExecutorAdoption.md @@ -0,0 +1,107 @@ +# 大栈任务执行器接入与跨版本并行 + +提案：[draft-large-stack-executor-and-cross-version-parallelism](../Evolutions/draft-large-stack-executor-and-cross-version-parallelism.md)。上游执行器本体：swift-demangling 提案 0014（`Documentations/StackSafety.md` 第八节）。本文记录落地后的形态、那些从签名上看不出来的决策，以及计时数据。 + +## 改了什么 + +| 之前 | 之后 | +|---|---| +| 打印路径每次 `printSemantic` / demangle / remangle 都由 `StackSafeExecutor` 探测调用线程剩余栈；协作线程 512 KB 永远不过，每次调用跳到 8 MB 池线程再用信号量停住（release 实测每次 8–21 µs） | 库的 async 入口用 `LargeStackTaskExecution.run` 把整段 task 放到 swift-demangling 0.6.3 的 16 MB `LargeStackTaskExecutor` 上；探针在每个入口都通过，全程原地执行，零跳转 | +| swift-demangling pin `0.6.0 ..< 0.7.0`（实际锁在 0.6.0，因为 0.6.1 的 QoS 改动慢 3–4 倍） | pin `0.6.3 ..< 0.7.0`，跳过 0.6.1 / 0.6.2 | +| `AnySwiftEvolutionInterfaceBuilder.prepare()` 逐版本串行 `await`；`DiffCommand` 先 old 后 new；`EvolutionCommand` 的 lineage 输入逐个加载 | `prepare(maximumConcurrentPreparations:)`（默认核数、`1` 即旧顺序）；`diff` / `evolution` 的输入按窗口并行索引，CLI 新增 `--jobs N` | +| 无通用的「限并发 map」 | `Utilities` 的 `Collection.concurrentMap(maximumConcurrency:_:)`：窗口化 task group，结果按源序，首错重抛 | +| `Node+.swift` 的注释引用上游已不存在的 `executeWithUncheckedSendability` | 改为如实描述 `execute` 与执行器路径的关系 | + +输出不变：渲染 A/B 逐字节一致（见「实测数据」）。 + +## 从签名看不出来的决策 + +### 执行器为什么不用改 demangler 的任何调用点 + +`StackSafeExecutor` 的探针（`currentThreadHasSufficientStack`）用 `pthread_get_stackaddr_np` / `pthread_get_stacksize_np` 算调用线程剩余栈是否 ≥ 2 MB，看的是**栈**不是**线程身份**。所以只要 task 跑在一条 16 MB 线程上，`execute` / `executeAsync` 的探针在每一层都通过、直接内联——同步被调方（`printSemantic`、`demangleAsNodeTransient`、remangle）一并受益，一个调用点都不用碰。这也是为什么本库这边的改动只有「包一层」：`withTaskExecutorPreference(StackSafeExecutor.taskExecutor) { body }`。 + +### 为什么在库入口自装，而不是让宿主装 + +宿主漏装一处就回到逐次跳转，而且每个宿主（RuntimeViewer、MachOKitUI、SymbolViewer、CLI）都得改。库入口自装后宿主零改动；想自己管执行器的宿主置 `LargeStackTaskExecution.isEnabled = false`。这是提案第二轮澄清时用户的选择。 + +### 包了哪些入口 + +| 模块 | 入口 | +|---|---| +| `SwiftIndexing` | `SwiftDeclarationIndexer.prepare()`（`updateConfiguration(_:)` 经它） | +| `SwiftInterface` | `SwiftInterfaceBuilder.prepare()` / `printRoot()`；`SwiftDiffableInterfaceBuilder.prepare()`；`AnySwiftEvolutionInterfaceBuilder.prepare(maximumConcurrentPreparations:)` / `printAnnotatedInterface()` / `annotatedBlocks()`（pack façade 委托）；`SwiftDiffableInterfaceRenderer.printAnnotatedInterface(format:)` / `annotatedDiffBlocks()` | +| `SwiftPrinting` | `SwiftDeclarationPrinter.printTypeDefinition` / `printProtocolDefinition` / `printExtensionDefinition` / `printDefinition`（RuntimeViewer 逐类型导出绕过 `printRoot` 的路径） | +| `SwiftDump` | 六个 `Dumpable.dump(using:in:)` 遵循者（`Struct` / `Class` / `Enum` / `Protocol` / `ProtocolConformance` / `AssociatedType`） | + +**嵌套免费**：`printRoot` 里再进 `printTypeDefinition`、父类型的嵌套子类型循环再进 `printTypeDefinition`，都是「已在执行器上再包一层」。SE-0417 的 `withTaskExecutorPreference` 在当前执行器就是目标执行器时不切换，`nestedRunsStayOnTheSameThread` 钉住线程不变。CLI 的 `dump` 循环按类型逐个调 `dump`，每个类型进出执行器各一跳，SwiftUICore 五千余类型约一万跳、总计零点几秒，远小于原来每个符号一跳，故 CLI 侧没有再包一层。 + +**壳 + 体的拆分**：`printExtensionDefinition` 与 `printDefinition` 原本函数体就是整段逻辑（后者还是 `@SemanticStringBuilder`），result builder 的函数体不能直接套一个 `run { }` 闭包再 return，所以拆成非 builder 的壳（过滤 + `run`）与 builder 的体（`printExtensionDefinitionContents` / `printDefinitionContents`），与提案 0016 拆 `printIncluded…` 的手法相同。注意 `printIncludedExtensionDefinition` 这个名字已被 0016 的 builder 体占用，新壳体用了 `…Contents` 后缀。 + +### 非结构化 `Task {}` 的核对结果 + +SE-0417：非结构化 `Task {}` 不继承执行器偏好。核对 `Sources/` 全部：库内**零处**非结构化 `Task`（唯一的 `withTaskGroup` 在 `TypeIndexing.TypeDatabase`，是结构化子任务，继承偏好）。`childTasksInheritTheExecutor` 钉住子任务继承这一前提，跨版本并行靠它。以后若在包裹的入口内起 `Task {}`，必须显式传 `executorPreference:`。 + +### 静默回退、`isEnabled` 与环境变量 + +`run` 里 `#available(macOS 15.0, iOS 18.0, tvOS 18.0, watchOS 11.0, visionOS 2.0, *)` 不满足、非 Darwin、或 `isEnabled == false`，都直接 `try await body()`——与接入前完全一致的行为（跳转照付）。`isEnabled` 的初值读环境变量 `MACHO_SWIFT_SECTION_LARGE_STACK_EXECUTOR`（等于 `"0"` 即关），用途是让渲染 A/B 与计时能用**同一个二进制**比较开 / 关两种状态，宿主不必为此重编。它是提案未列出的一个小补充。 + +### 主 actor 调用方 + +本包的库 target 没有开启 SE-0461 `NonisolatedNonsendingByDefault`（`Package.swift` 里定义了这些 upcoming feature 常量，但只有测试 target 接 `testSettings`，且它几乎为空），所以库的 async 入口是经典 `nonisolated`：从 `@MainActor` 调用会离开主 actor。接入前它落在协作线程（512 KB，逐次跳转），接入后落在执行器线程——对 RuntimeViewer 这类从主线程发起导出的宿主是纯收益。若将来开启 SE-0461，主 actor 调用会留在主线程（主 actor 有自己的 executor，偏好不生效），主线程栈 8 MB 探针本就通过，也无损失；从默认 actor 调用则仍在执行器上（SE-0417：默认 actor 继承偏好）。 + +### 跨版本并行为什么安全、上限为什么取核数 + +各版本是不同文件：`MachOFile.identifier` 按 LC_UUID 键控，五个 `SharedCache` 单例全部按此分片；描述符读取走 `MemoryMappedFile`；demangler 池按核数扩。2026-09-02 已用三个进程并行证明可行，进程内并行只多了 `SharedCache` 字典锁与 `PerImageCacheEvictionRegistry` 的 `NSLock`，都是短临界区。窗口上限取核数的原因来自上游契约：一个 `prepare` 是整段 task，占住执行器的一条线程直到结束；执行器每个 QoS 类的稳态额度是 `max(2, 核数)`，窗口超过它只会排队，不会更快。`parallelPreparationMatchesSerialPreparation` 钉住并行与串行的接口、结构流、evolution JSON 逐字节一致。 + +### 版本内并行为什么不做 + +MachOKit 自己的读取有上百处 `fileHandle.seek` + `read` 共用一个句柄（`MachOFile.swift`、`DyldCache.swift` 等），两个线程交错就读错位置；`index(in:)` 的 `guard !isIndexed` 是非原子的检查加赋值。前者不在本仓库，需要 MachOKit 改 mmap 读或每个分片独立 `MachOFile` 实例，另起提案。 + +### `concurrentMap(maximumConcurrency:)` 的错误语义 + +窗口化 `withThrowingTaskGroup`：先提交 `window` 个，每完成一个再提交一个，结果按源序落位。首个错误经 `group.next()` 抛出，task group 在作用域退出时取消并等待在飞的子任务（`prepare` 不检查取消，所以在飞的会跑完，结果丢弃），**尚未启动的元素永远不启动**（`theFirstFailureIsRethrownAndPendingElementsNeverStart`）。哪个错误先到是调度决定的——串行时固定是最旧版本的错误，并行时不一定；只影响错误报文，不影响成功路径。 + +### `--jobs` 在 CLI 校验而库端 clamp + +命令行上的 `--jobs 0` 是笔误，`ValidationError("--jobs must be at least 1.")` 立刻报；库 API 的 `maximumConcurrentPreparations` 小于 1 则按 1 处理（`preparationWindowIsClampedNotValidated`），因为宿主可能直接把「核数 - 1」之类的算式传进来，为一个下界抛错不值得。 + +## 实测数据 + +release 二进制，宿主 dyld cache（macOS 26.5.2，10 核 Apple Silicon），每项跑两次取两次的值；四个配置的输出逐字节一致（`cmp`）。 + +**单版本 `dump` / `interface`（`--uses-system-dyld-shared-cache -n `，秒）** + +| 配置 | SwiftUICore dump | SwiftUICore interface | SwiftUI dump | SwiftUI interface | +|---|---|---|---|---| +| swift-demangling 0.6.0（ABI 分支，接入前） | 48.8 / 48.6 | 56.4 / 57.0 | 79.4 / 79.9 | 87.5 / 89.4 | +| 0.6.3 仅抬 pin（接入前） | 48.5 / 48.4 | 55.8 / 56.2 | 77.2 / 79.7 | 89.3 / 89.6 | +| 0.6.3 + 执行器**关**（`MACHO_SWIFT_SECTION_LARGE_STACK_EXECUTOR=0`） | 48.6 / 48.7 | 56.4 / 55.7 | 78.0 / 79.7 | 88.4 / 89.7 | +| 0.6.3 + 执行器**开**（默认） | **40.6 / 40.4** | **47.0 / 47.1** | **61.3 / 60.2** | **71.2 / 70.9** | + +执行器带来 16–23% 的墙钟缩短（SwiftUICore −17% / −16%，SwiftUI −23% / −20%）；0.6.0 → 0.6.3 本身无差别（0.6.1 的回归已在 0.6.2 修掉）。剩下的时间是索引与打印本身——跳转只是每次调用的固定开销，它在 SwiftUI 这种符号更多的镜像上占比更大。 + +**三版本 `evolution`（SwiftUI，归档 cache 15.5 / 26.5.2 / 27.0-beta.6，秒）** + +| 配置 | `--interface` | lineage 报告 | +|---|---|---| +| 执行器关 + `--jobs 1`（接入前的形态） | 306.7 | 282.4 | +| 执行器开 + `--jobs 1` | 242.6 | 233.8 | +| 执行器开 + 默认并行（3 个版本同时） | **151.9** | **139.4** | + +并行本身 1.6–1.7×（三版本不等长，最慢的那个定墙钟），叠加执行器 2.0×；`--jobs 1` 与默认的输出逐字节一致。 + +**渲染 A/B**（`Scripts/run-rendering-ab-verification.py`，基线 = ABI 分支 `feature/self-contained-abi-layer`，候选 = 本分支；基线 swift-demangling 0.6.0、候选 0.6.3，其余 pin 一致）：执行器**开**与**关**（`MACHO_SWIFT_SECTION_LARGE_STACK_EXECUTOR=0`）各跑一轮，**两轮均 78 对输出逐字节一致，0 差异**。覆盖当前系统 dyld cache（归档 cache 目录名与脚本期望不符，按文档回退到系统 cache）、模拟器运行时 iOS 15.5 / 18.5 / 18.6 / 26.5、进程内 MachOImage 三条路径的 dump 与 interface。脚本自己记录的墙钟也印证了收益：候选侧 SwiftUI dump 60 s / interface 71 s 对基线 80 s / 88 s，关掉执行器后候选与基线持平（48 s vs 48 s）。 + +## 测试锚点 + +- `LargeStackTaskExecutionTests`（MachOSymbolsTests）：执行器线程栈 ≥ 16 MB 且线程名前缀 `swift-demangling.task-executor.`；`execute` / `executeAsync` 在体内不跳线程；嵌套 `run` 不换线程；子任务继承；`isEnabled = false` 时留在调用线程；值与错误透传。 +- `BoundedConcurrentMapTests`（SwiftInterfaceTests）：源序、窗口不超、窗口 1 严格串行、窗口内真并发（rendezvous，超时即失败）、首错语义、空输入。 +- `SwiftEvolutionInterfaceBuilderTests.parallelPreparationMatchesSerialPreparation` / `preparationWindowIsClampedNotValidated`。 +- `DiffCommandValidationTests` / `EvolutionCommandValidationTests`：`--jobs` 解析与下界校验。 + +## 已知限制 + +- 执行器只在 macOS 15 / iOS 18 起可用；以下系统行为不变。 +- `KnownIssues.md` #4（上游）在执行器路径上对打印器与 remangler 关闭，`TypeDecoder`（需约 30 MB）仍会先爆栈——它本就不经 `StackSafeExecutor`。 +- 事件 handler 的 stderr 输出在并行 `prepare` 时会交错。 +- 并行 `prepare` 期间同时常驻的索引器数等于窗口大小，内存峰值随之上升（`evolution --interface` 路径本来就全部常驻，lineage / JSON 路径从「逐版本释放」变为「窗口内同时常驻」）。 diff --git a/Documentations/Internal/Modules/SwiftInterface.md b/Documentations/Internal/Modules/SwiftInterface.md index 4fb95ae0..5c678f07 100644 --- a/Documentations/Internal/Modules/SwiftInterface.md +++ b/Documentations/Internal/Modules/SwiftInterface.md @@ -31,6 +31,8 @@ SwiftInterface 是接口生成的**编排层**（thin orchestrator）：它自 **`prepare()` 的顺序与失败语义**：先逐个 `extraDataProvider.setup()`（失败**降级**为 `renderingDegraded` 事件，不阻断——外挂数据源坏了不该毁掉整份接口），再 `indexer.prepare()`（失败**抛出**），最后 `collectModules()`（失败**抛出**）。全程用 `phaseTransition` 事件汇报阶段。 +**运行线程**：`prepare()` 与 `printRoot()` 的函数体都包在 `LargeStackTaskExecution.run` 里（提案 `large-stack-executor-and-cross-version-parallelism`），整段 task 跑在 demangler 的 16 MB 大栈执行器上，打印路径每个符号不再付一次线程往返；输出与执行器无关。`SwiftDiffableInterfaceBuilder.prepare()`、evolution builder 的 `prepare` / 渲染入口、diff renderer 的两个入口同样如此。见 [LargeStackTaskExecutorAdoption.md](../LargeStackTaskExecutorAdoption.md)。 + **`collectModules()`**：import 列表不来自 load command，而是扫全部符号的 demangle 树收 `.module` 节点——binary 里真正被引用的模块才进 import。过滤 `__C` / `__ObjC` / stdlib 三个伪模块；`internalModules`（`Swift`、`_Concurrency`、`_StringProcessing`、`_SwiftConcurrencyShims`）恒定并入。 **`printRoot()` 的段落与 catch 契约**：组成顺序是 header（提案 0008，flag-gated 默认缺席）→ imports → 全局变量 → 全局函数 → 根类型 → 特化变体（`specializedChildren` 挂在各 `TypeDefinition` 上，indexer 对用户驱动的特化保持无知，所以这里全量走查 `allTypeDefinitions`）→ 根协议 → **嵌套**协议的 default-implementation 扩展块（extension 不能嵌进父体，顶层补印；这个循环在提案 0007 之前是死代码）→ 四桶 extension（`isAttachedToProtocolDefinition` 的已附着定义被过滤，避免 issue #106 §5 的重复块）。贯穿全部段落的契约是**逐定义 catch**：一个定义打印抛错只丢它自己，绝不空掉整块（历史上块级 catch 让一个旧 binary 的全部类型被抹白；由 `LegacyDyldInfoBindTests` 与 `corruptNestedChildDropsOnlyItself` 钉住）。两个全局块例外地不带定义上下文——`printVariable`/`printFunction` 本身不抛、各自派发过失败事件，块级包裹只是保险带。 @@ -78,7 +80,7 @@ diff 与 evolution 两条比较渲染路径的公共结构核心。分工是这 N ≥ 2 版本渲染成**一份**并集接口，声明尾注生命周期注解。承重决策是**事实与文本的分工**： -- **注解事实只来自 `ABIEvolution`**：`prepare()` 逐版本索引 → 冻结 snapshot → `ABIEvolutionBuilder` 建 lineage 矩阵；渲染期经 `EvolutionAnnotationIndex` 按键查询，**查不到即是「全程在场、从未变化」的裁决**（`ABIEvolution` 只物化有变化的 lineage），渲染为无注解。策略自己绝不重推事件，所以注解接口、lineage 报告、JSON 三个视图永不打架。 +- **注解事实只来自 `ABIEvolution`**：`prepare(maximumConcurrentPreparations:)` **并行**索引各版本（窗口默认取核数、`1` 即旧的串行顺序；各版本是不同文件、缓存按 UUID 键控，结果与窗口无关，由 `parallelPreparationMatchesSerialPreparation` 钉住）→ 冻结 snapshot → `ABIEvolutionBuilder` 建 lineage 矩阵；渲染期经 `EvolutionAnnotationIndex` 按键查询，**查不到即是「全程在场、从未变化」的裁决**（`ABIEvolution` 只物化有变化的 lineage），渲染为无注解。策略自己绝不重推事件，所以注解接口、lineage 报告、JSON 三个视图永不打架。 - **渲染文本来自活模型**：每个声明由最后携带它的版本的 printer 渲染（modified 成员只显示最新一代，旧形态进注解短语 `modified in 26.0: old → new`；箭头两侧相同则塌回裸短语）。header 解析是「最新可渲染」：从新到旧找第一个渲染成功的版本，每次失败都在其版本自己的 dispatcher 上派发，全部失败才整体丢弃（diff 的 drop-whole 规则推广到 N 侧）。 - **公开面是两个类型**：`AnySwiftEvolutionInterfaceBuilder` 是类型擦除的 runtime-N 主力（同构数组 init + 异构 pack init 都全平台可用——pack 在*函数*位不需要可用性门槛），CLI 与宿主的用户选版场景都走它；`SwiftEvolutionInterfaceBuilder` 是 pack 泛型 façade（类型位的 pack 需要 Swift 5.9 运行时，故 `@available(macOS 14…)`；构造即擦除，行为逐字节一致，由 `packGenericFacadeMatchesTheErasedBuilder` 钉住）。工具链尚不支持 `repeat each MachO == M` 的同元素约束，所以数组 init 上不了 pack 类型——这是两个类型并存的直接原因。 - **格式层 `EvolutionMarking`**（+ `EvolutionContainerAssembler`）：legend 头两行（轴 + bitmap 位置对照）、注解列按块对齐、上限 72 列（超限换行缩一级）、锚点规则（成员注解锚**首行**——attribute 内联，computed property 的注解不能沉到 accessor 块闭括号；容器 header 锚**末行**——带 `{` 的那行）、镜像 `ABIEvolutionReporter` 措辞的 warnings 尾巴。与 `DiffMarking` 故意不合并：marker 按行、注解按 unit 锚定，是真不同语义。 diff --git a/Documentations/Internal/ProjectEvolutionLog.md b/Documentations/Internal/ProjectEvolutionLog.md index aa2ec878..43492d73 100644 --- a/Documentations/Internal/ProjectEvolutionLog.md +++ b/Documentations/Internal/ProjectEvolutionLog.md @@ -1271,6 +1271,29 @@ [TaskReports/2026-09-03-self-contained-abi-layer.md](TaskReports/2026-09-03-self-contained-abi-layer.md)。 - **对应版本**：0.18.0（破坏性 API 变更，见 Changelog）。 +## 2026-09-03 大栈任务执行器接入与跨版本并行（提案 large-stack-executor-and-cross-version-parallelism；节号落地时取） + +- **时间段**：2026-09-03。 +- **动机**：用户问「整个库都使用 async 环境是否可行」。调研结论：这个库的 async 是签名上的 async， + 真正的开销在打印路径——每次 `printSemantic` 都跳到 swift-demangling 的 8 MB 大栈线程再用信号量 + 停住协作线程（release 每次 8–21 µs，1.14–2.28×）；索引侧已用同步 `withLargeStack` 摊掉，打印循环 + 因为是 async 包不住。多版本 `prepare` 串行而各版本彼此独立。全库 async 化不是答案（撞 `deinit` / + getter / `Hashable` 不能 `await`、MachOKit 同步、`Node` 非 `Sendable`，且协作线程 512 KB 让探测 + 100% 不过）。 +- **关键决策**：执行器归上游（swift-demangling 提案 0014，随 0.6.3 发版：`StackSafeExecutor.taskExecutor`， + 16 MB 线程、与 8 MB 跳转池分池共码）；本库在库入口自装偏好（`MachOSymbols.LargeStackTaskExecution.run`）， + 宿主零改动；macOS 15 以下静默回退；跨版本并行默认开、窗口取核数（`prepare(maximumConcurrentPreparations:)`、 + CLI `--jobs`），版本内并行不做（MachOKit 共享 FileHandle）；pin 抬到 0.6.3 跳过 0.6.1。 +- **落地模块**：`MachOSymbols`（`LargeStackTaskExecution`）、`Utilities`（`concurrentMap(maximumConcurrency:)`）、 + `SwiftIndexing`、`SwiftInterface`、`SwiftPrinting`、`SwiftDump`、`swift-section`。 +- **关联文档**：[提案](../Evolutions/draft-large-stack-executor-and-cross-version-parallelism.md)、 + [LargeStackTaskExecutorAdoption.md](LargeStackTaskExecutorAdoption.md)、 + [TaskReports/2026-09-03-large-stack-executor-and-cross-version-parallelism.md](TaskReports/2026-09-03-large-stack-executor-and-cross-version-parallelism.md)、 + Glossary 新术语「large-stack executor（大栈执行器）」。 +- **对应版本**：0.19.0（依赖下限 swift-demangling ≥ 0.6.3，见 Changelog）。 + +--- + ## 维护约定 1. **每个非平凡批次结束时必须在本文追加/更新一节**（新工作弧新增一节；延续既有弧则在该节 diff --git a/Documentations/Internal/Reviews/2026-07-31-node-store-migration-review.md b/Documentations/Internal/Reviews/2026-07-31-node-store-migration-review.md index 2f43f4f1..da0004ec 100644 --- a/Documentations/Internal/Reviews/2026-07-31-node-store-migration-review.md +++ b/Documentations/Internal/Reviews/2026-07-31-node-store-migration-review.md @@ -126,6 +126,7 @@ libdispatch worker reported_size= 524 KB remaining= 523 KB >=2MB: no (HOP 每符号省 6.2 µs。值得注意的是跳转成本（615.7 ms）几乎与 demangle 本身（701.5 ms）等价——**近一半时间花在线程往返上**。主线程两组数据一致，再次确认探测通过时批量边界不产生任何作用，也印证上文对原实测表的更正。 - **`Node+.swift` 的 `printSemantic` 注释**（本次改动）：原注释断言 `print(_:options:)` "runs the recursion inline against a stack floor and pays for a worker only for a tree that actually reaches it"，与代码相反——它内部就是 `executeWithUncheckedSendability`，与 `execute` 是同一段逻辑，仅少了 `Sendable` 约束。注释已改为如实描述，并说明为何摊销点不在此处。**代码未动**。 - **渲染循环暂不处理**：打印侧的循环在 `SwiftDeclarationPrinter` 里，是 `async`，同步的 `withLargeStack` 无法包裹。真要摊销需要自定义一个跑在 8 MB 线程上的 `SerialExecutor`，或把打印批次改成同步——两者都是独立的重构。先量清楚真实导出中的调用次数与总开销，再决定是否值得。 + **已解决（2026-09-03，提案 `large-stack-executor-and-cross-version-parallelism`）**：走的是第三条路——上游 swift-demangling 0.6.3 提供 16 MB 线程的 `TaskExecutor`，本库的 async 入口经 `LargeStackTaskExecution.run` 把整个 task 放到它上面，探针在每个入口都通过，打印路径零跳转；见 `Documentations/Internal/LargeStackTaskExecutorAdoption.md`。 > 探针为一次性测量代码，测完已删除。若需长期守护该性质，应整理为正式 benchmark。 diff --git a/Documentations/Internal/TaskReports/2026-09-03-large-stack-executor-and-cross-version-parallelism.md b/Documentations/Internal/TaskReports/2026-09-03-large-stack-executor-and-cross-version-parallelism.md new file mode 100644 index 00000000..5240d42c --- /dev/null +++ b/Documentations/Internal/TaskReports/2026-09-03-large-stack-executor-and-cross-version-parallelism.md @@ -0,0 +1,50 @@ +# 2026-09-03 大栈任务执行器接入与跨版本并行 + +## 问题 + +用户问「整个库都使用 async 环境是否可行」。核实结果：306 个 async 函数、72 个公开 async 入口，但除 `TypeIndexing` 的两个 actor 外没有 actor，真正会挂起的地方只有三处；真正的开销在打印路径——每次 `printSemantic` 都由 swift-demangling 的 `StackSafeExecutor` 探测调用线程剩余栈，协作线程 512 KB 永远不过，每次调用跳到 8 MB 池线程再用信号量停住（release 每次 8–21 µs，1.14–2.28×）。索引侧已用同步 `withLargeStack` 摊掉；打印循环是 async，包不住。此外 diff / evolution 的多版本 `prepare` 串行，而各版本彼此独立。 + +## 调研 + +- 全库 async 化不可取：`deinit` / getter / `Hashable` / `for … where` 不能 `await`；MachOKit 与运行时调用同步；`Node` 与三个 Definition 是非 `Sendable` 的 class；async 化后协作线程 512 KB 让探测 100% 不通过；符号扫描改逐个 `await` 反而比批量内联慢。收益在「并行」和「让 async 代码跑在大栈上」。 +- 上游探测看的是**剩余栈**不是线程身份，所以 16 MB 线程上的 task 每个入口都内联——只需一个 `TaskExecutor`（SE-0417，macOS 15 / iOS 18 起）。上游已有 `LargeStackThreadPool`（按 QoS 分池、`pthread_attr_setstacksize`）可复用。 +- swift-demangling 版本线：0.6.1 的 QoS 改动慢 3–4 倍（2026-09-02 二分）；0.6.2 分池修复、用户实测恢复；0.6.3 含执行器（提案 0014，`StackSafeExecutor.taskExecutor`，`@_spi(Internals)`）。 +- 跨版本并行安全：`MachOFile.identifier` 按 LC_UUID 键控，`SharedCache` 全按此分片，描述符读取走 mmap；三进程并行实测约 2 倍。版本内并行卡在 MachOKit 共享 `FileHandle` 的 seek + read。 +- 库内非结构化 `Task {}`：零处。 + +## 澄清提问（完整档，四轮 + 收尾） + +1. 范围：执行器 + 跨版本并行；版本内并行另起提案。 +2. 执行器归上游 swift-demangling，进程内一个池（后改为分池共码）。 +3. 库入口自装偏好；macOS 15 以下静默回退；默认并行上限取核数。 +4. 上游提案由本人在 sibling 仓库起草；执行器线程 16 MB、跳转池 8 MB；`@_spi(Internals)`；发版 0.6.3。 +5. 收尾：用户确认「其他没问题」，0.6.2 已实测恢复；之后指示「基于上一个 PR 实现 async 提案」，视为 Accepted。 + +## 最终方案 + +见提案与实现说明。要点：`MachOSymbols.LargeStackTaskExecution.run` 包住索引器 `prepare`、interface builder 的 `prepare` / `printRoot`、diffable builder 的 `prepare`、evolution builder 的 `prepare` / 两个渲染入口、diff renderer 的两个入口、printer 的四个逐定义入口、六个 `Dumpable.dump`；`AnySwiftEvolutionInterfaceBuilder.prepare(maximumConcurrentPreparations:)`（默认核数）；`diff` / `evolution` 的输入按窗口并行，CLI `--jobs`；`Utilities.concurrentMap(maximumConcurrency:)`；pin 抬到 0.6.3。 + +## 实际执行 + +worktree `.worktrees/MachOSwiftSection-LargeStackExecutor`，分支 `feature/large-stack-executor-and-cross-version-parallelism`，基于 `feature/self-contained-abi-layer`（ABI 提案的分支，PR #121 未合并时堆叠）。 + +1. **抬 pin**：`"0.6.3" ..< "0.7.0"`，`Package.resolved` 解析到 0.6.3（`8f32e30`）；其余 pin 不动。同一份 `Package.resolved` 下先做 0.6.0 vs 0.6.3 的 release 计时（见下）。 +2. **`LargeStackTaskExecution`**（`Sources/MachOSymbols/LargeStackTaskExecution.swift`）：`isEnabled`（`@Mutex`，初值读 `MACHO_SWIFT_SECTION_LARGE_STACK_EXECUTOR`）、`isSupported`、`run`。`withTaskExecutorPreference` 在当前工具链接受非 `Sendable` 的 `operation`（用 `swiftc -typecheck -swift-version 6 -strict-concurrency=complete` 探针确认），所以 `run` 的 `body` 不必 `@Sendable`——各入口闭包捕获非 `Sendable` 的 Definition class 才能过编译。 +3. **逐入口包裹**：按提案清单。`printExtensionDefinition` / `printDefinition` 拆壳 + 体；`printIncludedExtensionDefinition` 名字已被提案 0016 的 builder 体占用（首次编译撞名），壳体改名 `…Contents`。 +4. **并行**：`Utilities/BoundedConcurrentMap.swift`；evolution builder 的 `prepare` 加参数；pack façade 跟随；`DiffCommand` 两侧、`EvolutionCommand` 两条路径都走 `concurrentMap`；`--jobs` 校验 ≥ 1。 +5. **测试**：`LargeStackTaskExecutionTests`（6）、`BoundedConcurrentMapTests`（7）、evolution builder 的并行等价 + clamp（2）、`DiffCommandValidationTests`（3）、`EvolutionCommandValidationTests` 加 `--jobs`（2）；CI filter 加入五个套件。首次跑：`EvolutionLine` 的属性是 `content` 不是 `text`，改后 32 个全过。 +6. **文档同批**：实现说明、术语表、AGENTS.md（`MachOSymbols` / `SwiftIndexing` / `SwiftInterface` 条目 + 测试环境节）、`Modules/SwiftInterface.md`、README 索引、评审记录待办标记已解决、`Node+.swift` 过期注释、演进账本、Changelog 0.19.0 + `Version.swift`。 + +## 验证 + +- 新增 / 改动的五个套件 32 个测试通过；全量 `swift test --skip IntegrationTests`：**1637 个测试、305 个套件全部通过**（385 s），含以往偶发的 `SharedCache` 并发墙钟测试。 +- 计时（release，宿主 cache；详表见实现说明）：SwiftUICore dump 48.6 → 40.5 s、interface 56 → 47 s；SwiftUI dump 79 → 61 s、interface 89 → 71 s（执行器关 → 开，−16% 到 −23%）；0.6.0 → 0.6.3 仅抬 pin 持平。三版本 SwiftUI `evolution --interface` 306.7 s（关 + `--jobs 1`）→ 242.6 s（开 + `--jobs 1`）→ 151.9 s（开 + 默认并行）；lineage 282.4 → 233.8 → 139.4 s。 +- 输出：单版本四种配置（0.6.0 / 0.6.3 pin-only / 执行器关 / 执行器开）的 dump 与 interface 逐字节一致；evolution 的 `--jobs 1` 与默认并行、执行器开与关逐字节一致。 +- 渲染 A/B（`Scripts/run-rendering-ab-verification.py`，基线 = ABI 分支，候选 = 本分支）：执行器开、关各一轮，**两轮均 78 对逐字节一致，0 差异**（当前系统 cache、模拟器运行时 iOS 15.5 / 18.5 / 18.6 / 26.5、进程内 MachOImage）。 + +## 与计划的偏差 + +- `isEnabled` 初值读环境变量（提案只有静态开关）——为 A/B 与计时服务。 +- 并行用通用 `concurrentMap(maximumConcurrency:)` 而非 `async let`。 +- CLI `dump` 循环不再额外包一层（逐类型跳转代价可忽略）。 +- 主 actor 段落：库 target 未开启 SE-0461，async 入口从 `@MainActor` 调用会离开主 actor 落到执行器线程（实现说明已按事实写）。 diff --git a/Documentations/README.md b/Documentations/README.md index 63ee6510..e754b097 100644 --- a/Documentations/README.md +++ b/Documentations/README.md @@ -88,6 +88,7 @@ required by `Version.swift`'s bump contract). | [FixtureTestingAndContinuousIntegration.md](Internal/FixtureTestingAndContinuousIntegration.md) | **Fixture 测试体系与 CI 的设计来历**（整合自已删除的 `docs/superpowers/` 四份 spec + CI 落地记录）：为什么 fixture 只用 SymbolTestsCore、`#filePath` 路径锚定陷阱、命名空间约定与边界规则、ABI 覆盖四支柱与 sentinel 信任危机（88/157 suite 失真）的收紧、CI 白名单 regex 与首轮五坑。现行操作规程仍以 AGENTS.md 为准。 | | [ClassMemberKeywordRecovery.md](Internal/ClassMemberKeywordRecovery.md) | `class` / `static` 成员关键字的还原：mangling 层面两者不可区分，判据是「类型级成员有 vtable method descriptor ⇒ 源码是 `class`」（`static` 隐式 final、不进 vtable）；模型侧 `isClassMember` 计算属性 + 三个 node printer 接线 + dump vtable 段落关键字，顺带消灭非法的 `override static` 输出；`final class func` 等四类 ABI 上与 `static` 完全一致，保守输出语义等价的 `static`。 | | [SelfContainedABILayer.md](Internal/SelfContainedABILayer.md) | ABI 层自包含（提案 `self-contained-abi-layer` 的实现说明）：`MachOSwiftSection` 只依赖新伞模块 `MachOBase`，描述符只暴露 `implementationOffset` / `implementationAddress(in:)`，符号归属上移 `SwiftInspection`，值类型下沉 `MachOResolving`；记录为什么多一个伞模块、为什么 `ReadingContext` 腿没有符号形态（旧腿在读机器码）、为什么不留 typealias、为什么 async 重载只能删；附下游迁移表。 | +| [LargeStackTaskExecutorAdoption.md](Internal/LargeStackTaskExecutorAdoption.md) | 大栈任务执行器接入与跨版本并行（提案 `large-stack-executor-and-cross-version-parallelism` 的实现说明）：为什么打印路径每个符号付一次线程往返、为什么执行器按剩余栈探测就能让整个 task 内联、`LargeStackTaskExecution.run` 包了哪些入口与嵌套为何免费、macOS 15 以下的静默回退、跨版本并行为什么安全而版本内并行为什么不做（MachOKit 共享 FileHandle）、`--jobs` 与 `concurrentMap(maximumConcurrency:)` 的语义；附 0.6.0 / 0.6.3 / 执行器开关的计时表与渲染 A/B 结论。 | | [ExtensionContainerUnification.md](Internal/ExtensionContainerUnification.md) | Extension 容器统一（提案 0007 的实现说明）：双产线重复（协议尾随 descriptor 副本 + 符号扫描桶副本）的「附着 + 打印抑制」消解——桶是 ABI 快照的直接输入故不可移除，附着对象留桶打标、顶层打印跳过；descriptor 合成降级为 fallback（ICF 地址上丢成员）；桶内同身份合并、空 requirement 签名桶折叠、`updateConfiguration` no-op 修复、嵌套协议扩展块死循环修复；裸头 typealias 块并存为 P1-9 残余（格式冻结约束下不合并）。 | | [FinalKeywordAndLazyAccessorTypeRecovery.md](Internal/FinalKeywordAndLazyAccessorTypeRecovery.md) | `final` 成员关键字还原与 lazy var 访问器类型修正（提案 0006 的实现说明）：核心是停止丢弃 `DefinitionBuilder` 已解析的 stored-var accessor→vtable 归属；三层证据门（非 actor class 有 vtable header / accessor 组确实 join 上 / `@objc` 排除）宁缺勿错；`memberJoinKey` 剥 `Tu` async-function-pointer 标记（顺带修复 async 成员一直缺失的 `override` 与 vtable 注释）；final class 的成员级 `final`、stored `let` 不标、dump 路径 lazy 保持存储真相等决策与降级。 | | [InterfaceHeaderAndExportStatusAnnotations.md](Internal/InterfaceHeaderAndExportStatusAnnotations.md) | Interface 文件头部与导出状态标注（提案 0008 的实现说明）：导出集为何必须在构建扫描里显式旁路收集（symtab 两腿只收本地符号、trie 腿建行有条件）；裸查实现符号在 evolution 构建上全量假阳性 → 派生符号形态查询（`Tj`/`Tq`/`Tu`/`TjTu`）；`override` / `@objc` 发射豁免与 conformance witness 故意不豁免的边界；头部组件的调用方传入 generator 身份、日期缺席字节稳定、evolution 行 detected/not detected 措辞；dump 路径的符号级语义收窄与已知残留。 | diff --git a/Sources/MachOSymbols/LargeStackTaskExecution.swift b/Sources/MachOSymbols/LargeStackTaskExecution.swift new file mode 100644 index 00000000..b6f40fb4 --- /dev/null +++ b/Sources/MachOSymbols/LargeStackTaskExecution.swift @@ -0,0 +1,77 @@ +import Foundation +import FoundationToolbox +@_spi(Internals) import Demangling + +/// Runs a library entry point on the demangler's large-stack task executor +/// (evolution proposal `large-stack-executor-and-cross-version-parallelism`). +/// +/// `StackSafeExecutor` decides per demangle / print / remangle whether to hop +/// to an 8MB pool thread by probing the CALLING thread's remaining stack, not +/// its identity. Swift Concurrency's cooperative threads (and libdispatch's) +/// carry 512KB, so on them the probe never passes and every call pays a +/// thread round trip plus a semaphore wait — 8–21 µs each in release, 1.14–2.28× +/// the work itself. The synchronous indexing sweep amortizes that with one +/// `withLargeStack` batch (`SymbolIndexStore.buildStorageImpl`); an `async` +/// print loop cannot be enclosed in a synchronous batch. The upstream answer +/// (swift-demangling proposal 0014, shipped in 0.6.3) is a `TaskExecutor` +/// whose threads carry 16MB: a task running on it passes the probe at every +/// entry point, synchronous callees included, so the whole pipeline runs +/// inline with zero hops. +/// +/// ``run(_:)`` is the one place this library adopts it. The async entry points +/// — indexer preparation, interface building and printing, the printer's +/// per-definition entries, diff / evolution preparation and rendering, and +/// the dump family — wrap their bodies in it, so a host gets the executor +/// without changing a line. Nesting is free: a task already on the executor +/// does not switch, so an entry point reached from another wrapped entry +/// point pays nothing. +/// +/// Output is identical either way; only the thread the work runs on differs. +/// Where the runtime has no task executors (below macOS 15 / iOS 18 / tvOS 18 +/// / watchOS 11 / visionOS 2, or off Darwin) the body runs unchanged on the +/// caller's executor — the pre-adoption behavior, hops included. +/// +/// Two upstream contracts the wrapping honors: an unstructured `Task {}` +/// does not inherit the preference (SE-0417), so the library starts none +/// inside a wrapped entry point (child tasks and default actors do inherit); +/// and a job that blocks its thread waiting on another job of the same +/// quality-of-service class can exhaust that class's workers, exactly as it +/// would exhaust the cooperative pool — which is why cross-version +/// parallelism caps its window at the processor count. +public enum LargeStackTaskExecution { + /// Process-wide switch. A host that manages its own executor sets it to + /// `false`. The environment variable + /// `MACHO_SWIFT_SECTION_LARGE_STACK_EXECUTOR=0` seeds it off for a process + /// that cannot be recompiled — the rendering A/B and the timing runs + /// compare the same binary with the executor on and off through it. + @Mutex + public static var isEnabled: Bool = ProcessInfo.processInfo.environment["MACHO_SWIFT_SECTION_LARGE_STACK_EXECUTOR"] != "0" + + /// Whether this process can run work on the executor at all: Darwin, on a + /// runtime with SE-0417 task executors. Independent of ``isEnabled``. + public static var isSupported: Bool { + #if canImport(Darwin) + if #available(macOS 15.0, iOS 18.0, tvOS 18.0, watchOS 11.0, visionOS 2.0, *) { + return true + } + #endif + return false + } + + /// Runs `body` with the large-stack executor as the task executor + /// preference when ``isEnabled`` and ``isSupported``; otherwise runs + /// `body` unchanged on the caller's executor. + /// + /// The preference governs nonisolated async code, child tasks and default + /// actors inside `body`. An actor with its own executor (the main actor) + /// keeps its executor — the main thread's 8MB stack already passes the + /// probe, so nothing is lost there. + public static func run(_ body: () async throws -> Success) async rethrows -> Success { + #if canImport(Darwin) + if isEnabled, #available(macOS 15.0, iOS 18.0, tvOS 18.0, watchOS 11.0, visionOS 2.0, *) { + return try await withTaskExecutorPreference(StackSafeExecutor.taskExecutor, operation: body) + } + #endif + return try await body() + } +} diff --git a/Sources/SwiftDeclarationRendering/Extensions/Node+.swift b/Sources/SwiftDeclarationRendering/Extensions/Node+.swift index 53ce6463..263794aa 100644 --- a/Sources/SwiftDeclarationRendering/Extensions/Node+.swift +++ b/Sources/SwiftDeclarationRendering/Extensions/Node+.swift @@ -76,24 +76,30 @@ extension DemanglingNode { /// overflow the stack where the identical `NodeReference` call would not. /// /// The guard is the engine's own `print(_:options:)`, which routes through - /// `StackSafeExecutor.executeWithUncheckedSendability` — the same probe - /// and the same 2MB floor as `execute`, only without the `Sendable` - /// checking a generic `Target` cannot satisfy. An earlier version of this - /// comment framed the two as a choice, claiming the engine's entry point - /// "runs the recursion inline and pays for a worker only for a tree that - /// actually reaches it"; it does not. Darwin gives every thread but the - /// main one a 512KB stack, so on a cooperative or libdispatch worker the - /// probe never passes and *every* call hops to a large-stack worker and - /// blocks on a semaphore — whichever wrapper the engine happens to use. + /// `StackSafeExecutor.execute` — probe the calling thread's remaining + /// stack against a 2MB floor, run inline when it passes, hop to an 8MB + /// pool worker and block on a semaphore when it does not. An earlier + /// version of this comment claimed the engine's entry point "runs the + /// recursion inline and pays for a worker only for a tree that actually + /// reaches it"; it does not. Darwin gives every thread but the main one a + /// 512KB stack, so on a cooperative or libdispatch worker the probe never + /// passes and *every* call hops. /// /// That cost is upstream's deliberate trade (`swift-demangling` 7b86137): /// before it, the printer recursed unguarded and a deeply nested generic /// really did overflow a 512KB worker. It is paid down at a *batch* /// boundary, never here — a `withLargeStack` around this single call would /// save exactly the one hop it adds. `SymbolIndexStore.buildStorageImpl` - /// is where the repo does that, because it owns a loop; the printer's own - /// loop lives in `SwiftDeclarationPrinter` and is `async`, which a - /// synchronous wrapper cannot enclose. + /// does that for the synchronous sweep, because it owns a loop; the + /// printer's loop lives in `SwiftDeclarationPrinter` and is `async`, which + /// a synchronous wrapper cannot enclose. The async side's batch boundary + /// is the TASK instead (evolution proposal + /// `large-stack-executor-and-cross-version-parallelism`): the library's + /// async entry points run on the demangler's 16MB `LargeStackTaskExecutor` + /// through `LargeStackTaskExecution.run`, where the probe passes at every + /// entry and this call runs inline. The hop is what happens OFF that path + /// — a host that disabled the executor, or a runtime without task + /// executors (below macOS 15). /// /// **Do not "modernize" this onto `runPrintWalk(using:)`.** Upstream added /// that protocol requirement as the dispatch hook behind @@ -105,11 +111,10 @@ extension DemanglingNode { /// it. For a non-`String` target the engine's static /// `DemanglingPrinter.print(_:options:)` is the only /// entry point, and will remain so. It is byte-for-byte unchanged across - /// the `runPrintWalk` introduction — including the - /// `StackSafeExecutor.executeWithUncheckedSendability` wrapper this comment - /// exists to explain (verified upstream on a deliberately 512KB-stacked - /// thread against 600 levels of nested generics, which survives only - /// because of that hop). + /// the `runPrintWalk` introduction — including the `StackSafeExecutor` + /// wrapper this comment exists to explain (verified upstream on a + /// deliberately 512KB-stacked thread against 600 levels of nested + /// generics, which survives only because of that hop). /// /// Worth noting *why* the shadowing hazard above is a recurring shape /// rather than a one-off: upstream hit the mirror image of it in the same diff --git a/Sources/SwiftDump/Dumpable/AssociatedType+Dumpable.swift b/Sources/SwiftDump/Dumpable/AssociatedType+Dumpable.swift index e1d99f9c..57debb70 100644 --- a/Sources/SwiftDump/Dumpable/AssociatedType+Dumpable.swift +++ b/Sources/SwiftDump/Dumpable/AssociatedType+Dumpable.swift @@ -1,6 +1,7 @@ import Foundation import MachOKit import MachOSwiftSection +import MachOFoundation import Semantic import Utilities import SwiftDeclarationRendering @@ -15,6 +16,8 @@ extension AssociatedType: ConformedDumpable { } public func dump(using configuration: DumperConfiguration, in machO: MachO) async throws -> SemanticString { - try await AssociatedTypeDumper(self, using: configuration, in: machO).body + try await LargeStackTaskExecution.run { + try await AssociatedTypeDumper(self, using: configuration, in: machO).body + } } } diff --git a/Sources/SwiftDump/Dumpable/Class+Dumpable.swift b/Sources/SwiftDump/Dumpable/Class+Dumpable.swift index 2b568528..09e705dc 100644 --- a/Sources/SwiftDump/Dumpable/Class+Dumpable.swift +++ b/Sources/SwiftDump/Dumpable/Class+Dumpable.swift @@ -2,6 +2,7 @@ import Semantic import Demangling import MachOKit import MachOSwiftSection +import MachOFoundation import Utilities import SwiftDeclarationRendering @@ -11,6 +12,8 @@ extension Class: NamedDumpable { } public func dump(using configuration: DumperConfiguration, in machO: MachO) async throws -> SemanticString { - try await ClassDumper(self, using: configuration, in: machO).body + try await LargeStackTaskExecution.run { + try await ClassDumper(self, using: configuration, in: machO).body + } } } diff --git a/Sources/SwiftDump/Dumpable/Enum+Dumpable.swift b/Sources/SwiftDump/Dumpable/Enum+Dumpable.swift index 3ab756e0..42c55fe5 100644 --- a/Sources/SwiftDump/Dumpable/Enum+Dumpable.swift +++ b/Sources/SwiftDump/Dumpable/Enum+Dumpable.swift @@ -1,6 +1,7 @@ import Foundation import MachOKit import MachOSwiftSection +import MachOFoundation import Semantic import Utilities import SwiftDeclarationRendering @@ -11,6 +12,8 @@ extension Enum: NamedDumpable { } public func dump(using configuration: DumperConfiguration, in machO: MachO) async throws -> SemanticString { - try await EnumDumper(self, using: configuration, in: machO).body + try await LargeStackTaskExecution.run { + try await EnumDumper(self, using: configuration, in: machO).body + } } } diff --git a/Sources/SwiftDump/Dumpable/Protocol+Dumpable.swift b/Sources/SwiftDump/Dumpable/Protocol+Dumpable.swift index 106a111d..25280ab2 100644 --- a/Sources/SwiftDump/Dumpable/Protocol+Dumpable.swift +++ b/Sources/SwiftDump/Dumpable/Protocol+Dumpable.swift @@ -1,6 +1,7 @@ import Foundation import MachOKit import MachOSwiftSection +import MachOFoundation import Semantic import Utilities import Demangling @@ -13,6 +14,8 @@ extension MachOSwiftSection.`Protocol`: NamedDumpable { } public func dump(using configuration: DumperConfiguration, in machO: MachO) async throws -> SemanticString { - try await ProtocolDumper(self, using: configuration, in: machO).body + try await LargeStackTaskExecution.run { + try await ProtocolDumper(self, using: configuration, in: machO).body + } } } diff --git a/Sources/SwiftDump/Dumpable/ProtocolConformance+Dumpable.swift b/Sources/SwiftDump/Dumpable/ProtocolConformance+Dumpable.swift index 33535a1b..bceeeadd 100644 --- a/Sources/SwiftDump/Dumpable/ProtocolConformance+Dumpable.swift +++ b/Sources/SwiftDump/Dumpable/ProtocolConformance+Dumpable.swift @@ -1,6 +1,7 @@ import Foundation import MachOKit import MachOSwiftSection +import MachOFoundation import Semantic import Demangling import Utilities @@ -17,6 +18,8 @@ extension ProtocolConformance: ConformedDumpable { } public func dump(using configuration: DumperConfiguration, in machO: MachO) async throws -> SemanticString { - try await ProtocolConformanceDumper(self, using: configuration, in: machO).body + try await LargeStackTaskExecution.run { + try await ProtocolConformanceDumper(self, using: configuration, in: machO).body + } } } diff --git a/Sources/SwiftDump/Dumpable/Struct+Dumpable.swift b/Sources/SwiftDump/Dumpable/Struct+Dumpable.swift index c06a21e3..3e687a6e 100644 --- a/Sources/SwiftDump/Dumpable/Struct+Dumpable.swift +++ b/Sources/SwiftDump/Dumpable/Struct+Dumpable.swift @@ -1,6 +1,7 @@ import Foundation import MachOKit import MachOSwiftSection +import MachOFoundation import Semantic import Utilities import SwiftDeclarationRendering @@ -11,6 +12,8 @@ extension Struct: NamedDumpable { } public func dump(using configuration: DumperConfiguration, in machO: MachO) async throws -> SemanticString { - try await StructDumper(self, using: configuration, in: machO).body + try await LargeStackTaskExecution.run { + try await StructDumper(self, using: configuration, in: machO).body + } } } diff --git a/Sources/SwiftIndexing/SwiftDeclarationIndexer.swift b/Sources/SwiftIndexing/SwiftDeclarationIndexer.swift index f1902561..ca09ec3f 100644 --- a/Sources/SwiftIndexing/SwiftDeclarationIndexer.swift +++ b/Sources/SwiftIndexing/SwiftDeclarationIndexer.swift @@ -263,9 +263,20 @@ public final class SwiftDeclarationIndexer SemanticString { let evolution = try requirePrepared() - let blocks = await makeRenderer(for: evolution).annotatedBlocks() - return EvolutionMarking.renderInterface(blocks: blocks, evolution: evolution) + return await LargeStackTaskExecution.run { + let blocks = await makeRenderer(for: evolution).annotatedBlocks() + return EvolutionMarking.renderInterface(blocks: blocks, evolution: evolution) + } } /// The structured line stream behind ``printAnnotatedInterface()``: the @@ -129,7 +152,10 @@ public final class AnySwiftEvolutionInterfaceBuilder: Sendable { /// renderer's `annotatedDiffBlocks()`. @_spi(Support) public func annotatedBlocks() async throws -> [[EvolutionLine]] { - try await makeRenderer(for: requirePrepared()).annotatedBlocks() + let evolution = try requirePrepared() + return await LargeStackTaskExecution.run { + await makeRenderer(for: evolution).annotatedBlocks() + } } private func makeRenderer(for evolution: ABIEvolution) -> SwiftEvolutionInterfaceRenderer { diff --git a/Sources/SwiftInterface/SwiftDiffableInterfaceBuilder.swift b/Sources/SwiftInterface/SwiftDiffableInterfaceBuilder.swift index f3b99cc8..573312f4 100644 --- a/Sources/SwiftInterface/SwiftDiffableInterfaceBuilder.swift +++ b/Sources/SwiftInterface/SwiftDiffableInterfaceBuilder.swift @@ -3,6 +3,7 @@ import SwiftDeclaration import SwiftDeclarationRendering import SwiftDiffing import MachOSwiftSection +import MachOSymbols /// The ABI-diff analogue of ``SwiftInterfaceBuilder``. /// @@ -38,7 +39,18 @@ public final class SwiftDiffableInterfaceBuilder: /// here. (`index(in:)` is `package`-scoped and idempotent, so this is safe /// and cheap to re-enter; it is callable because this builder lives in the /// same package as the model.) + /// + /// Runs on the demangler's large-stack task executor + /// (`LargeStackTaskExecution.run`), so the per-definition indexing pass + /// — the bulk of a diff's cost — demangles inline instead of hopping to + /// a pool thread per symbol. public func prepare() async throws { + try await LargeStackTaskExecution.run { + try await prepareContents() + } + } + + private func prepareContents() async throws { try await indexer.prepare() for typeDefinition in indexer.allTypeDefinitions.values { diff --git a/Sources/SwiftInterface/SwiftDiffableInterfaceRenderer.swift b/Sources/SwiftInterface/SwiftDiffableInterfaceRenderer.swift index c85140ee..1fa9d9d1 100644 --- a/Sources/SwiftInterface/SwiftDiffableInterfaceRenderer.swift +++ b/Sources/SwiftInterface/SwiftDiffableInterfaceRenderer.swift @@ -8,6 +8,7 @@ import MachOSwiftSection import Semantic import Demangling import OrderedCollections +import MachOSymbols /// Renders a **full Swift interface annotated with diff markers** — a git-diff /// style view of how the new binary's ABI surface differs from the old. @@ -58,8 +59,13 @@ public final class SwiftDiffableInterfaceRenderer< /// ``DiffFormat/inline``, the git-diff-style `+`/`-`/` ` markers with a /// one-space gutter). The classified stream comes from /// ``annotatedDiffBlocks()``; the format turns it into the final string. + /// + /// Runs on the demangler's large-stack task executor + /// (`LargeStackTaskExecution.run`), as does ``annotatedDiffBlocks()``. public func printAnnotatedInterface(format: DiffFormat = .inline) async -> SemanticString { - await format.render(annotatedDiffBlocks()) + await LargeStackTaskExecution.run { + await format.render(annotatedDiffBlocks()) + } } /// The full classified diff as a block-grouped, single-line-split stream: the @@ -70,7 +76,9 @@ public final class SwiftDiffableInterfaceRenderer< /// than a rendered string. @_spi(Support) public func annotatedDiffBlocks() async -> [[DiffLine]] { - await InterfaceUnionWalker(versions: versions, strategy: DiffUnionStrategy(versions: versions)).blocks() + await LargeStackTaskExecution.run { + await InterfaceUnionWalker(versions: versions, strategy: DiffUnionStrategy(versions: versions)).blocks() + } } } diff --git a/Sources/SwiftInterface/SwiftEvolutionInterfaceBuilder.swift b/Sources/SwiftInterface/SwiftEvolutionInterfaceBuilder.swift index 69b88673..60ece49f 100644 --- a/Sources/SwiftInterface/SwiftEvolutionInterfaceBuilder.swift +++ b/Sources/SwiftInterface/SwiftEvolutionInterfaceBuilder.swift @@ -57,9 +57,9 @@ public final class SwiftEvolutionInterfaceBuilder: Sendable /// - Building cross-reference maps for conformances and associated types /// - Collecting all required module imports /// + /// Runs on the demangler's large-stack task executor + /// (`LargeStackTaskExecution.run`, evolution proposal + /// `large-stack-executor-and-cross-version-parallelism`), as does + /// `printRoot()`: every demangle / print / remangle inside runs inline + /// instead of hopping to a pool thread per call. Output is independent + /// of where it runs. + /// /// - Throws: An error if indexing fails or if required data cannot be extracted. public func prepare() async throws { + try await LargeStackTaskExecution.run { + try await prepareContents() + } + } + + private func prepareContents() async throws { eventDispatcher.dispatch(.phaseTransition(phase: .preparation, state: .started)) for extraDataProvider in extraDataProviders { @@ -136,7 +149,9 @@ public final class SwiftInterfaceBuilder: Sendable if printer.configuration.printExportedDeclarationsOnly { printer.installExportFilterScope(types: indexer.allTypeDefinitions.values, protocols: indexer.allProtocolDefinitions.values) } - return try await printRootContents() + return try await LargeStackTaskExecution.run { + try await printRootContents() + } } @SemanticStringBuilder diff --git a/Sources/SwiftPrinting/SwiftDeclarationPrinter.swift b/Sources/SwiftPrinting/SwiftDeclarationPrinter.swift index 02525e75..d936fb44 100644 --- a/Sources/SwiftPrinting/SwiftDeclarationPrinter.swift +++ b/Sources/SwiftPrinting/SwiftDeclarationPrinter.swift @@ -168,9 +168,21 @@ public final class SwiftDeclarationPrinter: Sendab /// an empty result every enclosing `BlockList` / `NestedDeclaration` /// skips without a stray break. Ruled before the start event, so a /// filtered definition leaves no unpaired `definitionPrintStarted`. + /// + /// The four per-definition entries (`printTypeDefinition` / + /// `printProtocolDefinition` / `printExtensionDefinition` / + /// `printDefinition`) run on the demangler's large-stack task executor + /// (`LargeStackTaskExecution.run`, evolution proposal + /// `large-stack-executor-and-cross-version-parallelism`): a host printing + /// one declaration at a time — RuntimeViewer's per-type export bypasses + /// `printRoot` — gets inline demangling too. Reached from `printRoot` or + /// from a parent's nested-children loop the task is already on the + /// executor and the wrap is a no-op. public func printTypeDefinition(_ typeDefinition: TypeDefinition, level: Int = 1, displayParentName: Bool = false) async throws -> SemanticString { guard !isExcludedByExportFilter(typeDefinition) else { return SemanticString() } - return try await printIncludedTypeDefinition(typeDefinition, level: level, displayParentName: displayParentName) + return try await LargeStackTaskExecution.run { + try await printIncludedTypeDefinition(typeDefinition, level: level, displayParentName: displayParentName) + } } @SemanticStringBuilder @@ -252,7 +264,9 @@ public final class SwiftDeclarationPrinter: Sendab /// trailing default-implementation extensions go with it. public func printProtocolDefinition(_ protocolDefinition: ProtocolDefinition, level: Int = 1, displayParentName: Bool = false) async throws -> SemanticString { guard !isExcludedByExportFilter(protocolDefinition) else { return SemanticString() } - return try await printIncludedProtocolDefinition(protocolDefinition, level: level, displayParentName: displayParentName) + return try await LargeStackTaskExecution.run { + try await printIncludedProtocolDefinition(protocolDefinition, level: level, displayParentName: displayParentName) + } } @SemanticStringBuilder @@ -324,7 +338,12 @@ public final class SwiftDeclarationPrinter: Sendab /// intact on that path too. public func printExtensionDefinition(_ extensionDefinition: ExtensionDefinition, level: Int = 1) async throws -> SemanticString { guard !isExcludedByExportFilter(extensionDefinition) else { return SemanticString() } + return try await LargeStackTaskExecution.run { + try await printExtensionDefinitionContents(extensionDefinition, level: level) + } + } + private func printExtensionDefinitionContents(_ extensionDefinition: ExtensionDefinition, level: Int) async throws -> SemanticString { let printingContext = SwiftIndexEvents.PrintingContext(name: extensionDefinition.extensionName.name, kind: .extension) eventDispatcher.dispatch(.definitionPrintStarted(context: printingContext)) @@ -462,8 +481,14 @@ public final class SwiftDeclarationPrinter: Sendab } } - @SemanticStringBuilder public func printDefinition(_ definition: some Definition, level: Int = 1) async throws -> SemanticString { + try await LargeStackTaskExecution.run { + try await printDefinitionContents(definition, level: level) + } + } + + @SemanticStringBuilder + private func printDefinitionContents(_ definition: some Definition, level: Int) async throws -> SemanticString { if let mutableDefinition = definition as? MutableDefinition, !mutableDefinition.isIndexed { try await mutableDefinition.index(in: machO) } diff --git a/Sources/Utilities/BoundedConcurrentMap.swift b/Sources/Utilities/BoundedConcurrentMap.swift new file mode 100644 index 00000000..2e583b7b --- /dev/null +++ b/Sources/Utilities/BoundedConcurrentMap.swift @@ -0,0 +1,50 @@ +extension Collection where Element: Sendable { + /// Transforms every element with at most `maximumConcurrency` transforms + /// in flight, returning the results in source order. + /// + /// Submission is windowed: the first `maximumConcurrency` elements start + /// at once and each completion admits the next, so a bounded number of + /// transforms ever runs concurrently — the shape cross-version preparation + /// needs (evolution proposal + /// `large-stack-executor-and-cross-version-parallelism`), where each + /// in-flight transform holds an indexed image's memory and a thread. A + /// `maximumConcurrency` of 1 runs the elements strictly one after the + /// other, in order; values below 1 count as 1. + /// + /// The first failure is rethrown. Elements not yet started never start; + /// transforms already in flight run to completion first (a task group + /// waits for its children), and their results are discarded. + /// + /// Child tasks inherit the caller's task executor preference, so under + /// `LargeStackTaskExecution.run` every transform runs on the large-stack + /// executor. + public func concurrentMap( + maximumConcurrency: Int, + _ transform: @escaping @Sendable (Element) async throws -> Result + ) async throws -> [Result] { + let window = Swift.max(1, maximumConcurrency) + return try await withThrowingTaskGroup(of: (index: Int, result: Result).self) { group in + var results = [Result?](repeating: nil, count: count) + var pending = enumerated().makeIterator() + + var started = 0 + while started < window, let (index, element) = pending.next() { + group.addTask { (index, try await transform(element)) } + started += 1 + } + + while let (index, result) = try await group.next() { + results[index] = result + if let (nextIndex, nextElement) = pending.next() { + group.addTask { (nextIndex, try await transform(nextElement)) } + } + } + + return results.map { result in + // Every index is filled once `next()` returns nil without + // throwing: each submitted task reports exactly one index. + result! + } + } + } +} diff --git a/Sources/swift-section/Commands/DiffCommand.swift b/Sources/swift-section/Commands/DiffCommand.swift index e24f21cc..05385529 100644 --- a/Sources/swift-section/Commands/DiffCommand.swift +++ b/Sources/swift-section/Commands/DiffCommand.swift @@ -60,6 +60,16 @@ struct DiffCommand: AsyncParsableCommand { @Option(name: .shortAndLong, help: "Write the report to this path instead of stdout.", completion: .file()) var outputPath: String? + @Option(name: .long, help: "How many inputs to index at once (default: the processor count). Pass 1 to index the old side, then the new side.") + var jobs: Int? + + /// The concurrency window for indexing the two sides (evolution proposal + /// `large-stack-executor-and-cross-version-parallelism`): both inputs are + /// independent files, so by default they index in parallel. + private var maximumConcurrentPreparations: Int { + jobs ?? ProcessInfo.processInfo.activeProcessorCount + } + func run() async throws { let abiDiff: ABIDiff? if interface { @@ -78,13 +88,15 @@ struct DiffCommand: AsyncParsableCommand { // to its printers, so this is what puts a dropped declaration on // stderr instead of leaving it to `Dispatcher`'s os_log floor, which // a CLI operator never sees. - log("Indexing old binary…") let oldBuilder = SwiftDiffableInterfaceBuilder(eventHandlers: [ConsoleEventHandler()], in: oldMachO) - try await oldBuilder.prepare() - - log("Indexing new binary…") let newBuilder = SwiftDiffableInterfaceBuilder(eventHandlers: [ConsoleEventHandler()], in: newMachO) - try await newBuilder.prepare() + // Old side first in the window, so `--jobs 1` is the historical + // order; with a wider window the two index side by side and their + // diagnostics interleave on stderr. + log(maximumConcurrentPreparations > 1 ? "Indexing old and new binaries…" : "Indexing old binary, then new binary…") + _ = try await [oldBuilder, newBuilder].concurrentMap(maximumConcurrency: maximumConcurrentPreparations) { builder in + try await builder.prepare() + } // Only the `--fail-on-breaking` CI gate needs the ABI diff on the // annotated-interface path. @@ -109,8 +121,10 @@ struct DiffCommand: AsyncParsableCommand { // The change-list path is snapshot-based either way, so each side // may be a binary (indexed and frozen here) or a persisted // baseline (decoded, with its format version validated). - let oldDocument = try await loadDocument(at: oldPath) - let newDocument = try await loadDocument(at: newPath) + let documents = try await [oldPath, newPath].concurrentMap(maximumConcurrency: maximumConcurrentPreparations) { path in + try await loadDocument(at: path) + } + let (oldDocument, newDocument) = (documents[0], documents[1]) log("Diffing…") let diff = ABIDiffer().diff(old: oldDocument, new: newDocument) @@ -147,6 +161,9 @@ struct DiffCommand: AsyncParsableCommand { if format != nil, !interface { throw ValidationError("--format only applies to the annotated interface; pass --interface.") } + if let jobs, jobs < 1 { + throw ValidationError("--jobs must be at least 1.") + } if cacheImageName != nil, cacheImagePath != nil { throw ValidationError("--cache-image-name and --cache-image-path are mutually exclusive; pass only one.") } diff --git a/Sources/swift-section/Commands/EvolutionCommand.swift b/Sources/swift-section/Commands/EvolutionCommand.swift index 1ab510b6..673b87b3 100644 --- a/Sources/swift-section/Commands/EvolutionCommand.swift +++ b/Sources/swift-section/Commands/EvolutionCommand.swift @@ -51,6 +51,18 @@ struct EvolutionCommand: AsyncParsableCommand { @Option(name: .shortAndLong, help: "Write the report to this path instead of stdout.", completion: .file()) var outputPath: String? + @Option(name: .long, help: "How many inputs to index at once (default: the processor count). Pass 1 to index the versions one after the other, oldest first.") + var jobs: Int? + + /// The concurrency window for indexing the inputs (evolution proposal + /// `large-stack-executor-and-cross-version-parallelism`): every input is + /// an independent file, so by default up to one per processor index at + /// once. Each in-flight input holds its indexed image in memory, which is + /// what `--jobs` trades against. + private var maximumConcurrentPreparations: Int { + jobs ?? ProcessInfo.processInfo.activeProcessorCount + } + func run() async throws { let explicitLabels = try ABISnapshotInputLoader.parseLabels(labels, inputCount: inputPaths.count) @@ -59,9 +71,8 @@ struct EvolutionCommand: AsyncParsableCommand { return } - var documents: [ABISnapshotDocument] = [] - for (index, inputPath) in inputPaths.enumerated() { - let document = try await ABISnapshotInputLoader.loadDocument( + let documents = try await Array(inputPaths.enumerated()).concurrentMap(maximumConcurrency: maximumConcurrentPreparations) { index, inputPath in + try await ABISnapshotInputLoader.loadDocument( path: inputPath, architecture: architecture, isDyldSharedCache: isDyldSharedCache, @@ -70,7 +81,6 @@ struct EvolutionCommand: AsyncParsableCommand { label: explicitLabels[index], log: log ) - documents.append(document) } // Snapshot inputs may already carry a provenance label; binaries fall @@ -132,8 +142,8 @@ struct EvolutionCommand: AsyncParsableCommand { versions: machOFiles, labels: resolvedLabels ) - log("Indexing \(machOFiles.count) versions…") - try await builder.prepare() + log("Indexing \(machOFiles.count) versions (\(min(maximumConcurrentPreparations, machOFiles.count)) at a time)…") + try await builder.prepare(maximumConcurrentPreparations: maximumConcurrentPreparations) log("Rendering annotated interface…") let annotated = try await builder.printAnnotatedInterface() try emitInterface(annotated.string) @@ -208,6 +218,9 @@ struct EvolutionCommand: AsyncParsableCommand { if json, summaryOnly { throw ValidationError("--json and --summary-only are mutually exclusive.") } + if let jobs, jobs < 1 { + throw ValidationError("--jobs must be at least 1.") + } if cacheImageName != nil, cacheImagePath != nil { throw ValidationError("--cache-image-name and --cache-image-path are mutually exclusive; pass only one.") } diff --git a/Sources/swift-section/Version.swift b/Sources/swift-section/Version.swift index 645acd6c..d16524ba 100644 --- a/Sources/swift-section/Version.swift +++ b/Sources/swift-section/Version.swift @@ -2,5 +2,5 @@ // When bumping: also add Changelogs/.md, then tag the release with the same string. // Verified by .github/workflows/version-check.yml (PR) and .github/workflows/release.yml (tag). enum BundledVersion { - static let value = "0.18.0" + static let value = "0.19.0" } diff --git a/Tests/MachOSymbolsTests/LargeStackTaskExecutionTests.swift b/Tests/MachOSymbolsTests/LargeStackTaskExecutionTests.swift new file mode 100644 index 00000000..0d800878 --- /dev/null +++ b/Tests/MachOSymbolsTests/LargeStackTaskExecutionTests.swift @@ -0,0 +1,113 @@ +import Foundation +import Testing +@_spi(Internals) import Demangling +@testable import MachOSymbols + +/// Pins `LargeStackTaskExecution.run` (evolution proposal +/// `large-stack-executor-and-cross-version-parallelism`): on a runtime with +/// task executors the body runs on one of the demangler's 16MB executor +/// threads and every demangler entry inside it runs inline (no hop to the +/// 8MB pool); nested runs do not switch threads; disabled or unsupported, the +/// body runs where the caller was. The upstream executor's own behavior +/// (thread size, QoS class, fallbacks) is pinned by swift-demangling's +/// `LargeStackTaskExecutorTests`; these tests cover the adoption seam only. +/// +/// Serialized: `disabledRunsTheBodyOnTheCallersExecutor` flips the +/// process-wide switch, and a parallel sibling asserting on the executor +/// thread would read it mid-flip. +@Suite(.serialized) +struct LargeStackTaskExecutionTests { + /// The upstream executor names its workers after itself + /// (`swift-demangling.task-executor.`); the prefix is the observable + /// identity of "an executor thread" from this side of the module boundary. + private static let executorThreadNamePrefix = "swift-demangling.task-executor." + + private static func currentThreadName() -> String { + var buffer = [CChar](repeating: 0, count: 128) + pthread_getname_np(pthread_self(), &buffer, buffer.count) + return String(cString: buffer) + } + + private static func currentThread() -> mach_port_t { + pthread_mach_thread_np(pthread_self()) + } + + @Test func bodyRunsOnAnExecutorThreadWhenSupported() async { + guard LargeStackTaskExecution.isSupported else { return } + let (stackSize, threadName) = await LargeStackTaskExecution.run { + (pthread_get_stacksize_np(pthread_self()), Self.currentThreadName()) + } + #expect(stackSize >= 16 * 1024 * 1024, "stack was \(stackSize) bytes") + #expect(threadName.hasPrefix(Self.executorThreadNamePrefix), "ran on \(threadName)") + } + + /// The property the adoption exists for: inside the body, the demangler's + /// stack probe passes, so its blocking and suspending entries stay on the + /// task's thread instead of hopping to a pool worker. + @Test func demanglerEntriesInsideTheBodyDoNotHop() async { + guard LargeStackTaskExecution.isSupported else { return } + let observation = await LargeStackTaskExecution.run { + let taskThread = Self.currentThread() + let blockingCallThread: mach_port_t = StackSafeExecutor.execute { Self.currentThread() } + let suspendingCallThread: mach_port_t = await StackSafeExecutor.executeAsync { Self.currentThread() } + return (taskThread, blockingCallThread, suspendingCallThread) + } + #expect(observation.1 == observation.0, "execute hopped off the executor thread") + #expect(observation.2 == observation.0, "executeAsync hopped off the executor thread") + } + + /// Every wrapped entry point reached from another wrapped entry point + /// (`printRoot` → `printTypeDefinition`, a parent's nested-children loop) + /// nests a run inside a run; the inner one must not move the task. + @Test func nestedRunsStayOnTheSameThread() async { + guard LargeStackTaskExecution.isSupported else { return } + let (outerThread, innerThread) = await LargeStackTaskExecution.run { + let outer = Self.currentThread() + let inner = await LargeStackTaskExecution.run { Self.currentThread() } + return (outer, inner) + } + #expect(innerThread == outerThread) + } + + /// Child tasks inherit the preference (SE-0417) — the cross-version + /// parallel preparation relies on this. + @Test func childTasksInheritTheExecutor() async { + guard LargeStackTaskExecution.isSupported else { return } + let childThreadNames = await LargeStackTaskExecution.run { + await withTaskGroup(of: String.self) { group in + for _ in 0 ..< 3 { + group.addTask { Self.currentThreadName() } + } + return await group.reduce(into: [String]()) { $0.append($1) } + } + } + #expect(childThreadNames.count == 3) + for threadName in childThreadNames { + #expect(threadName.hasPrefix(Self.executorThreadNamePrefix), "child ran on \(threadName)") + } + } + + @Test func disabledRunsTheBodyOnTheCallersExecutor() async { + let wasEnabled = LargeStackTaskExecution.isEnabled + LargeStackTaskExecution.isEnabled = false + defer { LargeStackTaskExecution.isEnabled = wasEnabled } + + let callerThread = Self.currentThread() + let (bodyThread, threadName) = await LargeStackTaskExecution.run { (Self.currentThread(), Self.currentThreadName()) } + // No switch at all: a non-suspending body on the caller's executor + // completes on the very thread that entered it. + #expect(bodyThread == callerThread) + #expect(!threadName.hasPrefix(Self.executorThreadNamePrefix), "ran on \(threadName)") + } + + @Test func valuesAndErrorsPassThrough() async throws { + struct Failure: Error, Equatable {} + + let value = await LargeStackTaskExecution.run { 42 } + #expect(value == 42) + + await #expect(throws: Failure.self) { + try await LargeStackTaskExecution.run { throw Failure() } + } + } +} diff --git a/Tests/SwiftInterfaceTests/BoundedConcurrentMapTests.swift b/Tests/SwiftInterfaceTests/BoundedConcurrentMapTests.swift new file mode 100644 index 00000000..1e2cb997 --- /dev/null +++ b/Tests/SwiftInterfaceTests/BoundedConcurrentMapTests.swift @@ -0,0 +1,136 @@ +import Foundation +import Testing +import MachOFoundation + +/// Pins `concurrentMap(maximumConcurrency:_:)` (evolution proposal +/// `large-stack-executor-and-cross-version-parallelism`), the windowed +/// scheduler behind cross-version preparation: source-ordered results, a +/// window that is never exceeded, a window of 1 that is strictly serial, +/// genuine concurrency inside the window, and first-failure semantics that +/// never start the elements still pending. +/// +/// No wall-clock assertions: the ordering and window facts are recorded +/// through locks and rendezvous, so a loaded machine cannot fail them. +@Suite +struct BoundedConcurrentMapTests { + /// Records start/end events and the in-flight high-water mark. + private final class Ledger: @unchecked Sendable { + private let lock = NSLock() + private var inFlight = 0 + private(set) var maximumInFlight = 0 + private(set) var events: [String] = [] + + func start(_ element: Int) { + lock.withLock { + inFlight += 1 + maximumInFlight = max(maximumInFlight, inFlight) + events.append("start \(element)") + } + } + + func end(_ element: Int) { + lock.withLock { + inFlight -= 1 + events.append("end \(element)") + } + } + } + + /// A one-shot gate: `wait()` suspends until `open()` — a rendezvous that + /// completes only if the waiter and the opener run concurrently. + private actor Gate { + private var isOpen = false + private var waiters: [CheckedContinuation] = [] + + func wait() async { + if isOpen { return } + await withCheckedContinuation { waiters.append($0) } + } + + func open() { + isOpen = true + for waiter in waiters { waiter.resume() } + waiters.removeAll() + } + } + + @Test func resultsKeepSourceOrderWhateverTheCompletionOrder() async throws { + let elements = Array(0 ..< 12) + let results = try await elements.concurrentMap(maximumConcurrency: 4) { element in + // Later elements finish first when they can. + try await Task.sleep(nanoseconds: UInt64(12 - element) * 1_000_000) + return element * 10 + } + #expect(results == elements.map { $0 * 10 }) + } + + @Test func theWindowIsNeverExceeded() async throws { + let ledger = Ledger() + _ = try await Array(0 ..< 10).concurrentMap(maximumConcurrency: 3) { element in + ledger.start(element) + try await Task.sleep(nanoseconds: 5_000_000) + ledger.end(element) + } + #expect(ledger.maximumInFlight <= 3) + #expect(ledger.events.count == 20) + } + + @Test func aWindowOfOneIsStrictlySerialInSourceOrder() async throws { + let ledger = Ledger() + _ = try await Array(0 ..< 5).concurrentMap(maximumConcurrency: 1) { element in + ledger.start(element) + await Task.yield() + ledger.end(element) + } + let expected = (0 ..< 5).flatMap { ["start \($0)", "end \($0)"] } + #expect(ledger.events == expected) + #expect(ledger.maximumInFlight == 1) + } + + @Test func valuesBelowOneCountAsOne() async throws { + let ledger = Ledger() + _ = try await Array(0 ..< 3).concurrentMap(maximumConcurrency: 0) { element in + ledger.start(element) + await Task.yield() + ledger.end(element) + } + #expect(ledger.maximumInFlight == 1) + } + + /// Two elements in a window of two must run at the same time: the first + /// waits on a gate only the second opens. A serial scheduler would never + /// reach the opener — hence the time limit, which is the failure mode. + @Test(.timeLimit(.minutes(1))) func elementsInsideTheWindowRunConcurrently() async throws { + let gate = Gate() + let results = try await [0, 1].concurrentMap(maximumConcurrency: 2) { element in + if element == 0 { + await gate.wait() + } else { + await gate.open() + } + return element + } + #expect(results == [0, 1]) + } + + @Test func theFirstFailureIsRethrownAndPendingElementsNeverStart() async { + struct Failure: Error, Equatable { + let element: Int + } + let ledger = Ledger() + await #expect(throws: Failure(element: 1)) { + _ = try await Array(0 ..< 5).concurrentMap(maximumConcurrency: 1) { element in + ledger.start(element) + defer { ledger.end(element) } + if element == 1 { throw Failure(element: element) } + } + } + // Serial window: element 0 completed, element 1 threw, 2…4 never started. + #expect(ledger.events == ["start 0", "end 0", "start 1", "end 1"]) + } + + @Test func emptyInputYieldsEmptyOutput() async throws { + let results = try await [Int]().concurrentMap(maximumConcurrency: 4) { $0 } + #expect(results.isEmpty) + } +} diff --git a/Tests/SwiftInterfaceTests/SwiftEvolutionInterfaceBuilderTests.swift b/Tests/SwiftInterfaceTests/SwiftEvolutionInterfaceBuilderTests.swift index d640976c..a3311218 100644 --- a/Tests/SwiftInterfaceTests/SwiftEvolutionInterfaceBuilderTests.swift +++ b/Tests/SwiftInterfaceTests/SwiftEvolutionInterfaceBuilderTests.swift @@ -135,15 +135,52 @@ struct SwiftEvolutionInterfaceBuilderTests { } } - private func preparedBuilder() async throws -> AnySwiftEvolutionInterfaceBuilder { + private func preparedBuilder(maximumConcurrentPreparations: Int = ProcessInfo.processInfo.activeProcessorCount) async throws -> AnySwiftEvolutionInterfaceBuilder { let builder = try AnySwiftEvolutionInterfaceBuilder( versions: try loadFixtureMachOFiles(), labels: ["1.0", "2.0", "3.0"] ) - try await builder.prepare() + try await builder.prepare(maximumConcurrentPreparations: maximumConcurrentPreparations) return builder } + // MARK: - Cross-version parallel preparation + + /// `prepare(maximumConcurrentPreparations:)` (evolution proposal + /// `large-stack-executor-and-cross-version-parallelism`) indexes the + /// versions concurrently; the annotated interface, the structured stream + /// and the evolution's JSON must be byte-identical to the serial + /// (`1`) preparation — the window is a scheduling knob, never a semantic + /// one. + @Test func parallelPreparationMatchesSerialPreparation() async throws { + let serialBuilder = try await preparedBuilder(maximumConcurrentPreparations: 1) + let parallelBuilder = try await preparedBuilder(maximumConcurrentPreparations: 3) + + let serialInterface = try await serialBuilder.printAnnotatedInterface().string + let parallelInterface = try await parallelBuilder.printAnnotatedInterface().string + #expect(serialInterface == parallelInterface) + #expect(serialInterface.contains("removed in 2.0")) + + let serialBlocks = try await serialBuilder.annotatedBlocks() + let parallelBlocks = try await parallelBuilder.annotatedBlocks() + #expect(serialBlocks.map { $0.map(\.content.string) } == parallelBlocks.map { $0.map(\.content.string) }) + + let encoder = ABIJSON.encoder() + let serialEvolution = try encoder.encode(try #require(serialBuilder.evolution)) + let parallelEvolution = try encoder.encode(try #require(parallelBuilder.evolution)) + #expect(serialEvolution == parallelEvolution) + } + + /// A window wider than the version count and a window below 1 are both + /// clamped, not rejected. + @Test func preparationWindowIsClampedNotValidated() async throws { + let wideBuilder = try await preparedBuilder(maximumConcurrentPreparations: 64) + let narrowBuilder = try await preparedBuilder(maximumConcurrentPreparations: 0) + let wideInterface = try await wideBuilder.printAnnotatedInterface().string + let narrowInterface = try await narrowBuilder.printAnnotatedInterface().string + #expect(wideInterface == narrowInterface) + } + // MARK: - The annotated interface @Test func annotatedInterfaceCarriesEveryLifecycleAnnotation() async throws { diff --git a/Tests/SwiftSectionCommandTests/DiffCommandValidationTests.swift b/Tests/SwiftSectionCommandTests/DiffCommandValidationTests.swift new file mode 100644 index 00000000..8d46d7c9 --- /dev/null +++ b/Tests/SwiftSectionCommandTests/DiffCommandValidationTests.swift @@ -0,0 +1,45 @@ +import Foundation +import Testing +import ArgumentParser +@testable import swift_section + +/// Pins `swift-section diff`'s `--jobs` option (evolution proposal +/// `large-stack-executor-and-cross-version-parallelism`): the two sides index +/// concurrently by default, `--jobs 1` restores the old-then-new order, and a +/// window below 1 is rejected at validation time rather than clamped silently +/// — on the command line a zero is a typo, not a request. +@Suite +struct DiffCommandValidationTests { + private func expectValidationFailure(_ arguments: [String], messagePart: String) { + #expect( + "parsing \(arguments) should fail validation", + performing: { + _ = try DiffCommand.parse(arguments) + }, + throws: { error in + DiffCommand.message(for: error).contains(messagePart) + } + ) + } + + @Test func jobsBelowOneIsRejected() { + expectValidationFailure( + ["--jobs", "0", "old.dylib", "new.dylib"], + messagePart: "--jobs must be at least 1" + ) + } + + @Test func jobsParsesAndDefaultsToAbsent() throws { + let explicit = try DiffCommand.parse(["--jobs", "1", "old.dylib", "new.dylib"]) + #expect(explicit.jobs == 1) + let implicit = try DiffCommand.parse(["old.dylib", "new.dylib"]) + #expect(implicit.jobs == nil) + } + + @Test func jobsParsesAlongsideTheInterfaceOptions() throws { + let command = try DiffCommand.parse(["--interface", "--format", "unified", "--jobs", "2", "old.dylib", "new.dylib"]) + #expect(command.interface) + #expect(command.format == .unified) + #expect(command.jobs == 2) + } +} diff --git a/Tests/SwiftSectionCommandTests/EvolutionCommandValidationTests.swift b/Tests/SwiftSectionCommandTests/EvolutionCommandValidationTests.swift index 26148c24..88b937d2 100644 --- a/Tests/SwiftSectionCommandTests/EvolutionCommandValidationTests.swift +++ b/Tests/SwiftSectionCommandTests/EvolutionCommandValidationTests.swift @@ -41,6 +41,22 @@ struct EvolutionCommandValidationTests { ) } + // MARK: - `--jobs` (evolution proposal `large-stack-executor-and-cross-version-parallelism`) + + @Test func jobsBelowOneIsRejected() { + expectValidationFailure( + ["--jobs", "0", "old.dylib", "new.dylib"], + messagePart: "--jobs must be at least 1" + ) + } + + @Test func jobsParsesAndDefaultsToAbsent() throws { + let explicit = try EvolutionCommand.parse(["--jobs", "2", "old.dylib", "new.dylib"]) + #expect(explicit.jobs == 2) + let implicit = try EvolutionCommand.parse(["old.dylib", "new.dylib"]) + #expect(implicit.jobs == nil) + } + @Test func interfaceParsesAlongsideTheSharedOptions() throws { let command = try EvolutionCommand.parse([ "--interface", "--labels", "1.0,2.0", "--fail-on-breaking", "old.dylib", "new.dylib", From 0896a71af1b49b965169b34798bbf87432cee8f8 Mon Sep 17 00:00:00 2001 From: Mx-Iris Date: Thu, 3 Sep 2026 22:31:00 +0800 Subject: [PATCH 3/4] docs(evolution): record PR #122 on the large-stack executor proposal --- .../draft-large-stack-executor-and-cross-version-parallelism.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentations/Evolutions/draft-large-stack-executor-and-cross-version-parallelism.md b/Documentations/Evolutions/draft-large-stack-executor-and-cross-version-parallelism.md index 44549a83..ebf83d6a 100644 --- a/Documentations/Evolutions/draft-large-stack-executor-and-cross-version-parallelism.md +++ b/Documentations/Evolutions/draft-large-stack-executor-and-cross-version-parallelism.md @@ -6,7 +6,7 @@ - **最后更新**: 2026-09-03 - **所属愿景**: 无 - **关联提案**: swift-demangling 提案 0014「大栈 TaskExecutor」（上游前置，执行器本体在那边；本提案只做接入）；[draft-self-contained-abi-layer](draft-self-contained-abi-layer.md)（同一轮调研产物，互不依赖） -- **实现分支 / PR**: `feature/large-stack-executor-and-cross-version-parallelism`（基于 `feature/self-contained-abi-layer`，PR 待开） +- **实现分支 / PR**: `feature/large-stack-executor-and-cross-version-parallelism`，[PR #122](https://github.com/MxIris-Reverse-Engineering/MachOSwiftSection/pull/122)（堆叠在 ABI 提案的 PR #121 之上，base 随其合并切到 `next`） - **配套文档**: [LargeStackTaskExecutorAdoption.md](../Internal/LargeStackTaskExecutorAdoption.md)（实现说明）；任务报告 [2026-09-03-large-stack-executor-and-cross-version-parallelism.md](../Internal/TaskReports/2026-09-03-large-stack-executor-and-cross-version-parallelism.md) ## 摘要 From 94a3d1c1c997f631a4cbd84647365236661d96d0 Mon Sep 17 00:00:00 2001 From: Mx-Iris Date: Fri, 4 Sep 2026 14:43:00 +0800 Subject: [PATCH 4/4] =?UTF-8?q?fix(concurrency):=20review=20follow-ups=20?= =?UTF-8?q?=E2=80=94=20cancellation-aware=20concurrentMap,=20serialized=20?= =?UTF-8?q?event=20delivery,=20labeled=20diagnostics=20(proposal=20large-s?= =?UTF-8?q?tack-executor-and-cross-version-parallelism)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fifteen review findings on PR #122 (Roadmaps/2026-09-04-pr122-review- findings.md): five defects, one false positive, nine design/style calls. - concurrentMap(maximumConcurrency:) submits through addTaskUnlessCancelled and fails with CancellationError once the calling task is cancelled; the first version used addTask, which a cancelled group still accepts, so a cancelled multi-version preparation indexed every remaining version to the end (reproduced independently by the reviewer). Never a partial array — the unwrap below would trap. - LargeStackTaskExecutionTests guard isEnabled as well as isSupported: under MACHO_SWIFT_SECTION_LARGE_STACK_EXECUTOR=0 (the A/B configuration) the executor-thread assertions went red for a reason the tests do not control. - That variable now accepts 0 / false / no / off (case-insensitive) as off; the literal-"0" comparison let =false measure the executor twice. - parallelPreparationMatchesSerialPreparation prepares the parallel builder FIRST, so it meets cold caches; a three-way barrier pins that a window of three admits three. - SwiftIndexEvents.Dispatcher serializes handler invocation process-wide with a recursive lock: Handler has no Sendable requirement and parallel preparation shares one host handler across N dispatchers on N tasks. - ConsoleEventHandler(label:) prefixes [label]; diff tags old / new, evolution tags each version (AnySwiftEvolutionInterfaceBuilder gained eventHandlersPerVersion:), snapshot inputs use their label or file name. - The redundant import Utilities is gone. Every behavioral fix verified red under a mutated build. Adjudicated as A25–A33: the import MachOFoundation false positive, per-entry wraps, the concurrentMap name, #isolation forwarding, .serialized vs the process switch, the executor-off window, the default lineage window (kept, user's call), the unavoidable dual availability gate, and TypeDatabase's same-shaped addTask (deferred: no injection seam for a red test). --- AGENTS.md | 4 +- Changelogs/0.19.0.md | 5 +- ...-executor-and-cross-version-parallelism.md | 1 + .../LargeStackTaskExecutorAdoption.md | 16 ++++ .../Internal/ReviewAdjudications.md | 82 +++++++++++++++++++ ...-executor-and-cross-version-parallelism.md | 7 ++ Roadmaps/2026-09-04-pr122-review-findings.md | 37 +++++++++ .../LargeStackTaskExecution.swift | 21 ++++- .../Events/SwiftIndexEvents.swift | 18 +++- .../SwiftIndexEventsHandlers.swift | 45 +++++++--- .../AnySwiftEvolutionInterfaceBuilder.swift | 17 +++- Sources/Utilities/BoundedConcurrentMap.swift | 30 +++++-- .../swift-section/Commands/DiffCommand.swift | 11 +-- .../Commands/EvolutionCommand.swift | 2 +- .../Utilities/ABISnapshotInputLoader.swift | 11 ++- .../LargeStackTaskExecutionTests.swift | 34 +++++++- .../ConsoleEventHandlerLineTests.swift | 40 +++++++++ .../BoundedConcurrentMapTests.swift | 67 +++++++++++++++ .../EventDeliverySerializationTests.swift | 82 +++++++++++++++++++ .../SwiftEvolutionInterfaceBuilderTests.swift | 54 +++++++++++- 20 files changed, 540 insertions(+), 44 deletions(-) create mode 100644 Roadmaps/2026-09-04-pr122-review-findings.md create mode 100644 Tests/SwiftIndexingTests/ConsoleEventHandlerLineTests.swift create mode 100644 Tests/SwiftInterfaceTests/EventDeliverySerializationTests.swift diff --git a/AGENTS.md b/AGENTS.md index 8a0d1ff9..854a7e2f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -131,7 +131,7 @@ The interface generation is split into layered peer modules over a shared `Swift **SwiftDeclaration** - Shared declaration model (base layer for the Swift* modules) - `TypeDefinition`, `ProtocolDefinition`, `ExtensionDefinition`, `FunctionDefinition`, names, kinds, `DefinitionBuilder` - The model retains **descriptor references, not parsed wrappers** (evolution proposal 0002): `TypeDefinition.typeContextDescriptorWrapper`, `ExtensionDefinition.protocolConformanceDescriptor`, `ProtocolDefinition.protocolDescriptor`. The full wrappers (`TypeContextWrapper` / `ProtocolConformance` / `Protocol`, trailing objects included) are rebuilt on demand via `materializedTypeContext(in:)` / `materializedProtocolConformance(in:)` / `materializedProtocol(in:)`. **Materialization discipline**: at most one materialization per operation (index it / print it / specialize it), threaded through as a local variable; never a per-access computed property, and the result is never cached on the definition — caching would re-accumulate, in browse order, the memory the slimming reclaimed. `DeclarationModelInstanceSizeTests` pins the instance-size ceilings. -- `SwiftIndexEvents` - event namespace (Payload/Dispatcher/Handler) emitted by both indexer and printer. **Library code never writes to a process stream** (evolution proposal 0005): every degradation — a dropped definition, a skipped descriptor, a dependency that would not load — is dispatched as an event, and the *host* decides where it lands (a GUI attaches `OSLogEventHandler`, the CLI attaches `ConsoleEventHandler`, which reports on **stderr**; stdout carries the generated Swift / JSON, so writing there corrupts the product output — issue #102). `Dispatcher.dispatch` has a **floor**: with no handler attached it reports failures through `#log` rather than dropping them, so forgetting to attach a sink degrades to "somewhere findable" instead of silence. What counts as a failure is `Payload.unhandledFailureDescription`, an exhaustive `switch` on purpose — a new failure case must opt *in* explicitly, since the alternative silently escapes the floor. Two modules sit below the event layer and cannot reach it (`SwiftDeclaration` depends on `SwiftDeclarationRendering`, so naming the event types there would cycle): `Node+OpaqueType` takes an injected `OpaqueTypeDegradationReporter` closure, `MultiPayloadEnumDescriptorCache` logs directly; both land on the same `#log` floor. Never use `FileHandle.standardError/Output.write(_:)` — that overload raises an uncatchable ObjC exception on a closed or broken stream and aborts the host; use `fputs` / `fwrite` (`write(contentsOf:)` needs macOS 10.15.4, above this package's 10.15 floor). Pinned by a source scan (`PrintFailureEventTests.libraryModulesWriteToNoProcessStream`) carrying an explicit, shrink-only list of baseline offenders. See [Documentations/Internal/EventBasedDegradationReporting.md](Documentations/Internal/EventBasedDegradationReporting.md) +- `SwiftIndexEvents` - event namespace (Payload/Dispatcher/Handler) emitted by both indexer and printer. **Library code never writes to a process stream** (evolution proposal 0005): every degradation — a dropped definition, a skipped descriptor, a dependency that would not load — is dispatched as an event, and the *host* decides where it lands (a GUI attaches `OSLogEventHandler`, the CLI attaches `ConsoleEventHandler`, which reports on **stderr**; stdout carries the generated Swift / JSON, so writing there corrupts the product output — issue #102). `Dispatcher.dispatch` has a **floor**: with no handler attached it reports failures through `#log` rather than dropping them, so forgetting to attach a sink degrades to "somewhere findable" instead of silence. Handler invocation is **serialized process-wide** (one recursive lock across every dispatcher, evolution proposal `large-stack-executor-and-cross-version-parallelism`): `Handler` has no `Sendable` requirement and cross-version preparation shares one host handler across N dispatchers on N tasks, so the lock is what keeps a stateful handler single-threaded — delivery ORDER across dispatchers is still first-come. `ConsoleEventHandler(label:)` prefixes `[label]` for that same reason (the CLI tags `old` / `new` and each evolution version). What counts as a failure is `Payload.unhandledFailureDescription`, an exhaustive `switch` on purpose — a new failure case must opt *in* explicitly, since the alternative silently escapes the floor. Two modules sit below the event layer and cannot reach it (`SwiftDeclaration` depends on `SwiftDeclarationRendering`, so naming the event types there would cycle): `Node+OpaqueType` takes an injected `OpaqueTypeDegradationReporter` closure, `MultiPayloadEnumDescriptorCache` logs directly; both land on the same `#log` floor. Never use `FileHandle.standardError/Output.write(_:)` — that overload raises an uncatchable ObjC exception on a closed or broken stream and aborts the host; use `fputs` / `fwrite` (`write(contentsOf:)` needs macOS 10.15.4, above this package's 10.15 floor). Pinned by a source scan (`PrintFailureEventTests.libraryModulesWriteToNoProcessStream`) carrying an explicit, shrink-only list of baseline offenders. See [Documentations/Internal/EventBasedDegradationReporting.md](Documentations/Internal/EventBasedDegradationReporting.md) **SwiftIndexing** - Builds the `SwiftDeclaration` model from a Mach-O image - `SwiftDeclarationIndexer.prepare()` runs its body on the demangler's large-stack task executor through `LargeStackTaskExecution.run` (evolution proposal `large-stack-executor-and-cross-version-parallelism`; see the `MachOSymbols` entry) — as do `SwiftInterfaceBuilder.prepare()` / `printRoot()`, `SwiftDiffableInterfaceBuilder.prepare()`, the evolution builder's `prepare` / render entries, the diff renderer's two entries, the printer's four per-definition entries and the six `Dumpable.dump(using:in:)` conformers. Output is independent of the executor; the wrap is about where the task runs, not what it produces @@ -227,7 +227,7 @@ Printing and indexing are peers — neither depends on the other. - **MachOFoundation** - `MachOBase` plus `MachOSymbols` and `MachODependencies` — the umbrella for everything above the ABI model - **MachOReading** - File reading abstractions - **MachOResolving** - Address/offset resolution; also home of the symbol **value** types `Symbol` (offset + name + `isExternal`), `Symbols` (every name at one offset — identical code folding leaves several) and `SymbolOrElement` (a bind-table symbol or a resolved element). They carry no lookup behavior: `Symbols` is deliberately not `Resolvable`, because "the symbols at this offset" is a query against the symbol index, not a read -- **MachOSymbols** - Symbol table parsing and demangling — the symbol *index*, one layer above the ABI model; the value types it vends live in `MachOResolving`. Also home of `LargeStackTaskExecution` (evolution proposal `large-stack-executor-and-cross-version-parallelism`): `run(_:)` sets swift-demangling's 16 MB `LargeStackTaskExecutor` (`StackSafeExecutor.taskExecutor`, `@_spi(Internals)`, 0.6.3+) as the task executor preference around a library entry point. The demangler decides per call whether to hop to its 8 MB pool by probing the CALLING thread's remaining stack against a 2 MB floor — cooperative and libdispatch threads carry 512 KB, so an async print loop paid one thread round trip + semaphore wait per printed symbol (8–21 µs, 1.14–2.28×); on an executor thread the probe passes at every entry, synchronous callees included, so the whole pipeline runs inline (the effect of `withLargeStack` extended to a whole task). Nesting is a no-op (already on the executor = no switch), an unstructured `Task {}` does NOT inherit the preference (SE-0417 — never start one inside a wrapped entry; child tasks and default actors do inherit), the main actor keeps its own executor (its 8 MB stack passes the probe anyway), and below macOS 15 / iOS 18 (or off Darwin) `run` executes the body unchanged. Process-wide off switch `isEnabled` (hosts with their own executor) seeded by `MACHO_SWIFT_SECTION_LARGE_STACK_EXECUTOR=0` (the A/B and timing runs compare one binary both ways). See [Documentations/Internal/LargeStackTaskExecutorAdoption.md](Documentations/Internal/LargeStackTaskExecutorAdoption.md). Index-backed lookups: `symbols(offset:)` on `MachORepresentableWithCache`, `Symbol.resolve(from:in:)` and the process-wide `Symbol.resolvesSymbolUsingIndexStore` switch. `SymbolIndexStore`'s offset and member indexes hold their row lists in `SymbolRowBucket` (evolution proposal 0003): the dominant single-row case stays inline in the dictionary slot, only a bucket that collects a second row allocates an array; iteration order is insertion order, so query output is byte-identical to the former `[UInt32]` buckets. The member/typeInfo/thunk-attribute indexes are keyed by printed type name FIRST and interned context node SECOND, and the name key is NOT injective — it is printed with `.interfaceTypeBuilderOnly`, which strips private discriminators, so same-named private types from different files share one name bucket (issue #115: the dump path's name-only lookups merged both types' members into each declaration). Any consumer resolving *one* type's members/info/attributes must use the node-taking overloads (`memberSymbols(of:for:node:in:)`, `methodDescriptorMemberSymbols(of:for:node:in:)`, `typeInfo(for:node:in:)`, `thunkAttributeMembers(of:for:node:in:)`); the name-only forms deliberately flatten every sub-bucket and are only for "all types printing as this name" aggregation. See [Documentations/Internal/PrivateTypeMemberAttribution.md](Documentations/Internal/PrivateTypeMemberAttribution.md). `Storage` also carries the image's **export facts** (evolution proposal 0008): both symtab collection legs filter on `!nlist.isExternal`, so exported symbols' rows come only from the export-trie leg, whose row minting is conditional — trie membership is therefore collected explicitly in the same pass (a per-row bitmap plus a fallback name set for offset-less re-exports), backing the tri-state `isExported(name:in:)` (`nil` = the image has no export information; never annotate then) and `isExportedIncludingDerivedSymbols(name:in:)` (extends over the `Tj`/`Tq`/`Tu`/`TjTu` appended-suffix forms) +- **MachOSymbols** - Symbol table parsing and demangling — the symbol *index*, one layer above the ABI model; the value types it vends live in `MachOResolving`. Also home of `LargeStackTaskExecution` (evolution proposal `large-stack-executor-and-cross-version-parallelism`): `run(_:)` sets swift-demangling's 16 MB `LargeStackTaskExecutor` (`StackSafeExecutor.taskExecutor`, `@_spi(Internals)`, 0.6.3+) as the task executor preference around a library entry point. The demangler decides per call whether to hop to its 8 MB pool by probing the CALLING thread's remaining stack against a 2 MB floor — cooperative and libdispatch threads carry 512 KB, so an async print loop paid one thread round trip + semaphore wait per printed symbol (8–21 µs, 1.14–2.28×); on an executor thread the probe passes at every entry, synchronous callees included, so the whole pipeline runs inline (the effect of `withLargeStack` extended to a whole task). Nesting is a no-op (already on the executor = no switch), an unstructured `Task {}` does NOT inherit the preference (SE-0417 — never start one inside a wrapped entry; child tasks and default actors do inherit), the main actor keeps its own executor (its 8 MB stack passes the probe anyway), and below macOS 15 / iOS 18 (or off Darwin) `run` executes the body unchanged. Process-wide off switch `isEnabled` (hosts with their own executor) seeded by `MACHO_SWIFT_SECTION_LARGE_STACK_EXECUTOR` (`0` / `false` / `no` / `off` = off, anything else or unset = on; the A/B and timing runs compare one binary both ways). `concurrentMap(maximumConcurrency:)` submits through `addTaskUnlessCancelled`: cancelling the caller stops pending submissions and fails the call with `CancellationError`, never a partial array. See [Documentations/Internal/LargeStackTaskExecutorAdoption.md](Documentations/Internal/LargeStackTaskExecutorAdoption.md). Index-backed lookups: `symbols(offset:)` on `MachORepresentableWithCache`, `Symbol.resolve(from:in:)` and the process-wide `Symbol.resolvesSymbolUsingIndexStore` switch. `SymbolIndexStore`'s offset and member indexes hold their row lists in `SymbolRowBucket` (evolution proposal 0003): the dominant single-row case stays inline in the dictionary slot, only a bucket that collects a second row allocates an array; iteration order is insertion order, so query output is byte-identical to the former `[UInt32]` buckets. The member/typeInfo/thunk-attribute indexes are keyed by printed type name FIRST and interned context node SECOND, and the name key is NOT injective — it is printed with `.interfaceTypeBuilderOnly`, which strips private discriminators, so same-named private types from different files share one name bucket (issue #115: the dump path's name-only lookups merged both types' members into each declaration). Any consumer resolving *one* type's members/info/attributes must use the node-taking overloads (`memberSymbols(of:for:node:in:)`, `methodDescriptorMemberSymbols(of:for:node:in:)`, `typeInfo(for:node:in:)`, `thunkAttributeMembers(of:for:node:in:)`); the name-only forms deliberately flatten every sub-bucket and are only for "all types printing as this name" aggregation. See [Documentations/Internal/PrivateTypeMemberAttribution.md](Documentations/Internal/PrivateTypeMemberAttribution.md). `Storage` also carries the image's **export facts** (evolution proposal 0008): both symtab collection legs filter on `!nlist.isExternal`, so exported symbols' rows come only from the export-trie leg, whose row minting is conditional — trie membership is therefore collected explicitly in the same pass (a per-row bitmap plus a fallback name set for offset-less re-exports), backing the tri-state `isExported(name:in:)` (`nil` = the image has no export information; never annotate then) and `isExportedIncludingDerivedSymbols(name:in:)` (extends over the `Tj`/`Tq`/`Tu`/`TjTu` appended-suffix forms) - **MachOPointers** - Pointer types (relative, indirect, etc.), including `SymbolOrElementPointer` (absorbed the former `MachOSymbolPointers` target): an indirectable target that resolves either to a bind-table symbol (`Symbol`, via `MachOBindRebaseResolving`) or to an element - **MachOCaches** - dyld shared cache support - **MachODependencies** - The one dependency-resolution implementation every feature shares (evolution proposal 0017; re-exported by `MachOFoundation`, so nothing above it needs an extra import). `DependencyClosure` walks a root's `LC_LOAD_DYLIB`-family load commands — `.direct` (the root's own list, load-command order) or `.transitive` (breadth-first, so a lazily indexing consumer meets the root's direct dependencies first) — deduplicated by **bare image name** (`DependencyLoadName.bareImageName(of:)`: last path component, FIRST extension stripped — `libobjc.A.dylib` → `libobjc`; the exact rule `MachOImage(name:)` matches on, which is why handing that lookup a raw load path resolves nothing — the bug `SwiftInterfaceBuilderDependencies`'s image initializer carried until this module), root excluded, misses recorded in `unresolvedLoadNames` rather than dropped. Locators: `InProcessDependencyLocator` (active dyld) and `FileDependencyLocator` (`DependencySearchPath`s: explicit files + dyld caches; **exact install path first, ranked bare name second** via `DyldCacheImageSearchMode.matchRank`, so a macOS cache's `/System/iOSSupport` Catalyst SwiftUI never shadows the native one; each cache indexed ONCE lazily, since a per-lookup `machOFile(by:)` scan is `O(dependencies × cache size)`; fat explicit files contribute the root's architecture; unopenable search paths land in `searchPathLoadFailures`, never thrown). Consumers: `SwiftLayout.ImageUniverse` (transitive), `SwiftInterface.SwiftInterfaceBuilderDependencies` (DIRECT on purpose — TypeIndexing generates one SourceKit interface per dependency module, so the transitive closure would bring back the whole-SDK generation that once disabled the target; it dispatches `searchPathLoadFailures` as `renderingDegraded(.dependencyLoad)` events and exposes `unresolvedLoadNames`), `swift-section interface --resolve-c-module-names`. This module sits below the event layer: failures are data, never logged here. See [Documentations/Internal/Modules/MachODependencies.md](Documentations/Internal/Modules/MachODependencies.md) diff --git a/Changelogs/0.19.0.md b/Changelogs/0.19.0.md index a60b7e5a..5c346ab4 100644 --- a/Changelogs/0.19.0.md +++ b/Changelogs/0.19.0.md @@ -6,7 +6,10 @@ A minor release on top of `0.18.0`. Two related changes, both output-neutral: th 1. **Async entry points run on a 16 MB task executor.** The demangler decides per call whether to hop to its 8 MB pool by probing the calling thread's remaining stack; Swift Concurrency's cooperative threads carry 512 KB, so an async print loop paid one thread round trip per printed symbol. `MachOSymbols.LargeStackTaskExecution.run` sets swift-demangling 0.6.3's `LargeStackTaskExecutor` as the task executor preference around every library entry — indexer `prepare()`, `SwiftInterfaceBuilder.prepare()` / `printRoot()`, `SwiftDiffableInterfaceBuilder.prepare()`, the evolution builder's `prepare` / render entries, the diff renderer, the printer's four per-definition entries and the six `Dumpable.dump(using:in:)` conformers — so the whole pipeline demangles, prints and remangles inline. Hosts change nothing. Requires macOS 15 / iOS 18 / tvOS 18 / watchOS 11 / visionOS 2 at runtime; below that (or off Darwin) the body runs exactly as before. `LargeStackTaskExecution.isEnabled` (seeded by `MACHO_SWIFT_SECTION_LARGE_STACK_EXECUTOR=0`) turns it off process-wide. 2. **Cross-version preparation is parallel.** `AnySwiftEvolutionInterfaceBuilder.prepare(maximumConcurrentPreparations:)` (and the pack-generic façade) indexes the versions concurrently, defaulting to the processor count; `1` is the former oldest-first serial order. `swift-section diff` and `swift-section evolution` index their inputs the same way and gained `--jobs N`. The result is byte-identical to serial preparation; only event delivery interleaves. Intra-version parallelism is out of scope (MachOKit's file reads share one `FileHandle`). -3. **`Collection.concurrentMap(maximumConcurrency:_:)`** (`Utilities`): a windowed task-group map — source-ordered results, first failure rethrown, pending elements never started. +3. **`Collection.concurrentMap(maximumConcurrency:_:)`** (`Utilities`): a windowed task-group map — source-ordered results, first failure rethrown, pending elements never started; cancelling the calling task stops submission and fails the call with `CancellationError`. +4. **Event delivery is serialized process-wide.** `SwiftIndexEvents.Dispatcher` takes one recursive lock around handler invocation, so a host handler shared by concurrently prepared versions is never entered concurrently (`Handler` still has no `Sendable` requirement). +5. **Labeled console diagnostics.** `ConsoleEventHandler(label:)` prints `[label]` after the timestamp; `diff` tags its sides `old` / `new`, `evolution` tags each version with its axis label (`AnySwiftEvolutionInterfaceBuilder.init` gained `eventHandlersPerVersion:`), snapshot inputs use their provenance label or file name. stderr only. +6. `MACHO_SWIFT_SECTION_LARGE_STACK_EXECUTOR` accepts `0` / `false` / `no` / `off` (case-insensitive) as off; anything else, or unset, is on. ## Measured diff --git a/Documentations/Evolutions/draft-large-stack-executor-and-cross-version-parallelism.md b/Documentations/Evolutions/draft-large-stack-executor-and-cross-version-parallelism.md index 69cf5964..ad74395f 100644 --- a/Documentations/Evolutions/draft-large-stack-executor-and-cross-version-parallelism.md +++ b/Documentations/Evolutions/draft-large-stack-executor-and-cross-version-parallelism.md @@ -194,4 +194,5 @@ AGENTS.md（`SwiftInterface` / `SwiftIndexing` 条目补执行器一句；测试 | 2026-09-03 | 落地偏差：CLI 的 `dump` 循环不再额外包一层 | 六个 `Dumpable.dump` 已各自包裹，CLI 逐类型进出执行器约一万跳、零点几秒，远小于原来逐符号跳转 | | 2026-09-03 | 核对：库内零处非结构化 `Task {}` | 唯一的 `withTaskGroup` 在 `TypeIndexing.TypeDatabase`，结构化、继承偏好；落地步骤 2 无需改动 | | 2026-09-03 | 验证：全量 1637 测试通过；计时 −16% ~ −23%（单版本）、2.0×（三版本 evolution）；四种配置输出逐字节一致 | 数据见实现说明「实测数据」；0.6.0 → 0.6.3 仅抬 pin 持平，证明 0.6.1 的回归未带入 | +| 2026-09-04 | Review 修复批次（PR #122 review，15 条：真缺陷 5、误报 1、取舍 9） | 修：`concurrentMap` 取消语义（`addTaskUnlessCancelled` + 抛 `CancellationError`）、执行器测试同时挡 `isEnabled`、环境变量接受 `0/false/no/off`、并行等价测试先跑并行、三方 barrier 钉窗口宽度；用户裁定：F2 用 Dispatcher 进程级递归锁串行化投递（不改 `Handler` API）、F6 保持核数、F7 加 `ConsoleEventHandler(label:)` 与 `eventHandlersPerVersion`；F9 重复门判为不可消除（A32）；其余登记 A25–A33。清单见 `Roadmaps/2026-09-04-pr122-review-findings.md` | | 2026-09-03 | 收尾判断：写实现说明；「大栈执行器」入术语表 | 实现说明记录探测机制为何免改调用点、入口清单与嵌套免费、回退与开关、并行安全性与不做版本内并行的原因、计时表；术语在 AGENTS.md / 提案 / 实现说明 / 账本多处出现，登记 `Documentations/Glossary.md` | diff --git a/Documentations/Internal/LargeStackTaskExecutorAdoption.md b/Documentations/Internal/LargeStackTaskExecutorAdoption.md index 8b4682e0..3ec4f955 100644 --- a/Documentations/Internal/LargeStackTaskExecutorAdoption.md +++ b/Documentations/Internal/LargeStackTaskExecutorAdoption.md @@ -61,6 +61,22 @@ MachOKit 自己的读取有上百处 `fileHandle.seek` + `read` 共用一个句 窗口化 `withThrowingTaskGroup`：先提交 `window` 个，每完成一个再提交一个，结果按源序落位。首个错误经 `group.next()` 抛出，task group 在作用域退出时取消并等待在飞的子任务（`prepare` 不检查取消，所以在飞的会跑完，结果丢弃），**尚未启动的元素永远不启动**（`theFirstFailureIsRethrownAndPendingElementsNeverStart`）。哪个错误先到是调度决定的——串行时固定是最旧版本的错误，并行时不一定；只影响错误报文，不影响成功路径。 +### 取消语义 + +`concurrentMap` 用 `addTaskUnlessCancelled` 提交：调用方的 task 被取消后，尚未启动的元素永远不启动，调用抛 `CancellationError`，绝不返回残缺数组（第一版用 `addTask`，取消后的 group 照收子任务，一次被取消的多版本准备会把剩余版本全部索引到底；审查者独立编译复现了这一点）。在飞的 transform 会跑完——索引链路本身不检查取消——结果丢弃。取消发生在最后一个元素提交之后则什么也不改变，结果照常返回。`TypeIndexing.TypeDatabase.index` 的 task group 是同形的旧代码，缺注入缝写不出复现测试，延后（裁决 A33）。 + +### 事件投递串行化 + +`SwiftIndexEvents.Handler` 没有 `Sendable` 约束，而并行准备把宿主传进来的同一个 handler 实例塞给 N 个版本的 dispatcher、在 N 个任务上调用。`Dispatcher.dispatch` 现在用一把**进程级** `NSRecursiveLock` 包住 handler 调用：有状态的宿主 handler 保持单线程，console handler 的一行不会被截断，代价是每个事件一次无竞争锁；递归锁让 handler 里再 dispatch 不死锁。跨 dispatcher 的投递**顺序**仍由谁先到决定。这是 F2 三个选项（加 `Sendable` 约束是破坏性 API 变更、只写文档等于把风险留给宿主）里用户选的。 + +### stderr 归属 + +`ConsoleEventHandler(label:)` 在时间戳后加 `[label]`：`diff` 用 `old` / `new`，`evolution` 每个版本用它的 axis label（通过新 init 参数 `eventHandlersPerVersion(versionIndex, label)`，共享 handler 照旧），snapshot 输入用 provenance label 或文件名。只改 stderr，不碰产品输出。 + +### 环境变量的接受值 + +`MACHO_SWIFT_SECTION_LARGE_STACK_EXECUTOR` 的 `0` / `false` / `no` / `off`（大小写与首尾空白不敏感）为关，其余（含未设置）为开——第一版只比对字面量 `"0"`，`=false` 会把执行器测两遍然后得出「执行器不要钱」的错误结论。`LargeStackTaskExecutionTests` 的执行器线程断言同时挡 `isSupported` 与 `isEnabled`，在该变量关闭的环境下跳过而不是假红。 + ### `--jobs` 在 CLI 校验而库端 clamp 命令行上的 `--jobs 0` 是笔误，`ValidationError("--jobs must be at least 1.")` 立刻报；库 API 的 `maximumConcurrentPreparations` 小于 1 则按 1 处理（`preparationWindowIsClampedNotValidated`），因为宿主可能直接把「核数 - 1」之类的算式传进来，为一个下界抛错不值得。 diff --git a/Documentations/Internal/ReviewAdjudications.md b/Documentations/Internal/ReviewAdjudications.md index c3beccb5..4e9c8bb1 100644 --- a/Documentations/Internal/ReviewAdjudications.md +++ b/Documentations/Internal/ReviewAdjudications.md @@ -293,3 +293,85 @@ - **为什么延后**：14 处分布在三个模块的热路径上，每处都要过一遍渲染 A/B 才敢合；与本 PR 的目标（分层）无关，捆进来只会拖大 review 面。已确认不产生功能分裂：两种写法等价，分裂只是「读代码时要认两种形状」。 - **复审条件**：单独开一个「收编 `implementationOffset`」的清理批次（轻量档提案即可），逐字节 A/B 后合入；届时本条关闭。 +--- + +## A25 — 「六个 `import MachOFoundation` 冗余」（PR #122 review 发现 F11，**误报**） + +- **裁决**：误报（2026-09-04）。 +- **发现**：审查者依据「`Sources/MachOSwiftSection/Exported.swift` 是 `@_exported import MachOFoundation`」判定 `SwiftDump/Dumpable/*+Dumpable.swift` 六处新增的 `import MachOFoundation` 多余。 +- **为什么是误报**：提案 0018 把 ABI 层的再导出收窄到 `MachOBase`（注释原文 "the ABI layer deliberately stops here"），`LargeStackTaskExecution` 住在 `MachOSymbols`，只有 `MachOFoundation` 再导出它；去掉那六行编译不过。审查者读的是 0018 之前的状态。 +- **附带子主张**：① `MachOSymbols` target 未声明 `FoundationToolbox` product 而 `LargeStackTaskExecution.swift` import 它——属实但基线既有（`SymbolIndexStore.swift` / `Symbol.swift` 同样如此），归 A23 同类清理批次；② `AnySwiftEvolutionInterfaceBuilder.swift` 新加的 `import Utilities` 冗余（`MachOSwiftSection → MachOBase → Utilities`）——属实，已删。 +- **复审条件**：无。 + +--- + +## A26 — 打印器四个逐定义入口每次调用都包一层 `LargeStackTaskExecution.run`（PR #122 review 发现 F13） + +- **裁决**：不修（2026-09-04）。 +- **发现**：`printTypeDefinition` / `printProtocolDefinition` / `printExtensionDefinition` / `printDefinition` 每次调用读一次 `isEnabled`（`@Mutex`，`os_unfair_lock`）并进一次 `withTaskExecutorPreference`；全仓 26 个 `run` 调用点，新增入口要同步维护。 +- **为什么不修**：效率论据被本 PR 自己的数据推翻——无竞争 `os_unfair_lock` 约 20 ns，十万次合计约 2 ms，而实测整体快 16–23%；已在执行器上的嵌套 `withTaskExecutorPreference` 不切换（`nestedRunsStayOnTheSameThread` 钉住）。包在逐定义入口是有意的：RuntimeViewer 逐类型导出绕过 `printRoot`，只包 `printRoot` 会漏掉它。 +- **复审条件**：profiling 显示 `run` 的开销在某条路径上可观；或出现第 27 个入口时考虑把「入口 = 包裹」写成 lint 检查。 + +--- + +## A27 — 新 `Collection.concurrentMap(maximumConcurrency:)` 与既有 `Array.concurrentMap(_:)` 同名而语义不同（PR #122 review 发现 F10） + +- **裁决**：不修（2026-09-04）。 +- **发现**：`Sources/Utilities/ConcurrentMap.swift` 的 `concurrentMap(_:)` 是 `DispatchQueue.concurrentPerform` 的同步阻塞版；新函数是 async、窗口化、可抛错。参数标签不同、无重载歧义，纯可读性。 +- **为什么不修**：两者的调用形态（`await` + `try` + `maximumConcurrency:` 标签）已把区别写在调用点上；改名或合并文件是纯搬动。既有同步版零调用方（2026-09-03 调研已记录），更合适的动作是下次清理批次删掉它。 +- **复审条件**：同步版被删或被重新启用时一并统一命名。 + +--- + +## A28 — `LargeStackTaskExecution.run` 未转发 `isolation: isolated (any Actor)? = #isolation`（PR #122 review 发现 F12） + +- **裁决**：不修（2026-09-04）。 +- **发现**：标准的「透传隔离」写法会加一个 `#isolation` 参数再转给 `withTaskExecutorPreference`；`run` 没有。 +- **为什么不修**：`body` 是非 `@Sendable` 闭包，在 actor 隔离上下文里字面量继承调用方隔离，实现说明里「主 actor 保持自己的 executor」仍然成立；差别只是多一次跳转。库 target 未开 SE-0461，从 `@MainActor` 调用本就离开主 actor（见实现说明「主 actor 调用方」）。 +- **复审条件**：库 target 开启 `NonisolatedNonsendingByDefault` 时重议。 + +--- + +## A29 — `@Suite(.serialized)` 不足以保护进程级开关 `isEnabled`（PR #122 review 发现 F4） + +- **裁决**：不修（2026-09-04）。 +- **发现**：`disabledRunsTheBodyOnTheCallersExecutor` 翻转 `LargeStackTaskExecution.isEnabled`，`.serialized` 只序列化本套件；其他套件与之并行时在翻转窗口内静默失去执行器。 +- **为什么不修**：只影响那几毫秒里其他套件跑在哪条线程上，不影响任何断言的正确性（全仓其他套件对执行器不敏感，AGENTS.md Test Environment 节写明）。审查者称「测试 trap 会跳过 defer」不成立——Swift Testing 的 `#expect` 失败不 trap，`defer` 正常恢复。横向排查：35 个 `.serialized` 套件里只有这一个翻转进程级开关。 +- **复审条件**：出现第二个断言线程身份的套件。 + +--- + +## A30 — 执行器关闭 / 不可用时并行窗口 = 核数会占满协作线程池（PR #122 review 发现 F8） + +- **裁决**：不修（2026-09-04）。 +- **发现**：`StackSafeExecutor` 探测失败时用 `DispatchSemaphore.wait()` 阻塞调用线程；macOS 14 或宿主关掉开关时，N 个并行 `prepare` 同时阻塞 N 条协作线程，宿主其他 async 工作会饿住（不会死锁：8 MB 跳转池是另一个池）。 +- **为什么不修**：这正是接入前每一次 `prepare` 的行为，并行只是把它乘以窗口；关掉执行器是宿主的显式选择（A/B 与计时配置），macOS 14 以下的用户面很小。宿主可用 `maximumConcurrentPreparations: 1` / `--jobs 1` 回到旧形态。 +- **复审条件**：有 macOS 14 宿主反馈饿死；届时可让 `run` 在不支持时把窗口自动收窄到 1。 + +--- + +## A31 — lineage / JSON 路径默认并行窗口取核数，峰值内存从 1 个索引镜像变为核数个（PR #122 review 发现 F6） + +- **裁决**：保持（2026-09-04，用户裁定）。 +- **发现**：`ABISnapshotInputLoader.loadDocument` 索引完即丢 builder，旧循环峰值一个镜像；新默认 `min(N, 核数)` 个（约 32 MB / 版本）。 +- **为什么保持**：提案第二轮澄清用户选「默认并行上限取核数」，`--jobs` 帮助文本写明代价；`--interface` 路径本来就全部常驻。 +- **复审条件**：出现内存受限的宿主场景（例如 CI 上几十个版本）时给 lineage 路径单独的默认值。 + +--- + +## A32 — `run` 与 `isSupported` 各写一遍平台 + 可用性门（PR #122 review 发现 F9） + +- **裁决**：不修（2026-09-04）。 +- **发现**：`#if canImport(Darwin)` + `#available(macOS 15…)` 在两处重复。 +- **为什么不修**：`#available` 必须在使用 `StackSafeExecutor.taskExecutor` 的词法位置出现，编译器不接受「`isSupported` 为真」作为可用性证明；把 `isSupported` 加进 `run` 的条件只是第三次重复。两处各有必要，已在 `run` 的注释说明。 +- **复审条件**：Swift 提供可用性谓词的抽象手段。 + +--- + +## A33 — `TypeIndexing.TypeDatabase.index` 的 task group 用 `addTask`，取消后仍提交剩余模块（PR #122 review 发现 1 的横向同类） + +- **裁决**：延后（2026-09-04）。 +- **发现**：`Sources/TypeIndexing/TypeDatabase.swift:76` 与 `concurrentMap` 修复前同形；基线既有，非本 PR 引入。 +- **为什么延后**：正确修法是 `addTaskUnlessCancelled` + 注册前 `Task.checkCancellation()`（否则取消会把残缺索引静默登记进去），而 `index(dependencies:moduleFilter:)` 直接构造 `SDKIndexer` / `ModuleInterfaceIndexer`（需要 SourceKit 与 SDK），没有注入缝可以写单元级复现测试；按「修复必带能变红的测试」规则，先补注入缝再修。 +- **复审条件**：`TypeDatabase` 获得 indexer 注入缝时一并修，或 GUI 宿主报告取消 `--resolve-c-module-names` 后 CPU 仍被占用。 + diff --git a/Documentations/Internal/TaskReports/2026-09-03-large-stack-executor-and-cross-version-parallelism.md b/Documentations/Internal/TaskReports/2026-09-03-large-stack-executor-and-cross-version-parallelism.md index 5240d42c..7a007678 100644 --- a/Documentations/Internal/TaskReports/2026-09-03-large-stack-executor-and-cross-version-parallelism.md +++ b/Documentations/Internal/TaskReports/2026-09-03-large-stack-executor-and-cross-version-parallelism.md @@ -48,3 +48,10 @@ worktree `.worktrees/MachOSwiftSection-LargeStackExecutor`，分支 `feature/lar - 并行用通用 `concurrentMap(maximumConcurrency:)` 而非 `async let`。 - CLI `dump` 循环不再额外包一层（逐类型跳转代价可忽略）。 - 主 actor 段落：库 target 未开启 SE-0461，async 入口从 `@MainActor` 调用会离开主 actor 落到执行器线程（实现说明已按事实写）。 + +## Review 修复批次（2026-09-04） + +并行 review 会话对 PR #122 给出 15 条发现（原文与处置见 [Roadmaps/2026-09-04-pr122-review-findings.md](../../../Roadmaps/2026-09-04-pr122-review-findings.md)）。最严重的一条已被审查者独立复现：`concurrentMap` 用 `addTask`，取消后剩余元素全部启动。修复：`addTaskUnlessCancelled`、被拒即抛 `CancellationError`（不能返回残缺数组，`result!` 会崩）。其余四条真缺陷是测试层面的：执行器线程断言没挡 `isEnabled`、环境变量只认 `"0"`、并行等价测试先串行焐热缓存、窗口断言退化成串行也绿。用户三项裁定：Dispatcher 进程级递归锁串行化 handler 调用、lineage 默认窗口保持核数、stderr 加输入标签（`ConsoleEventHandler(label:)` + `eventHandlersPerVersion`）。审查者对 F9「重复门」的修法在实际中不可行（`#available` 必须出现在使用点），登记 A32；`TypeDatabase` 的同形 `addTask` 缺注入缝写不出复现测试，登记 A33。 + +验证：受影响 7 个套件 43 个测试通过；突变检查——把 `addTaskUnlessCancelled` 改回 `addTask`、去掉 dispatcher 锁、测试 guard 只看 `isSupported`，并在 `MACHO_SWIFT_SECTION_LARGE_STACK_EXECUTOR=0` 下跑对应四个测试：`cancellationStopsSubmittingPendingElements`（元素 1 启动、不抛错）、`aHandlerSharedByConcurrentDispatchersIsNeverEnteredConcurrently`（handler 被并发进入）、`bodyRunsOnAnExecutorThreadWhenSupported` 与 `demanglerEntriesInsideTheBodyDoNotHop`（环境变量关闭时假红）四个测试共 7 处失败；改回后同一环境下全绿。全量 `swift test --skip IntegrationTests` 结果见下一行。全量 1649 测试 / 307 套件，仅 `SharedCache.resolve under Swift Concurrency` 的两个墙钟并行度断言在全量并行时假失败（已知），单独重跑通过。 + diff --git a/Roadmaps/2026-09-04-pr122-review-findings.md b/Roadmaps/2026-09-04-pr122-review-findings.md new file mode 100644 index 00000000..c080f991 --- /dev/null +++ b/Roadmaps/2026-09-04-pr122-review-findings.md @@ -0,0 +1,37 @@ +# PR #122 review findings（大栈执行器接入与跨版本并行，2026-09-04） + +并行 review 会话（machoswiftsection-cf，xhigh code-review）对 PR #122（`feature/large-stack-executor-and-cross-version-parallelism` → `feature/self-contained-abi-layer`）的 15 条发现，已按四问逐条裁决：真缺陷 5、误报 1、设计取舍 / 风格 9。本表是原始清单与处置状态；「不修 / 误报 / 延后」的终审条目收录进 [ReviewAdjudications.md](../Documentations/Internal/ReviewAdjudications.md)（A25–A33）。 + +用户裁定：5 条真缺陷全修，另修重复门（后判为不可消除，A32）与冗余 import；F2 用 Dispatcher 进程级锁；F6 保持核数；F7 加输入标签。 + +## 真缺陷（5 条） + +| # | 位置 | 问题 | 状态 | +|---|---|---|---| +| 1 | `Utilities/BoundedConcurrentMap.swift` | `addTask` 不响应取消：宿主取消后剩余元素全部启动（审查者独立编译复现：8 元素窗口 2 全部启动） | **已修**：`addTaskUnlessCancelled`，被拒即抛 `CancellationError`（不返回残缺数组——否则 `result!` 崩溃）；`cancellationStopsSubmittingPendingElements` 修复前红（元素 1 启动且不抛错） | +| 2 | `LargeStackTaskExecutionTests` | 四个执行器线程测试只挡 `isSupported`，`MACHO_SWIFT_SECTION_LARGE_STACK_EXECUTOR=0` 下必红 | **已修**：`executorIsActive` 同时看 `isEnabled`；修复前该环境变量下两个测试红，修复后绿 | +| 3 | `LargeStackTaskExecution.swift` | 环境变量只认字面量 `"0"`，`=false` 静默保持开启 | **已修**：`isEnabled(fromEnvironmentValue:)`——`0` / `false` / `no` / `off`（大小写、首尾空白不敏感）为关，其余为开；参数化测试 | +| 4 | `SwiftEvolutionInterfaceBuilderTests` | 先串行再并行，并行那轮全是缓存命中 | **已修**：并行先跑 | +| 5 | `BoundedConcurrentMapTests` | `maximumInFlight <= 3` 退化成串行也绿 | **已修**：新增三方 barrier 测试 `theWindowAdmitsItsFullWidth`（窗口不足 3 即挂起、超时失败） | + +## 误报（1 条） + +| # | 位置 | 结论 | 状态 | +|---|---|---|---| +| F11 | `SwiftDump/Dumpable/*` 六处 `import MachOFoundation` | 审查者按 0018 之前的再导出状态判断；实际必需 | **误报**，见 A25（附带子主张：`FoundationToolbox` 未声明属基线既有归 A23；`AnySwiftEvolutionInterfaceBuilder` 的 `import Utilities` 冗余已删） | + +## 设计取舍 / 风格（9 条） + +| # | 位置 | 问题 | 状态 | +|---|---|---|---| +| F2 | `AnySwiftEvolutionInterfaceBuilder` | 同一 handler 被 N 个版本的任务并发调用，`Handler` 无 `Sendable` | **已修**：`Dispatcher.dispatch` 用进程级 `NSRecursiveLock` 串行化 handler 调用（`EventDeliverySerializationTests`：修复前 4 个 dispatcher 并发投递时 handler 被并发进入，修复后最大在飞 1；可重入不死锁） | +| F6 | `EvolutionCommand` lineage 路径 | 默认窗口取核数，峰值内存倍增 | **保持**，见 A31 | +| F7 | `DiffCommand` / `EvolutionCommand` | stderr 诊断交错、无输入归属 | **已修**：`ConsoleEventHandler(label:)`，`diff` 用 `old` / `new`，`evolution` 每版本用其 label（新 init 参数 `eventHandlersPerVersion`），snapshot 输入用 label 或文件名；`ConsoleEventHandlerLineTests` | +| F8 | 执行器关闭时窗口占满协作线程池 | 属实 | **不修**，见 A30 | +| F4 | `.serialized` 不保护进程级开关 | 属实但无正确性影响 | **不修**，见 A29 | +| F9 | `run` / `isSupported` 双重可用性门 | 属实但不可消除 | **不修**，见 A32 | +| F12 | `run` 未转发 `#isolation` | 惯用法 | **不修**，见 A28 | +| F10 | 与既有 `concurrentMap(_:)` 同名 | 可读性 | **不修**，见 A27 | +| F13 | 逐定义入口每次包一层 | 维护性 | **不修**，见 A26 | + +横向同类：`TypeDatabase.swift:76` 的裸 `addTask`（基线既有）延后，见 A33。 diff --git a/Sources/MachOSymbols/LargeStackTaskExecution.swift b/Sources/MachOSymbols/LargeStackTaskExecution.swift index b6f40fb4..d0e15367 100644 --- a/Sources/MachOSymbols/LargeStackTaskExecution.swift +++ b/Sources/MachOSymbols/LargeStackTaskExecution.swift @@ -41,11 +41,24 @@ import FoundationToolbox public enum LargeStackTaskExecution { /// Process-wide switch. A host that manages its own executor sets it to /// `false`. The environment variable - /// `MACHO_SWIFT_SECTION_LARGE_STACK_EXECUTOR=0` seeds it off for a process - /// that cannot be recompiled — the rendering A/B and the timing runs - /// compare the same binary with the executor on and off through it. + /// `MACHO_SWIFT_SECTION_LARGE_STACK_EXECUTOR` seeds it for a process that + /// cannot be recompiled — the rendering A/B and the timing runs compare + /// the same binary with the executor on and off through it. See + /// ``isEnabled(fromEnvironmentValue:)`` for the accepted spellings. @Mutex - public static var isEnabled: Bool = ProcessInfo.processInfo.environment["MACHO_SWIFT_SECTION_LARGE_STACK_EXECUTOR"] != "0" + public static var isEnabled: Bool = isEnabled(fromEnvironmentValue: ProcessInfo.processInfo.environment["MACHO_SWIFT_SECTION_LARGE_STACK_EXECUTOR"]) + + /// The verdict `MACHO_SWIFT_SECTION_LARGE_STACK_EXECUTOR` seeds: + /// `0` / `false` / `no` / `off` (case-insensitive, surrounding whitespace + /// ignored) turn the executor off, every other value — including an + /// unset variable — leaves it on. The first version compared the raw + /// value against the literal `"0"`, so `=false` silently measured the + /// executor twice. + public static func isEnabled(fromEnvironmentValue value: String?) -> Bool { + guard let value else { return true } + let normalized = value.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() + return !["0", "false", "no", "off"].contains(normalized) + } /// Whether this process can run work on the executor at all: Darwin, on a /// runtime with SE-0417 task executors. Independent of ``isEnabled``. diff --git a/Sources/SwiftDeclaration/Events/SwiftIndexEvents.swift b/Sources/SwiftDeclaration/Events/SwiftIndexEvents.swift index e422b5a2..8fd74817 100644 --- a/Sources/SwiftDeclaration/Events/SwiftIndexEvents.swift +++ b/Sources/SwiftDeclaration/Events/SwiftIndexEvents.swift @@ -99,6 +99,18 @@ public enum SwiftIndexEvents { handlers.removeAll() } + /// Handler invocation is serialized PROCESS-WIDE, across every + /// dispatcher: `Handler` carries no `Sendable` requirement, and since + /// cross-version preparation runs in parallel (evolution proposal + /// `large-stack-executor-and-cross-version-parallelism`) one host + /// handler instance is shared by N dispatchers on N tasks. The lock + /// keeps a stateful handler single-threaded and a console handler's + /// lines whole, at one uncontended lock per event. Recursive, so a + /// handler that dispatches from inside `handle` does not deadlock. + /// Delivery ORDER across dispatchers is still whichever task gets + /// there first. + private static let deliveryLock = NSRecursiveLock() + public func dispatch(_ event: Payload) { // Read the handler list once. `@Mutex` takes the lock per access, so // testing emptiness and then iterating would be two separate critical @@ -106,8 +118,10 @@ public enum SwiftIndexEvents { // reach neither the handlers nor the fallback. let currentHandlers = handlers guard currentHandlers.isEmpty else { - for handler in currentHandlers { - handler.handle(event: event) + Self.deliveryLock.withLock { + for handler in currentHandlers { + handler.handle(event: event) + } } return } diff --git a/Sources/SwiftIndexing/SwiftIndexEventsHandlers.swift b/Sources/SwiftIndexing/SwiftIndexEventsHandlers.swift index 20b67b71..24897790 100644 --- a/Sources/SwiftIndexing/SwiftIndexEventsHandlers.swift +++ b/Sources/SwiftIndexing/SwiftIndexEventsHandlers.swift @@ -193,50 +193,69 @@ public struct OSLogEventHandler: SwiftIndexEvents.Handler { /// the reason `SwiftIndexEvents.Dispatcher`'s own floor (os_log) is only a /// floor. Issue #102 reported the CI case directly. public struct ConsoleEventHandler: SwiftIndexEvents.Handler { - public init() {} + /// An input label (`old` / `new`, a version label, a file name) printed + /// after the timestamp on every line. Set it when several inputs report + /// at once — `diff` and `evolution` index their inputs concurrently + /// (evolution proposal `large-stack-executor-and-cross-version-parallelism`), + /// so an unlabeled line cannot be attributed to an input. + public let label: String? + + public init(label: String? = nil) { + self.label = label + } public func handle(event: SwiftIndexEvents.Payload) { let timestamp = DateFormatter.localizedString(from: Date(), dateStyle: .none, timeStyle: .medium) + if let line = line(for: event, timestamp: timestamp) { + report(line) + } + } + + /// The line `handle(event:)` writes for `event`, or `nil` for an event the + /// console does not report. `[timestamp] [label] [LEVEL] message`; the + /// label bracket is absent when there is no label. + package func line(for event: SwiftIndexEvents.Payload, timestamp: String) -> String? { + let prefix = "[\(timestamp)]" + (label.map { " [\($0)]" } ?? "") switch event { case .extractionCompleted(let result): - report("[\(timestamp)] [INFO] Extracted \(result.count) \(sectionName(result.section))") + return "\(prefix) [INFO] Extracted \(result.count) \(sectionName(result.section))" case .typeIndexingCompleted(let result): - report("[\(timestamp)] [INFO] Types: \(result.successful) successful, \(result.failed) failed, \(result.cImportedSkipped) C-imported skipped, \(result.nestedTypes) nested, \(result.extensionTypes) in extensions") + return "\(prefix) [INFO] Types: \(result.successful) successful, \(result.failed) failed, \(result.cImportedSkipped) C-imported skipped, \(result.nestedTypes) nested, \(result.extensionTypes) in extensions" case .protocolIndexingCompleted(let result): - report("[\(timestamp)] [INFO] Protocols: \(result.successful) successful, \(result.failed) failed") + return "\(prefix) [INFO] Protocols: \(result.successful) successful, \(result.failed) failed" case .conformanceIndexingCompleted(let result): - report("[\(timestamp)] [INFO] Conformances: \(result.extensionCount) extensions, \(result.failedConformances + result.failedAssociatedTypes + result.failedExtensions) failed") + return "\(prefix) [INFO] Conformances: \(result.extensionCount) extensions, \(result.failedConformances + result.failedAssociatedTypes + result.failedExtensions) failed" case .extensionIndexingCompleted(let result): - report("[\(timestamp)] [INFO] Extensions: \(result.typeExtensions) type, \(result.protocolExtensions) protocol, \(result.typeAliasExtensions) typealias, \(result.failed) failed") + return "\(prefix) [INFO] Extensions: \(result.typeExtensions) type, \(result.protocolExtensions) protocol, \(result.typeAliasExtensions) typealias, \(result.failed) failed" case .moduleCollectionCompleted(let result): - report("[\(timestamp)] [INFO] Found \(result.moduleCount) modules to import") + return "\(prefix) [INFO] Found \(result.moduleCount) modules to import" case .phaseTransition(let phase, let state): let phaseName = phaseName(phase) switch state { case .completed: - report("[\(timestamp)] [SUCCESS] \(phaseName.capitalized) completed") + return "\(prefix) [SUCCESS] \(phaseName.capitalized) completed" case .failed(let error): - report("[\(timestamp)] [ERROR] \(phaseName.capitalized) failed: \(String(describing: error))") + return "\(prefix) [ERROR] \(phaseName.capitalized) failed: \(String(describing: error))" case .started: - break // Ignore started events for console output + return nil // Ignore started events for console output } case .definitionPrintFailed(let context, let error): - report("[\(timestamp)] [ERROR] Failed to print \(context.kind.description) '\(context.name)': \(String(describing: error))") + return "\(prefix) [ERROR] Failed to print \(context.kind.description) '\(context.name)': \(String(describing: error))" case .renderingDegraded(let context, let error): let subject = context.subject.map { " for \($0)" } ?? "" - report("[\(timestamp)] [ERROR] Degraded \(context.source)\(subject): \(String(describing: error))") + return "\(prefix) [ERROR] Degraded \(context.source)\(subject): \(String(describing: error))" default: - break // Ignore other detailed events + return nil // Ignore other detailed events } } diff --git a/Sources/SwiftInterface/AnySwiftEvolutionInterfaceBuilder.swift b/Sources/SwiftInterface/AnySwiftEvolutionInterfaceBuilder.swift index 4d0b5320..81c116e1 100644 --- a/Sources/SwiftInterface/AnySwiftEvolutionInterfaceBuilder.swift +++ b/Sources/SwiftInterface/AnySwiftEvolutionInterfaceBuilder.swift @@ -7,7 +7,6 @@ import MachOSwiftSection import Semantic import SwiftStdlibToolbox import MachOSymbols -import Utilities /// Renders one module's ABI across N ≥ 2 ordered binary versions as a single /// **union interface with lifecycle annotations** — the N-way, human-readable @@ -47,17 +46,29 @@ public final class AnySwiftEvolutionInterfaceBuilder: Sendable { /// Homogeneous construction: N versions of the same reader type, count /// decided at runtime. /// + /// `eventHandlers` are shared by every version; `eventHandlersPerVersion` + /// adds handlers built for one version (index and label), so a host can + /// attribute the diagnostics of concurrently prepared versions — the CLI + /// attaches a `ConsoleEventHandler(label:)` per version through it. + /// Handler invocation is serialized process-wide by the dispatcher, so a + /// shared handler is never called concurrently. + /// /// - Throws: `ABIEvolutionError.fewerThanTwoVersions` / /// `.labelCountMismatch` on invalid input shapes. public init( configuration: SwiftDeclarationIndexConfiguration = .init(), eventHandlers: [SwiftIndexEvents.Handler] = [], + eventHandlersPerVersion: ((_ versionIndex: Int, _ label: String) -> [SwiftIndexEvents.Handler])? = nil, versions: [MachO], labels: [String] ) throws { try Self.validate(versionCount: versions.count, labelCount: labels.count) - self.versionUnits = versions.map { - InterfaceVersionUnit(configuration: configuration, eventHandlers: eventHandlers, machO: $0) + self.versionUnits = versions.enumerated().map { versionIndex, machO in + InterfaceVersionUnit( + configuration: configuration, + eventHandlers: eventHandlers + (eventHandlersPerVersion?(versionIndex, labels[versionIndex]) ?? []), + machO: machO + ) } self.labels = labels } diff --git a/Sources/Utilities/BoundedConcurrentMap.swift b/Sources/Utilities/BoundedConcurrentMap.swift index 2e583b7b..9c0e6523 100644 --- a/Sources/Utilities/BoundedConcurrentMap.swift +++ b/Sources/Utilities/BoundedConcurrentMap.swift @@ -11,9 +11,15 @@ extension Collection where Element: Sendable { /// `maximumConcurrency` of 1 runs the elements strictly one after the /// other, in order; values below 1 count as 1. /// - /// The first failure is rethrown. Elements not yet started never start; - /// transforms already in flight run to completion first (a task group - /// waits for its children), and their results are discarded. + /// Two things stop the submission of elements not yet started, and in + /// both the pending elements never start: the first failure, which is + /// rethrown; and cancellation of the calling task, after which the call + /// throws `CancellationError` — never a partial array. Transforms already + /// in flight run to completion first (a task group waits for its + /// children, and the transforms this library passes do not observe + /// cancellation), and their results are discarded. A cancellation that + /// arrives after the last element was submitted changes nothing: the + /// work is done, so the results are returned. /// /// Child tasks inherit the caller's task executor preference, so under /// `LargeStackTaskExecution.run` every transform runs on the large-stack @@ -27,22 +33,34 @@ extension Collection where Element: Sendable { var results = [Result?](repeating: nil, count: count) var pending = enumerated().makeIterator() + // `addTaskUnlessCancelled`, not `addTask`: a cancelled group still + // accepts children through `addTask`, which is how a cancelled + // multi-version preparation used to index every remaining version + // to the end. A refused submission means the calling task was + // cancelled; throwing here is what makes the group cancel and + // drain its in-flight children and the call fail as a whole + // (returning `results` with holes would trap on the unwrap below). var started = 0 while started < window, let (index, element) = pending.next() { - group.addTask { (index, try await transform(element)) } + guard group.addTaskUnlessCancelled(operation: { (index, try await transform(element)) }) else { + throw CancellationError() + } started += 1 } while let (index, result) = try await group.next() { results[index] = result if let (nextIndex, nextElement) = pending.next() { - group.addTask { (nextIndex, try await transform(nextElement)) } + guard group.addTaskUnlessCancelled(operation: { (nextIndex, try await transform(nextElement)) }) else { + throw CancellationError() + } } } return results.map { result in // Every index is filled once `next()` returns nil without - // throwing: each submitted task reports exactly one index. + // throwing: each submitted task reports exactly one index, and + // a refused submission threw above. result! } } diff --git a/Sources/swift-section/Commands/DiffCommand.swift b/Sources/swift-section/Commands/DiffCommand.swift index 05385529..2b56c27b 100644 --- a/Sources/swift-section/Commands/DiffCommand.swift +++ b/Sources/swift-section/Commands/DiffCommand.swift @@ -88,8 +88,8 @@ struct DiffCommand: AsyncParsableCommand { // to its printers, so this is what puts a dropped declaration on // stderr instead of leaving it to `Dispatcher`'s os_log floor, which // a CLI operator never sees. - let oldBuilder = SwiftDiffableInterfaceBuilder(eventHandlers: [ConsoleEventHandler()], in: oldMachO) - let newBuilder = SwiftDiffableInterfaceBuilder(eventHandlers: [ConsoleEventHandler()], in: newMachO) + let oldBuilder = SwiftDiffableInterfaceBuilder(eventHandlers: [ConsoleEventHandler(label: "old")], in: oldMachO) + let newBuilder = SwiftDiffableInterfaceBuilder(eventHandlers: [ConsoleEventHandler(label: "new")], in: newMachO) // Old side first in the window, so `--jobs 1` is the historical // order; with a wider window the two index side by side and their // diagnostics interleave on stderr. @@ -121,8 +121,8 @@ struct DiffCommand: AsyncParsableCommand { // The change-list path is snapshot-based either way, so each side // may be a binary (indexed and frozen here) or a persisted // baseline (decoded, with its format version validated). - let documents = try await [oldPath, newPath].concurrentMap(maximumConcurrency: maximumConcurrentPreparations) { path in - try await loadDocument(at: path) + let documents = try await [(path: oldPath, side: "old"), (path: newPath, side: "new")].concurrentMap(maximumConcurrency: maximumConcurrentPreparations) { input in + try await loadDocument(at: input.path, consoleLabel: input.side) } let (oldDocument, newDocument) = (documents[0], documents[1]) @@ -195,7 +195,7 @@ struct DiffCommand: AsyncParsableCommand { /// Loads one change-list-path input: a snapshot JSON is decoded, a binary /// is indexed and frozen (with provenance stamped). - private func loadDocument(at path: String) async throws -> ABISnapshotDocument { + private func loadDocument(at path: String, consoleLabel: String) async throws -> ABISnapshotDocument { try await ABISnapshotInputLoader.loadDocument( path: path, architecture: architecture, @@ -203,6 +203,7 @@ struct DiffCommand: AsyncParsableCommand { cacheImageName: cacheImageName, cacheImagePath: cacheImagePath, label: nil, + consoleLabel: consoleLabel, log: log ) } diff --git a/Sources/swift-section/Commands/EvolutionCommand.swift b/Sources/swift-section/Commands/EvolutionCommand.swift index 673b87b3..1885859e 100644 --- a/Sources/swift-section/Commands/EvolutionCommand.swift +++ b/Sources/swift-section/Commands/EvolutionCommand.swift @@ -138,7 +138,7 @@ struct EvolutionCommand: AsyncParsableCommand { // builder, because the version count is a runtime value here — the // pack-generic SwiftEvolutionInterfaceBuilder's arity is compile-time. let builder = try AnySwiftEvolutionInterfaceBuilder( - eventHandlers: [ConsoleEventHandler()], + eventHandlersPerVersion: { _, label in [ConsoleEventHandler(label: label)] }, versions: machOFiles, labels: resolvedLabels ) diff --git a/Sources/swift-section/Utilities/ABISnapshotInputLoader.swift b/Sources/swift-section/Utilities/ABISnapshotInputLoader.swift index e5e86903..5f2220c7 100644 --- a/Sources/swift-section/Utilities/ABISnapshotInputLoader.swift +++ b/Sources/swift-section/Utilities/ABISnapshotInputLoader.swift @@ -27,7 +27,10 @@ enum ABISnapshotInputLoader { /// Load one input as a frozen snapshot document. A JSON path decodes (with /// the format-version check); a binary path is loaded, indexed, and frozen, /// with provenance stamped from the load parameters. `label` overrides the - /// document's provenance label either way. + /// document's provenance label either way. `consoleLabel` tags the + /// input's stderr diagnostics (inputs index concurrently under `--jobs`, + /// so an unlabeled line cannot be attributed); it defaults to `label`, + /// then to the input's file name. static func loadDocument( path: String, architecture: Architecture?, @@ -35,6 +38,7 @@ enum ABISnapshotInputLoader { cacheImageName: String?, cacheImagePath: String?, label: String?, + consoleLabel: String? = nil, log: (String) -> Void ) async throws -> ABISnapshotDocument { if try isSnapshotDocument(atPath: path) { @@ -59,7 +63,10 @@ enum ABISnapshotInputLoader { ) // Shared by `snapshot` / `evolution` / `diff`'s snapshot inputs, so this // is where those three get a stderr sink for anything indexing drops. - let builder = SwiftDiffableInterfaceBuilder(eventHandlers: [ConsoleEventHandler()], in: machOFile) + let builder = SwiftDiffableInterfaceBuilder( + eventHandlers: [ConsoleEventHandler(label: consoleLabel ?? label ?? defaultLabel(forPath: path))], + in: machOFile + ) try await builder.prepare() let cacheImageSuffix = [cacheImageName, cacheImagePath].compactMap { $0 }.first.map { " (\($0))" } ?? "" let provenance = ABIProvenance( diff --git a/Tests/MachOSymbolsTests/LargeStackTaskExecutionTests.swift b/Tests/MachOSymbolsTests/LargeStackTaskExecutionTests.swift index 0d800878..0bfc5cd9 100644 --- a/Tests/MachOSymbolsTests/LargeStackTaskExecutionTests.swift +++ b/Tests/MachOSymbolsTests/LargeStackTaskExecutionTests.swift @@ -32,8 +32,18 @@ struct LargeStackTaskExecutionTests { pthread_mach_thread_np(pthread_self()) } + /// `run` puts the body on the executor only when the runtime supports it + /// AND the process-wide switch is on — a run under + /// `MACHO_SWIFT_SECTION_LARGE_STACK_EXECUTOR=0` (the A/B and timing + /// configuration) takes the pass-through path, so the executor-thread + /// assertions below would be false for a reason the test does not + /// control. Guard both, not just support. + private static var executorIsActive: Bool { + LargeStackTaskExecution.isSupported && LargeStackTaskExecution.isEnabled + } + @Test func bodyRunsOnAnExecutorThreadWhenSupported() async { - guard LargeStackTaskExecution.isSupported else { return } + guard Self.executorIsActive else { return } let (stackSize, threadName) = await LargeStackTaskExecution.run { (pthread_get_stacksize_np(pthread_self()), Self.currentThreadName()) } @@ -45,7 +55,7 @@ struct LargeStackTaskExecutionTests { /// stack probe passes, so its blocking and suspending entries stay on the /// task's thread instead of hopping to a pool worker. @Test func demanglerEntriesInsideTheBodyDoNotHop() async { - guard LargeStackTaskExecution.isSupported else { return } + guard Self.executorIsActive else { return } let observation = await LargeStackTaskExecution.run { let taskThread = Self.currentThread() let blockingCallThread: mach_port_t = StackSafeExecutor.execute { Self.currentThread() } @@ -60,7 +70,7 @@ struct LargeStackTaskExecutionTests { /// (`printRoot` → `printTypeDefinition`, a parent's nested-children loop) /// nests a run inside a run; the inner one must not move the task. @Test func nestedRunsStayOnTheSameThread() async { - guard LargeStackTaskExecution.isSupported else { return } + guard Self.executorIsActive else { return } let (outerThread, innerThread) = await LargeStackTaskExecution.run { let outer = Self.currentThread() let inner = await LargeStackTaskExecution.run { Self.currentThread() } @@ -72,7 +82,7 @@ struct LargeStackTaskExecutionTests { /// Child tasks inherit the preference (SE-0417) — the cross-version /// parallel preparation relies on this. @Test func childTasksInheritTheExecutor() async { - guard LargeStackTaskExecution.isSupported else { return } + guard Self.executorIsActive else { return } let childThreadNames = await LargeStackTaskExecution.run { await withTaskGroup(of: String.self) { group in for _ in 0 ..< 3 { @@ -100,6 +110,22 @@ struct LargeStackTaskExecutionTests { #expect(!threadName.hasPrefix(Self.executorThreadNamePrefix), "ran on \(threadName)") } + /// The environment seed: the four off spellings, case-insensitive and + /// whitespace-tolerant, turn the executor off; anything else — including + /// an unset variable — leaves it on. The first version compared against + /// the literal `"0"`, so `=false` silently kept the executor on. + @Test(arguments: [ + ("0", false), ("false", false), ("FALSE", false), ("no", false), ("off", false), (" 0 ", false), ("Off\n", false), + ("1", true), ("true", true), ("yes", true), ("on", true), ("", true), ("2", true), ("disabled", true), + ]) + func environmentValueParsing(value: String, expected: Bool) { + #expect(LargeStackTaskExecution.isEnabled(fromEnvironmentValue: value) == expected) + } + + @Test func unsetEnvironmentValueLeavesTheExecutorOn() { + #expect(LargeStackTaskExecution.isEnabled(fromEnvironmentValue: nil)) + } + @Test func valuesAndErrorsPassThrough() async throws { struct Failure: Error, Equatable {} diff --git a/Tests/SwiftIndexingTests/ConsoleEventHandlerLineTests.swift b/Tests/SwiftIndexingTests/ConsoleEventHandlerLineTests.swift new file mode 100644 index 00000000..a09b8e91 --- /dev/null +++ b/Tests/SwiftIndexingTests/ConsoleEventHandlerLineTests.swift @@ -0,0 +1,40 @@ +import Foundation +import Testing +import SwiftDeclaration +@testable import SwiftIndexing + +/// Pins `ConsoleEventHandler`'s line format, in particular the input label +/// (evolution proposal `large-stack-executor-and-cross-version-parallelism`): +/// `diff` / `evolution` index their inputs concurrently, so every stderr line +/// carries `[label]` after the timestamp when a label is set, and nothing +/// extra when it is not. +@Suite +struct ConsoleEventHandlerLineTests { + private let completedExtraction = SwiftIndexEvents.Payload.extractionCompleted( + result: SwiftIndexEvents.ExtractionResult(section: .swiftTypes, count: 42) + ) + + @Test func labeledLinesCarryTheLabelAfterTheTimestamp() { + let handler = ConsoleEventHandler(label: "26.0") + #expect(handler.line(for: completedExtraction, timestamp: "12:00:00") == "[12:00:00] [26.0] [INFO] Extracted 42 Swift types") + } + + @Test func unlabeledLinesAreUnchanged() { + let handler = ConsoleEventHandler() + #expect(handler.label == nil) + #expect(handler.line(for: completedExtraction, timestamp: "12:00:00") == "[12:00:00] [INFO] Extracted 42 Swift types") + } + + @Test func unreportedEventsProduceNoLine() { + let handler = ConsoleEventHandler(label: "old") + #expect(handler.line(for: .moduleCollectionStarted, timestamp: "12:00:00") == nil) + #expect(handler.line(for: .phaseTransition(phase: .indexing, state: .started), timestamp: "12:00:00") == nil) + } + + @Test func failuresKeepTheLabelToo() { + struct Failure: Error, CustomStringConvertible { var description: String { "boom" } } + let handler = ConsoleEventHandler(label: "new") + let line = handler.line(for: .phaseTransition(phase: .indexing, state: .failed(Failure())), timestamp: "12:00:00") + #expect(line == "[12:00:00] [new] [ERROR] Indexing failed: boom") + } +} diff --git a/Tests/SwiftInterfaceTests/BoundedConcurrentMapTests.swift b/Tests/SwiftInterfaceTests/BoundedConcurrentMapTests.swift index 1e2cb997..4f639e97 100644 --- a/Tests/SwiftInterfaceTests/BoundedConcurrentMapTests.swift +++ b/Tests/SwiftInterfaceTests/BoundedConcurrentMapTests.swift @@ -36,6 +36,29 @@ struct BoundedConcurrentMapTests { } } + /// A rendezvous for a fixed number of participants: `arrive()` suspends + /// until every participant has arrived, so it completes only if that many + /// transforms are in flight at the same time. + private actor Barrier { + private let participantCount: Int + private var arrivedCount = 0 + private var waiters: [CheckedContinuation] = [] + + init(participantCount: Int) { + self.participantCount = participantCount + } + + func arrive() async { + arrivedCount += 1 + if arrivedCount >= participantCount { + for waiter in waiters { waiter.resume() } + waiters.removeAll() + return + } + await withCheckedContinuation { waiters.append($0) } + } + } + /// A one-shot gate: `wait()` suspends until `open()` — a rendezvous that /// completes only if the waiter and the opener run concurrently. private actor Gate { @@ -113,6 +136,50 @@ struct BoundedConcurrentMapTests { #expect(results == [0, 1]) } + /// The window admits exactly its width, not fewer: three elements in a + /// window of three must all be in flight at once (a three-way barrier that + /// only releases when all three have arrived). A window that admitted two + /// would leave the third pending forever — hence the time limit. + @Test(.timeLimit(.minutes(1))) func theWindowAdmitsItsFullWidth() async throws { + let barrier = Barrier(participantCount: 3) + let results = try await [0, 1, 2].concurrentMap(maximumConcurrency: 3) { element in + await barrier.arrive() + return element + } + #expect(results == [0, 1, 2]) + } + + /// Cancelling the calling task stops the submission of pending elements + /// and fails the call with `CancellationError` — never a partial array. + /// Element 0 holds the (width-1) window open until the test has cancelled + /// the task; element 1 must then never start. The first version used + /// `addTask`, which a cancelled group still accepts, so every remaining + /// version of a cancelled multi-version preparation indexed to the end. + @Test(.timeLimit(.minutes(1))) func cancellationStopsSubmittingPendingElements() async { + let ledger = Ledger() + let elementZeroStarted = Gate() + let elementZeroMayFinish = Gate() + + let task = Task { + try await Array(0 ..< 4).concurrentMap(maximumConcurrency: 1) { element in + ledger.start(element) + if element == 0 { + await elementZeroStarted.open() + await elementZeroMayFinish.wait() + } + ledger.end(element) + } + } + + await elementZeroStarted.wait() + task.cancel() + await elementZeroMayFinish.open() + + let outcome = await task.result + #expect(throws: CancellationError.self) { try outcome.get() } + #expect(ledger.events == ["start 0", "end 0"]) + } + @Test func theFirstFailureIsRethrownAndPendingElementsNeverStart() async { struct Failure: Error, Equatable { let element: Int diff --git a/Tests/SwiftInterfaceTests/EventDeliverySerializationTests.swift b/Tests/SwiftInterfaceTests/EventDeliverySerializationTests.swift new file mode 100644 index 00000000..ddb2bab6 --- /dev/null +++ b/Tests/SwiftInterfaceTests/EventDeliverySerializationTests.swift @@ -0,0 +1,82 @@ +import Foundation +import Testing +import SwiftDeclaration + +/// Pins the dispatcher's process-wide serialization of handler invocation +/// (evolution proposal `large-stack-executor-and-cross-version-parallelism`): +/// `SwiftIndexEvents.Handler` carries no `Sendable` requirement, and +/// cross-version preparation shares one host handler across N dispatchers on +/// N tasks, so without the lock a stateful handler is entered concurrently. +@Suite +struct EventDeliverySerializationTests { + /// Counts concurrent entries into `handle`; a short spin inside makes an + /// overlap near-certain when delivery is not serialized. + private final class OverlapDetectingHandler: SwiftIndexEvents.Handler, @unchecked Sendable { + private let lock = NSLock() + private var inFlight = 0 + private(set) var maximumInFlight = 0 + private(set) var eventCount = 0 + + func handle(event: SwiftIndexEvents.Payload) { + lock.withLock { + inFlight += 1 + maximumInFlight = max(maximumInFlight, inFlight) + eventCount += 1 + } + usleep(50) + lock.withLock { inFlight -= 1 } + } + } + + @Test func aHandlerSharedByConcurrentDispatchersIsNeverEnteredConcurrently() async { + let handler = OverlapDetectingHandler() + let dispatcherCount = 4 + let eventsPerDispatcher = 200 + let dispatchers = (0 ..< dispatcherCount).map { _ in + let dispatcher = SwiftIndexEvents.Dispatcher() + dispatcher.addHandler(handler) + return dispatcher + } + + await withTaskGroup(of: Void.self) { group in + for dispatcher in dispatchers { + group.addTask { + for _ in 0 ..< eventsPerDispatcher { + dispatcher.dispatch(.moduleCollectionStarted) + } + } + } + } + + #expect(handler.eventCount == dispatcherCount * eventsPerDispatcher) + #expect(handler.maximumInFlight == 1, "handler was entered \(handler.maximumInFlight) times concurrently") + } + + /// The lock is recursive: a handler that dispatches from inside `handle` + /// (a host forwarding events into its own dispatcher) must not deadlock. + private final class ReentrantHandler: SwiftIndexEvents.Handler, @unchecked Sendable { + let inner = SwiftIndexEvents.Dispatcher() + private let lock = NSLock() + private(set) var outerCount = 0 + + func handle(event: SwiftIndexEvents.Payload) { + lock.withLock { outerCount += 1 } + if case .moduleCollectionStarted = event { + inner.dispatch(.moduleCollectionCompleted(result: .init(moduleCount: 0, modules: []))) + } + } + } + + @Test(.timeLimit(.minutes(1))) func reentrantDispatchFromAHandlerDoesNotDeadlock() { + let handler = ReentrantHandler() + let innerHandler = OverlapDetectingHandler() + handler.inner.addHandler(innerHandler) + let outer = SwiftIndexEvents.Dispatcher() + outer.addHandler(handler) + + outer.dispatch(.moduleCollectionStarted) + + #expect(handler.outerCount == 1) + #expect(innerHandler.eventCount == 1) + } +} diff --git a/Tests/SwiftInterfaceTests/SwiftEvolutionInterfaceBuilderTests.swift b/Tests/SwiftInterfaceTests/SwiftEvolutionInterfaceBuilderTests.swift index a3311218..f4b73e2d 100644 --- a/Tests/SwiftInterfaceTests/SwiftEvolutionInterfaceBuilderTests.swift +++ b/Tests/SwiftInterfaceTests/SwiftEvolutionInterfaceBuilderTests.swift @@ -3,6 +3,7 @@ import Testing import MachOKit import Semantic import SwiftDiffing +import SwiftDeclaration @testable import MachOSwiftSection @_spi(Support) @testable import SwiftInterface @@ -152,9 +153,14 @@ struct SwiftEvolutionInterfaceBuilderTests { /// and the evolution's JSON must be byte-identical to the serial /// (`1`) preparation — the window is a scheduling knob, never a semantic /// one. + /// + /// The PARALLEL builder prepares first: the per-image caches key on the + /// file's identity, so a serial run first would leave the parallel run + /// with warm caches and no concurrent cold build — the one thing this + /// test exists to exercise. @Test func parallelPreparationMatchesSerialPreparation() async throws { - let serialBuilder = try await preparedBuilder(maximumConcurrentPreparations: 1) let parallelBuilder = try await preparedBuilder(maximumConcurrentPreparations: 3) + let serialBuilder = try await preparedBuilder(maximumConcurrentPreparations: 1) let serialInterface = try await serialBuilder.printAnnotatedInterface().string let parallelInterface = try await parallelBuilder.printAnnotatedInterface().string @@ -171,6 +177,52 @@ struct SwiftEvolutionInterfaceBuilderTests { #expect(serialEvolution == parallelEvolution) } + /// A recording handler: which version labels reported through it, and + /// how many events it saw. Reference type on purpose — the assertion is + /// about the instance each version was handed. + private final class RecordingHandler: SwiftIndexEvents.Handler, @unchecked Sendable { + private let lock = NSLock() + private(set) var eventCount = 0 + let label: String + + init(label: String) { + self.label = label + } + + func handle(event: SwiftIndexEvents.Payload) { + lock.withLock { eventCount += 1 } + } + } + + /// `eventHandlersPerVersion` hands each version its own handlers (built + /// from the version's index and label), on top of the shared ones — the + /// seam the CLI uses to label each version's stderr diagnostics. + @Test func perVersionEventHandlersReachTheirOwnVersion() async throws { + let shared = RecordingHandler(label: "shared") + var perVersion: [RecordingHandler] = [] + let perVersionLock = NSLock() + let builder = try AnySwiftEvolutionInterfaceBuilder( + eventHandlers: [shared], + eventHandlersPerVersion: { versionIndex, label in + let handler = RecordingHandler(label: "\(versionIndex):\(label)") + perVersionLock.withLock { perVersion.append(handler) } + return [handler] + }, + versions: try loadFixtureMachOFiles(), + labels: ["1.0", "2.0", "3.0"] + ) + try await builder.prepare(maximumConcurrentPreparations: 3) + + #expect(perVersion.map(\.label).sorted() == ["0:1.0", "1:2.0", "2:3.0"]) + for handler in perVersion { + #expect(handler.eventCount > 0, "\(handler.label) saw no events") + } + // The shared handler hears every version: at least the sum of what + // the per-version handlers saw individually. + #expect(shared.eventCount >= perVersion.map(\.eventCount).max() ?? 0) + #expect(shared.eventCount == perVersion.map(\.eventCount).reduce(0, +)) + } + /// A window wider than the version count and a window below 1 are both /// clamped, not rejected. @Test func preparationWindowIsClampedNotValidated() async throws {