diff --git a/.github/scripts/test-merge-candidate-workflows.sh b/.github/scripts/test-merge-candidate-workflows.sh index 0e3aa903e3..d9b8c53e17 100755 --- a/.github/scripts/test-merge-candidate-workflows.sh +++ b/.github/scripts/test-merge-candidate-workflows.sh @@ -701,6 +701,10 @@ for workflow in "$STAGING_WORKFLOW" "$PREPARE"; do grep -Fq 'run: bash scripts/dev-shell.sh npm ci --no-audit --no-fund' \ <<<"$root_install_step" || fail "$(basename "$workflow") materialization validation must install the root esbuild dependency" + materialization_step="$(step_block "$workflow" "Test binary materialization flow")" + grep -Fq 'bash scripts/dev-shell.sh bash scripts/build-fork-instrument-tool.sh' \ + <<<"$materialization_step" || + fail "$(basename "$workflow") materialization validation must build the fork contract inventory tool" done grep -Fq 'cleanup-merge-candidates.sh' "$CLEANUP_WORKFLOW" || \ diff --git a/.github/workflows/prepare-merge.yml b/.github/workflows/prepare-merge.yml index 0612384e54..46b1b9f8fc 100644 --- a/.github/workflows/prepare-merge.yml +++ b/.github/workflows/prepare-merge.yml @@ -1335,6 +1335,10 @@ jobs: - name: Test binary materialization flow if: env.BINARY_MATERIALIZATION_CHANGED == 'true' run: | + # WHY: artifact-guard integration tests query the ABI 43 binary + # contract inventory. This source-only job does not download the + # package-toolchain artifact, so prepare its host tool explicitly. + bash scripts/dev-shell.sh bash scripts/build-fork-instrument-tool.sh bash scripts/dev-shell.sh npx --prefix host vitest run --root . tests/package-system test-suite-early: diff --git a/.github/workflows/staging-build.yml b/.github/workflows/staging-build.yml index db113440e8..0148f5945c 100644 --- a/.github/workflows/staging-build.yml +++ b/.github/workflows/staging-build.yml @@ -852,6 +852,10 @@ jobs: - name: Test binary materialization flow if: env.BINARY_MATERIALIZATION_CHANGED == 'true' run: | + # WHY: artifact-guard integration tests query the ABI 43 binary + # contract inventory. This source-only job does not download the + # package-toolchain artifact, so prepare its host tool explicitly. + bash scripts/dev-shell.sh bash scripts/build-fork-instrument-tool.sh bash scripts/dev-shell.sh npx --prefix host vitest run --root . tests/package-system test-suite-early: diff --git a/Cargo.lock b/Cargo.lock index dece0e0c20..d603c8835f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -404,6 +404,7 @@ version = "0.1.0" dependencies = [ "anyhow", "clap", + "sha2", "walrus", "wasm-posix-shared", "wasmparser 0.247.0", @@ -1523,9 +1524,9 @@ checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" [[package]] name = "walrus" -version = "0.26.1" +version = "0.26.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e151599d689dac80e85c66a7cfa6ffd1b2ab79220517f9161040a87a5041aee3" +checksum = "3bfa49767bb3a9e1afb02aa95bbcbde8d82f2db4ca377afae94d688f14f62378" dependencies = [ "anyhow", "gimli", diff --git a/abi/snapshot.json b/abi/snapshot.json index 3f84edf3e8..1f14930e44 100644 --- a/abi/snapshot.json +++ b/abi/snapshot.json @@ -1,5 +1,5 @@ { - "abi_version": 42, + "abi_version": 43, "channel_buffers": { "data_offset": 72, "data_size": 65536, @@ -36,10 +36,22 @@ "offset": 64, "size": 4, "type": "i32" + }, + { + "name": "request_flags", + "offset": 68, + "size": 4, + "type": "u32" } ], "size": 72 }, + "channel_request_flags": [ + { + "bit": 1, + "name": "defer_signal_delivery" + } + ], "channel_signal_area": { "base": 65560, "slots": [ @@ -88,7 +100,15 @@ } ], "custom_sections": [ + "kandelo.wpk_fork.capabilities", + "kandelo.wpk_fork.exception_codec", + "kandelo.wpk_fork.gc_codec", + "kandelo.wpk_fork.imported_globals", + "kandelo.wpk_fork.imported_tables", "kandelo.wpk_fork.linked_frames", + "kandelo.wpk_fork.module_state", + "kandelo.wpk_fork.static_root_catalog", + "kandelo.wpk_fork.unwind_transport", "wasm-posix-abi" ], "export_deny": { @@ -114,7 +134,7 @@ }, "host_adapter": { "manifest": { - "abi_version": 42, + "abi_version": 43, "channel_data_offset": 72, "channel_data_size": 65536, "channel_header_size": 72, @@ -3217,6 +3237,270 @@ }, "program_artifact": { "fork_instrumentation": { + "capabilities": { + "flags": [ + { + "bit": 1, + "name": "side_entry" + }, + { + "bit": 2, + "name": "dylink_main" + }, + { + "bit": 4, + "name": "activation_state_safe" + } + ], + "known_mask": 7, + "required_flags": 4, + "section": "kandelo.wpk_fork.capabilities", + "version": 1 + }, + "exception_codec": { + "activation_import": { + "module": "env", + "mutable": false, + "name": "__wpk_fork_module_activation", + "type": "i32" + }, + "header_size": 8, + "section": "kandelo.wpk_fork.exception_codec", + "tag_record_size": 16, + "version": 1 + }, + "gc_codec": { + "field_record": { + "fields": [ + { + "name": "storage", + "offset": 0, + "size": 1 + }, + { + "name": "flags", + "offset": 1, + "size": 1 + }, + { + "name": "reserved", + "offset": 2, + "size": 2 + }, + { + "name": "scalar_offset_or_none", + "offset": 4, + "size": 4 + }, + { + "name": "reference_ordinal_or_none", + "offset": 8, + "size": 4 + } + ], + "size": 12 + }, + "header_size": 16, + "layout_record": { + "fields": [ + { + "name": "layout_id", + "offset": 0, + "size": 4 + }, + { + "name": "type_ordinal", + "offset": 4, + "size": 4 + }, + { + "name": "kind", + "offset": 8, + "size": 1 + }, + { + "name": "constructor", + "offset": 9, + "size": 1 + }, + { + "name": "flags", + "offset": 10, + "size": 2 + }, + { + "name": "snapshot_scalar_len_or_stride", + "offset": 12, + "size": 4 + }, + { + "name": "field_start", + "offset": 16, + "size": 4 + }, + { + "name": "field_count", + "offset": 20, + "size": 4 + }, + { + "name": "super_type_ordinal_or_none", + "offset": 24, + "size": 4 + }, + { + "name": "base_layout_id", + "offset": 28, + "size": 4 + }, + { + "name": "auxiliary", + "offset": 32, + "size": 4 + }, + { + "name": "provenance_scalar_len", + "offset": 36, + "size": 4 + }, + { + "name": "provenance_ref_count", + "offset": 40, + "size": 4 + } + ], + "size": 44 + }, + "magic_bytes": [ + 75, + 70, + 71, + 67 + ], + "section": "kandelo.wpk_fork.gc_codec", + "transit_table": { + "element": "anyref", + "maximum": null, + "minimum": 1, + "module": "env", + "name": "__wpk_fork_ref_gc_transit", + "table64": false + }, + "version": 1 + }, + "imported_globals": { + "header_size": 16, + "known_flags": 3, + "magic_bytes": [ + 75, + 70, + 73, + 71 + ], + "mutable_flag": 1, + "record_fields": [ + { + "name": "record_size", + "offset": 0, + "size": 4 + }, + { + "name": "owner", + "offset": 4, + "size": 4 + }, + { + "name": "value_type", + "offset": 8, + "size": 1 + }, + { + "name": "flags", + "offset": 9, + "size": 1 + }, + { + "name": "reserved", + "offset": 10, + "size": 2 + }, + { + "name": "module_name_length", + "offset": 12, + "size": 4 + }, + { + "name": "field_name_length", + "offset": 16, + "size": 4 + }, + { + "name": "import_ordinal", + "offset": 20, + "size": 4 + } + ], + "record_header_size": 24, + "section": "kandelo.wpk_fork.imported_globals", + "shared_flag": 2, + "version": 1 + }, + "imported_tables": { + "header_size": 16, + "known_flags": 1, + "magic_bytes": [ + 75, + 70, + 73, + 84 + ], + "record_fields": [ + { + "name": "record_size", + "offset": 0, + "size": 4 + }, + { + "name": "owner", + "offset": 4, + "size": 4 + }, + { + "name": "element_type", + "offset": 8, + "size": 1 + }, + { + "name": "flags", + "offset": 9, + "size": 1 + }, + { + "name": "reserved", + "offset": 10, + "size": 2 + }, + { + "name": "module_name_length", + "offset": 12, + "size": 4 + }, + { + "name": "field_name_length", + "offset": 16, + "size": 4 + }, + { + "name": "import_ordinal", + "offset": 20, + "size": 4 + } + ], + "record_header_size": 24, + "section": "kandelo.wpk_fork.imported_tables", + "table64_flag": 1, + "version": 1 + }, "linked_frame_descriptor": { "alignment": 8, "descriptor_size": 24, @@ -3252,7 +3536,574 @@ "section": "kandelo.wpk_fork.linked_frames", "version": 1 }, + "module_state": { + "arena": { + "chunk_flags": [ + { + "bit": 1, + "name": "root" + }, + { + "bit": 2, + "name": "sealed" + } + ], + "chunk_magic_bytes": [ + 75, + 70, + 77, + 67 + ], + "known_chunk_flags": 3, + "pointer_widths": [ + { + "bytes": 4, + "chunk_header_size": 40 + }, + { + "bytes": 8, + "chunk_header_size": 56 + } + ], + "record": { + "alignment": 8, + "header_size": 24, + "kinds": [ + { + "name": "module", + "number": 1 + }, + { + "name": "reference_recipe", + "number": 2 + }, + { + "name": "mutable_global", + "number": 3 + }, + { + "name": "table", + "number": 4 + }, + { + "name": "table_page", + "number": 5 + }, + { + "name": "element_segments", + "number": 6 + }, + { + "name": "data_segments", + "number": 7 + }, + { + "name": "replay_events", + "number": 8 + }, + { + "name": "imported_global_bindings", + "number": 9 + }, + { + "name": "activation_continuations", + "number": 10 + }, + { + "name": "imported_table_bindings", + "number": 11 + }, + { + "name": "reference_recipe_segment", + "number": 12 + }, + { + "name": "replay_event_segment", + "number": 13 + } + ], + "magic_bytes": [ + 75, + 70, + 77, + 82 + ], + "version": 1 + }, + "version": 1 + }, + "descriptor": { + "alignment": 8, + "descriptor_size": 24, + "flags": [ + { + "bit": 1, + "name": "root_prefix_pointer" + }, + { + "bit": 2, + "name": "explicit_owners" + }, + { + "bit": 4, + "name": "sparse_tables" + } + ], + "known_flags": 7, + "magic_bytes": [ + 75, + 70, + 77, + 68 + ], + "required_flags": 7, + "root_pointer_word_offset": 1, + "section": "kandelo.wpk_fork.module_state", + "version": 1 + }, + "record_payloads": { + "activation_continuations": { + "entry_fields": [ + { + "name": "activation_id", + "offset": 0, + "size": 4 + }, + { + "name": "flags", + "offset": 4, + "size": 4 + }, + { + "name": "root", + "offset": 8, + "size": 8 + } + ], + "entry_known_flags": 0, + "entry_size": 16, + "header_size": 24, + "known_flags": 0, + "magic_bytes": [ + 75, + 70, + 65, + 67 + ], + "owner": 3, + "version": 1 + }, + "data_segments": { + "header_size": 8 + }, + "element_segments": { + "header_size": 8 + }, + "imported_global_bindings": { + "binding_kinds": [ + { + "name": "raw_number", + "number": 1 + }, + { + "name": "raw_bigint", + "number": 2 + }, + { + "name": "raw_reference", + "number": 3 + }, + { + "name": "activation_global", + "number": 4 + }, + { + "name": "base_import", + "number": 5 + } + ], + "entry_fields": [ + { + "name": "consumer_activation", + "offset": 0, + "size": 4 + }, + { + "name": "consumer_owner", + "offset": 4, + "size": 4 + }, + { + "name": "source_activation", + "offset": 8, + "size": 4 + }, + { + "name": "source_owner", + "offset": 12, + "size": 4 + }, + { + "name": "reserved", + "offset": 16, + "size": 4 + }, + { + "name": "recipe_id", + "offset": 20, + "size": 4 + }, + { + "name": "raw_bits", + "offset": 24, + "size": 8 + }, + { + "name": "binding_kind", + "offset": 32, + "size": 1 + }, + { + "name": "import_flags", + "offset": 33, + "size": 1 + }, + { + "name": "value_type", + "offset": 34, + "size": 1 + }, + { + "name": "reserved", + "offset": 35, + "size": 5 + } + ], + "entry_size": 40, + "header_size": 24, + "known_flags": 0, + "magic_bytes": [ + 75, + 70, + 66, + 71 + ], + "owner": 2, + "version": 1 + }, + "imported_table_bindings": { + "binding_kinds": [ + { + "name": "activation_table", + "number": 1 + }, + { + "name": "base_import", + "number": 2 + } + ], + "entry_fields": [ + { + "name": "consumer_activation", + "offset": 0, + "size": 4 + }, + { + "name": "consumer_owner", + "offset": 4, + "size": 4 + }, + { + "name": "source_activation", + "offset": 8, + "size": 4 + }, + { + "name": "source_owner", + "offset": 12, + "size": 4 + }, + { + "name": "reserved", + "offset": 16, + "size": 4 + }, + { + "name": "binding_kind", + "offset": 20, + "size": 1 + }, + { + "name": "reserved", + "offset": 21, + "size": 3 + } + ], + "entry_size": 24, + "header_size": 24, + "known_flags": 0, + "magic_bytes": [ + 75, + 70, + 66, + 84 + ], + "owner": 4, + "version": 1 + }, + "module": { + "known_flags": 0, + "payload_size": 40, + "template_id_size": 32 + }, + "mutable_global": { + "header_size": 8, + "value_types": [ + { + "bytes": 4, + "name": "i32", + "number": 1 + }, + { + "bytes": 8, + "name": "i64", + "number": 2 + }, + { + "bytes": 4, + "name": "f32", + "number": 3 + }, + { + "bytes": 8, + "name": "f64", + "number": 4 + }, + { + "bytes": 16, + "name": "v128", + "number": 5 + }, + { + "bytes": 4, + "name": "funcref_recipe", + "number": 6 + }, + { + "bytes": 4, + "name": "externref_recipe", + "number": 7 + }, + { + "bytes": 4, + "name": "exnref_recipe", + "number": 8 + }, + { + "bytes": 4, + "name": "anyref_recipe", + "number": 9 + } + ] + }, + "reference_transaction": { + "known_flags": 1, + "magic": [ + 75, + 70, + 82, + 86 + ], + "manifest_size": 96, + "node_record_size": 48, + "owner": 1, + "sealed_flag": 1, + "sections": [ + { + "name": "nodes", + "number": 1 + }, + { + "name": "edges", + "number": 2 + }, + { + "name": "scalars", + "number": 3 + }, + { + "name": "vector_index", + "number": 4 + }, + { + "name": "vector_entries", + "number": 5 + } + ], + "segment_header_size": 40, + "segment_known_flags": 0, + "segment_magic": [ + 75, + 70, + 82, + 83 + ], + "vector_index_size": 16, + "version": 2 + }, + "replay_events": { + "entry_size": 8, + "header_size": 40, + "known_flags": 0, + "magic": [ + 75, + 70, + 82, + 69 + ], + "owner": 1, + "segment_capacity": 4080, + "segment_header_size": 24, + "segment_known_flags": 0, + "segment_version": 1, + "version": 2 + }, + "table": { + "baseline_fingerprint_size": 32, + "descriptor_payload_size": 56, + "flags": [ + { + "bit": 1, + "name": "sparse_overrides" + } + ], + "known_flags": 1, + "max_page_shift": 20, + "min_page_shift": 4, + "page_header_size": 16, + "page_shift": 10, + "run_header_size": 8 + } + } + }, "required_exports": [ + { + "kind": "func", + "name": "__wpk_fork_exception_materialize", + "params": [ + "i32" + ], + "results": [] + }, + { + "kind": "func", + "name": "__wpk_fork_ref_decode_exnref", + "params": [ + "i32" + ], + "results": [ + "exnref" + ] + }, + { + "kind": "func", + "name": "__wpk_fork_ref_encode_exnref", + "params": [ + "exnref" + ], + "results": [ + "i32" + ] + }, + { + "kind": "func", + "name": "__wpk_fork_ref_exn_abort", + "params": [], + "results": [] + }, + { + "kind": "func", + "name": "__wpk_fork_ref_exn_clear", + "params": [], + "results": [] + }, + { + "kind": "func", + "name": "__wpk_fork_ref_exn_encode_ingress", + "params": [ + "i32" + ], + "results": [ + "i32" + ] + }, + { + "kind": "func", + "name": "__wpk_fork_ref_exn_throw_recipe", + "params": [ + "i32" + ], + "results": [] + }, + { + "kind": "func", + "name": "__wpk_fork_ref_exn_throw_slot", + "params": [ + "i32" + ], + "results": [] + }, + { + "kind": "func", + "name": "__wpk_fork_ref_gc_allocate", + "params": [ + "i32" + ], + "results": [] + }, + { + "kind": "func", + "name": "__wpk_fork_ref_gc_encode_slot", + "params": [ + "i32" + ], + "results": [ + "i32" + ] + }, + { + "kind": "func", + "name": "__wpk_fork_ref_gc_fill", + "params": [ + "i32" + ], + "results": [] + }, + { + "kind": "func", + "name": "__wpk_fork_ref_gc_probe", + "params": [ + "i32" + ], + "results": [ + "i64" + ] + }, + { + "kind": "func", + "name": "__wpk_fork_ref_gc_publish_externref", + "params": [ + "i32", + "externref" + ], + "results": [] + }, + { + "kind": "func", + "name": "__wpk_fork_static_root_harvest", + "params": [], + "results": [] + }, { "kind": "func", "name": "wpk_fork_abort_begin", @@ -3267,6 +4118,58 @@ "params": [], "results": [] }, + { + "kind": "func", + "name": "wpk_fork_module_bootstrap", + "params": [], + "results": [] + }, + { + "kind": "func", + "name": "wpk_fork_module_state_finish_restore", + "params": [ + "i32" + ], + "results": [] + }, + { + "kind": "func", + "name": "wpk_fork_module_state_restore", + "params": [ + "i32" + ], + "results": [] + }, + { + "kind": "func", + "name": "wpk_fork_module_state_save", + "params": [ + "i32" + ], + "results": [] + }, + { + "kind": "func", + "name": "wpk_fork_module_table_state_restore", + "params": [ + "i32" + ], + "results": [] + }, + { + "kind": "func", + "name": "wpk_fork_module_table_state_save", + "params": [ + "i32" + ], + "results": [] + }, + { + "kind": "func", + "name": "wpk_fork_module_thread_bootstrap", + "params": [], + "results": [] + }, { "kind": "func", "name": "wpk_fork_rewind_begin", @@ -3325,6 +4228,17 @@ "ptr" ] }, + { + "kind": "func", + "module": "env", + "name": "__wpk_fork_frame_peek", + "params": [ + "ptr" + ], + "results": [ + "ptr" + ] + }, { "kind": "func", "module": "env", @@ -3335,8 +4249,525 @@ "results": [ "ptr" ] - } - ] + }, + { + "kind": "func", + "module": "env", + "name": "__wpk_fork_module_state_record_commit", + "params": [ + "ptr" + ], + "results": [] + }, + { + "kind": "func", + "module": "env", + "name": "__wpk_fork_module_state_record_find", + "params": [ + "i32", + "i32", + "i32", + "i32" + ], + "results": [ + "ptr" + ] + }, + { + "kind": "func", + "module": "env", + "name": "__wpk_fork_module_state_record_reserve", + "params": [ + "i32", + "i32", + "i32", + "ptr" + ], + "results": [ + "ptr" + ] + }, + { + "kind": "func", + "module": "env", + "name": "__wpk_fork_module_state_table_dirty_count", + "params": [ + "i32" + ], + "results": [ + "i32" + ] + }, + { + "kind": "func", + "module": "env", + "name": "__wpk_fork_module_state_table_dirty_mark", + "params": [ + "i32", + "i64", + "i64" + ], + "results": [] + }, + { + "kind": "func", + "module": "env", + "name": "__wpk_fork_module_state_table_dirty_page", + "params": [ + "i32", + "i32" + ], + "results": [ + "i64" + ] + }, + { + "kind": "func", + "module": "env", + "name": "__wpk_fork_module_state_table_mutation_abort", + "params": [], + "results": [] + }, + { + "kind": "func", + "module": "env", + "name": "__wpk_fork_module_state_table_mutation_begin", + "params": [], + "results": [ + "i64" + ] + }, + { + "kind": "func", + "module": "env", + "name": "__wpk_fork_module_state_table_mutation_commit", + "params": [ + "i32", + "i64", + "i64" + ], + "results": [] + }, + { + "kind": "func", + "module": "env", + "name": "__wpk_fork_module_state_table_reconcile", + "params": [], + "results": [ + "i64" + ] + }, + { + "kind": "func", + "module": "env", + "name": "__wpk_fork_module_state_table_state_owned", + "params": [ + "i32" + ], + "results": [ + "i32" + ] + }, + { + "kind": "func", + "module": "env", + "name": "__wpk_fork_ref_decode_funcref", + "params": [ + "i32" + ], + "results": [ + "funcref" + ] + }, + { + "kind": "func", + "module": "env", + "name": "__wpk_fork_ref_encode_funcref", + "params": [ + "funcref" + ], + "results": [ + "i32" + ] + }, + { + "kind": "func", + "module": "env", + "name": "__wpk_fork_ref_exn_broker_encode", + "params": [ + "i32" + ], + "results": [ + "i32" + ] + }, + { + "kind": "func", + "module": "env", + "name": "__wpk_fork_ref_exn_broker_throw_recipe", + "params": [ + "i32" + ], + "results": [] + }, + { + "kind": "func", + "module": "env", + "name": "__wpk_fork_ref_exn_cache_index", + "params": [ + "i32" + ], + "results": [ + "i32" + ] + }, + { + "kind": "func", + "module": "env", + "name": "__wpk_fork_ref_exn_claim", + "params": [ + "i32" + ], + "results": [ + "i32" + ] + }, + { + "kind": "func", + "module": "env", + "name": "__wpk_fork_ref_exn_define", + "params": [ + "i32", + "i32", + "i32", + "i32", + "ptr", + "i32", + "ptr", + "i32" + ], + "results": [] + }, + { + "kind": "func", + "module": "env", + "name": "__wpk_fork_ref_exn_ingress_throw", + "params": [ + "i32" + ], + "results": [] + }, + { + "kind": "func", + "module": "env", + "name": "__wpk_fork_ref_exn_load", + "params": [ + "i32", + "i32", + "i32", + "i32", + "ptr", + "i32", + "ptr", + "i32" + ], + "results": [ + "i32" + ] + }, + { + "kind": "func", + "module": "env", + "name": "__wpk_fork_ref_exn_lookup", + "params": [ + "i32" + ], + "results": [ + "i32" + ] + }, + { + "kind": "func", + "module": "env", + "name": "__wpk_fork_ref_exn_route", + "params": [ + "i32", + "i32" + ], + "results": [ + "i32" + ] + }, + { + "kind": "func", + "module": "env", + "name": "__wpk_fork_ref_gc_broker_encode", + "params": [ + "i32" + ], + "results": [ + "i32" + ] + }, + { + "kind": "func", + "module": "env", + "name": "__wpk_fork_ref_gc_capture_layout", + "params": [ + "i32", + "i32", + "i32" + ], + "results": [ + "i32" + ] + }, + { + "kind": "func", + "module": "env", + "name": "__wpk_fork_ref_gc_claim", + "params": [ + "i32" + ], + "results": [ + "i32" + ] + }, + { + "kind": "func", + "module": "env", + "name": "__wpk_fork_ref_gc_define", + "params": [ + "i32", + "i32", + "i32", + "i32", + "i32", + "ptr", + "i32", + "i32" + ], + "results": [] + }, + { + "kind": "func", + "module": "env", + "name": "__wpk_fork_ref_gc_i31", + "params": [ + "i32" + ], + "results": [ + "i32" + ] + }, + { + "kind": "func", + "module": "env", + "name": "__wpk_fork_ref_gc_load", + "params": [ + "i32", + "i32", + "i32", + "i32", + "i32", + "ptr", + "i32" + ], + "results": [ + "i32" + ] + }, + { + "kind": "func", + "module": "env", + "name": "__wpk_fork_ref_gc_lookup", + "params": [ + "i32" + ], + "results": [ + "i32" + ] + }, + { + "kind": "func", + "module": "env", + "name": "__wpk_fork_ref_gc_payload_len", + "params": [ + "i32", + "i32", + "i32" + ], + "results": [ + "i32" + ] + }, + { + "kind": "func", + "module": "env", + "name": "__wpk_fork_ref_gc_provenance_begin", + "params": [ + "i32", + "i32", + "i32", + "i32", + "i64", + "i64", + "i32" + ], + "results": [ + "i32" + ] + }, + { + "kind": "func", + "module": "env", + "name": "__wpk_fork_ref_gc_provenance_end", + "params": [ + "i32" + ], + "results": [] + }, + { + "kind": "func", + "module": "env", + "name": "__wpk_fork_ref_gc_provenance_ref", + "params": [ + "i32", + "i32", + "i32" + ], + "results": [] + }, + { + "kind": "func", + "module": "env", + "name": "__wpk_fork_ref_gc_route", + "params": [ + "i32", + "i32" + ], + "results": [ + "i32" + ] + }, + { + "kind": "func", + "module": "env", + "name": "__wpk_fork_ref_scratch_release", + "params": [ + "ptr", + "ptr" + ], + "results": [] + }, + { + "kind": "func", + "module": "env", + "name": "__wpk_fork_ref_scratch_reserve", + "params": [ + "ptr" + ], + "results": [ + "ptr" + ] + }, + { + "kind": "func", + "module": "env", + "name": "__wpk_fork_ref_vector_append", + "params": [ + "i32", + "i32" + ], + "results": [] + }, + { + "kind": "func", + "module": "env", + "name": "__wpk_fork_ref_vector_begin", + "params": [ + "i32" + ], + "results": [ + "i32" + ] + }, + { + "kind": "func", + "module": "env", + "name": "__wpk_fork_ref_vector_finish", + "params": [ + "i32" + ], + "results": [ + "i32" + ] + }, + { + "kind": "func", + "module": "env", + "name": "__wpk_fork_ref_vector_get", + "params": [ + "i32", + "i32" + ], + "results": [ + "i32" + ] + }, + { + "kind": "func", + "module": "env", + "name": "__wpk_fork_resume_peek", + "params": [ + "i32" + ], + "results": [ + "i32" + ] + }, + { + "element": "anyref", + "kind": "table", + "maximum": null, + "minimum": 1, + "module": "env", + "name": "__wpk_fork_ref_gc_transit", + "table64": false + }, + { + "element": "funcref", + "kind": "table", + "maximum": null, + "minimum": 1, + "module": "env", + "name": "__wpk_fork_resume_table", + "table64": false + } + ], + "static_root_catalog": { + "export": "__wpk_fork_static_root_catalog", + "harvest_export": "__wpk_fork_static_root_harvest", + "header_size": 12, + "magic_bytes": [ + 75, + 70, + 83, + 82 + ], + "section": "kandelo.wpk_fork.static_root_catalog", + "version": 1 + }, + "unwind_transport": { + "import": { + "kind": "tag", + "module": "env", + "name": "__wpk_fork_unwind" + }, + "payload_arity": 0, + "section": "kandelo.wpk_fork.unwind_transport", + "version": 1 + } } }, "syscall_arg_descriptors": { diff --git a/apps/browser-demos/test/fixtures/static-root-gc.wat b/apps/browser-demos/test/fixtures/static-root-gc.wat new file mode 100644 index 0000000000..b90942a860 --- /dev/null +++ b/apps/browser-demos/test/fixtures/static-root-gc.wat @@ -0,0 +1,17 @@ +(module + (type $pair (struct (field i32))) + (global $root (ref $pair) + (struct.new $pair (i32.const 41))) + (table $catalog (export "catalog") 1 1 (ref null any)) + + (func (export "harvest") + i32.const 0 + global.get $root + table.set $catalog) + + (func (export "matches_root") + (param (ref null $pair)) + (result i32) + local.get 0 + global.get $root + ref.eq)) diff --git a/apps/browser-demos/test/fork-continuation.spec.ts b/apps/browser-demos/test/fork-continuation.spec.ts index 480dc0246d..c3a323713a 100644 --- a/apps/browser-demos/test/fork-continuation.spec.ts +++ b/apps/browser-demos/test/fork-continuation.spec.ts @@ -1,7 +1,12 @@ import { expect, test, type Page } from "@playwright/test"; +import { execFileSync } from "node:child_process"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { dirname, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { resolveBinary } from "../../../host/src/binary-resolver"; +import { + RAW_GC_REFERENCE_STATE_FRESH_WORKER_HEX, +} from "../../../host/test/fixtures/gc-reference-state-fresh-worker-bytes"; const __dirname = dirname(fileURLToPath(import.meta.url)); const browserKernelModulePath = resolve( @@ -12,6 +17,18 @@ const memoryFsModulePath = resolve( __dirname, "../../../host/src/vfs/memory-fs.ts", ); +const catchRefFixtureSource = resolve( + __dirname, + "../../../host/test/fixtures/catch-ref-fresh-worker.wat", +); +const referenceCatchPayloadFixtureSource = resolve( + __dirname, + "../../../host/test/fixtures/reference-catch-payload-fresh-worker.wat", +); +const forkInstrumenterPath = resolve( + __dirname, + "../../../tools/bin/wasm-fork-instrument", +); interface BrowserFixtureResult { exitCode: number; @@ -122,7 +139,7 @@ test("Chromium grows and replays a continuation beyond ABI 41's fixed reserve", "p_10_deep_linked_continuation", ); - expect(result.exitCode).toBe(0); + expect(result.exitCode, JSON.stringify(result, null, 2)).toBe(0); expect(result.stdout).toContain("PRE_DEEP_FORK"); expect(result.stdout).toContain("DEEP_CHILD: ok"); expect(result.stdout).toContain("DEEP_PARENT: child="); @@ -163,3 +180,133 @@ test("Chromium preserves the parent across root and later continuation ENOMEM", expect(result.stderr).toBe(""); expect(result.diagnostics).toEqual([]); }); + +test("Chromium reconstructs CatchRef state in a fresh child worker", async ({ + page, + baseURL, + browserName, +}) => { + test.skip(browserName !== "chromium", "the aggregate browser gate uses Chromium"); + test.setTimeout(180_000); + expect(baseURL).toBeTruthy(); + + const workDir = mkdtempSync( + // Vite deliberately refuses to serve arbitrary host temporary paths. + // Keep this generated fixture under the checked-out test tree so the + // browser receives bytes from this exact worktree's allow-listed root. + resolve(__dirname, ".catch-ref-fresh-worker-"), + ); + try { + const rawPath = resolve(workDir, "catch-ref-fresh-worker.raw.wasm"); + const programPath = resolve(workDir, "catch-ref-fresh-worker.wasm"); + execFileSync("wat2wasm", [ + "--enable-exceptions", + "--enable-threads", + catchRefFixtureSource, + "-o", + rawPath, + ]); + execFileSync(forkInstrumenterPath, [rawPath, "-o", programPath]); + + // The parent waits for the child, whose exit 91 means CatchRef payload + // reconstruction failed after the browser worker instantiated a fresh + // module. The parent reports that wait failure as exit 92. + const result = await runBrowserFixture( + page, + baseURL!, + programPath, + "catch-ref-fresh-worker", + ); + expect(result.exitCode).toBe(0); + expect(result.stderr).toBe(""); + expect(result.diagnostics).toEqual([]); + } finally { + rmSync(workDir, { recursive: true, force: true }); + } +}); + +test("Chromium reconstructs reference-bearing catches in fresh child workers", async ({ + page, + baseURL, + browserName, +}) => { + test.skip(browserName !== "chromium", "the aggregate browser gate uses Chromium"); + test.setTimeout(180_000); + expect(baseURL).toBeTruthy(); + + const workDir = mkdtempSync( + resolve(__dirname, ".reference-catch-payload-fresh-worker-"), + ); + try { + const rawPath = resolve( + workDir, + "reference-catch-payload-fresh-worker.raw.wasm", + ); + const programPath = resolve( + workDir, + "reference-catch-payload-fresh-worker.wasm", + ); + execFileSync("wat2wasm", [ + "--enable-exceptions", + "--enable-threads", + referenceCatchPayloadFixtureSource, + "-o", + rawPath, + ]); + execFileSync(forkInstrumenterPath, [rawPath, "-o", programPath]); + + // One fresh child calls the reconstructed non-null funcref; a second + // verifies the nullable externref path. Either child exits nonzero if its + // caught exception recipe depended on the parent's module instance. + const result = await runBrowserFixture( + page, + baseURL!, + programPath, + "reference-catch-payload-fresh-worker", + ); + expect(result.exitCode).toBe(0); + expect(result.stderr).toBe(""); + expect(result.diagnostics).toEqual([]); + } finally { + rmSync(workDir, { recursive: true, force: true }); + } +}); + +test("Chromium reconstructs aliased Wasm GC state in a fresh child worker", async ({ + page, + baseURL, + browserName, +}) => { + test.skip(browserName !== "chromium", "the aggregate browser gate uses Chromium"); + test.setTimeout(180_000); + expect(baseURL).toBeTruthy(); + + const workDir = mkdtempSync( + resolve(__dirname, ".gc-reference-state-fresh-worker-"), + ); + try { + const rawPath = resolve(workDir, "gc-reference-state.raw.wasm"); + const programPath = resolve(workDir, "gc-reference-state.wasm"); + writeFileSync( + rawPath, + Buffer.from(RAW_GC_REFERENCE_STATE_FRESH_WORKER_HEX, "hex"), + ); + execFileSync(forkInstrumenterPath, [rawPath, "-o", programPath]); + + // The child verifies one cyclic identity through a live parameter, + // operand-stack carryover, mutable reference global, and mutated typed + // table. Any fresh-instance alias break exits 91; its waiting parent then + // exits 92. + const result = await runBrowserFixture( + page, + baseURL!, + programPath, + "gc-reference-state-fresh-worker", + ); + expect(result.exitCode).toBe(0); + expect(result.stderr).toBe(""); + expect(result.diagnostics).toEqual([]); + } finally { + rmSync(workDir, { recursive: true, force: true }); + } +}); diff --git a/apps/browser-demos/test/sjlj-noexcept-boundary.spec.ts b/apps/browser-demos/test/sjlj-noexcept-boundary.spec.ts index 580f572f52..72b04589ec 100644 --- a/apps/browser-demos/test/sjlj-noexcept-boundary.spec.ts +++ b/apps/browser-demos/test/sjlj-noexcept-boundary.spec.ts @@ -19,7 +19,6 @@ const fixturePaths = { repoRoot, "local-binaries/test-fixtures/wasm64/sjlj_noexcept_boundary.raw.wasm", ), - instrumented: resolveBinary("programs/sjlj_noexcept_boundary.wasm"), sigchld: resolveBinary("programs/sigchld_sjlj.wasm"), }; @@ -76,11 +75,7 @@ test("Chromium preserves the SjLj controls and positive SIGCHLD path", async ({ "sjlj_noexcept_boundary", "--noexcept", ]), - instrumented: await run(fixtureUrls.instrumented, [ - "sjlj_noexcept_boundary", - "--noexcept", - ]), - permissive: await run(fixtureUrls.instrumented, [ + permissive: await run(fixtureUrls.rawWasm32, [ "sjlj_noexcept_boundary", "--permissive", ]), @@ -99,7 +94,6 @@ test("Chromium preserves the SjLj controls and positive SIGCHLD path", async ({ for (const control of [ results.rawWasm32, - results.instrumented, results.rawWasm64, ]) { expect(control.exitCode).toBe(128 + 6); diff --git a/apps/browser-demos/test/wasm-gc-reference-transport.spec.ts b/apps/browser-demos/test/wasm-gc-reference-transport.spec.ts new file mode 100644 index 0000000000..2f15f61429 --- /dev/null +++ b/apps/browser-demos/test/wasm-gc-reference-transport.spec.ts @@ -0,0 +1,100 @@ +import { expect, test } from "@playwright/test"; +import { + FORK_ANYREF_TRANSIT_IMPORT, + forkAnyrefTransitProviderBytes, +} from "../../../host/src/fork-anyref-transit"; + +// WHY: the dev shell's WABT release cannot parse typed Wasm GC references. +// This is the Rust `wat` crate's deterministic encoding of the adjacent +// fixtures/static-root-gc.wat source. +const FIXTURE_WASM_HEX = [ + "0061736d01000000010e035f017f0060000060016300017f03030201020405016e", + "010101060a016400004129fb00000b07240307636174616c6f6701000768617276", + "65737400000c6d6174636865735f726f6f7400010a120208004100230026000b07", + "0020002300d30b0023046e616d65040701000470616972050a010007636174616c", + "6f670707010004726f6f74", +].join(""); + +function fixtureBytes(): number[] { + return Array.from(Buffer.from(FIXTURE_WASM_HEX, "hex")); +} + +test("browser preserves GC identity through weak harvest and anyref transit", async ({ + page, + baseURL, +}) => { + const bytes = fixtureBytes(); + const providerBytes = Array.from(forkAnyrefTransitProviderBytes()); + await page.goto(new URL("/trap-signal-test.html", baseURL!).href); + const result = await page.evaluate(async ({ + moduleBytes, + transitProviderBytes, + transitExport, + }) => { + const module = await WebAssembly.compile(new Uint8Array(moduleBytes)); + const transitProviderModule = await WebAssembly.compile( + new Uint8Array(transitProviderBytes), + ); + const parent = await WebAssembly.instantiate(module); + const child = await WebAssembly.instantiate(module); + const transitProvider = await WebAssembly.instantiate(transitProviderModule); + const parentExports = parent.exports as { + catalog: WebAssembly.Table; + harvest: () => void; + matches_root: (value: unknown) => number; + }; + const childExports = child.exports as typeof parentExports; + const transit = transitProvider.exports[transitExport] as WebAssembly.Table; + const clearTransit = transitProvider.exports[ + `${transitExport}_clear` + ] as () => void; + + parentExports.harvest(); + childExports.harvest(); + const parentRoot = parentExports.catalog.get(0); + const childRoot = childExports.catalog.get(0); + const repeatedReadIsIdentical = + childExports.catalog.get(0) === childRoot; + const freshInstancesDiffer = parentRoot !== childRoot; + + // The host creates the ABI transit table from an audited provider module + // for this exact WebKit compatibility boundary. + transit.set(0, childRoot); + transit.grow(2); + transit.set(1, parentRoot); + transit.set(2, childRoot); + const transported = transit.get(0); + const jsTransitIsIdentical = transported === childRoot; + const wasmTransitIsIdentical = + childExports.matches_root(transported) === 1; + + parentExports.catalog.set(0, null); + childExports.catalog.set(0, null); + clearTransit(); + return { + repeatedReadIsIdentical, + freshInstancesDiffer, + jsTransitIsIdentical, + wasmTransitIsIdentical, + harvestCleared: + parentExports.catalog.get(0) === null + && childExports.catalog.get(0) === null + && Array.from( + { length: transit.length }, + (_, index) => transit.get(index), + ).every((value) => value === null), + }; + }, { + moduleBytes: bytes, + transitProviderBytes: providerBytes, + transitExport: FORK_ANYREF_TRANSIT_IMPORT, + }); + + expect(result).toEqual({ + repeatedReadIsIdentical: true, + freshInstancesDiffer: true, + jsTransitIsIdentical: true, + wasmTransitIsIdentical: true, + harvestCleared: true, + }); +}); diff --git a/crates/fork-instrument/Cargo.toml b/crates/fork-instrument/Cargo.toml index 499050811b..0d122a7f46 100644 --- a/crates/fork-instrument/Cargo.toml +++ b/crates/fork-instrument/Cargo.toml @@ -19,10 +19,14 @@ wasm-posix-shared = { path = "../shared" } # walrus — typed wasm IR with validator. The primary workhorse for # parsing, manipulating, and emitting instrumented modules. -walrus = "0.26" +# 0.26.4 pre-registers implicit multi-value legacy-catch block types. Earlier +# 0.26 releases panic while parsing valid `[tag params] -> [try results]` +# handlers, before the legacy-to-modern normalization pass can run. +walrus = "0.26.4" # anyhow — ergonomic error chaining for a CLI / build tool. anyhow = "1" +sha2 = "0.10" # clap — argument parsing. `derive` feature for #[derive(Parser)]. clap = { version = "4", features = ["derive"] } diff --git a/crates/fork-instrument/README.md b/crates/fork-instrument/README.md index ecdda3e0b4..f321c50391 100644 --- a/crates/fork-instrument/README.md +++ b/crates/fork-instrument/README.md @@ -24,13 +24,33 @@ for the current design, ABI, save-buffer layout, and operating limits. wasm-fork-instrument -o [--entry kernel.kernel_fork] ``` +Artifact publication guards use the same wasmparser-backed binary decoder +instead of depending on a text disassembler understanding every proposal used +by the transformed module: + +```sh +wasm-fork-instrument --contract-inventory +wasm-fork-instrument --fork-capability-hex +wasm-fork-instrument --linked-frame-descriptor-hex +``` + +The inventory is one stable tab-separated row covering fork imports, control +exports, metadata counts, memory width, and ABI signature mismatches. The two +metadata modes require exactly one matching custom section and fail on missing +or duplicate sections. + ## Status -PR #307 (`fierce-wire`) replaces the old Binaryen Asyncify fork path in the -build scripts. The tool instruments direct + indirect fork-path callers, -spills scalar and supported ref-typed locals, survives modern `try_table` -catch-handler rewind, and preserves module validity. Remaining unsupported -patterns are documented in `docs/fork-instrumentation.md`. +The tool instruments direct, indirect, reference-call, tail-call, exception, +and cross-module fork paths. ABI 43 stores scalars in linked activation frames +and reconstructs reference locals/carryovers, typed GC graphs, complete +exceptions, mutable reference globals, tables, and dynamic-link activations +from versioned process-owned recipes in copied linear memory. It emits no +module-static reference stash. `Catch`, `CatchRef`, `CatchAll`, and +`CatchAllRef` replay through fresh-instance exceptions. Engine proposal +availability, stale ABI artifacts, and userspace stack-switching primitives +are documented as platform boundaries in +`docs/fork-instrumentation.md`. ## Build diff --git a/crates/fork-instrument/fuzz/Cargo.lock b/crates/fork-instrument/fuzz/Cargo.lock index d876c1d7cc..4de46e7419 100644 --- a/crates/fork-instrument/fuzz/Cargo.lock +++ b/crates/fork-instrument/fuzz/Cargo.lock @@ -185,6 +185,8 @@ dependencies = [ "anyhow", "clap", "walrus", + "wasm-posix-shared", + "wasmparser 0.247.0", ] [[package]] @@ -493,6 +495,10 @@ dependencies = [ "wasmparser 0.247.0", ] +[[package]] +name = "wasm-posix-shared" +version = "0.1.0" + [[package]] name = "wasmparser" version = "0.245.1" diff --git a/crates/fork-instrument/fuzz/fuzz_targets/generator.rs b/crates/fork-instrument/fuzz/fuzz_targets/generator.rs index a054b633dd..1cc24de915 100644 --- a/crates/fork-instrument/fuzz/fuzz_targets/generator.rs +++ b/crates/fork-instrument/fuzz/fuzz_targets/generator.rs @@ -4,27 +4,24 @@ //! Every generator output is a syntactically well-formed module that //! imports `kernel.kernel_fork` so the instrumenter has work to do. //! -//! Covers: single or nested try_tables, all four catch-clause shapes, +//! Covers: single or nested try_tables, tagged Catch/CatchRef clauses, //! and 0..=4 scalar locals of varying numeric type. Nested shape wraps //! an inner try_table in an outer try_table of the *same* clause //! variant to keep block result types trivially lined up. use arbitrary::{Arbitrary, Unstructured}; -/// Which catch-clause shape the generated try_table uses. Covers all -/// four `try_table` catch variants so the instrumenter's rewrite of -/// `call $fork` inside an exception-handled region is exercised across -/// ref-returning and non-ref clauses. +/// Which supported tagged catch shape the generated try_table uses. +/// +/// This compact generator focuses on tagged Catch and CatchRef shapes. +/// Deterministic tests separately cover CatchAll and CatchAllRef through the +/// complete-exception reconstruction recipe. #[derive(Debug, Clone, Copy, arbitrary::Arbitrary)] enum ClauseVariant { /// (catch_ref $exn $handler) — handler receives exnref; try_table result is exnref. CatchRef, - /// (catch_all_ref $handler) — handler receives exnref; try_table result is exnref. - CatchAllRef, /// (catch $exn $handler) — handler receives nothing (tag has no params); try_table empty result. Catch, - /// (catch_all $handler) — handler receives nothing; try_table empty result. - CatchAll, } impl ClauseVariant { @@ -41,14 +38,7 @@ impl ClauseVariant { "ref.null exn", "drop", ), - ClauseVariant::CatchAllRef => ( - format!("(catch_all_ref {label})"), - "(result (ref null exn))", - "ref.null exn", - "drop", - ), ClauseVariant::Catch => (format!("(catch {tag} {label})"), "", "", ""), - ClauseVariant::CatchAll => (format!("(catch_all {label})"), "", "", ""), } } } @@ -73,26 +63,6 @@ impl ScalarLocalTy { } } -/// Reference type used for a generated local declaration. Exercises -/// Phase 4f aux-table spill handling (for funcref/externref) and Phase -/// 6's `captured_exnref_K` non-spill invariant (for exnref). -#[derive(Debug, Clone, Copy, arbitrary::Arbitrary)] -enum RefLocalTy { - FuncRef, - ExternRef, - ExnRef, -} - -impl RefLocalTy { - fn as_wat(&self) -> &'static str { - match self { - RefLocalTy::FuncRef => "(ref null func)", - RefLocalTy::ExternRef => "(ref null extern)", - RefLocalTy::ExnRef => "(ref null exn)", - } - } -} - /// One generated program. Keep fields private so future generator /// extensions don't require downstream changes. #[derive(Debug)] @@ -112,9 +82,6 @@ pub struct WatProgram { /// variant ensures the block result types line up trivially — /// mixed families are not generated here. wrap_in_outer: bool, - /// 0..=2 ref-typed locals. Exercises Phase 4f aux-table spill and - /// Phase 6's captured_exnref_K non-spill invariant. - ref_locals: Vec, /// When true, adds a second function `$inner_fork` called via /// call_indirect through a funcref table, instead of calling /// `$fork` directly. Exercises indirect-call closure (Phase 3a/3b). @@ -128,17 +95,11 @@ impl<'a> Arbitrary<'a> for WatProgram { for _ in 0..count { scalar_locals.push(ScalarLocalTy::arbitrary(u)?); } - let ref_count = (u8::arbitrary(u)? & 0b11).min(2); // 0..=2 - let mut ref_locals = Vec::with_capacity(ref_count as usize); - for _ in 0..ref_count { - ref_locals.push(RefLocalTy::arbitrary(u)?); - } Ok(Self { scalar_locals, has_memory_grow: bool::arbitrary(u)?, clause_variant: ClauseVariant::arbitrary(u)?, wrap_in_outer: bool::arbitrary(u)?, - ref_locals, has_indirect_call: bool::arbitrary(u)?, }) } @@ -155,12 +116,6 @@ impl WatProgram { .map(|ty| format!("(local {}) ", ty.as_wat())) .collect(); - let ref_locals_wat: String = self - .ref_locals - .iter() - .map(|ty| format!("(local {}) ", ty.as_wat())) - .collect(); - let mem_grow = if self.has_memory_grow { "i32.const 0 memory.grow drop" } else { @@ -212,7 +167,7 @@ impl WatProgram { (tag $exn) {extra_decls} (func $caller (export "caller") (result i32) - {locals_wat}{ref_locals_wat} + {locals_wat} {body} i32.const 0) (memory 1)) diff --git a/crates/fork-instrument/src/call_graph.rs b/crates/fork-instrument/src/call_graph.rs index cc58e4971e..53e3b7ff8f 100644 --- a/crates/fork-instrument/src/call_graph.rs +++ b/crates/fork-instrument/src/call_graph.rs @@ -4,15 +4,17 @@ //! `kernel.kernel_fork`), computes the set of functions in the module //! that can transitively reach the seed via calls. //! -//! Discovery follows direct calls and table-aware indirect calls. An -//! indirect call can only reach functions that may inhabit the same -//! table as that `call_indirect` instruction, with the same signature. +//! Discovery follows direct calls, table-aware indirect calls, and typed +//! function-reference calls to a fixed point. Tail calls are transparent +//! edges: execution can reach the seed through them, but their eliminated +//! caller activation is not reported as live at the suspension point. -use std::collections::{HashMap, HashSet, VecDeque}; +use std::collections::{BTreeMap, HashMap, HashSet, VecDeque}; +use anyhow::{Result, bail}; use walrus::ir::{ - self, dfs_in_order, BinaryOp, Call, Instr, InstrLocId, InstrSeqId, ReturnCall, TableCopy, - TableFill, TableGrow, TableInit, TableSet, Visitor, + self, BinaryOp, Call, Instr, InstrLocId, InstrSeqId, TableCopy, TableFill, TableGrow, + TableInit, TableSet, Visitor, dfs_in_order, }; use walrus::{ ConstExpr, ElementId, ElementItems, ElementKind, FunctionId, ImportKind, LocalFunction, Module, @@ -23,19 +25,139 @@ use walrus::{ /// `"kernel.kernel_fork"`). Returns `None` if the module has no such /// import or if the import exists but isn't a function. pub fn find_import_func(module: &Module, qualified_name: &str) -> Option { - let (mod_name, field) = qualified_name.split_once('.')?; + find_import_funcs(module, qualified_name).into_iter().next() +} + +/// Look up every function import with the qualified name. +/// +/// WebAssembly permits more than one import declaration to use the same +/// module/name pair (including declarations with distinct function types). +/// Fork reachability must seed all of them: selecting only the first could +/// leave a live caller activation outside the continuation. +pub fn find_import_funcs(module: &Module, qualified_name: &str) -> Vec { + let Some((mod_name, field)) = qualified_name.split_once('.') else { + return Vec::new(); + }; + let mut functions = Vec::new(); for import in module.imports.iter() { if import.module == mod_name && import.name == field { if let ImportKind::Function(id) = import.kind { - return Some(id); + functions.push(id); } } } - None + functions +} + +/// Every function import in deterministic module order. +/// +/// A dynamically linked side module can call back into the main image or a +/// different side module through any unresolved function import. The callee +/// may eventually fork even when this module does not itself import +/// `env.fork`, so side-boundary analysis uses all of these functions as roots. +pub fn imported_functions(module: &Module) -> Vec { + module + .imports + .iter() + .filter_map(|import| match import.kind { + ImportKind::Function(id) => Some(id), + _ => None, + }) + .collect() +} + +fn is_dynamic_linker_function_import(module: &str, name: &str) -> bool { + module == "env" + && matches!( + name, + "__wasm_dlopen" + | "__wasm_dlopen_main" + | "__wasm_dlopen_prepare" + | "__wasm_dlopen_next" + | "__wasm_dlopen_commit" + | "__wasm_dlsym" + | "__wasm_dlclose" + | "__wasm_dlerror" + ) +} + +fn is_reentrant_dynamic_linker_function_import(module: &str, name: &str) -> bool { + module == "env" && name == "__wasm_dlopen" +} + +/// Legacy host dynamic-linker calls that can synchronously enter guest +/// side-module initialization code. +/// +/// ABI 43's prepare/next/commit imports never enter Wasm. Its libc-owned +/// `call_indirect` is analyzed as the real cross-module suspension boundary. +pub fn dynamic_linker_imported_functions(module: &Module) -> Vec { + module + .imports + .iter() + .filter_map(|import| { + if !is_reentrant_dynamic_linker_function_import( + &import.module, + &import.name, + ) { + return None; + } + match import.kind { + ImportKind::Function(id) => Some(id), + _ => None, + } + }) + .collect() +} + +/// The tail-call instruction that needs an ordinary resumable landing before +/// the fork transform can preserve that control-flow edge. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub enum TailCallKind { + Direct, + Indirect, + Ref, +} + +/// A suspension-capable tail-call site. +/// +/// Tail calls do not contribute an activation to [`ReachingAnalysis::activations`]. +/// They are reported separately for diagnostics and coverage. Replay preserves +/// them as tail calls and routes directly to the next committed activation, +/// rather than materializing a caller frame that did not exist at capture time. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct TailCallSite { + pub caller: FunctionId, + pub sequence: InstrSeqId, + pub instruction_index: usize, + pub kind: TailCallKind, } -/// Walks a single local function, collecting every `Call` target -/// and every indirect-call site. +/// Semantic fork reachability plus tail sites that require transform work. +#[derive(Debug)] +pub struct ReachingAnalysis { + /// Functions whose activations can still be live when `seed` executes. + /// The seed itself is retained for the existing reporting contract. + pub activations: HashSet, + /// Every function through which control can reach `seed`, including + /// transparent tail callers whose activations do not survive. + /// + /// Instrumentation uses this set to recognize ordinary call sites that can + /// suspend even when the lexical callee first traverses a tail-call chain. + pub control_reachable: HashSet, + /// Tail edges on a path to the seed. Their caller activation has already + /// been eliminated, so these sites are not implicitly activations. + pub tail_call_landings: Vec, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +struct ProgramPoint { + sequence: InstrSeqId, + instruction_index: usize, +} + +/// Walks a single local function, collecting ordinary direct calls and table +/// operations. Dispatch and tail sites need lexical provenance, so they are +/// collected by [`collect_dispatch_calls`] below. #[derive(Default)] struct CollectCalls { direct: HashSet, @@ -49,10 +171,6 @@ impl<'a> Visitor<'a> for CollectCalls { self.direct.insert(instr.func); } - fn visit_return_call(&mut self, instr: &ReturnCall) { - self.direct.insert(instr.func); - } - fn visit_table_init(&mut self, instr: &TableInit) { self.table_inits.push((instr.elem, instr.table)); } @@ -74,11 +192,15 @@ impl<'a> Visitor<'a> for CollectCalls { } } -/// Per-function analysis: what it directly calls and what -/// indirect calls/table operations it uses. +/// Per-function analysis: activation-preserving calls, transparent tail calls, +/// and table operations. struct FuncProfile { direct: HashSet, indirect: HashSet, + refs: HashSet, + tail_direct: Vec<(FunctionId, ProgramPoint)>, + tail_indirect: Vec<(IndirectCall, ProgramPoint)>, + tail_refs: Vec<(RefCall, ProgramPoint)>, table_inits: Vec<(ElementId, TableId)>, table_copies: Vec<(TableId, TableId)>, dynamic_table_writes: HashSet, @@ -89,12 +211,16 @@ fn profile_functions(module: &Module) -> HashMap { for (id, func) in module.funcs.iter_local() { let mut collector = CollectCalls::default(); dfs_in_order(&mut collector, func, func.entry_block()); - let indirect = collect_indirect_calls(func); + let dispatch = collect_dispatch_calls(func); profiles.insert( id, FuncProfile { direct: collector.direct, - indirect, + indirect: dispatch.indirect, + refs: dispatch.refs, + tail_direct: dispatch.tail_direct, + tail_indirect: dispatch.tail_indirect, + tail_refs: dispatch.tail_refs, table_inits: collector.table_inits, table_copies: collector.table_copies, dynamic_table_writes: collector.dynamic_table_writes, @@ -104,9 +230,11 @@ fn profile_functions(module: &Module) -> HashMap { profiles } -/// Build the reverse call graph: a map from callee to set of direct -/// callers. Only includes edges originating from local (non-imported) -/// functions, since imported functions have no body to scan. +/// Build the reverse activation graph for ordinary direct calls. +/// +/// Tail calls are intentionally absent: their caller frame no longer exists +/// while the callee executes. Use [`analyze_reaching_closure`] when transparent +/// tail traversal is also required. pub fn build_reverse_call_graph(module: &Module) -> HashMap> { let mut reverse: HashMap> = HashMap::new(); for (caller_id, profile) in profile_functions(module) { @@ -117,24 +245,47 @@ pub fn build_reverse_call_graph(module: &Module) -> HashMap HashSet { - let reverse = build_reverse_call_graph(module); - let mut result = HashSet::new(); + let profiles = profile_functions(module); + let mut reverse: HashMap> = HashMap::new(); + let mut reverse_tail: HashMap> = HashMap::new(); + for (caller, profile) in &profiles { + for &callee in &profile.direct { + reverse.entry(callee).or_default().insert(*caller); + } + for &(callee, _) in &profile.tail_direct { + reverse_tail.entry(callee).or_default().insert(*caller); + } + } + + let mut activations = HashSet::new(); + let mut reachable = HashSet::new(); let mut queue = VecDeque::new(); - result.insert(seed); + activations.insert(seed); + reachable.insert(seed); queue.push_back(seed); while let Some(f) = queue.pop_front() { if let Some(callers) = reverse.get(&f) { for &caller in callers { - if result.insert(caller) { + activations.insert(caller); + if reachable.insert(caller) { + queue.push_back(caller); + } + } + } + if let Some(callers) = reverse_tail.get(&f) { + for &caller in callers { + if reachable.insert(caller) { queue.push_back(caller); } } } } - result + activations } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] @@ -150,54 +301,97 @@ enum IndexProof { Unknown, } -fn collect_indirect_calls(func: &LocalFunction) -> HashSet { - let mut calls = HashSet::new(); - collect_indirect_calls_seq(func, func.entry_block(), &mut calls); +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +struct RefCall { + ty: TypeId, + target: RefTargetProof, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +enum RefTargetProof { + Func(FunctionId), + Null, + Unknown, +} + +#[derive(Default)] +struct DispatchCalls { + indirect: HashSet, + refs: HashSet, + tail_direct: Vec<(FunctionId, ProgramPoint)>, + tail_indirect: Vec<(IndirectCall, ProgramPoint)>, + tail_refs: Vec<(RefCall, ProgramPoint)>, +} + +fn collect_dispatch_calls(func: &LocalFunction) -> DispatchCalls { + let mut calls = DispatchCalls::default(); + collect_dispatch_calls_seq(func, func.entry_block(), &mut calls); calls } -fn collect_indirect_calls_seq( - func: &LocalFunction, - seq_id: InstrSeqId, - calls: &mut HashSet, -) { +fn collect_dispatch_calls_seq(func: &LocalFunction, seq_id: InstrSeqId, calls: &mut DispatchCalls) { let instrs = &func.block(seq_id).instrs; for (idx, (instr, _)) in instrs.iter().enumerate() { + let point = ProgramPoint { + sequence: seq_id, + instruction_index: idx, + }; match instr { Instr::CallIndirect(call) => { - calls.insert(IndirectCall { + calls.indirect.insert(IndirectCall { table: call.table, ty: call.ty, index: infer_call_indirect_index(&instrs[..idx]), }); } Instr::ReturnCallIndirect(call) => { - calls.insert(IndirectCall { - table: call.table, + calls.tail_indirect.push(( + IndirectCall { + table: call.table, + ty: call.ty, + index: infer_call_indirect_index(&instrs[..idx]), + }, + point, + )); + } + Instr::CallRef(call) => { + calls.refs.insert(RefCall { ty: call.ty, - index: infer_call_indirect_index(&instrs[..idx]), + target: infer_call_ref_target(&instrs[..idx]), }); } + Instr::ReturnCallRef(call) => { + calls.tail_refs.push(( + RefCall { + ty: call.ty, + target: infer_call_ref_target(&instrs[..idx]), + }, + point, + )); + } + Instr::ReturnCall(call) => { + calls.tail_direct.push((call.func, point)); + } Instr::Block(ir::Block { seq }) | Instr::Loop(ir::Loop { seq }) => { - collect_indirect_calls_seq(func, *seq, calls); + collect_dispatch_calls_seq(func, *seq, calls); } Instr::IfElse(ir::IfElse { consequent, alternative, }) => { - collect_indirect_calls_seq(func, *consequent, calls); - collect_indirect_calls_seq(func, *alternative, calls); + collect_dispatch_calls_seq(func, *consequent, calls); + collect_dispatch_calls_seq(func, *alternative, calls); } Instr::TryTable(ir::TryTable { seq, .. }) => { - collect_indirect_calls_seq(func, *seq, calls); + collect_dispatch_calls_seq(func, *seq, calls); } Instr::Try(ir::Try { seq, catches }) => { - collect_indirect_calls_seq(func, *seq, calls); + collect_dispatch_calls_seq(func, *seq, calls); for catch in catches { match catch { ir::LegacyCatch::Catch { handler, .. } | ir::LegacyCatch::CatchAll { handler } => { - collect_indirect_calls_seq(func, *handler, calls); + collect_dispatch_calls_seq(func, *handler, calls); } ir::LegacyCatch::Delegate { .. } => {} } @@ -208,6 +402,30 @@ fn collect_indirect_calls_seq( } } +fn infer_call_ref_target(prefix: &[(Instr, InstrLocId)]) -> RefTargetProof { + infer_ref_expr(prefix, prefix.len()) + .map(|(proof, _)| proof) + .unwrap_or(RefTargetProof::Unknown) +} + +fn infer_ref_expr(instrs: &[(Instr, InstrLocId)], end: usize) -> Option<(RefTargetProof, usize)> { + if end == 0 { + return None; + } + + let idx = end - 1; + match &instrs[idx].0 { + Instr::RefFunc(reference) => Some((RefTargetProof::Func(reference.func), idx)), + Instr::RefNull(_) => Some((RefTargetProof::Null, idx)), + // These instructions preserve the identity of the single reference + // operand. Recovering through them avoids whole-signature fallback for + // the common typed-ref lowering without pretending local/global values + // have lexical provenance. + Instr::RefAsNonNull(_) | Instr::RefCast(_) => infer_ref_expr(instrs, idx), + _ => Some((RefTargetProof::Unknown, idx)), + } +} + fn infer_call_indirect_index(prefix: &[(Instr, InstrLocId)]) -> IndexProof { infer_i32_expr(prefix, prefix.len()) .map(|(proof, _)| proof) @@ -553,11 +771,10 @@ fn function_type_id(module: &Module, id: FunctionId) -> TypeId { module.funcs.get(id).ty() } -/// Check whether two type ids refer to structurally identical -/// function types (same params, same results). For modern wasm with -/// type indices the ids usually match exactly when two functions -/// share a signature, but we compare structurally to be robust to -/// modules where the same signature has multiple type-section entries. +/// Check whether two type ids refer to structurally identical function types. +/// +/// Type ids usually match when two functions share a signature, but separate +/// type-section entries can encode the same non-recursive signature. fn types_match(module: &Module, a: TypeId, b: TypeId) -> bool { if a == b { return true; @@ -567,7 +784,66 @@ fn types_match(module: &Module, a: TypeId, b: TypeId) -> bool { ta.params() == tb.params() && ta.results() == tb.results() } -const MAX_INDIRECT_DEPTH: u8 = 2; +/// Whether `candidate` can satisfy a dispatch instruction expecting +/// `expected`. +/// +/// Typed function references allow a function type to be a declared subtype +/// of the call's expected type. Structural equality remains accepted for +/// duplicate non-recursive type-section entries, matching the historical +/// call-indirect behavior. +fn function_type_is_subtype(module: &Module, candidate: TypeId, expected: TypeId) -> bool { + let mut current = Some(candidate); + let mut seen = HashSet::new(); + while let Some(ty) = current { + if !seen.insert(ty) { + // Valid Wasm cannot contain a supertype cycle. Keep malformed + // internal modules finite rather than turning graph discovery into + // an unbounded walk. + return false; + } + if types_match(module, ty, expected) { + return true; + } + current = module.types.get(ty).supertype; + } + false +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +struct FunctionSignature { + params: Vec, + results: Vec, +} + +fn function_signature(module: &Module, ty: TypeId) -> FunctionSignature { + let ty = module.types.get(ty); + FunctionSignature { + params: ty.params().to_vec(), + results: ty.results().to_vec(), + } +} + +fn compatible_dispatch_types( + module: &Module, + candidate: TypeId, + dispatch_types_by_signature: &HashMap>, +) -> Vec { + let mut compatible = HashSet::new(); + let mut current = Some(candidate); + let mut seen = HashSet::new(); + while let Some(ty) = current { + if !seen.insert(ty) { + break; + } + if let Some(expected_types) = + dispatch_types_by_signature.get(&function_signature(module, ty)) + { + compatible.extend(expected_types.iter().copied()); + } + current = module.types.get(ty).supertype; + } + compatible.into_iter().collect() +} /// Whether this module can resolve and invoke functions installed by Kandelo's /// dynamic linker after static call-graph analysis has completed. @@ -577,146 +853,353 @@ const MAX_INDIRECT_DEPTH: u8 = 2; /// closure below and the artifact claim consumed by the host runtime. pub fn has_dynamic_linker_imports(module: &Module) -> bool { module.imports.iter().any(|import| { - import.module == "env" - && matches!(import.kind, ImportKind::Function(_)) - && matches!( - import.name.as_str(), - "__wasm_dlopen" | "__wasm_dlsym" | "__wasm_dlclose" | "__wasm_dlerror" - ) + matches!(import.kind, ImportKind::Function(_)) + && is_dynamic_linker_function_import(&import.module, &import.name) }) } -/// Compute the transitive closure of functions that reach `seed` via -/// direct calls, plus a bounded number of table/function-pointer dispatches. +/// Compute semantic fork reachability to a fixed point. /// -/// A function `F` reaches `seed` if any of these hold: +/// A function activation `F` can be live at `seed` if any of these hold: /// (1) `F == seed` -/// (2) `F` directly calls some function `G` that reaches `seed` -/// (3) `F` executes `call_indirect` of type `T`, and some -/// function `G` of type `T` reaches `seed` and may inhabit the -/// same table that `F` indexes +/// (2) `F` ordinarily calls a function whose execution reaches `seed` +/// (3) `F` executes `call_indirect` that can dispatch to such a function +/// (4) `F` executes `call_ref` whose proven or type-compatible target can +/// reach `seed` /// -/// Rule 3 is intentionally bounded. Functions discovered through indirect -/// edges still pull in their direct callers, but after `MAX_INDIRECT_DEPTH` -/// indirect hops they do not become new indirect roots. Depth 2 covers the -/// common C/POSIX callback cases plus QuickJS's C-function trampoline -/// (`JS_CallInternal -> js_call_c_function -> js_os_exec`) while avoiding -/// whole-runtime closure in dynamic interpreters where a generic dispatcher -/// can theoretically call thousands of same-table, same-signature callbacks. -pub fn reaching_closure(module: &Module, seed: FunctionId) -> HashSet { +/// Direct, indirect, and reference edges participate in one worklist until it +/// reaches a fixed point. `return_call*` edges make their callers +/// control-reachable but not activation-live; traversal continues through +/// those transparent nodes so an older ordinary caller is still discovered. +pub fn analyze_reaching_closure(module: &Module, seed: FunctionId) -> ReachingAnalysis { + analyze_reaching_closure_from_seeds(module, [seed], has_dynamic_linker_imports(module)) +} + +/// Compute semantic fork reachability from every supplied suspension boundary. +/// +/// `external_dynamic_dispatch` means an unresolved `call_indirect` or +/// `call_ref` may enter another module that can fork. This is true for +/// dlopen-capable main modules and for every dynamically linked side module. +/// It broadens instrumentation only; ordinary valid Wasm is never rejected. +pub fn analyze_reaching_closure_from_seeds( + module: &Module, + seeds: impl IntoIterator, + external_dynamic_dispatch: bool, +) -> ReachingAnalysis { let profiles = profile_functions(module); let table_targets = table_targets(module, &profiles); - // A dlsym result can be installed into the main module's table only after - // static analysis. Every call_indirect in a dlopen-capable main module is - // therefore a possible boundary above a fork-capable side-module frame. - // Keep this opt-in to the dynamic-linker imports so ordinary programs - // retain the precise table-target closure below. - let has_dynamic_linker_imports = has_dynamic_linker_imports(module); - - // Reverse direct-call graph: `callee -> set of callers`. + + // Reverse ordinary and transparent direct-call graphs. let mut reverse_direct: HashMap> = HashMap::new(); + let mut reverse_tail_direct: HashMap> = + HashMap::new(); for (caller, profile) in &profiles { for callee in &profile.direct { reverse_direct.entry(*callee).or_default().insert(*caller); } + for &(callee, point) in &profile.tail_direct { + reverse_tail_direct + .entry(callee) + .or_default() + .push((*caller, point)); + } } - // Reverse indirect-call graph: `(table, call_indirect type T) -> - // callers that index that table with type T`. We compare types - // structurally (§types_match); TypeId is still stored and compared - // at lookup time rather than forcing exact type-index equality. - let indirect_callers: Vec<(IndirectCall, FunctionId)> = profiles - .iter() - .flat_map(|(caller, profile)| { - profile - .indirect - .iter() - .map(move |indirect| (*indirect, *caller)) - }) - .collect(); + // Index dynamic dispatch sites by their expected type. Compatibility is + // computed once per reached candidate type rather than rescanning every + // site for every function. + let mut indirect_callers: HashMap> = HashMap::new(); + let mut tail_indirect_callers: HashMap> = + HashMap::new(); + let mut unknown_ref_callers: HashMap> = HashMap::new(); + let mut tail_unknown_ref_callers: HashMap> = + HashMap::new(); + let mut precise_ref_callers: HashMap> = HashMap::new(); + let mut tail_precise_ref_callers: HashMap> = + HashMap::new(); + let mut dispatch_types = HashSet::new(); - // First compute the direct-only closure. Every function in this set - // reaches the seed without crossing a function-pointer dispatch, so it - // is safe to use as an indirect root below. - let mut result = HashSet::new(); - let mut direct_queue = VecDeque::new(); - result.insert(seed); - direct_queue.push_back(seed); - while let Some(g) = direct_queue.pop_front() { - if let Some(callers) = reverse_direct.get(&g) { - for &caller in callers { - if result.insert(caller) { - direct_queue.push_back(caller); + for (&caller, profile) in &profiles { + for &indirect in &profile.indirect { + dispatch_types.insert(indirect.ty); + indirect_callers + .entry(indirect.ty) + .or_default() + .push((indirect, caller)); + } + for &(indirect, point) in &profile.tail_indirect { + dispatch_types.insert(indirect.ty); + tail_indirect_callers + .entry(indirect.ty) + .or_default() + .push((indirect, caller, point)); + } + for &reference in &profile.refs { + dispatch_types.insert(reference.ty); + match reference.target { + RefTargetProof::Func(target) => precise_ref_callers + .entry(target) + .or_default() + .push((reference.ty, caller)), + RefTargetProof::Null => {} + RefTargetProof::Unknown => { + unknown_ref_callers + .entry(reference.ty) + .or_default() + .insert(caller); } } } + for &(reference, point) in &profile.tail_refs { + dispatch_types.insert(reference.ty); + match reference.target { + RefTargetProof::Func(target) => tail_precise_ref_callers + .entry(target) + .or_default() + .push((reference.ty, caller, point)), + RefTargetProof::Null => {} + RefTargetProof::Unknown => tail_unknown_ref_callers + .entry(reference.ty) + .or_default() + .push((caller, point)), + } + } + } + let mut dispatch_types_by_signature: HashMap> = HashMap::new(); + for &ty in &dispatch_types { + dispatch_types_by_signature + .entry(function_signature(module, ty)) + .or_default() + .push(ty); } - let direct_roots = result.clone(); - let mut best_indirect_depth: HashMap = - direct_roots.iter().map(|&id| (id, 0)).collect(); - let mut worklist: VecDeque<(FunctionId, u8)> = direct_roots.iter().map(|&id| (id, 0)).collect(); + let mut activations = HashSet::new(); + let mut control_reachable = HashSet::new(); + let mut tail_call_landings = HashSet::new(); + let mut worklist = VecDeque::new(); + for seed in seeds { + activations.insert(seed); + if control_reachable.insert(seed) { + worklist.push_back(seed); + } + } - fn enqueue( - func: FunctionId, - indirect_depth: u8, - best_indirect_depth: &mut HashMap, - result: &mut HashSet, - worklist: &mut VecDeque<(FunctionId, u8)>, + fn discover( + caller: FunctionId, + activation_survives: bool, + activations: &mut HashSet, + control_reachable: &mut HashSet, + worklist: &mut VecDeque, ) { - let should_enqueue = match best_indirect_depth.get(&func) { - Some(&old_depth) => indirect_depth < old_depth, - None => true, - }; - if should_enqueue { - best_indirect_depth.insert(func, indirect_depth); - result.insert(func); - worklist.push_back((func, indirect_depth)); + if activation_survives { + activations.insert(caller); + } + if control_reachable.insert(caller) { + worklist.push_back(caller); } } - if has_dynamic_linker_imports { + if external_dynamic_dispatch { for (&caller, profile) in &profiles { if !profile.indirect.is_empty() { - enqueue( + discover( + caller, + true, + &mut activations, + &mut control_reachable, + &mut worklist, + ); + } + if profile + .refs + .iter() + .any(|reference| reference.target == RefTargetProof::Unknown) + { + // WHY: a funcref received from another module has no local + // FunctionId. Treat the call site itself as the boundary so + // the live caller is activation-owned before control crosses + // the instance boundary. + discover( caller, - 1, - &mut best_indirect_depth, - &mut result, + true, + &mut activations, + &mut control_reachable, + &mut worklist, + ); + } + for &(_, point) in &profile.tail_indirect { + // A side-module function installed after instrumentation is + // not present in any static target set. A tail dispatch can + // still reach its fork path, but the caller frame is gone. + tail_call_landings.insert(TailCallSite { + caller, + sequence: point.sequence, + instruction_index: point.instruction_index, + kind: TailCallKind::Indirect, + }); + discover( + caller, + false, + &mut activations, + &mut control_reachable, + &mut worklist, + ); + } + for &(reference, point) in &profile.tail_refs { + if reference.target != RefTargetProof::Unknown { + continue; + } + tail_call_landings.insert(TailCallSite { + caller, + sequence: point.sequence, + instruction_index: point.instruction_index, + kind: TailCallKind::Ref, + }); + discover( + caller, + false, + &mut activations, + &mut control_reachable, &mut worklist, ); } } } - while let Some((g, indirect_depth)) = worklist.pop_front() { - // (2) Direct-reverse: who calls g directly? + let mut compatible_type_cache: HashMap> = HashMap::new(); + while let Some(g) = worklist.pop_front() { + // Ordinary direct callers retain an activation. if let Some(callers) = reverse_direct.get(&g) { for &caller in callers { - enqueue( + discover( + caller, + true, + &mut activations, + &mut control_reachable, + &mut worklist, + ); + } + } + + // A true tail caller is a transparent control-flow node. Record the + // exact site for selective lowering, and continue walking through it + // without claiming that its eliminated frame survives. + if let Some(callers) = reverse_tail_direct.get(&g) { + for &(caller, point) in callers { + tail_call_landings.insert(TailCallSite { + caller, + sequence: point.sequence, + instruction_index: point.instruction_index, + kind: TailCallKind::Direct, + }); + discover( caller, - indirect_depth, - &mut best_indirect_depth, - &mut result, + false, + &mut activations, + &mut control_reachable, &mut worklist, ); } } - // (3) Indirect-reverse: every function that does - // `call_indirect` with g's signature against a table that can - // contain g might be reaching g. Add those callers. - if indirect_depth < MAX_INDIRECT_DEPTH { - let g_ty = function_type_id(module, g); - for &(indirect, caller) in &indirect_callers { - if table_targets.table_can_dispatch(indirect, g) - && types_match(module, indirect.ty, g_ty) - { - enqueue( + let g_ty = function_type_id(module, g); + + // A statically proven ref.func target does not need the all-compatible + // fallback. The type check remains explicit so hand-built walrus + // modules cannot manufacture an impossible edge. + if let Some(callers) = precise_ref_callers.get(&g) { + for &(expected, caller) in callers { + if function_type_is_subtype(module, g_ty, expected) { + discover( caller, - indirect_depth + 1, - &mut best_indirect_depth, - &mut result, + true, + &mut activations, + &mut control_reachable, + &mut worklist, + ); + } + } + } + if let Some(callers) = tail_precise_ref_callers.get(&g) { + for &(expected, caller, point) in callers { + if function_type_is_subtype(module, g_ty, expected) { + tail_call_landings.insert(TailCallSite { + caller, + sequence: point.sequence, + instruction_index: point.instruction_index, + kind: TailCallKind::Ref, + }); + discover( + caller, + false, + &mut activations, + &mut control_reachable, + &mut worklist, + ); + } + } + } + + let compatible_types = compatible_type_cache.entry(g_ty).or_insert_with(|| { + compatible_dispatch_types(module, g_ty, &dispatch_types_by_signature) + }); + for expected in compatible_types.iter().copied() { + if let Some(callers) = indirect_callers.get(&expected) { + for &(indirect, caller) in callers { + if table_targets.table_can_dispatch(indirect, g) { + discover( + caller, + true, + &mut activations, + &mut control_reachable, + &mut worklist, + ); + } + } + } + if let Some(callers) = tail_indirect_callers.get(&expected) { + for &(indirect, caller, point) in callers { + if table_targets.table_can_dispatch(indirect, g) { + tail_call_landings.insert(TailCallSite { + caller, + sequence: point.sequence, + instruction_index: point.instruction_index, + kind: TailCallKind::Indirect, + }); + discover( + caller, + false, + &mut activations, + &mut control_reachable, + &mut worklist, + ); + } + } + } + if let Some(callers) = unknown_ref_callers.get(&expected) { + for &caller in callers { + discover( + caller, + true, + &mut activations, + &mut control_reachable, + &mut worklist, + ); + } + } + if let Some(callers) = tail_unknown_ref_callers.get(&expected) { + for &(caller, point) in callers { + tail_call_landings.insert(TailCallSite { + caller, + sequence: point.sequence, + instruction_index: point.instruction_index, + kind: TailCallKind::Ref, + }); + discover( + caller, + false, + &mut activations, + &mut control_reachable, &mut worklist, ); } @@ -724,7 +1207,122 @@ pub fn reaching_closure(module: &Module, seed: FunctionId) -> HashSet = tail_call_landings.into_iter().collect(); + tail_call_landings.sort_by_key(|site| { + ( + site.caller.index(), + site.sequence.index(), + site.instruction_index, + site.kind, + ) + }); + ReachingAnalysis { + activations, + control_reachable, + tail_call_landings, + } +} + +/// Lower only fork-reaching tail calls to ordinary calls followed by `return`. +/// +/// This is the bridge between semantic analysis and frame instrumentation: +/// analysis first reports the caller as eliminated, this pass creates a real +/// resumable landing at the suspension-capable edge, and a second analysis +/// then includes that newly materialized activation. Tail calls outside +/// `sites` retain their original stack and performance semantics. +/// +/// `sites` must come from [`analyze_reaching_closure`] for this module before +/// any other mutation. Stale or mismatched instruction kinds fail explicitly. +pub fn lower_tail_call_landings(module: &mut Module, sites: &[TailCallSite]) -> Result<()> { + let mut grouped: BTreeMap>> = + BTreeMap::new(); + let mut unique = HashSet::new(); + for &site in sites { + if unique.insert(site) { + grouped + .entry(site.caller) + .or_default() + .entry(site.sequence) + .or_default() + .push(site); + } + } + + for (caller, sequences) in grouped { + let caller_name = func_display_name(module, caller); + let function = module.funcs.get_mut(caller); + let walrus::FunctionKind::Local(local) = &mut function.kind else { + bail!( + "tail-call landing for `{caller_name}` names an imported \ + function; reachability metadata is stale" + ); + }; + + for (sequence, mut sequence_sites) in sequences { + // Inserting after a site shifts later indexes only. Descending + // mutation preserves every original program point in this seq. + sequence_sites.sort_by_key(|site| std::cmp::Reverse(site.instruction_index)); + let instrs = &mut local.block_mut(sequence).instrs; + for site in sequence_sites { + let Some((instruction, location)) = instrs.get(site.instruction_index).cloned() + else { + bail!( + "tail-call landing for `{caller_name}` points past the \ + end of sequence {:?}; reachability metadata is stale", + sequence + ); + }; + + let (replacement, actual_kind) = match instruction { + Instr::ReturnCall(call) => ( + Instr::Call(ir::Call { func: call.func }), + TailCallKind::Direct, + ), + Instr::ReturnCallIndirect(call) => ( + Instr::CallIndirect(ir::CallIndirect { + ty: call.ty, + table: call.table, + }), + TailCallKind::Indirect, + ), + Instr::ReturnCallRef(call) => ( + Instr::CallRef(ir::CallRef { ty: call.ty }), + TailCallKind::Ref, + ), + other => { + bail!( + "tail-call landing for `{caller_name}` points at \ + non-tail instruction {other:?}; reachability \ + metadata is stale" + ); + } + }; + if actual_kind != site.kind { + bail!( + "tail-call landing for `{caller_name}` expected {:?} \ + but found {actual_kind:?}; reachability metadata is stale", + site.kind + ); + } + + instrs.splice( + site.instruction_index..=site.instruction_index, + [ + (replacement, location), + (Instr::Return(ir::Return {}), location), + ], + ); + } + } + } + Ok(()) +} + +/// Compute the set of activation-live functions that need fork frame +/// instrumentation. Use [`analyze_reaching_closure`] when the transform also +/// needs the exact suspension-capable tail sites. +pub fn reaching_closure(module: &Module, seed: FunctionId) -> HashSet { + analyze_reaching_closure(module, seed).activations } /// Human-readable name for a function, for logging and JSON output. diff --git a/crates/fork-instrument/src/contract_inventory.rs b/crates/fork-instrument/src/contract_inventory.rs new file mode 100644 index 0000000000..819468b822 --- /dev/null +++ b/crates/fork-instrument/src/contract_inventory.rs @@ -0,0 +1,614 @@ +//! Structural inventory for the fork-artifact publication guards. +//! +//! The fork-contract inventory inspects only sections. Artifact identity also +//! verifies the exact constant ABI thunk, but decodes only that function and +//! its optional delegate: large package executables should not need a full text +//! disassembly or full-module instruction decode for publication checks. + +use anyhow::{Context, Result, bail}; +use std::fmt::{self, Write}; +use wasm_posix_shared::abi::{ + ABI_KERNEL_EXPORT, WPK_FORK_CAPABILITIES_SECTION, WPK_FORK_EXPORT_ABORT_BEGIN, + WPK_FORK_EXPORT_ABORT_END, WPK_FORK_EXPORT_REWIND_BEGIN, WPK_FORK_EXPORT_REWIND_END, + WPK_FORK_EXPORT_STATE, WPK_FORK_EXPORT_UNWIND_BEGIN, WPK_FORK_EXPORT_UNWIND_END, + WPK_FORK_FRAME_IMPORT_COMMIT, WPK_FORK_FRAME_IMPORT_MODULE, WPK_FORK_FRAME_IMPORT_NEXT, + WPK_FORK_FRAME_IMPORT_RESERVE, WPK_FORK_LINKED_FRAME_FORMAT_SECTION, +}; +use wasmparser::{ + CompositeInnerType, Encoding, ExternalKind, FuncType, FunctionBody, Operator, Parser, Payload, + TypeRef, ValType, +}; + +/// The exact tab-separated inventory consumed by `wasm-artifact-guards.sh`. +#[derive(Debug, Default, Clone, PartialEq, Eq)] +pub struct ForkContractInventory { + pub relocatable: usize, + pub imports_kernel_fork: usize, + pub frame_reserve: usize, + pub frame_commit: usize, + pub frame_next: usize, + pub linked_descriptor: usize, + pub fork_capability: usize, + pub abort_begin: usize, + pub abort_end: usize, + pub rewind_begin: usize, + pub rewind_end: usize, + pub state: usize, + pub unwind_begin: usize, + pub unwind_end: usize, + pub memory_count: usize, + pub memory64_count: usize, + pub signature_mismatch: usize, + pub legacy_dlopen: usize, + pub native_start: usize, +} + +impl fmt::Display for ForkContractInventory { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}", + self.relocatable, + self.imports_kernel_fork, + self.frame_reserve, + self.frame_commit, + self.frame_next, + self.linked_descriptor, + self.fork_capability, + self.abort_begin, + self.abort_end, + self.rewind_begin, + self.rewind_end, + self.state, + self.unwind_begin, + self.unwind_end, + self.memory_count, + self.memory64_count, + self.signature_mismatch, + self.legacy_dlopen, + self.native_start, + ) + } +} + +/// Strict status of the optional `__abi_version` artifact export. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ArtifactAbiVersion { + Missing, + Invalid, + Present(u32), +} + +/// Structural identity needed by executable publication guards. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ArtifactIdentity { + pub relocatable: usize, + pub memory_count: usize, + pub memory64_count: usize, + pub abi_version: ArtifactAbiVersion, + pub imports_kernel_fork: usize, + pub has_fork_exports: usize, +} + +impl fmt::Display for ArtifactIdentity { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let (status, version) = match self.abi_version { + ArtifactAbiVersion::Missing => ("missing", "-".to_string()), + ArtifactAbiVersion::Invalid => ("invalid", "-".to_string()), + ArtifactAbiVersion::Present(version) => ("present", version.to_string()), + }; + write!( + f, + "{}\t{}\t{}\t{status}\t{}\t{}\t{}", + self.relocatable, + self.memory_count, + self.memory64_count, + version, + self.imports_kernel_fork, + self.has_fork_exports, + ) + } +} + +#[derive(Debug, Clone, Copy)] +enum ExpectedSignature { + PointerToPointer, + PointerToNil, + NilToNil, + NilToI32, +} + +/// Inspect the binary structure used to decide whether a module carries one +/// complete fork-instrumentation contract. +pub fn fork_contract_inventory(bytes: &[u8]) -> Result { + let mut inventory = ForkContractInventory::default(); + let mut types: Vec> = Vec::new(); + let mut function_type_indices: Vec = Vec::new(); + let mut checked_functions: Vec<(u32, ExpectedSignature)> = Vec::new(); + + for payload in Parser::new(0).parse_all(bytes) { + match payload.context("parsing wasm structure for fork-contract inventory")? { + Payload::Version { encoding, .. } => { + if encoding != Encoding::Module { + bail!("fork-contract inventory requires a core wasm module"); + } + } + Payload::TypeSection(groups) => { + for group in groups { + let group = group.context("parsing wasm type section")?; + types.extend(group.into_types().map( + |subtype| match subtype.composite_type.inner { + CompositeInnerType::Func(function) => Some(function), + CompositeInnerType::Array(_) + | CompositeInnerType::Struct(_) + | CompositeInnerType::Cont(_) => None, + }, + )); + } + } + Payload::ImportSection(imports) => { + for import in imports.into_imports() { + let import = import.context("parsing wasm import section")?; + match import.ty { + TypeRef::Func(type_index) | TypeRef::FuncExact(type_index) => { + let function_index = function_type_indices.len() as u32; + function_type_indices.push(type_index); + if import.module == "kernel" && import.name == "kernel_fork" { + inventory.imports_kernel_fork = 1; + } + if import.module == "env" && import.name == "__wasm_dlopen" { + inventory.legacy_dlopen += 1; + } + if import.module == WPK_FORK_FRAME_IMPORT_MODULE { + let expected = match import.name { + WPK_FORK_FRAME_IMPORT_RESERVE => { + inventory.frame_reserve += 1; + Some(ExpectedSignature::PointerToPointer) + } + WPK_FORK_FRAME_IMPORT_COMMIT => { + inventory.frame_commit += 1; + Some(ExpectedSignature::PointerToNil) + } + WPK_FORK_FRAME_IMPORT_NEXT => { + inventory.frame_next += 1; + Some(ExpectedSignature::PointerToPointer) + } + _ => None, + }; + if let Some(expected) = expected { + checked_functions.push((function_index, expected)); + } + } + } + TypeRef::Memory(memory) => { + inventory.memory_count += 1; + inventory.memory64_count += usize::from(memory.memory64); + } + TypeRef::Table(_) | TypeRef::Global(_) | TypeRef::Tag(_) => {} + } + } + } + Payload::FunctionSection(functions) => { + for type_index in functions { + function_type_indices + .push(type_index.context("parsing wasm function section")?); + } + } + Payload::MemorySection(memories) => { + for memory in memories { + let memory = memory.context("parsing wasm memory section")?; + inventory.memory_count += 1; + inventory.memory64_count += usize::from(memory.memory64); + } + } + Payload::StartSection { .. } => { + inventory.native_start += 1; + } + Payload::ExportSection(exports) => { + for export in exports { + let export = export.context("parsing wasm export section")?; + if export.kind != ExternalKind::Func { + continue; + } + let expected = match export.name { + WPK_FORK_EXPORT_ABORT_BEGIN => { + inventory.abort_begin += 1; + Some(ExpectedSignature::PointerToNil) + } + WPK_FORK_EXPORT_ABORT_END => { + inventory.abort_end += 1; + Some(ExpectedSignature::NilToNil) + } + WPK_FORK_EXPORT_REWIND_BEGIN => { + inventory.rewind_begin += 1; + Some(ExpectedSignature::PointerToNil) + } + WPK_FORK_EXPORT_REWIND_END => { + inventory.rewind_end += 1; + Some(ExpectedSignature::NilToNil) + } + WPK_FORK_EXPORT_STATE => { + inventory.state += 1; + Some(ExpectedSignature::NilToI32) + } + WPK_FORK_EXPORT_UNWIND_BEGIN => { + inventory.unwind_begin += 1; + Some(ExpectedSignature::PointerToNil) + } + WPK_FORK_EXPORT_UNWIND_END => { + inventory.unwind_end += 1; + Some(ExpectedSignature::NilToNil) + } + _ => None, + }; + if let Some(expected) = expected { + checked_functions.push((export.index, expected)); + } + } + } + Payload::CustomSection(section) => match section.name() { + "linking" => inventory.relocatable = 1, + name if name.starts_with("reloc.") => inventory.relocatable = 1, + WPK_FORK_LINKED_FRAME_FORMAT_SECTION => inventory.linked_descriptor += 1, + WPK_FORK_CAPABILITIES_SECTION => inventory.fork_capability += 1, + _ => {} + }, + _ => {} + } + } + + let pointer = if inventory.memory_count == 1 && inventory.memory64_count == 1 { + ValType::I64 + } else { + ValType::I32 + }; + for (function_index, expected) in checked_functions { + let signature = function_type_indices + .get(function_index as usize) + .and_then(|type_index| types.get(*type_index as usize)) + .and_then(Option::as_ref); + if !signature.is_some_and(|signature| signature_matches(signature, expected, pointer)) { + inventory.signature_mismatch += 1; + } + } + + Ok(inventory) +} + +/// Inspect the object kind, memory width, and strict constant ABI export in +/// one wasmparser-backed CLI request. +/// +/// WHY: ABI 43's generated reference/exception helpers use proposal features +/// that older WABT releases cannot disassemble. Publication must not confuse a +/// text-decoder limitation with an unsafe artifact, and it must not weaken the +/// exact constant-return ABI contract to work around that limitation. +pub fn artifact_identity(bytes: &[u8]) -> Result { + let contract = fork_contract_inventory(bytes)?; + let has_fork_exports = usize::from( + contract.abort_begin + + contract.abort_end + + contract.rewind_begin + + contract.rewind_end + + contract.state + + contract.unwind_begin + + contract.unwind_end + != 0, + ); + Ok(ArtifactIdentity { + relocatable: contract.relocatable, + memory_count: contract.memory_count, + memory64_count: contract.memory64_count, + abi_version: artifact_abi_version(bytes)?, + imports_kernel_fork: contract.imports_kernel_fork, + has_fork_exports, + }) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum AbiOperator { + I32Const(i32), + Call(u32), + Return, + End, +} + +#[derive(Debug, Clone, Copy)] +struct AbiBody { + operators: [Option; 3], + operator_count: usize, + exact: bool, +} + +impl AbiBody { + fn pure_constant(self) -> Option { + let value = match (self.exact, self.operator_count, self.operators) { + ( + true, + 2, + [Some(AbiOperator::I32Const(value)), Some(AbiOperator::End), None], + ) + | ( + true, + 3, + [ + Some(AbiOperator::I32Const(value)), + Some(AbiOperator::Return), + Some(AbiOperator::End), + ], + ) => value, + _ => return None, + }; + u32::try_from(value).ok() + } +} + +fn parse_abi_body(body: FunctionBody<'_>) -> Result { + for local in body + .get_locals_reader() + .context("reading ABI candidate locals")? + { + local.context("reading ABI candidate local")?; + } + + let mut operators = [None; 3]; + let mut operator_count = 0usize; + let mut exact = true; + let mut reader = body + .get_operators_reader() + .context("reading ABI candidate operators")?; + while !reader.eof() { + let operator = match reader + .read() + .context("decoding ABI candidate operator")? + { + Operator::I32Const { value } => Some(AbiOperator::I32Const(value)), + Operator::Call { function_index } => Some(AbiOperator::Call(function_index)), + Operator::Return => Some(AbiOperator::Return), + Operator::End => Some(AbiOperator::End), + _ => None, + }; + if operator_count < operators.len() { + operators[operator_count] = operator; + } else { + exact = false; + } + exact &= operator.is_some(); + operator_count += 1; + } + Ok(AbiBody { + operators, + operator_count, + exact, + }) +} + +fn function_signature<'a>( + types: &'a [Option], + function_type_indices: &[u32], + function_index: u32, +) -> Option<&'a FuncType> { + function_type_indices + .get(function_index as usize) + .and_then(|type_index| types.get(*type_index as usize)) + .and_then(Option::as_ref) +} + +fn signature_is(signature: Option<&FuncType>, params: &[ValType], results: &[ValType]) -> bool { + signature.is_some_and(|signature| { + signature.params() == params && signature.results() == results + }) +} + +fn body_for( + bytes: &[u8], + imported_function_count: usize, + function_index: u32, +) -> Result> { + let Some(local_index) = (function_index as usize).checked_sub(imported_function_count) else { + return Ok(None); + }; + let mut current_local_index = 0usize; + for payload in Parser::new(0).parse_all(bytes) { + if let Payload::CodeSectionEntry(body) = + payload.context("parsing wasm structure for ABI candidate")? + { + if current_local_index == local_index { + return parse_abi_body(body).map(Some); + } + current_local_index += 1; + } + } + Ok(None) +} + +fn artifact_abi_version(bytes: &[u8]) -> Result { + let mut types: Vec> = Vec::new(); + let mut function_type_indices = Vec::new(); + let mut imported_function_count = 0usize; + let mut code_body_count = None; + let mut abi_export_count = 0usize; + let mut abi_function = None; + + for payload in Parser::new(0).parse_all(bytes) { + match payload.context("parsing wasm structure for artifact identity")? { + Payload::Version { encoding, .. } => { + if encoding != Encoding::Module { + bail!("artifact identity requires a core wasm module"); + } + } + Payload::TypeSection(groups) => { + for group in groups { + let group = group.context("parsing artifact type section")?; + types.extend(group.into_types().map( + |subtype| match subtype.composite_type.inner { + CompositeInnerType::Func(function) => Some(function), + CompositeInnerType::Array(_) + | CompositeInnerType::Struct(_) + | CompositeInnerType::Cont(_) => None, + }, + )); + } + } + Payload::ImportSection(imports) => { + for import in imports.into_imports() { + let import = import.context("parsing artifact import section")?; + if let TypeRef::Func(type_index) | TypeRef::FuncExact(type_index) = import.ty { + function_type_indices.push(type_index); + imported_function_count += 1; + } + } + } + Payload::FunctionSection(functions) => { + for type_index in functions { + function_type_indices + .push(type_index.context("parsing artifact function section")?); + } + } + Payload::ExportSection(exports) => { + for export in exports { + let export = export.context("parsing artifact export section")?; + if export.name != ABI_KERNEL_EXPORT { + continue; + } + abi_export_count += 1; + if export.kind == ExternalKind::Func { + abi_function = Some(export.index); + } + } + } + Payload::CodeSectionStart { count, .. } => { + if code_body_count.replace(count as usize).is_some() { + return Ok(ArtifactAbiVersion::Invalid); + } + } + _ => {} + } + } + + if abi_export_count == 0 { + return Ok(ArtifactAbiVersion::Missing); + } + if abi_export_count != 1 { + return Ok(ArtifactAbiVersion::Invalid); + } + let Some(target) = abi_function else { + return Ok(ArtifactAbiVersion::Invalid); + }; + if code_body_count.unwrap_or(0) + imported_function_count != function_type_indices.len() + || !signature_is( + function_signature(&types, &function_type_indices, target), + &[], + &[ValType::I32], + ) + { + return Ok(ArtifactAbiVersion::Invalid); + } + + // WHY: publication checks run over very large package executables. The + // export and signatures are known before the code section, so decode only + // the ABI thunk rather than every unrelated compiler-generated body. + let Some(target_body) = body_for(bytes, imported_function_count, target)? else { + return Ok(ArtifactAbiVersion::Invalid); + }; + if let Some(version) = target_body.pure_constant() { + return Ok(ArtifactAbiVersion::Present(version)); + } + + match ( + target_body.exact, + target_body.operator_count, + target_body.operators, + ) { + ( + true, + 3, + [ + Some(AbiOperator::Call(leading)), + Some(AbiOperator::I32Const(version)), + Some(AbiOperator::End), + ], + ) if signature_is( + function_signature(&types, &function_type_indices, leading), + &[], + &[], + ) => Ok(u32::try_from(version) + .map(ArtifactAbiVersion::Present) + .unwrap_or(ArtifactAbiVersion::Invalid)), + ( + true, + 3, + [ + Some(AbiOperator::Call(leading)), + Some(AbiOperator::Call(delegate)), + Some(AbiOperator::End), + ], + ) if signature_is( + function_signature(&types, &function_type_indices, leading), + &[], + &[], + ) && signature_is( + function_signature(&types, &function_type_indices, delegate), + &[], + &[ValType::I32], + ) => Ok(body_for(bytes, imported_function_count, delegate)? + .and_then(AbiBody::pure_constant) + .map(ArtifactAbiVersion::Present) + .unwrap_or(ArtifactAbiVersion::Invalid)), + _ => Ok(ArtifactAbiVersion::Invalid), + } +} + +/// Return the raw custom-section payload used by `wasm-objdump -s -j`, +/// including the encoded section name before its data. +pub fn fork_capability_section_hex(bytes: &[u8]) -> Result { + unique_custom_section_hex(bytes, WPK_FORK_CAPABILITIES_SECTION) +} + +/// Return the raw custom-section payload used by `wasm-objdump -s -j`, +/// including the encoded section name before its data. +pub fn linked_frame_descriptor_section_hex(bytes: &[u8]) -> Result { + unique_custom_section_hex(bytes, WPK_FORK_LINKED_FRAME_FORMAT_SECTION) +} + +fn unique_custom_section_hex(bytes: &[u8], expected_name: &str) -> Result { + let mut found = None; + for payload in Parser::new(0).parse_all(bytes) { + match payload.context("parsing wasm structure for custom-section inventory")? { + Payload::Version { encoding, .. } => { + if encoding != Encoding::Module { + bail!("custom-section inventory requires a core wasm module"); + } + } + Payload::CustomSection(section) if section.name() == expected_name => { + if found.is_some() { + bail!("found duplicate `{expected_name}` custom sections"); + } + found = Some(section.range()); + } + _ => {} + } + } + + let range = found.with_context(|| format!("missing `{expected_name}` custom section"))?; + let section = bytes + .get(range) + .context("custom-section range falls outside the wasm binary")?; + let mut hex = String::with_capacity(section.len() * 2); + for byte in section { + write!(&mut hex, "{byte:02x}").expect("writing to a String cannot fail"); + } + Ok(hex) +} + +fn signature_matches(signature: &FuncType, expected: ExpectedSignature, pointer: ValType) -> bool { + let (params, results): (&[ValType], &[ValType]) = match expected { + ExpectedSignature::PointerToPointer => ( + std::slice::from_ref(&pointer), + std::slice::from_ref(&pointer), + ), + ExpectedSignature::PointerToNil => (std::slice::from_ref(&pointer), &[]), + ExpectedSignature::NilToNil => (&[], &[]), + ExpectedSignature::NilToI32 => (&[], std::slice::from_ref(&ValType::I32)), + }; + signature.params() == params && signature.results() == results +} diff --git a/crates/fork-instrument/src/instrument.rs b/crates/fork-instrument/src/instrument.rs index b78a7ca863..d294c9501d 100644 --- a/crates/fork-instrument/src/instrument.rs +++ b/crates/fork-instrument/src/instrument.rs @@ -22,8 +22,8 @@ //! ;; --- PREAMBLE (runs only when state == REWINDING) --- //! (if (i32.eq (global.get $_wpk_fork_state) (i32.const 2)) //! (then -//! ;; pop frame from save buffer, then restore catch_region_id, -//! ;; exnref_slot, scalar locals, and arg-spill locals +//! ;; pop frame from save buffer, then restore catch_selector, +//! ;; reserved catch metadata, scalar locals, and arg-spill locals //! )) //! //! ;; --- DISPATCH + WRAPPER + NESTED POST LABELS --- @@ -43,7 +43,7 @@ //! ) ;; end $POST_0 — also the br_table landing for call_idx==0 //! //! (call $callee_0) ;; or call_indirect -//! +//! ;; catch capture already selected the exact dynamic arm //! (global.get $_wpk_fork_state) (i32.const 1) (i32.eq) //! (if (then //! ;; frame.call_index = 0 @@ -55,7 +55,7 @@ //! ) ;; end $POST_{N-1} //! //! (call $callee_{N-1}) -//! +//! ;; catch capture already selected the exact dynamic arm //! (if state == UNWINDING: //! frame.call_index = N-1 //! br $unwind_save) @@ -65,28 +65,31 @@ //! //! ;; --- POSTAMBLE (runs only when branched-to via br $unwind_save) --- //! ;; push frame header fields except call_index, save scalar user locals, -//! ;; save arg-spill locals, spill ref-typed user locals to aux tables, -//! ;; advance current_pos, push defaults for the function's result types +//! ;; save arg-spill locals, advance current_pos, push defaults for the +//! ;; function's result types //! ) //! ``` //! -//! ## MVP scope +//! ## Supported replay surface //! -//! - **Top-level fork-path calls only.** A fork-path call nested -//! inside a `block`/`loop`/`if`/`try_table` causes `br_table` to be -//! unable to land at its site (wasm semantics forbid branching into -//! a block from outside). The tool panics with a diagnostic in -//! that case; the function must be restructured or the tool -//! extended. -//! - **Fork from modern `try_table` catches is supported.** Plain-catch -//! arm identity and scalar payloads are frame-backed per activation; -//! catch_ref values use the exnref auxiliary table. Legacy `try` -//! catch handlers remain unsupported. -//! - **Scalar args only for fork-path calls.** If a fork-path call -//! has a ref-typed argument, we'd need to spill it through an aux -//! table (not currently wired up). Panic in that case. +//! - **Top-level and nested fork-path calls.** Top-level calls use the +//! function switch-dispatch. Calls nested in structured control flow use a +//! per-block switch-dispatch so rewind never branches into a block from +//! outside it. +//! - **Fork from statically tagged modern `try_table` catches is +//! supported.** Catch and CatchRef arm identity and scalar tag +//! payloads are frame-backed per activation. Rewind throws the tag +//! again so the fresh module instance creates a fresh exnref. +//! Fork-reachable legacy `try` handlers are normalized to this same modern +//! activation-owned representation before instrumentation. +//! - **Abstract function and external references use activation-owned +//! recipes.** Live locals, parameters, call operands, and operand-stack +//! carryovers are encoded to deterministic recipe IDs in a call-specific +//! process vector and decoded against the fresh child instance. +//! Definitely-null references need no recipe. Statically tagged CatchRef +//! state is reconstructed by rethrowing its saved payload inside Wasm. //! -//! ## Frame layout (unchanged from the previous transform) +//! ## Frame layout //! //! All offsets are relative to the frame's base address. //! @@ -94,37 +97,45 @@ //! |---------------|------|-------------------| //! | 0 | 4 | `func_index` | //! | 4 | 4 | `call_index` | -//! | 8 | 4 | `catch_region_id` | -//! | 12 | 4 | `exnref_slot` | -//! | 16.. | var | scalar locals (user, arg spills, plain-catch state) | +//! | 8 | 4 | `catch_selector` | +//! | 12 | 4 | process reference-vector ordinal | +//! | 16.. | var | scalar locals (user, arg spills, tagged-catch state) | //! -//! Ref-typed user locals are routed through module-level auxiliary -//! tables; their storage is outside the frame. +//! There is deliberately no module-instance auxiliary reference storage: +//! workers reconstruct a child from linear memory in a fresh Wasm instance. //! //! ## What's preserved verbatim //! //! - `crates/fork-instrument/src/call_graph.rs` — fork-path closure //! discovery (direct + indirect). -//! - `crates/fork-instrument/src/runtime.rs` — state machine, five +//! - `crates/fork-instrument/src/runtime.rs` — state machine, seven //! exported control functions, save-buffer layout, saved-globals //! handling. -//! - Phase 4f aux-table injection for ref-typed user locals. -//! - Phase 6a–6d plumbing for `try_table` / catch-handler resume. +//! - Phase 6a–6d plumbing for `try_table` / tagged-catch resume. -use std::collections::{HashMap, HashSet}; +use anyhow::Result; +use std::collections::{BTreeMap, HashMap, HashSet}; use walrus::{ - AbstractHeapType, ExportItem, FunctionId, FunctionKind, HeapType, LocalFunction, LocalId, - MemoryId, Module, RefType, TableId, TagId, TypeId, ValType, + AbstractHeapType, ElementItems, ElementKind, ExportItem, FunctionBuilder, FunctionId, + FunctionKind, HeapType, LocalFunction, LocalId, MemoryId, Module, RawCustomSection, RefType, + TableId, TagId, TypeId, ValType, ir::{ AtomicWidth, BinaryOp, Binop, Block, Br, BrTable, Call, CallIndirect, Const, GlobalGet, IfElse, Instr, InstrLocId, InstrSeqId, InstrSeqType, LegacyCatch, LoadKind, LocalGet, - LocalSet, LocalTee, Loop, MemArg, RefAsNonNull, RefNull, Return, StoreKind, TableGet, - TableSet, Throw, ThrowRef, TryTable, TryTableCatch, UnaryOp, Value, + LocalSet, LocalTee, Loop, MemArg, RefAsNonNull, RefNull, Return, StoreKind, Throw, + TryTable, TryTableCatch, UnaryOp, Unreachable, Value, }, }; -use crate::runtime::{self, Runtime}; +use crate::{ + call_graph::{TailCallKind, TailCallSite}, + reference_analysis::{ + FunctionReferenceAnalysis, OriginalCallKind, ReferenceNullability, + analyze_function_references, + }, + runtime::{self, ReferenceCodecClass as RefClass, Runtime}, +}; const HOST_PARSED_MARKER_EXPORTS: &[&str] = &[ "__abi_version", @@ -132,6 +143,14 @@ const HOST_PARSED_MARKER_EXPORTS: &[&str] = &[ "__get_channel_base_addr", ]; +pub const RESUME_CATALOG_EXPORT: &str = "__wpk_fork_resume_catalog"; +pub const RESUME_CATALOG_SECTION: &str = "kandelo.wpk_fork.resume_catalog"; +pub const RESUME_START_EXPORT: &str = "wpk_fork_resume_start"; +pub const RESUME_THREAD_EXPORT: &str = "wpk_fork_resume_thread"; +const RESUME_CATALOG_MAGIC: [u8; 4] = *b"KFRC"; +const RESUME_CATALOG_VERSION: u16 = 1; +const RESUME_CATALOG_HEADER_SIZE: u16 = 12; + fn is_host_parsed_marker_function(module: &Module, id: FunctionId) -> bool { module.exports.iter().any(|export| { HOST_PARSED_MARKER_EXPORTS.contains(&export.name.as_str()) @@ -139,6 +158,119 @@ fn is_host_parsed_marker_function(module: &Module, id: FunctionId) -> bool { }) } +/// Verify that every fork-reachable reference shape has a typed owner. +/// +/// WHY this runs before any rewriting: the host creates fork children by +/// copying linear memory into a newly instantiated module. The complete Wasm +/// reference hierarchy is routed to a generated codec class here; mutable +/// globals, tables, and segment lifetime are owned by the KFMS guest helpers, +/// while activation references are owned by frame recipe IDs. Errors from +/// this pass indicate malformed/stale transformation metadata, not a policy +/// that excludes otherwise-valid reference-bearing programs. +pub fn validate_activation_state(module: &Module, fork_path: &HashSet) -> Result<()> { + validate_activation_state_with_targets(module, fork_path, fork_path) +} + +/// Validate surviving activations while selecting suspension-capable call +/// sites from the larger semantic control-reachability closure. +/// +/// A function traversed only by `return_call*` is intentionally absent from +/// `activations`: its frame no longer exists at the fork point. It remains in +/// `fork_path_targets` so an older live caller recognizes that an ordinary +/// call into the transparent tail chain is a replay landing. +pub fn validate_activation_state_with_targets( + module: &Module, + activations: &HashSet, + fork_path_targets: &HashSet, +) -> Result<()> { + if activations.is_empty() { + return Ok(()); + } + + let mut targets: Vec = activations.iter().copied().collect(); + targets.sort(); + for func_id in targets { + let function = module.funcs.get(func_id); + let FunctionKind::Local(_) = &function.kind else { + continue; + }; + if is_host_parsed_marker_function(module, func_id) { + continue; + } + let name = function.name.as_deref().unwrap_or(""); + + for (local_id, ty) in collect_user_locals(module, func_id) { + let ValType::Ref(reference) = ty else { + continue; + }; + validate_reference_shape( + module, + reference, + &format!("fork-reachable function `{name}` local/parameter {local_id:?}"), + )?; + } + + let signature = module.types.get(function.ty()); + for reference in signature + .params() + .iter() + .chain(signature.results()) + .filter_map(|ty| match ty { + ValType::Ref(reference) => Some(*reference), + _ => None, + }) + { + validate_reference_shape( + module, + reference, + &format!("fork-reachable function `{name}` signature"), + )?; + } + + let reference_analysis = analyze_function_references(module, func_id, fork_path_targets)?; + validate_reference_call_state(module, name, &reference_analysis)?; + } + + Ok(()) +} + +fn validate_reference_shape(module: &Module, reference: RefType, _owner: &str) -> Result<()> { + // Every WebAssembly reference hierarchy has a typed recipe provider. + // Concrete function/GC types are upcast for encoding and cast back after + // decoding in the fresh instance. + let _ = RefClass::of(module, reference); + Ok(()) +} + +fn validate_reference_call_state( + module: &Module, + function_name: &str, + analysis: &FunctionReferenceAnalysis, +) -> Result<()> { + for site in &analysis.call_sites { + for operand in site + .reference_arguments + .iter() + .chain(site.reference_carryovers.iter()) + { + validate_reference_shape( + module, + operand.ty, + &format!( + "fork-reachable function `{function_name}` call {:?} operand {}", + site.id, operand.index + ), + )?; + } + + if site.has_reference_callee { + // call_ref's concrete callee type is statically recovered with a + // Wasm ref.cast after decoding the abstract funcref recipe. + } + } + Ok(()) +} + /// Instrument every function in `fork_path` that we can instrument. /// /// Returns the set of function IDs that were actually rewritten. @@ -147,6 +279,53 @@ pub fn instrument_functions( runtime: &Runtime, fork_path: &HashSet, plain_catch_plan: &PlainCatchPlan, +) -> HashSet { + instrument_functions_with_targets_and_tail_sites( + module, + runtime, + fork_path, + fork_path, + &[], + plain_catch_plan, + ) +} + +/// Instrument only activation-live functions, selecting their replay +/// landings from the full semantic fork-reachability closure. +pub fn instrument_functions_with_targets( + module: &mut Module, + runtime: &Runtime, + activations: &HashSet, + fork_path_targets: &HashSet, + plain_catch_plan: &PlainCatchPlan, +) -> HashSet { + instrument_functions_with_targets_and_tail_sites( + module, + runtime, + activations, + fork_path_targets, + &[], + plain_catch_plan, + ) +} + +/// Instrument activation-live functions after making every fork boundary use +/// the private exception transport. +/// +/// Ordinary direct calls to rewritten local functions need no shim: their +/// generated postamble throws `__wpk_fork_unwind`. Imported fork entries and +/// dynamic dispatch can instead return normally after setting +/// `STATE_UNWINDING`, so those operations are moved into short generated +/// helpers which check the state before exposing any result to the source +/// activation. Fork-reaching tail sites tail-call the same helpers, retaining +/// bounded-stack semantics for transparent tail chains. +pub fn instrument_functions_with_targets_and_tail_sites( + module: &mut Module, + runtime: &Runtime, + activations: &HashSet, + fork_path_targets: &HashSet, + tail_call_sites: &[TailCallSite], + plain_catch_plan: &PlainCatchPlan, ) -> HashSet { let runtime_funcs: HashSet = [ runtime.unwind_begin, @@ -158,7 +337,7 @@ pub fn instrument_functions( .into_iter() .collect(); - let mut targets: Vec = fork_path + let mut targets: Vec = activations .iter() .copied() .filter(|id| !runtime_funcs.contains(id)) @@ -166,304 +345,816 @@ pub fn instrument_functions( .filter(|id| matches!(module.funcs.get(*id).kind, FunctionKind::Local(_))) .collect(); targets.sort(); + let materialized_activations: HashSet = + targets.iter().copied().collect(); - let (aux_tables, ref_plan, catch_plans) = plan_and_inject_aux_tables(module, &targets); + let catch_plans = plan_catch_regions(module, &targets); + let transport_helpers = inject_unwind_transport_helpers( + module, + runtime, + &targets, + fork_path_targets, + tail_call_sites, + ); + rewrite_activation_unwind_boundaries(module, &targets, tail_call_sites, &transport_helpers); + let unwind_frame_select = emit_unwind_frame_select_helper(module, runtime); + let mut transformed_call_targets = fork_path_targets.clone(); + transformed_call_targets.extend(transport_helpers.values().copied()); + + // Analyze every target before rewriting the first body. Stable original + // program points are the ownership boundary: synthetic dispatch locals + // must never make an otherwise-dead guest reference look live. + let reference_analyses: HashMap = targets + .iter() + .copied() + .map(|id| { + let analysis = analyze_function_references(module, id, &transformed_call_targets) + .unwrap_or_else(|error| panic!("fork reference analysis failed: {error:#}")); + (id, analysis) + }) + .collect(); let empty_plain_catches: Vec<(InstrSeqId, Vec)> = Vec::new(); let mut instrumented = HashSet::new(); + let mut resume_thunks = Vec::with_capacity(targets.len()); for (ordinal, id) in targets.iter().enumerate() { - let empty_plan: Vec = Vec::new(); - let this_plan = ref_plan.get(id).unwrap_or(&empty_plan); let empty_catch_plan: Vec = Vec::new(); let this_catch_plan = catch_plans.get(id).unwrap_or(&empty_catch_plan); let this_plain_catches = plain_catch_plan .per_function .get(id) .unwrap_or(&empty_plain_catches); - instrument_one_function( + let thunk = instrument_one_function( module, *id, runtime, - fork_path, + &materialized_activations, + &transformed_call_targets, ordinal as u32, - &aux_tables, - this_plan, this_catch_plan, this_plain_catches, + &reference_analyses[id], + unwind_frame_select, ); + resume_thunks.push(thunk); instrumented.insert(*id); } + emit_resume_catalog(module, &resume_thunks); + emit_fixed_resume_boundaries(module, runtime); instrumented } -// ---------------------------------------------------------------------- -// Frame layout constants -// ---------------------------------------------------------------------- - -const HEADER_SIZE: u32 = 16; -const FUNC_INDEX_OFFSET: u64 = 0; -const CALL_INDEX_OFFSET: u64 = 4; -const CATCH_REGION_OFFSET: u64 = 8; -const EXNREF_SLOT_OFFSET: u64 = 12; -const LOCALS_START_OFFSET: u32 = HEADER_SIZE; - -// ---------------------------------------------------------------------- -// Per-function pipeline -// ---------------------------------------------------------------------- - -/// Classification of a top-level fork-path call site. -#[derive(Debug, Clone, Copy)] -enum CallTarget { +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +enum UnwindTransportKey { Direct(FunctionId), - Indirect { table: TableId }, -} - -/// A top-level call site awaiting dispatch-structure emission. -struct CallSiteInfo { - target: CallTarget, - sig_ty: TypeId, - loc: InstrLocId, + Indirect { table: TableId, ty: TypeId }, + Ref { ty: TypeId }, } -#[derive(Debug, Clone, Copy)] -struct CatchStateLocals { - catch_region_id: LocalId, - exnref_slot: LocalId, -} - -#[derive(Debug, Clone, Copy)] -struct AbortDispatch { - live_frame: LocalId, - restart_loop: InstrSeqId, +impl UnwindTransportKey { + fn stable_sort_key(self) -> (u8, usize, usize) { + match self { + Self::Direct(function) => (0, function.index(), 0), + Self::Indirect { table, ty } => (1, table.index(), ty.index()), + Self::Ref { ty } => (2, ty.index(), 0), + } + } } -#[allow(clippy::too_many_arguments)] -fn instrument_one_function( - module: &mut Module, - func_id: FunctionId, - runtime: &Runtime, - fork_path: &HashSet, - func_ordinal: u32, - aux_tables: &AuxTables, - ref_plan: &[RefLocalSlot], - catch_plan: &[CatchRegionPlan], - plain_catches: &[(InstrSeqId, Vec)], -) { - // Choose scheme based on call-site topology. Post-commit-4 - // (2026-05-14) there are TWO live schemes (guard-dispatch was - // deleted; legacy catch-handler forks still panic defensively): - // - // instrument_one_function_switch — top-level fork-path calls - // only. Body is restructured so a top-level `br_table` jumps - // directly to the resumed call site, skipping all code in - // between. Per-call operand-stack carryovers (LLVM `*(sp+K) = - // call(...)` shapes) are absorbed via per-call spill locals - // (sub-commit 2.4c) — formerly forced guard-dispatch. - // - // instrument_one_function_nested_switch — fork-path calls - // nested inside Block/IfElse/Loop/TryTable bodies. Cascading - // POST_K blocks plus per-region br_tables route REWIND through - // each enclosing instruction's own dispatch. Sub-commits 2.5/2.6 - // added carryover spilling at nested direct-call landings, - // nested-Loop-with-carryover (side benefit), and multi-value- - // params SubRegion body-input-param prespill. - // - // Catch-handler bodies live inside a nested try_table; nested - // switch-dispatch handles them via the rewind-throw stub + - // capture block mechanism (see Phase 6 + B1 stages 1+2 docs). - // - // Both schemes: - // - share the same fork-resume contract (state machine, frame - // layout, aux-table ref-typed spills, throw_ref catch resume). - // - skip body chunks before the chosen POST_K on REWIND, so - // non-fork-path calls and side-effect ops in those chunks run - // exactly once on NORMAL — no per-op gating needed (the - // pre-2.5/2.6 Phase 4g machinery was deleted with guard- - // dispatch in commit 4). - if has_nested_fork_calls(module, func_id, fork_path) { - // Nested per-block switch-dispatch: if classify_nested_pattern - // accepts the function's nesting shape, use the cascading - // POST_K + per-region br_table transform. Sub-commits 2.5/2.6 - // expanded "supported" to cover Loops/TryTables/legacy Try - // bodies/multi-value-params/carryovers; only fork-from-legacy- - // catch remains a panic-defensive fallback. - let nested_status = classify_nested_pattern(module, func_id, fork_path); - if nested_status.is_supported() { - instrument_one_function_nested_switch( - module, - func_id, - runtime, - fork_path, - func_ordinal, - aux_tables, - ref_plan, - catch_plan, - plain_catches, - ); - return; - } - // Commit 3 (2026-05-14): the only remaining - // `NestedSupportStatus` rejection is fork-from-legacy-catch. - // Sub-commits 2.5c/2.6c closed `UnsupportedCarryover` and - // `UnsupportedMultiValueParams` respectively; legacy Try bodies - // now use the same nested-switch route as TryTable bodies. If - // we reach this branch on a shipping binary, the fork-path call - // is in a legacy catch handler, which still needs exception - // state reconstruction. - let func = func_name(module, func_id); - if has_fork_call_in_catch_handler(module, func_id, fork_path) { - panic!( - "fork-instrument: function `{func}` has a fork-path call inside a \ - try_table catch-handler body. This pattern is currently \ - unsupported end-to-end (B1 stages 1+2 shipped machinery but the \ - C1 fixture still hangs). See \ - memory/fork-instrument-b1-followup.md and the C1 fixture in \ - programs/cpp_eh_fork_from_catch_test.cpp." - ); - } - match nested_status { - NestedSupportStatus::UnsupportedLegacyTry => panic!( - "fork-instrument: function `{func}` triggered `UnsupportedLegacyTry` \ - — a fork-path call inside a legacy `catch` handler. Legacy `try` \ - bodies are supported by nested switch-dispatch, but legacy catch \ - handlers still need exception-state reconstruction before REWIND \ - can re-enter the handler path." - ), - NestedSupportStatus::UnsupportedCarryover => panic!( - "fork-instrument: function `{func}` has a nested fork-path call with \ - an operand-stack carryover shape the nested-switch analyser cannot \ - type. Extend `compute_nested_carryover_types` / \ - `analyze_subregion_spill_types` for the specific producer." - ), - NestedSupportStatus::UnsupportedMultiValueParams => panic!( - "fork-instrument: function `{func}` has unsupported multi-value \ - params in nested fork-path control flow." - ), - NestedSupportStatus::Supported => unreachable!(), +fn collect_unwind_transport_keys( + module: &Module, + targets: &[FunctionId], + fork_path_targets: &HashSet, + tail_call_sites: &[TailCallSite], +) -> Vec { + fn visit( + module: &Module, + local: &LocalFunction, + seq: InstrSeqId, + fork_path_targets: &HashSet, + keys: &mut HashSet, + ) { + for (instruction, _) in &local.block(seq).instrs { + match instruction { + Instr::Call(call) + if fork_path_targets.contains(&call.func) + && matches!(module.funcs.get(call.func).kind, FunctionKind::Import(_)) => + { + keys.insert(UnwindTransportKey::Direct(call.func)); + } + Instr::CallIndirect(call) => { + keys.insert(UnwindTransportKey::Indirect { + table: call.table, + ty: call.ty, + }); + } + Instr::CallRef(call) => { + keys.insert(UnwindTransportKey::Ref { ty: call.ty }); + } + _ => {} + } + for child in nested_seqs(instruction) { + visit(module, local, child, fork_path_targets, keys); + } } } - if has_top_level_stack_carryovers(module, func_id, fork_path) { - // Sub-commit 2.4c (2026-05-14): switch-dispatch absorbs - // top-level carryovers via in-place spill/reload at the call - // site. The compute_carryover_types Option refactor - // (sub-commit 9-followup) made the analyser succeed for any - // shape whose carryover values are statically typed — and - // unknown-type values consumed before any fork-path call are - // also tolerated. If the analyser still returns None here, a - // shipping binary has an unknown-type value AS a carryover at - // a fork-path call (genuinely rare LLVM output). Panic loudly - // for the same reason as the LegacyTry case above. - if compute_carryover_types(module, func_id, fork_path).is_some() { - instrument_one_function_switch( - module, - func_id, - runtime, - fork_path, - func_ordinal, - aux_tables, - ref_plan, - catch_plan, - plain_catches, - ); - return; - } - let func = func_name(module, func_id); - panic!( - "fork-instrument: function `{func}` has a top-level fork-path call \ - whose operand-stack carryover contains a value of a type the \ - analyser can't statically determine or cannot scalar-spill \ - (ref-typed producer, non-fork-path CallIndirect or CallRef, \ - or ref-typed structured-control result). The 2.6c push-before \ - emission can spill this carryover only if its type is known. \ - Extend `compute_carryover_types` to handle the specific producer, \ - or change the source to avoid the pattern." + let mut keys = HashSet::new(); + for &target in targets { + let FunctionKind::Local(local) = &module.funcs.get(target).kind else { + continue; + }; + visit( + module, + local, + local.entry_block(), + fork_path_targets, + &mut keys, ); } - instrument_one_function_switch( - module, - func_id, - runtime, - fork_path, - func_ordinal, - aux_tables, - ref_plan, - catch_plan, - plain_catches, - ); + for &site in tail_call_sites { + let FunctionKind::Local(local) = &module.funcs.get(site.caller).kind else { + panic!("fork-reaching tail site belongs to a non-local function"); + }; + let Some((instruction, _)) = local + .block(site.sequence) + .instrs + .get(site.instruction_index) + else { + panic!("fork-reaching tail site points past its instruction sequence"); + }; + let key = match (site.kind, instruction) { + (TailCallKind::Direct, Instr::ReturnCall(call)) + if matches!(module.funcs.get(call.func).kind, FunctionKind::Import(_)) => + { + Some(UnwindTransportKey::Direct(call.func)) + } + (TailCallKind::Direct, Instr::ReturnCall(_)) => None, + (TailCallKind::Indirect, Instr::ReturnCallIndirect(call)) => { + Some(UnwindTransportKey::Indirect { + table: call.table, + ty: call.ty, + }) + } + (TailCallKind::Ref, Instr::ReturnCallRef(call)) => { + Some(UnwindTransportKey::Ref { ty: call.ty }) + } + _ => panic!("fork-reaching tail-site metadata disagrees with the original instruction"), + }; + if let Some(key) = key { + keys.insert(key); + } + } + + let mut keys: Vec<_> = keys.into_iter().collect(); + keys.sort_by_key(|key| key.stable_sort_key()); + keys } -/// Switch-dispatch transform: fork-path calls are hoisted out of the -/// function body and reached during REWIND via a top-level `br_table` -/// that lands directly at the post-active-call-site label. Chunks -/// between calls run only on the NORMAL fall-through path. -#[allow(clippy::too_many_arguments)] -fn instrument_one_function_switch( +fn emit_unwind_transport_helper( module: &mut Module, - func_id: FunctionId, runtime: &Runtime, - fork_path: &HashSet, - func_ordinal: u32, - aux_tables: &AuxTables, - ref_plan: &[RefLocalSlot], - catch_plan: &[CatchRegionPlan], - plain_catches: &[(InstrSeqId, Vec)], -) { - // Pre-existing user locals (args + referenced in body). Scalars - // live in the frame; ref-typed locals go through aux tables. - let all_user_locals = collect_user_locals(module, func_id); - let user_scalar_locals: Vec<(LocalId, ValType)> = all_user_locals + key: UnwindTransportKey, +) -> FunctionId { + let (mut params, results, name) = match key { + UnwindTransportKey::Direct(function) => { + let signature = module.types.get(module.funcs.get(function).ty()); + ( + signature.params().to_vec(), + signature.results().to_vec(), + format!("__wpk_fork_unwind_transport_direct_{}", function.index()), + ) + } + UnwindTransportKey::Indirect { table, ty } => { + let signature = module.types.get(ty); + let mut params = signature.params().to_vec(); + params.push(if module.tables.get(table).table64 { + ValType::I64 + } else { + ValType::I32 + }); + ( + params, + signature.results().to_vec(), + format!( + "__wpk_fork_unwind_transport_indirect_{}_{}", + table.index(), + ty.index() + ), + ) + } + UnwindTransportKey::Ref { ty } => { + let signature = module.types.get(ty); + let mut params = signature.params().to_vec(); + params.push(ValType::Ref(RefType::FUNCREF)); + ( + params, + signature.results().to_vec(), + format!("__wpk_fork_unwind_transport_ref_{}", ty.index()), + ) + } + }; + let arguments: Vec<_> = params.drain(..).map(|ty| module.locals.add(ty)).collect(); + let helper_params: Vec<_> = arguments .iter() - .copied() - .filter(|(_, ty)| is_scalar(*ty)) + .map(|argument| module.locals.get(*argument).ty()) .collect(); + let mut builder = FunctionBuilder::new(&mut module.types, &helper_params, &results); + builder.name(name); + let helper = builder.finish(arguments.clone(), &mut module.funcs); - // Sub-commit 2.4c: compute carryover types BEFORE taking the - // original body, since `compute_carryover_types` reads the body - // through `module.funcs.get(func_id)`. Computing it after `take` - // would see an empty body and report no carryovers. - let carryover_types_pre_take = compute_carryover_types(module, func_id, fork_path); - - // Take the original entry body; we rebuild it wholesale. - let entry_id = local_mut(module, func_id).entry_block(); - let original_body: Vec<(Instr, InstrLocId)> = - std::mem::take(&mut local_mut(module, func_id).block_mut(entry_id).instrs); - - // Partition the body at top-level fork-path call sites. - let (mut chunks, call_sites) = partition_body(&original_body, fork_path, module); - let n_calls = call_sites.len(); - - // Allocate per-function synthetic locals. + let local = local_mut(module, helper); + let throws_unwind = local + .builder_mut() + .dangling_instr_seq(InstrSeqType::Simple(None)) + .id(); + let normal_return = local + .builder_mut() + .dangling_instr_seq(InstrSeqType::Simple(None)) + .id(); + { + let out = &mut local.block_mut(throws_unwind).instrs; + push_instr( + out, + Instr::Throw(Throw { + tag: runtime + .unwind_tag + .expect("unwind transport helper requires private tag"), + }), + ); + } + { + let out = &mut local.block_mut(local.entry_block()).instrs; + for &argument in &arguments { + push_instr(out, Instr::LocalGet(LocalGet { local: argument })); + } + match key { + UnwindTransportKey::Direct(function) => { + push_instr(out, Instr::Call(Call { func: function })); + } + UnwindTransportKey::Indirect { table, ty } => { + push_instr(out, Instr::CallIndirect(CallIndirect { ty, table })); + } + UnwindTransportKey::Ref { ty } => { + push_instr( + out, + Instr::RefCast(walrus::ir::RefCast { + nullable: false, + heap_type: HeapType::Concrete(ty), + }), + ); + push_instr(out, Instr::CallRef(walrus::ir::CallRef { ty })); + } + } + // WHY: results deliberately remain below this zero-result test only + // inside the short helper. The source activation receives them only + // after UNWINDING has been converted to the private tag, so engines + // never need result-spill scratch in every recursive source frame. + push_instr( + out, + Instr::GlobalGet(GlobalGet { + global: runtime.state_global, + }), + ); + push_instr( + out, + Instr::Const(Const { + value: Value::I32(runtime::STATE_UNWINDING), + }), + ); + push_instr( + out, + Instr::Binop(Binop { + op: BinaryOp::I32Eq, + }), + ); + push_instr( + out, + Instr::IfElse(IfElse { + consequent: throws_unwind, + alternative: normal_return, + }), + ); + } + helper +} + +fn inject_unwind_transport_helpers( + module: &mut Module, + runtime: &Runtime, + targets: &[FunctionId], + fork_path_targets: &HashSet, + tail_call_sites: &[TailCallSite], +) -> HashMap { + collect_unwind_transport_keys(module, targets, fork_path_targets, tail_call_sites) + .into_iter() + .map(|key| (key, emit_unwind_transport_helper(module, runtime, key))) + .collect() +} + +/// Emit the cold unwind-only frame-selection path once per module. +/// +/// Source activations pass only their constant frame size and static call +/// index. Keeping reserve, null-result handling, abort-scratch selection, and +/// the header write here avoids multiplying that sequence by every lexical +/// call site without adding a local to ordinary recursive activations. +fn emit_unwind_frame_select_helper(module: &mut Module, runtime: &Runtime) -> FunctionId { + let memory = first_memory(module); + let ptr_ty = runtime.buf_type; + let frame_size = module.locals.add(ptr_ty); + let call_index = module.locals.add(ValType::I32); + let mut builder = + FunctionBuilder::new(&mut module.types, &[ptr_ty, ValType::I32], &[ValType::I32]); + builder.name("__wpk_fork_select_unwind_frame".into()); + let helper = builder.finish(vec![frame_size, call_index], &mut module.funcs); + + let local = local_mut(module, helper); + let reserve_succeeded = local + .builder_mut() + .dangling_instr_seq(InstrSeqType::Simple(Some(ValType::I32))) + .id(); + let reserve_failed = local + .builder_mut() + .dangling_instr_seq(InstrSeqType::Simple(Some(ValType::I32))) + .id(); + + { + let out = &mut local.block_mut(reserve_failed).instrs; + // WHY: `frame_reserve == 0` synchronously moves the host runtime to + // abort replay. The descriptor's fixed prefix is therefore the only + // module-owned frame scratch that remains valid for selecting the + // failing live activation. + push_instr( + out, + Instr::GlobalGet(GlobalGet { + global: runtime.buf_global, + }), + ); + push_instr( + out, + Instr::GlobalGet(GlobalGet { + global: runtime.buf_global, + }), + ); + push_instr(out, ptr_const(ptr_ty, runtime.frames_start_offset as i64)); + push_instr( + out, + Instr::Binop(Binop { + op: ptr_add(ptr_ty), + }), + ); + push_instr(out, store_ptr(memory, ptr_ty, 0)); + push_current_frame_ptr(out, runtime, memory, ptr_ty); + push_instr(out, Instr::LocalGet(LocalGet { local: call_index })); + push_instr(out, store_i32(memory, CALL_INDEX_OFFSET)); + push_instr( + out, + Instr::Const(Const { + value: Value::I32(0), + }), + ); + } + { + let out = &mut local.block_mut(reserve_succeeded).instrs; + push_current_frame_ptr(out, runtime, memory, ptr_ty); + push_instr(out, Instr::LocalGet(LocalGet { local: call_index })); + push_instr(out, store_i32(memory, CALL_INDEX_OFFSET)); + push_instr( + out, + Instr::Const(Const { + value: Value::I32(1), + }), + ); + } + { + let out = &mut local.block_mut(local.entry_block()).instrs; + if let Some(frame_reserve) = runtime.frame_reserve { + push_instr( + out, + Instr::GlobalGet(GlobalGet { + global: runtime.buf_global, + }), + ); + push_instr(out, Instr::LocalGet(LocalGet { local: frame_size })); + push_instr( + out, + Instr::Call(Call { + func: frame_reserve, + }), + ); + push_instr(out, store_ptr(memory, ptr_ty, 0)); + + push_instr( + out, + Instr::GlobalGet(GlobalGet { + global: runtime.buf_global, + }), + ); + push_instr(out, load_ptr(memory, ptr_ty, 0)); + push_instr( + out, + Instr::Unop(walrus::ir::Unop { + op: match ptr_ty { + ValType::I32 => UnaryOp::I32Eqz, + ValType::I64 => UnaryOp::I64Eqz, + other => unreachable!("unsupported pointer type {other:?}"), + }, + }), + ); + push_instr( + out, + Instr::IfElse(IfElse { + consequent: reserve_failed, + alternative: reserve_succeeded, + }), + ); + } else { + // The legacy contiguous runtime already points at its active + // frame, so only the static call-index write is required. + push_instr( + out, + Instr::Block(Block { + seq: reserve_succeeded, + }), + ); + } + } + helper +} + +fn rewrite_activation_unwind_boundaries( + module: &mut Module, + targets: &[FunctionId], + tail_call_sites: &[TailCallSite], + helpers: &HashMap, +) { + fn rewrite_seq( + local: &mut LocalFunction, + seq: InstrSeqId, + helpers: &HashMap, + ) { + let original = std::mem::take(&mut local.block_mut(seq).instrs); + let mut rewritten = Vec::with_capacity(original.len()); + for (instruction, location) in original { + for child in nested_seqs(&instruction) { + rewrite_seq(local, child, helpers); + } + let instruction = match instruction { + Instr::Call(call) => helpers + .get(&UnwindTransportKey::Direct(call.func)) + .map_or(Instr::Call(call), |&helper| { + Instr::Call(Call { func: helper }) + }), + Instr::CallIndirect(call) => { + let helper = helpers[&UnwindTransportKey::Indirect { + table: call.table, + ty: call.ty, + }]; + Instr::Call(Call { func: helper }) + } + Instr::CallRef(call) => { + let helper = helpers[&UnwindTransportKey::Ref { ty: call.ty }]; + Instr::Call(Call { func: helper }) + } + other => other, + }; + rewritten.push((instruction, location)); + } + local.block_mut(seq).instrs = rewritten; + } + + for &target in targets { + let FunctionKind::Local(local) = &mut module.funcs.get_mut(target).kind else { + continue; + }; + rewrite_seq(local, local.entry_block(), helpers); + } + + for &site in tail_call_sites { + let FunctionKind::Local(local) = &mut module.funcs.get_mut(site.caller).kind else { + panic!("fork-reaching tail site belongs to a non-local function"); + }; + let Some((instruction, _)) = local + .block_mut(site.sequence) + .instrs + .get_mut(site.instruction_index) + else { + panic!("fork-reaching tail site points past its instruction sequence"); + }; + let helper = match (site.kind, &*instruction) { + (TailCallKind::Direct, Instr::ReturnCall(call)) => { + helpers.get(&UnwindTransportKey::Direct(call.func)).copied() + } + (TailCallKind::Indirect, Instr::ReturnCallIndirect(call)) => Some( + helpers[&UnwindTransportKey::Indirect { + table: call.table, + ty: call.ty, + }], + ), + (TailCallKind::Ref, Instr::ReturnCallRef(call)) => { + Some(helpers[&UnwindTransportKey::Ref { ty: call.ty }]) + } + _ => { + panic!("fork-reaching tail-site metadata disagrees with the rewritten instruction") + } + }; + if let Some(helper) = helper { + *instruction = Instr::ReturnCall(walrus::ir::ReturnCall { func: helper }); + } + } +} + +// ---------------------------------------------------------------------- +// Frame layout constants +// ---------------------------------------------------------------------- + +const HEADER_SIZE: u32 = 16; +const FUNC_INDEX_OFFSET: u64 = 0; +const CALL_INDEX_OFFSET: u64 = 4; +const CATCH_SELECTOR_OFFSET: u64 = 8; +const REFERENCE_VECTOR_OFFSET: u64 = 12; +const LOCALS_START_OFFSET: u32 = HEADER_SIZE; + +/// One reference value addressable from a call-specific recipe vector. +#[derive(Debug, Clone, Copy)] +struct ReferenceFrameSlot { + local: LocalId, + ty: RefType, + class: RefClass, + /// Stable vector position for values present at every landing. Resume + /// thunks need this for function parameters before the original preamble + /// has consumed the frame. + universal_position: Option, +} + +/// Per-call reference state derived from the original IR before rewriting. +/// +/// A slot is emitted only when at least one call landing needs it. Each call +/// names the exact slots to encode/decode; definitely-null live locals instead +/// receive a direct `ref.null` restore and consume no frame bytes or host +/// recipe entry. +#[derive(Debug, Clone)] +struct ReferenceFramePlan { + slots: Vec, + slots_by_call: Vec>, + null_locals_by_call: Vec>, +} + +#[derive(Debug, Clone, Copy)] +struct ResumeThunk { + func_ordinal: u32, + function: FunctionId, +} + +impl ReferenceFramePlan { + fn frame_end(&self, start: u32) -> u32 { + // Reference recipes live in the process transaction's compact vector + // log. The frame owns only its vector ordinal in reserved header word + // 12, independent of function-wide reference liveness. + start + } +} + +type TypedSpillLocal = (LocalId, ValType); + +// ---------------------------------------------------------------------- +// Per-function pipeline +// ---------------------------------------------------------------------- + +/// Classification of a top-level fork-path call site. +#[derive(Debug, Clone, Copy)] +enum CallTarget { + Direct(FunctionId), + Indirect { table: TableId }, + Ref, +} + +/// A top-level call site awaiting dispatch-structure emission. +struct CallSiteInfo { + target: CallTarget, + /// A direct lexical callee whose activation owns the next replay frame. + /// + /// Such calls can enter the original function without an intervening + /// resume thunk. The callee's frame-next import still validates the exact + /// process event before consuming it. + direct_activation: bool, + sig_ty: TypeId, + resume_ty: Option, + loc: InstrLocId, +} + +#[derive(Debug, Clone, Copy)] +struct CatchStateLocals { + /// Zero before any caught edge, otherwise the function-local ordinal of + /// the exact `(try_table region, catch arm)` pair most recently selected + /// by this activation's dynamic execution. + catch_selector: LocalId, +} + +#[derive(Debug, Clone, Copy)] +struct AbortDispatch { + /// Partial allocation failure branches back to the dispatch loop after + /// writing its static call index into the module-owned abort scratch. + /// + /// The replay preamble intentionally lives outside this loop: fresh + /// parent/child replay consumes a committed frame once, while the still- + /// live failing activation restarts directly at its selected call without + /// a per-activation selector/flag local. + restart_loop: InstrSeqId, + /// Cold module helper which reserves/selects the frame and writes the + /// statically supplied call index, returning one on reservation success. + frame_select: FunctionId, +} + +#[allow(clippy::too_many_arguments)] +fn instrument_one_function( + module: &mut Module, + func_id: FunctionId, + runtime: &Runtime, + activations: &HashSet, + fork_path: &HashSet, + func_ordinal: u32, + catch_plan: &[CatchRegionPlan], + plain_catches: &[(InstrSeqId, Vec)], + reference_analysis: &FunctionReferenceAnalysis, + unwind_frame_select: FunctionId, +) -> ResumeThunk { + // Choose scheme based on call-site topology. Post-commit-4 + // (2026-05-14) there are TWO live schemes (guard-dispatch was deleted): + // + // instrument_one_function_switch — top-level fork-path calls + // only. Body is restructured so a top-level `br_table` jumps + // directly to the resumed call site, skipping all code in + // between. Per-call operand-stack carryovers (LLVM `*(sp+K) = + // call(...)` shapes) are absorbed via per-call spill locals + // (sub-commit 2.4c) — formerly forced guard-dispatch. + // + // instrument_one_function_nested_switch — fork-path calls + // nested inside Block/IfElse/Loop/TryTable bodies. Cascading + // POST_K blocks plus per-region br_tables route REWIND through + // each enclosing instruction's own dispatch. Sub-commits 2.5/2.6 + // added carryover spilling at nested direct-call landings, + // nested-Loop-with-carryover (side benefit), and multi-value- + // params SubRegion body-input-param prespill. + // + // Catch-handler bodies live inside a nested try_table; nested + // switch-dispatch handles them via the rewind-throw stub + + // capture block mechanism (see Phase 6 + B1 stages 1+2 docs). + // + // Both schemes: + // - share the same fork-resume contract (state machine, linked + // activation frames, and deterministic tagged-catch rethrow). + // - skip body chunks before the chosen POST_K on REWIND, so + // non-fork-path calls and side-effect ops in those chunks run + // exactly once on NORMAL — no per-op gating needed (the + // pre-2.5/2.6 Phase 4g machinery was deleted with guard- + // dispatch in commit 4). + if has_nested_fork_calls(module, func_id, fork_path) { + // Nested per-block switch-dispatch uses the cascading POST_K + + // per-region br_table transform for every validated Wasm shape. + // Classification below is only an internal typed-stack consistency + // check; it is not an artifact support policy. + let nested_status = classify_nested_pattern(module, func_id, fork_path); + if nested_status.is_supported() { + return instrument_one_function_nested_switch( + module, + func_id, + runtime, + activations, + fork_path, + func_ordinal, + catch_plan, + plain_catches, + reference_analysis, + unwind_frame_select, + ); + } + // Every Walrus producer is typed by `typed_instruction_pushes`. + // Reaching this branch means those exhaustive stack effects disagree + // with validated IR, which is an instrumenter bug rather than a + // reference/control shape the artifact is forbidden to contain. + let func = func_name(module, func_id); + match nested_status { + NestedSupportStatus::AnalysisInvariantFailed => panic!( + "fork-instrument internal error: typed nested-stack analysis \ + disagrees with validated function `{func}`; every valid Wasm \ + reference, GC, EH, and multi-value producer must have an \ + activation-owned carryover type" + ), + NestedSupportStatus::Supported => unreachable!(), + } + } + + if has_top_level_stack_carryovers(module, func_id, fork_path) { + // Switch-dispatch absorbs every typed top-level carryover through + // in-place spill/reload. `None` can now mean only that the exhaustive + // Walrus stack model disagreed with validated IR. + if compute_carryover_types(module, func_id, fork_path).is_some() { + return instrument_one_function_switch( + module, + func_id, + runtime, + activations, + fork_path, + func_ordinal, + catch_plan, + plain_catches, + reference_analysis, + unwind_frame_select, + ); + } + let func = func_name(module, func_id); + panic!( + "fork-instrument internal error: typed top-level stack analysis \ + disagrees with validated function `{func}`; every valid Wasm \ + reference, GC, EH, and multi-value producer must have an \ + activation-owned carryover type" + ); + } + + instrument_one_function_switch( + module, + func_id, + runtime, + activations, + fork_path, + func_ordinal, + catch_plan, + plain_catches, + reference_analysis, + unwind_frame_select, + ) +} + +/// Switch-dispatch transform: fork-path calls are hoisted out of the +/// function body and reached during REWIND via a top-level `br_table` +/// that lands directly at the post-active-call-site label. Chunks +/// between calls run only on the NORMAL fall-through path. +#[allow(clippy::too_many_arguments)] +fn instrument_one_function_switch( + module: &mut Module, + func_id: FunctionId, + runtime: &Runtime, + activations: &HashSet, + fork_path: &HashSet, + func_ordinal: u32, + catch_plan: &[CatchRegionPlan], + plain_catches: &[(InstrSeqId, Vec)], + reference_analysis: &FunctionReferenceAnalysis, + unwind_frame_select: FunctionId, +) -> ResumeThunk { + // Pre-existing user locals (args + referenced in body). Validation + // guarantees that every one is scalar and therefore frame-owned. + let all_user_locals = collect_user_locals(module, func_id); + let user_scalar_locals: Vec<(LocalId, ValType)> = all_user_locals + .iter() + .copied() + .filter(|(_, ty)| is_scalar(*ty)) + .collect(); + + // Sub-commit 2.4c: compute carryover types BEFORE taking the + // original body, since `compute_carryover_types` reads the body + // through `module.funcs.get(func_id)`. Computing it after `take` + // would see an empty body and report no carryovers. + let carryover_types_pre_take = compute_carryover_types(module, func_id, fork_path); + + // Take the original entry body; we rebuild it wholesale. + let entry_id = local_mut(module, func_id).entry_block(); + let original_body: Vec<(Instr, InstrLocId)> = + std::mem::take(&mut local_mut(module, func_id).block_mut(entry_id).instrs); + + // Partition the body at top-level fork-path call sites. + let (mut chunks, mut call_sites) = partition_body(&original_body, fork_path, module); + for site in &mut call_sites { + site.direct_activation = matches!( + site.target, + CallTarget::Direct(target) if activations.contains(&target) + ); + let results = module.types.get(site.sig_ty).results().to_vec(); + site.resume_ty = Some(module.types.add(&[], &results)); + } + let n_calls = call_sites.len(); + assert_reference_call_alignment(reference_analysis, &call_sites); + + // Allocate per-function synthetic locals. let catch_state_locals = if catch_plan.is_empty() && plain_catches.is_empty() { None } else { Some(CatchStateLocals { - catch_region_id: module.locals.add(ValType::I32), - exnref_slot: module.locals.add(ValType::I32), + catch_selector: module.locals.add(ValType::I32), }) }; - let abort_live_frame = module.locals.add(ValType::I32); - // Per-call argument materialization. The default is the existing - // spill-local path; a conservative pure scalar suffix can instead - // be replayed after POST_K and needs no frame-backed arg locals. + // spill-local path; a conservative side-effect-free suffix can instead + // be replayed after POST_K and needs no frame-backed arg locals. Reference + // local.get operands are saved directly in the call's recipe vector. let pending_arg_materializations: Vec = call_sites .iter() .enumerate() .map(|(site_idx, cs)| { let arg_types = call_arg_types(module, cs); - for ty in &arg_types { - if !is_scalar(*ty) { - let name = func_name(module, func_id); - panic!( - "fork-instrument: function `{name}` has a fork-path call with a ref-typed \ - argument ({ty:?}). Ref-typed call arguments need aux-table spilling, \ - which the MVP switch-dispatch transform does not yet support.", - ); - } - } plan_call_arg_materialization(module, &chunks[site_idx], arg_types) }) .collect(); @@ -486,11 +1177,11 @@ fn instrument_one_function_switch( Some(v) if v.len() == n_calls => v, _ => vec![Vec::new(); n_calls], }; - let mut carryover_spills: Vec> = Vec::with_capacity(n_calls); + let mut carryover_spills: Vec> = Vec::with_capacity(n_calls); for site_carryovers in &carryover_types { - let spills: Vec = site_carryovers + let spills: Vec = site_carryovers .iter() - .map(|&ty| module.locals.add(ty)) + .map(|&ty| (module.locals.add(spill_storage_type(ty)), ty)) .collect(); carryover_spills.push(spills); } @@ -508,18 +1199,36 @@ fn instrument_one_function_switch( .iter() .zip(arg_types.iter()) { - frame_scalars.push((lid, ty)); + if is_scalar(ty) { + frame_scalars.push((lid, ty)); + } } } - for (site_idx, cr_types) in carryover_types.iter().enumerate() { - for (&lid, &ty) in carryover_spills[site_idx].iter().zip(cr_types.iter()) { - frame_scalars.push((lid, ty)); + for spills in &carryover_spills { + for &(lid, ty) in spills { + if is_scalar(ty) { + frame_scalars.push((lid, ty)); + } } } - append_plain_catch_frame_scalars(&mut frame_scalars, &plain_catch_state); - let locals_with_offsets = assign_local_offsets(&frame_scalars, LOCALS_START_OFFSET); - let frame_size = HEADER_SIZE + user_locals_size(&frame_scalars); + let ordinary_scalar_end = HEADER_SIZE + user_locals_size(&frame_scalars); + let catch_scalar_frame = plan_plain_catch_scalar_frame(&plain_catch_state, ordinary_scalar_end); + let scalar_end = catch_scalar_frame.frame_end(ordinary_scalar_end); + let mut per_call_references = vec![Vec::new(); n_calls]; + for call_idx in 0..call_sites.len() { + arg_materializations[call_idx] + .append_reference_inputs(module, &mut per_call_references[call_idx]); + for &(local, ty) in &carryover_spills[call_idx] { + if let Some(reference) = supported_reference(ty) { + per_call_references[call_idx].push((local, reference)); + } + } + } + append_resume_parameter_references(module, func_id, &mut per_call_references); + append_plain_catch_frame_references(&mut per_call_references, &plain_catch_state); + let reference_frame = plan_reference_frame(module, reference_analysis, per_call_references); + let frame_size = reference_frame.frame_end(scalar_end); let result_types: Vec = { let ty_id = module.funcs.get(func_id).ty(); @@ -527,29 +1236,22 @@ fn instrument_one_function_switch( }; let restart_loop_ty = InstrSeqType::new(&mut module.types, &[], &result_types); - // Plan catch-handler entry-capture (Phase 6d). We allocate in_catch - // and captured_exnref locals now; the IR rewrite is applied later, - // after the body has been rebuilt. - let catch_handlers = - plan_catch_ref_handlers(module, func_id, catch_plan, aux_tables, &plain_catch_state); + let catch_handlers = plan_catch_handlers(catch_plan, &plain_catch_state); // Build the new body: preamble-if + Block($unwind_save) + postamble. let memory = first_memory(module); let ptr_ty = runtime.buf_type; - // Phase 6c rewind-throw stubs: prepended to each fork-path - // try_table body. Phase 6 covers catch_ref / catch_all_ref. - // B1 Stage 2 (Task 2.3) extends the same stub with a plain-catch - // dispatch when `plain_catches` lists arms for the region. - if !catch_plan.is_empty() && aux_tables.exnref.is_some() { + // Rewind rethrows the frame-restored tag and scalar payload. CatchRef + // clauses then manufacture a fresh instance-local exnref. + if !plain_catch_state.is_empty() { let catch_state = - catch_state_locals.expect("exnref catch plan requires catch-state locals"); + catch_state_locals.expect("tagged catch plan requires catch-state locals"); inject_rewind_throw_stubs( module, func_id, runtime, - catch_state.catch_region_id, - aux_tables, + catch_state.catch_selector, catch_plan, &plain_catch_state, ); @@ -583,9 +1285,20 @@ fn instrument_one_function_switch( .id(); let restart_loop = local.builder_mut().dangling_instr_seq(restart_loop_ty).id(); let abort = AbortDispatch { - live_frame: abort_live_frame, restart_loop, + frame_select: unwind_frame_select, }; + let catch_scalar_restore_dispatch = catch_state_locals.and_then(|catch_state| { + build_plain_catch_scalar_dispatch( + local, + runtime, + memory, + ptr_ty, + catch_state.catch_selector, + &catch_scalar_frame, + PlainCatchScalarIo::Restore, + ) + }); let post_seqs: Vec = (0..n_calls) .map(|_| { local @@ -604,8 +1317,8 @@ fn instrument_one_function_switch( ptr_ty, catch_state_locals, &locals_with_offsets, - ref_plan, - aux_tables, + catch_scalar_restore_dispatch, + &reference_frame, frame_size, ); @@ -627,9 +1340,23 @@ fn instrument_one_function_switch( ); // Postamble lives outside $unwind_save, in the entry block, right - // after the Block($unwind_save) instruction. Built as a flat list - // of instructions. + // after the Block($unwind_save) instruction. It commits this + // activation and throws the private unwind tag; no function result + // is fabricated merely to walk the caller stack. let mut postamble: Vec<(Instr, InstrLocId)> = Vec::new(); + let catch_scalar_save_dispatch = catch_state_locals.and_then(|catch_state| { + build_plain_catch_scalar_dispatch( + local, + runtime, + memory, + ptr_ty, + catch_state.catch_selector, + &catch_scalar_frame, + PlainCatchScalarIo::Save, + ) + }); + let reference_save_dispatch = + build_reference_save_dispatch(local, runtime, memory, ptr_ty, &reference_frame); populate_postamble( &mut postamble, runtime, @@ -637,17 +1364,17 @@ fn instrument_one_function_switch( ptr_ty, catch_state_locals, &locals_with_offsets, - ref_plan, - aux_tables, + catch_scalar_save_dispatch, + reference_save_dispatch, frame_size, func_ordinal, - &result_types, ); - // Rebuild the function body around a result-typed restart loop. A partial - // allocation failure branches here from the still-live activation; fresh - // inner activations keep abort_live_frame=0 and restore committed nodes. - let entry_seq = &mut local.block_mut(restart_loop).instrs; + // The preamble is outside the result-typed live-restart loop. Fresh + // parent/child replay consumes its committed frame once; a synchronous + // reservation failure branches straight back to the selected call inside + // the loop without restoring over the still-live activation. + let entry_seq = &mut local.block_mut(entry_id).instrs; push_instr( entry_seq, Instr::GlobalGet(GlobalGet { @@ -666,24 +1393,6 @@ fn instrument_one_function_switch( op: BinaryOp::I32GeU, }), ); - push_instr( - entry_seq, - Instr::LocalGet(LocalGet { - local: abort_live_frame, - }), - ); - push_instr( - entry_seq, - Instr::Unop(walrus::ir::Unop { - op: UnaryOp::I32Eqz, - }), - ); - push_instr( - entry_seq, - Instr::Binop(Binop { - op: BinaryOp::I32And, - }), - ); push_instr( entry_seq, Instr::IfElse(IfElse { @@ -691,33 +1400,41 @@ fn instrument_one_function_switch( alternative: preamble_else, }), ); - push_instr(entry_seq, Instr::Block(Block { seq: unwind_save })); - entry_seq.extend(postamble); - let entry_seq = &mut local.block_mut(entry_id).instrs; - entry_seq.clear(); push_instr(entry_seq, Instr::Loop(Loop { seq: restart_loop })); + let restart_seq = &mut local.block_mut(restart_loop).instrs; + push_instr(restart_seq, Instr::Block(Block { seq: unwind_save })); + restart_seq.extend(postamble); - // Phase 6d application: replaces each fork-path try_table with - // an $outer/$capture wrap so caught exnrefs are stashed and the - // original handler is re-entered via `br`. Runs after body rebuild - // so it finds the try_tables at their new locations inside chunks. - apply_catch_ref_handlers(module, func_id, &catch_handlers, aux_tables); - - // Stage 2 (B1) plain-catch capture-block emission: per-arm - // captures intercept plain catch dispatch so the operand tuple - // can be saved at unwind time. Runs AFTER Phase 6 so it finds - // try_tables at their post-Phase-6 locations. + // Per-arm captures intercept both Catch and CatchRef dispatch after the + // body rebuild, save only transferable state, and forward the original + // handler operands. if let Some(catch_state) = catch_state_locals { + shield_private_unwind_from_user_catches(module, func_id, runtime); apply_plain_catch_handlers( module, func_id, - catch_state.catch_region_id, + catch_state.catch_selector, &plain_catch_state, - catch_plan, &catch_handlers, ); } else { debug_assert!(plain_catches.is_empty()); + shield_private_unwind_from_user_catches(module, func_id, runtime); + } + + ResumeThunk { + func_ordinal, + function: emit_resume_thunk( + module, + func_id, + runtime, + memory, + ptr_ty, + frame_size, + &locals_with_offsets, + &reference_frame, + func_ordinal, + ), } } @@ -758,7 +1475,7 @@ fn has_nested_fork_calls( return; } } - Instr::CallIndirect(_) => { + Instr::CallIndirect(_) | Instr::CallRef(_) => { if depth > 0 { *found = true; return; @@ -797,15 +1514,11 @@ fn has_nested_fork_calls( /// saving the per-instruction typed-stack walk on functions that /// don't need it). /// -/// The walk is conservative: if we encounter an instruction whose -/// stack effect we can't statically determine (wasm-GC ops, legacy -/// exception `try`, …), we report `true` so the caller invokes -/// `compute_carryover_types`. That analyser may itself return `None` -/// (forcing the post-commit-3 panic) if an unknown-type slot reaches -/// a carryover; otherwise switch-dispatch handles it. -/// Likewise for stack underflows — which shouldn't happen in valid -/// wasm, but we defensively route to the post-commit-3 panic path if -/// the input is malformed in a way we can't analyze. +/// `top_level_stack_effect` is exhaustive over Walrus instructions, +/// including Wasm GC and legacy/modern EH. A depth underflow therefore +/// indicates malformed IR or an instrumenter bug; the exact typed walk will +/// diagnose that invariant rather than treating a valid source shape as +/// unsupported. fn has_top_level_stack_carryovers( module: &Module, func_id: FunctionId, @@ -834,6 +1547,7 @@ fn has_top_level_stack_carryovers( // +1 for the table index on top of the signature's params. Some(module.types.get(ci.ty).params().len() + 1) } + Instr::CallRef(call) => Some(module.types.get(call.ty).params().len() + 1), _ => None, }; if let Some(expected) = expected_args { @@ -860,10 +1574,6 @@ fn has_top_level_stack_carryovers( // any fork-path call there is dead code. return false; } - StackEffect::Unknown => { - // Can't analyze — play safe. - return true; - } } } @@ -873,14 +1583,13 @@ fn has_top_level_stack_carryovers( enum StackEffect { Delta { pops: usize, pushes: usize }, Terminator, - Unknown, } /// Compute the stack effect of a single instruction assuming it is /// reachable (i.e., not sitting in a polymorphic post-terminator /// region). Only used by `has_top_level_stack_carryovers`. fn top_level_stack_effect(module: &Module, local: &LocalFunction, instr: &Instr) -> StackEffect { - use StackEffect::{Delta, Terminator, Unknown}; + use StackEffect::{Delta, Terminator}; let block_params_results = |seq_id: InstrSeqId| -> (usize, usize) { let seq = local.block(seq_id); @@ -966,12 +1675,14 @@ fn top_level_stack_effect(module: &Module, local: &LocalFunction, instr: &Instr) // br_if pops its condition; the target's expected args remain // on the stack on fall-through, so static delta is just pop 1. Instr::BrIf(_) => Delta { pops: 1, pushes: 0 }, - // br_on_null / br_on_non_null / br_on_cast / br_on_cast_fail: - // all pop 1 ref and push back on the non-branching path. - Instr::BrOnNull(_) - | Instr::BrOnNonNull(_) - | Instr::BrOnCast(_) - | Instr::BrOnCastFail(_) => Delta { pops: 1, pushes: 1 }, + // br_on_null refines and preserves the non-null fallthrough value. + // br_on_cast* likewise preserves either the source or target value on + // fallthrough. br_on_non_null consumes the known-null fallthrough + // value; its non-null value is carried only on the branch edge. + Instr::BrOnNull(_) | Instr::BrOnCast(_) | Instr::BrOnCastFail(_) => { + Delta { pops: 1, pushes: 1 } + } + Instr::BrOnNonNull(_) => Delta { pops: 1, pushes: 0 }, // --- Nested blocks --- Instr::Block(b) => { @@ -1035,70 +1746,43 @@ fn top_level_stack_effect(module: &Module, local: &LocalFunction, instr: &Instr) | Instr::ThrowRef(_) | Instr::Rethrow(_) => Terminator, - // --- Wasm-GC: not produced by our LLVM toolchain today. Report - // Unknown so we conservatively force the post-commit-3 panic - // path if any ever appears. --- - Instr::StructNew(_) - | Instr::StructNewDefault(_) - | Instr::StructGet(_) - | Instr::StructGetS(_) - | Instr::StructGetU(_) - | Instr::StructSet(_) - | Instr::ArrayNew(_) - | Instr::ArrayNewDefault(_) - | Instr::ArrayNewFixed(_) - | Instr::ArrayNewData(_) - | Instr::ArrayNewElem(_) - | Instr::ArrayGet(_) - | Instr::ArrayGetS(_) - | Instr::ArrayGetU(_) - | Instr::ArraySet(_) - | Instr::ArrayLen(_) - | Instr::ArrayFill(_) - | Instr::ArrayCopy(_) - | Instr::ArrayInitData(_) - | Instr::ArrayInitElem(_) => Unknown, - } -} - -fn seq_scalar_result_types( - module: &Module, - local: &LocalFunction, - seq_id: InstrSeqId, -) -> Option> { - match local.block(seq_id).ty { - InstrSeqType::Simple(None) => Some(Vec::new()), - InstrSeqType::Simple(Some(ty)) if is_scalar(ty) => Some(vec![ty]), - InstrSeqType::Simple(Some(_)) => None, - InstrSeqType::MultiValue(ty_id) => { - let results = module.types.get(ty_id).results(); - if results.iter().all(|&ty| is_scalar(ty)) { - Some(results.to_vec()) - } else { - None - } + // --- Wasm-GC --- + Instr::StructNew(new) => Delta { + pops: module.types.get(new.ty).kind().unwrap_struct().fields.len(), + pushes: 1, + }, + Instr::StructNewDefault(_) => Delta { pops: 0, pushes: 1 }, + Instr::StructGet(_) | Instr::StructGetS(_) | Instr::StructGetU(_) => { + Delta { pops: 1, pushes: 1 } + } + Instr::StructSet(_) => Delta { pops: 2, pushes: 0 }, + Instr::ArrayNew(_) => Delta { pops: 2, pushes: 1 }, + Instr::ArrayNewDefault(_) => Delta { pops: 1, pushes: 1 }, + Instr::ArrayNewFixed(new) => Delta { + pops: new.len as usize, + pushes: 1, + }, + Instr::ArrayNewData(_) | Instr::ArrayNewElem(_) => Delta { pops: 2, pushes: 1 }, + Instr::ArrayGet(_) | Instr::ArrayGetS(_) | Instr::ArrayGetU(_) => { + Delta { pops: 2, pushes: 1 } } + Instr::ArraySet(_) => Delta { pops: 3, pushes: 0 }, + Instr::ArrayLen(_) => Delta { pops: 1, pushes: 1 }, + Instr::ArrayFill(_) => Delta { pops: 4, pushes: 0 }, + Instr::ArrayCopy(_) => Delta { pops: 5, pushes: 0 }, + Instr::ArrayInitData(_) | Instr::ArrayInitElem(_) => Delta { pops: 4, pushes: 0 }, } } -fn push_structured_results( - stack: &mut Vec>, +fn seq_result_types( module: &Module, local: &LocalFunction, seq_id: InstrSeqId, - fallback_pushes: usize, -) { - match seq_scalar_result_types(module, local, seq_id) { - Some(types) => { - for ty in types { - stack.push(Some(ty)); - } - } - None => { - for _ in 0..fallback_pushes { - stack.push(None); - } - } +) -> Option> { + match local.block(seq_id).ty { + InstrSeqType::Simple(None) => Some(Vec::new()), + InstrSeqType::Simple(Some(ty)) => Some(vec![ty]), + InstrSeqType::MultiValue(ty_id) => Some(module.types.get(ty_id).results().to_vec()), } } @@ -1245,7 +1929,7 @@ fn atomic_width_pushes(width: AtomicWidth) -> ValType { fn select_pushes(explicit: Option, pre_stack: &[Option]) -> Option { if let Some(ty) = explicit { - return is_scalar(ty).then_some(ty); + return Some(ty); } if pre_stack.len() < 3 { return None; @@ -1253,13 +1937,20 @@ fn select_pushes(explicit: Option, pre_stack: &[Option]) -> Op let lhs = pre_stack[pre_stack.len() - 3]; let rhs = pre_stack[pre_stack.len() - 2]; match (lhs, rhs) { - (Some(a), Some(b)) if a == b && is_scalar(a) => Some(a), - (Some(a), None) if is_scalar(a) => Some(a), - (None, Some(b)) if is_scalar(b) => Some(b), + (Some(a), Some(b)) if a == b => Some(a), + (Some(a), None) => Some(a), + (None, Some(b)) => Some(b), _ => None, } } +fn concrete_non_null_ref(ty: TypeId) -> ValType { + ValType::Ref(RefType { + nullable: false, + heap_type: HeapType::Concrete(ty), + }) +} + fn typed_single_push( module: &Module, instr: &Instr, @@ -1277,6 +1968,70 @@ fn typed_single_push( Some(module.locals.get(*l).ty()) } Instr::GlobalGet(GlobalGet { global: g }) => Some(module.globals.get(*g).ty), + Instr::TableGet(table_get) => { + Some(ValType::Ref(module.tables.get(table_get.table).element_ty)) + } + Instr::RefNull(reference) => Some(ValType::Ref(reference.ty)), + Instr::RefFunc(reference) => Some(ValType::Ref(RefType { + nullable: false, + heap_type: HeapType::Concrete(module.funcs.get(reference.func).ty()), + })), + Instr::BrOnNull(_) => pre_stack.last().copied().flatten().and_then(|ty| { + let ValType::Ref(mut reference) = ty else { + return None; + }; + reference.nullable = false; + Some(ValType::Ref(reference)) + }), + Instr::BrOnCast(cast) => Some(ValType::Ref(RefType { + nullable: cast.from_nullable, + heap_type: cast.from_heap_type, + })), + Instr::BrOnCastFail(cast) => Some(ValType::Ref(RefType { + nullable: cast.to_nullable, + heap_type: cast.to_heap_type, + })), + Instr::RefAsNonNull(_) => pre_stack.last().copied().flatten().map(|ty| match ty { + ValType::Ref(mut reference) => { + reference.nullable = false; + ValType::Ref(reference) + } + other => other, + }), + Instr::RefI31(_) => Some(ValType::Ref(RefType { + nullable: false, + heap_type: HeapType::Abstract(AbstractHeapType::I31), + })), + Instr::RefCast(cast) => Some(ValType::Ref(RefType { + nullable: cast.nullable, + heap_type: cast.heap_type, + })), + Instr::AnyConvertExtern(_) => Some(ValType::Ref(RefType::ANYREF)), + Instr::ExternConvertAny(_) => Some(ValType::Ref(RefType::EXTERNREF)), + Instr::StructNew(new) => Some(concrete_non_null_ref(new.ty)), + Instr::StructNewDefault(new) => Some(concrete_non_null_ref(new.ty)), + Instr::StructGet(get) => Some( + module.types.get(get.ty).kind().unwrap_struct().fields[get.field as usize] + .element_type + .unpack(), + ), + Instr::StructGetS(_) | Instr::StructGetU(_) => Some(ValType::I32), + Instr::ArrayNew(new) => Some(concrete_non_null_ref(new.ty)), + Instr::ArrayNewDefault(new) => Some(concrete_non_null_ref(new.ty)), + Instr::ArrayNewFixed(new) => Some(concrete_non_null_ref(new.ty)), + Instr::ArrayNewData(new) => Some(concrete_non_null_ref(new.ty)), + Instr::ArrayNewElem(new) => Some(concrete_non_null_ref(new.ty)), + Instr::ArrayGet(get) => Some( + module + .types + .get(get.ty) + .kind() + .unwrap_array() + .field + .element_type + .unpack(), + ), + Instr::ArrayGetS(_) | Instr::ArrayGetU(_) | Instr::ArrayLen(_) => Some(ValType::I32), Instr::Load(load) => Some(load_pushes(&load.kind)), Instr::LoadSimd(_) => Some(ValType::V128), Instr::Binop(b) => Some(binop_pushes(&b.op)), @@ -1291,6 +2046,7 @@ fn typed_single_push( | Instr::TableSize(_) | Instr::TableGrow(_) | Instr::RefIsNull(_) + | Instr::RefTest(_) | Instr::RefEq(_) | Instr::I31GetS(_) | Instr::I31GetU(_) => Some(ValType::I32), @@ -1299,6 +2055,34 @@ fn typed_single_push( } } +fn typed_instruction_pushes( + module: &Module, + local: &LocalFunction, + instr: &Instr, + pre_stack: &[Option], +) -> Option> { + let types = match instr { + Instr::Call(call) => module + .types + .get(module.funcs.get(call.func).ty()) + .results() + .to_vec(), + Instr::CallIndirect(call) => module.types.get(call.ty).results().to_vec(), + Instr::CallRef(call) => module.types.get(call.ty).results().to_vec(), + Instr::Block(block) => seq_result_types(module, local, block.seq)?, + Instr::Loop(loop_) => seq_result_types(module, local, loop_.seq)?, + Instr::IfElse(if_else) => seq_result_types(module, local, if_else.consequent)?, + Instr::TryTable(try_table) => seq_result_types(module, local, try_table.seq)?, + Instr::Try(try_) => seq_result_types(module, local, try_.seq)?, + Instr::I64Add128 { .. } + | Instr::I64Sub128 { .. } + | Instr::I64MulWideS { .. } + | Instr::I64MulWideU { .. } => vec![ValType::I64, ValType::I64], + _ => vec![typed_single_push(module, instr, pre_stack)?], + }; + Some(types) +} + /// Compute the operand-stack carryover types for each top-level /// fork-path call site in the function. /// @@ -1312,21 +2096,13 @@ fn typed_single_push( /// - `Some(per_call_carryovers)` where `per_call_carryovers[K]` is the /// list of carryover ValTypes (deepest stack slot first) at call K. /// Empty vec if call K has no carryover. -/// - `None` if any producer instruction pushes a value of a type we -/// can't statically determine AND that value ends up in a carryover. -/// Post-commit-3, caller panics in this case — sub-commit 9-followup's -/// `Vec>` refinement made the analyser succeed for -/// any shape whose unknown slots are consumed before a fork-path -/// call's carryover, so `None` should be vanishingly rare in -/// shipping wasm. If it does fire, the panic message names the -/// function so the specific producer can be added to the typed- -/// producer list. +/// - `None` only when the exhaustive typed stack model disagrees with +/// already-validated Wasm IR (underflow, count mismatch, or impossible +/// producer typing). Callers treat that as an instrumenter bug. /// -/// Statically-typed producers handled here: -/// `Const`, `LocalGet`, `LocalTee`, `GlobalGet`, `Load` (all kinds), -/// `Binop` (encoded by op-name prefix), direct `Call` (signature -/// results), `MemorySize`/`TableSize` (i32). Anything else triggers -/// `None`. +/// `typed_instruction_pushes` covers every value-producing Walrus +/// instruction, including references, GC, EH control, indirect/ref calls, +/// structured multi-value results, and SIMD. /// /// Used by `instrument_one_function`'s dispatch decision: if this /// returns Some, switch-dispatch can absorb the carryover by spilling @@ -1343,18 +2119,9 @@ fn compute_carryover_types( }; let entry = local.entry_block(); - // Typed operand stack — bottom-to-top. Sub-commit 9-followup - // (2026-05-14): tracked as `Vec>` so unknown - // producers (ref-typed producers, non-fork-path CallIndirect/ - // CallRef, ref-typed structured-control results) push `None` - // without aborting. Failure is only triggered when a `None` slot - // ends up in a fork-path call's carryover. This mirrors - // `walk_seq_for_carryovers`'s 2.5c policy and closes the - // second `instrument_one_function_guard_dispatch` caller in - // `instrument_one_function` (since deleted by commit 4) for the - // case where a top-level fork-path call HAS no carryover but - // the function body still contains unknown-type producers - // consumed before the call. + // Typed operand stack, bottom-to-top. `Option` remains as a defensive + // assertion channel for analyzer bugs; every valid producer has an exact + // `ValType`, including reference and GC producers. let mut stack: Vec> = Vec::new(); let mut carryovers: Vec> = Vec::new(); @@ -1379,6 +2146,20 @@ fn compute_carryover_types( } continue; } + Instr::CallRef(call) => { + let sig = module.types.get(call.ty); + let n_args = sig.params().len() + 1; + if stack.len() < n_args { + return None; + } + let n_cr = stack.len() - n_args; + carryovers.push(snapshot(&stack[..n_cr])?); + stack.truncate(n_cr); + for &ty in sig.results() { + stack.push(Some(ty)); + } + continue; + } Instr::CallIndirect(ci) => { let sig = module.types.get(ci.ty); let n_args = sig.params().len() + 1; // +1 for table index @@ -1407,60 +2188,11 @@ fn compute_carryover_types( if pushes == 0 { continue; } - // Determine pushed type(s). Multi-push instructions - // are only Call / CallIndirect / CallRef; non-fork- - // path indirect calls go through here as `None` slots - // (the conservative `return None` from the legacy - // code only fired AFTER pops had already happened, so - // emitting None preserves stack-depth accounting). - match instr { - Instr::Call(c) => { - let sig = module.types.get(module.funcs.get(c.func).ty()); - for &ty in sig.results() { - stack.push(Some(ty)); - } - continue; - } - Instr::CallIndirect(ci) => { - let sig = module.types.get(ci.ty); - for _ in sig.results() { - stack.push(None); - } - continue; - } - Instr::CallRef(cr) => { - let sig = module.types.get(cr.ty); - for _ in sig.results() { - stack.push(None); - } - continue; - } - Instr::Block(b) => { - push_structured_results(&mut stack, module, local, b.seq, pushes); - continue; - } - Instr::Loop(l) => { - push_structured_results(&mut stack, module, local, l.seq, pushes); - continue; - } - Instr::IfElse(ie) => { - push_structured_results(&mut stack, module, local, ie.consequent, pushes); - continue; - } - Instr::TryTable(t) => { - push_structured_results(&mut stack, module, local, t.seq, pushes); - continue; - } - Instr::Try(t) => { - push_structured_results(&mut stack, module, local, t.seq, pushes); - continue; - } - _ => {} + let produced = typed_instruction_pushes(module, local, instr, &pre_stack)?; + if produced.len() != pushes { + return None; } - // Single-push, non-call, non-structured-control - // producers. - debug_assert_eq!(pushes, 1, "multi-push non-Call should not appear"); - stack.push(typed_single_push(module, instr, &pre_stack)); + stack.extend(produced.into_iter().map(Some)); } StackEffect::Terminator => { // Post-terminator code in the same seq is unreachable @@ -1480,7 +2212,7 @@ fn compute_carryover_types( Instr::Call(c) if fork_path.contains(&c.func) => { carryovers.push(Vec::new()); } - Instr::CallIndirect(_) => { + Instr::CallIndirect(_) | Instr::CallRef(_) => { carryovers.push(Vec::new()); } _ => {} @@ -1488,7 +2220,6 @@ fn compute_carryover_types( } break; } - StackEffect::Unknown => return None, } } @@ -1512,12 +2243,9 @@ fn compute_carryover_types( /// of carryover ValTypes (deepest stack slot first) for that call /// site; an empty vec means the call has no carryover. /// -/// Returns `None` if any producer instruction in any walked seq -/// pushes a value whose type can't be determined statically (e.g. a -/// non-fork-path `CallIndirect` / `CallRef`, a wasm-GC ref, a -/// multi-value or ref-typed `Block`/`Loop`/`IfElse`/`TryTable` -/// result). The caller (sub-commit 2.5c) keeps the existing rejection -/// in `seq_has_unsupported_carryover` for the `None` case. +/// Returns `None` only if the exhaustive stack model disagrees with +/// validated IR; reference, GC, EH, indirect/ref-call, and multi-value +/// producers all have exact types. fn compute_nested_carryover_types( module: &Module, func_id: FunctionId, @@ -1549,9 +2277,9 @@ fn compute_nested_carryover_types( for (&seq_id, direct_idxs) in &direct_idxs_per_seq { let per_seq = walk_seq_for_carryovers(module, local, seq_id, fork_path)?; if per_seq.len() != direct_idxs.len() { - // Mismatch implies the walk terminated early (e.g., hit a - // terminator before the last fork-path call). Conservative - // fallback: report unanalyzable. + // Mismatch means discovery and reachable typed walking disagree. + // Dead suffixes are excluded before this activation reaches the + // transform; a mismatch here is an internal invariant failure. return None; } for (cr, &idx) in per_seq.into_iter().zip(direct_idxs.iter()) { @@ -1568,16 +2296,10 @@ fn compute_nested_carryover_types( /// instructions are treated as opaque — see /// `compute_nested_carryover_types`. /// -/// Stack values are tracked as `Option`: producers we can -/// type statically push `Some(ty)`; producers we can't scalar-spill -/// (e.g. ref-typed producers or non-fork-path CallRef results) push -/// `None`. -/// `None` slots are tolerated as long as they're consumed before -/// the next fork-path call; only `None` slots that end up IN A -/// carryover force `walk_seq_for_carryovers` to fail conservatively -/// (returning `None`). This makes the analyser succeed for any -/// fork-bearing seq with no carryover at all, regardless of the -/// producer instructions it contains. +/// Stack values retain `Option` as an internal consistency channel. +/// Every valid producer, including reference/GC values and CallRef results, +/// pushes `Some(ty)`; `None` can therefore reach a snapshot only through an +/// analyzer bug. fn walk_seq_for_carryovers( module: &Module, f: &LocalFunction, @@ -1600,8 +2322,8 @@ fn walk_seq_for_carryovers( }; let mut carryovers: Vec> = Vec::new(); - // Helper: materialise the typed-carryover slice. Returns None if - // any `None` slot would be captured. + // Materialize the exact typed carryover. A `None` slot means the + // exhaustive producer model failed its internal invariant. fn snapshot_carryover(slots: &[Option]) -> Option> { slots.iter().copied().collect::>>() } @@ -1637,6 +2359,20 @@ fn walk_seq_for_carryovers( } continue; } + Instr::CallRef(call) => { + let sig = module.types.get(call.ty); + let n_args = sig.params().len() + 1; + if stack.len() < n_args { + return None; + } + let n_cr = stack.len() - n_args; + carryovers.push(snapshot_carryover(&stack[..n_cr])?); + stack.truncate(n_cr); + for &ty in sig.results() { + stack.push(Some(ty)); + } + continue; + } _ => {} } @@ -1650,67 +2386,11 @@ fn walk_seq_for_carryovers( if pushes == 0 { continue; } - // Determine pushed type(s). Multi-push instructions are - // only Call / CallIndirect / CallRef and structured - // control flow with a multi-value result. We type the - // single-result cases precisely; everything else - // contributes `None` slots so the seq can still proceed - // as long as the unknown slot is consumed before any - // carryover snapshot. - match instr { - Instr::Call(c) => { - let sig = module.types.get(module.funcs.get(c.func).ty()); - for &ty in sig.results() { - stack.push(Some(ty)); - } - continue; - } - Instr::CallIndirect(ci) => { - // Non-fork-path CallIndirect (fork-path is - // handled above). Unknown ref-typed result? - // Push None slots — caller may or may not - // observe them as a carryover. - let sig = module.types.get(ci.ty); - for _ in sig.results() { - stack.push(None); - } - continue; - } - Instr::CallRef(cr) => { - let sig = module.types.get(cr.ty); - for _ in sig.results() { - stack.push(None); - } - continue; - } - Instr::Block(b) => { - push_structured_results(&mut stack, module, f, b.seq, pushes); - continue; - } - Instr::Loop(l) => { - push_structured_results(&mut stack, module, f, l.seq, pushes); - continue; - } - Instr::IfElse(ie) => { - push_structured_results(&mut stack, module, f, ie.consequent, pushes); - continue; - } - Instr::TryTable(t) => { - push_structured_results(&mut stack, module, f, t.seq, pushes); - continue; - } - Instr::Try(t) => { - push_structured_results(&mut stack, module, f, t.seq, pushes); - continue; - } - _ => {} + let produced = typed_instruction_pushes(module, f, instr, &pre_stack)?; + if produced.len() != pushes { + return None; } - // Single-push, non-call, non-block-typed producers. - debug_assert_eq!( - pushes, 1, - "multi-push non-call/non-structured-control should not reach here" - ); - stack.push(typed_single_push(module, instr, &pre_stack)); + stack.extend(produced.into_iter().map(Some)); } StackEffect::Terminator => { // Post-terminator code in this seq is unreachable. @@ -1721,7 +2401,6 @@ fn walk_seq_for_carryovers( // `carryovers`.) return Some(carryovers); } - StackEffect::Unknown => return None, } } @@ -1753,7 +2432,9 @@ fn partition_body( let sig_ty = module.funcs.get(c.func).ty(); calls.push(CallSiteInfo { target: CallTarget::Direct(c.func), + direct_activation: false, sig_ty, + resume_ty: None, loc: *loc, }); chunks.push(Vec::new()); @@ -1761,7 +2442,19 @@ fn partition_body( Instr::CallIndirect(ci) => { calls.push(CallSiteInfo { target: CallTarget::Indirect { table: ci.table }, + direct_activation: false, sig_ty: ci.ty, + resume_ty: None, + loc: *loc, + }); + chunks.push(Vec::new()); + } + Instr::CallRef(call) => { + calls.push(CallSiteInfo { + target: CallTarget::Ref, + direct_activation: false, + sig_ty: call.ty, + resume_ty: None, loc: *loc, }); chunks.push(Vec::new()); @@ -1777,11 +2470,42 @@ fn partition_body( (chunks, calls) } +fn assert_reference_call_alignment(analysis: &FunctionReferenceAnalysis, calls: &[CallSiteInfo]) { + assert_eq!( + analysis.call_sites.len(), + calls.len(), + "original reference analysis and top-level transform discovered different call counts" + ); + for (reference, call) in analysis.call_sites.iter().zip(calls) { + let aligned = match (reference.kind, call.target) { + (OriginalCallKind::Direct(expected), CallTarget::Direct(actual)) => expected == actual, + ( + OriginalCallKind::Indirect { + table: expected_table, + ty: expected_ty, + }, + CallTarget::Indirect { + table: actual_table, + }, + ) => expected_table == actual_table && expected_ty == call.sig_ty, + (OriginalCallKind::Ref { ty }, CallTarget::Ref) => ty == call.sig_ty, + _ => false, + }; + assert!( + aligned, + "reference analysis call {:?} does not align with transformed call target {:?}", + reference.kind, call.target + ); + } +} + fn call_arg_types(module: &Module, cs: &CallSiteInfo) -> Vec { let params = module.types.get(cs.sig_ty).params().to_vec(); let mut arg_types = params; - if matches!(cs.target, CallTarget::Indirect { .. }) { - arg_types.push(ValType::I32); + match cs.target { + CallTarget::Indirect { .. } => arg_types.push(ValType::I32), + CallTarget::Ref => arg_types.push(ValType::Ref(RefType::FUNCREF)), + CallTarget::Direct(_) => {} } arg_types } @@ -1801,6 +2525,7 @@ enum PendingCallArgMaterialization { enum CallArgMaterialization { Spill { locals: Vec, + types: Vec, }, PureTail { tail: Vec<(Instr, InstrLocId)>, @@ -1811,7 +2536,7 @@ enum CallArgMaterialization { impl CallArgMaterialization { fn spill_locals(&self) -> &[LocalId] { match self { - Self::Spill { locals } => locals, + Self::Spill { locals, .. } => locals, Self::PureTail { .. } => &[], } } @@ -1822,6 +2547,34 @@ impl CallArgMaterialization { Self::PureTail { tail_len, .. } => *tail_len, } } + + fn append_reference_inputs(&self, module: &Module, references: &mut Vec<(LocalId, RefType)>) { + match self { + Self::Spill { locals, types } => { + for (&local, &ty) in locals.iter().zip(types) { + if let Some(reference) = supported_reference(ty) { + references.push((local, reference)); + } + } + } + Self::PureTail { tail, .. } => { + // WHY: replaying a reference local.get is side-effect-free, + // but only if that exact local is itself activation-owned. + // Recording it here avoids a per-call reference spill local + // while ensuring the preamble restores it before reissuing + // the pure argument suffix. + for (instr, _) in tail { + let Instr::LocalGet(LocalGet { local }) = instr else { + continue; + }; + let ValType::Ref(reference) = module.locals.get(*local).ty() else { + continue; + }; + references.push((*local, reference)); + } + } + } + } } fn plan_call_arg_materialization( @@ -1829,7 +2582,7 @@ fn plan_call_arg_materialization( chunk: &[(Instr, InstrLocId)], arg_types: Vec, ) -> PendingCallArgMaterialization { - if let Some((tail_len, tail)) = split_pure_scalar_tail(module, chunk, &arg_types) { + if let Some((tail_len, tail)) = split_pure_replay_tail(module, chunk, &arg_types) { PendingCallArgMaterialization::PureTail { tail, tail_len } } else { PendingCallArgMaterialization::Spill { arg_types } @@ -1842,8 +2595,14 @@ fn allocate_call_arg_materialization( ) -> CallArgMaterialization { match pending { PendingCallArgMaterialization::Spill { arg_types } => { - let locals = arg_types.iter().map(|&ty| module.locals.add(ty)).collect(); - CallArgMaterialization::Spill { locals } + let locals = arg_types + .iter() + .map(|&ty| module.locals.add(spill_storage_type(ty))) + .collect(); + CallArgMaterialization::Spill { + locals, + types: arg_types, + } } PendingCallArgMaterialization::PureTail { tail, tail_len } => { CallArgMaterialization::PureTail { tail, tail_len } @@ -1859,7 +2618,7 @@ fn truncate_materialized_tail(chunk: &mut Vec<(Instr, InstrLocId)>, tail_len: us chunk.truncate(chunk.len() - tail_len); } -fn split_pure_scalar_tail( +fn split_pure_replay_tail( module: &Module, chunk: &[(Instr, InstrLocId)], expected_outputs: &[ValType], @@ -1870,7 +2629,7 @@ fn split_pure_scalar_tail( for start in 0..chunk.len() { let tail = &chunk[start..]; - if let Some(outputs) = pure_scalar_tail_outputs(module, tail) { + if let Some(outputs) = pure_replay_tail_outputs(module, tail) { if outputs == expected_outputs { return Some((tail.len(), tail.to_vec())); } @@ -1880,16 +2639,13 @@ fn split_pure_scalar_tail( None } -fn pure_scalar_tail_outputs(module: &Module, tail: &[(Instr, InstrLocId)]) -> Option> { +fn pure_replay_tail_outputs(module: &Module, tail: &[(Instr, InstrLocId)]) -> Option> { let mut stack: Vec = Vec::new(); for (instr, _) in tail { match instr { Instr::Const(c) => stack.push(pure_const_type(c)?), Instr::LocalGet(LocalGet { local }) => { let ty = module.locals.get(*local).ty(); - if !is_scalar(ty) { - return None; - } stack.push(ty); } Instr::Unop(u) => { @@ -2341,7 +3097,7 @@ fn populate_dispatch_structure( chunks: &[Vec<(Instr, InstrLocId)>], call_sites: &[CallSiteInfo], arg_materializations: &[CallArgMaterialization], - carryover_spills: &[Vec], + carryover_spills: &[Vec], catch_handlers: &[CatchHandlerInfo], runtime: &Runtime, memory: MemoryId, @@ -2401,7 +3157,7 @@ fn emit_dispatch_node( chunks: &[Vec<(Instr, InstrLocId)>], call_sites: &[CallSiteInfo], arg_materializations: &[CallArgMaterialization], - carryover_spills: &[Vec], + carryover_spills: &[Vec], catch_handlers: &[CatchHandlerInfo], runtime: &Runtime, memory: MemoryId, @@ -2487,7 +3243,7 @@ fn emit_internal_dispatch( chunks: &[Vec<(Instr, InstrLocId)>], call_sites: &[CallSiteInfo], arg_materializations: &[CallArgMaterialization], - carryover_spills: &[Vec], + carryover_spills: &[Vec], catch_handlers: &[CatchHandlerInfo], runtime: &Runtime, memory: MemoryId, @@ -2610,7 +3366,7 @@ fn emit_leaf_dispatch( chunks: &[Vec<(Instr, InstrLocId)>], call_sites: &[CallSiteInfo], arg_materializations: &[CallArgMaterialization], - carryover_spills: &[Vec], + carryover_spills: &[Vec], catch_handlers: &[CatchHandlerInfo], runtime: &Runtime, memory: MemoryId, @@ -2733,171 +3489,82 @@ fn emit_leaf_dispatch( for (instr, loc) in &chunks[n_calls_total] { s.push((instr.clone(), *loc)); } - push_instr(s, Instr::Return(Return {})); - } else { - let s = &mut local.block_mut(exit_seq).instrs; - for (instr, loc) in &chunks[leaf_end] { - s.push((instr.clone(), *loc)); - } - emit_spill_call_tail( - s, - &arg_materializations[leaf_end], - &carryover_spills[leaf_end], - ); - } -} - -/// Spill the arg values off the operand stack into the per-call -/// spill locals. Args are spilled in reverse (top-of-stack first), -/// so the deepest arg ends up in `spills[0]`. -/// -/// When `carryovers` is non-empty (sub-commit 2.4c), the operand -/// stack at the call site is `[..., carryover_0, ..., carryover_{n-1}, -/// arg_0, ..., arg_{m-1}]` (bottom-to-top). After popping all args, -/// we keep popping into `carryovers` (also reverse-order), so -/// `carryovers[0]` ends up holding the deepest carryover slot. -fn emit_spill_args(out: &mut Vec<(Instr, InstrLocId)>, spills: &[LocalId], carryovers: &[LocalId]) { - for &local in spills.iter().rev() { - push_instr(out, Instr::LocalSet(LocalSet { local })); - } - for &local in carryovers.iter().rev() { - push_instr(out, Instr::LocalSet(LocalSet { local })); - } -} - -fn emit_spill_call_tail( - out: &mut Vec<(Instr, InstrLocId)>, - arg_materialization: &CallArgMaterialization, - carryovers: &[LocalId], -) { - emit_spill_args(out, arg_materialization.spill_locals(), carryovers); -} - -fn emit_materialized_call_args( - out: &mut Vec<(Instr, InstrLocId)>, - arg_materialization: &CallArgMaterialization, -) { - match arg_materialization { - CallArgMaterialization::Spill { locals } => { - for &l in locals.iter() { - push_instr(out, Instr::LocalGet(LocalGet { local: l })); - } - } - CallArgMaterialization::PureTail { tail, .. } => { - out.extend(tail.iter().cloned()); - } - } -} - -/// Emit Phase 6e writes inline. Must be called with mutable access to -/// the function (so dangling seqs can be allocated for each handler's -/// if-branch). -fn emit_phase_6e_writes( - local: &mut LocalFunction, - seq_id: InstrSeqId, - catch_handlers: &[CatchHandlerInfo], - catch_state_locals: Option, -) { - if catch_handlers.is_empty() { - return; - } - let catch_state = catch_state_locals.expect("catch handlers require catch-state locals"); - { - let s = &mut local.block_mut(seq_id).instrs; - push_instr( - s, - Instr::Const(Const { - value: Value::I32(0), - }), - ); - push_instr( - s, - Instr::LocalSet(LocalSet { - local: catch_state.catch_region_id, - }), - ); - push_instr( - s, - Instr::Const(Const { - value: Value::I32(0), - }), - ); - push_instr( - s, - Instr::LocalSet(LocalSet { - local: catch_state.exnref_slot, - }), - ); - } - for info in catch_handlers { - let if_ty = InstrSeqType::Simple(None); - let ih_then = local.builder_mut().dangling_instr_seq(if_ty).id(); - let ih_else = local.builder_mut().dangling_instr_seq(if_ty).id(); - { - let s = &mut local.block_mut(ih_then).instrs; - push_instr( - s, - Instr::Const(Const { - value: Value::I32(info.catch_region_id as i32), - }), - ); - push_instr( - s, - Instr::LocalSet(LocalSet { - local: catch_state.catch_region_id, - }), - ); - push_instr( - s, - Instr::Const(Const { - value: Value::I32(info.exnref_slot as i32), - }), - ); - push_instr( - s, - Instr::LocalSet(LocalSet { - local: catch_state.exnref_slot, - }), - ); - } - let s = &mut local.block_mut(seq_id).instrs; - push_instr( - s, - Instr::LocalGet(LocalGet { - local: info.in_catch_local, - }), - ); - push_instr( + push_instr(s, Instr::Return(Return {})); + } else { + let s = &mut local.block_mut(exit_seq).instrs; + for (instr, loc) in &chunks[leaf_end] { + s.push((instr.clone(), *loc)); + } + emit_spill_call_tail( s, - Instr::IfElse(IfElse { - consequent: ih_then, - alternative: ih_else, - }), + &arg_materializations[leaf_end], + &carryover_spills[leaf_end], ); } } -fn emit_call_index_store_and_unwind_branch( +/// Spill the arg values off the operand stack into the per-call +/// spill locals. Args are spilled in reverse (top-of-stack first), +/// so the deepest arg ends up in `spills[0]`. +/// +/// When `carryovers` is non-empty (sub-commit 2.4c), the operand +/// stack at the call site is `[..., carryover_0, ..., carryover_{n-1}, +/// arg_0, ..., arg_{m-1}]` (bottom-to-top). After popping all args, +/// we keep popping into `carryovers` (also reverse-order), so +/// `carryovers[0]` ends up holding the deepest carryover slot. +fn emit_spill_args( + out: &mut Vec<(Instr, InstrLocId)>, + spills: &[LocalId], + carryovers: &[TypedSpillLocal], +) { + for &local in spills.iter().rev() { + push_instr(out, Instr::LocalSet(LocalSet { local })); + } + for &(local, _ty) in carryovers.iter().rev() { + push_instr(out, Instr::LocalSet(LocalSet { local })); + } +} + +fn emit_spill_call_tail( + out: &mut Vec<(Instr, InstrLocId)>, + arg_materialization: &CallArgMaterialization, + carryovers: &[TypedSpillLocal], +) { + emit_spill_args(out, arg_materialization.spill_locals(), carryovers); +} + +fn emit_materialized_call_args( + out: &mut Vec<(Instr, InstrLocId)>, + arg_materialization: &CallArgMaterialization, +) { + match arg_materialization { + CallArgMaterialization::Spill { locals, types } => { + for (&local, &ty) in locals.iter().zip(types) { + push_typed_local_get(out, local, ty); + } + } + CallArgMaterialization::PureTail { tail, .. } => { + out.extend(tail.iter().cloned()); + } + } +} + +/// Handle the private unwind tag at one statically known call site. +/// +/// Successful reservation records the static call index and branches to the +/// common frame postamble. A synchronous allocation failure instead selects +/// the header-sized abort scratch, records the same index, and restarts the +/// live activation at the dispatch loop. Since the replay preamble is outside +/// that loop, no activation-local selector/flag is required. +fn emit_static_call_unwind_handler( local: &mut LocalFunction, seq_id: InstrSeqId, - runtime: &Runtime, - memory: MemoryId, ptr_ty: ValType, frame_size: u32, call_idx: u32, unwind_save: InstrSeqId, - catch_handlers: &[CatchHandlerInfo], - catch_state_locals: Option, abort: AbortDispatch, ) { - let unwind_then = local - .builder_mut() - .dangling_instr_seq(InstrSeqType::Simple(None)) - .id(); - let normal_else = local - .builder_mut() - .dangling_instr_seq(InstrSeqType::Simple(None)) - .id(); let reserve_succeeded = local .builder_mut() .dangling_instr_seq(InstrSeqType::Simple(None)) @@ -2908,106 +3575,31 @@ fn emit_call_index_store_and_unwind_branch( .id(); { - let s = &mut local.block_mut(unwind_then).instrs; - if let Some(frame_reserve) = runtime.frame_reserve { - // This is the first frame write on the unwind path. Reserve the - // complete node before publishing call_index or any postamble - // scalar/reference state. - push_instr( - s, - Instr::GlobalGet(GlobalGet { - global: runtime.buf_global, - }), - ); - push_instr(s, ptr_const(ptr_ty, frame_size as i64)); - push_instr( - s, - Instr::Call(Call { - func: frame_reserve, - }), - ); - push_instr(s, store_ptr(memory, ptr_ty, 0)); - - push_instr( - s, - Instr::GlobalGet(GlobalGet { - global: runtime.buf_global, - }), - ); - push_instr(s, load_ptr(memory, ptr_ty, 0)); - push_instr( - s, - Instr::Unop(walrus::ir::Unop { - op: match ptr_ty { - ValType::I32 => UnaryOp::I32Eqz, - ValType::I64 => UnaryOp::I64Eqz, - other => unreachable!("unsupported pointer type {other:?}"), - }, - }), - ); - push_instr( - s, - Instr::IfElse(IfElse { - consequent: reserve_failed, - alternative: reserve_succeeded, - }), - ); - } else { - push_instr( - s, - Instr::Block(Block { - seq: reserve_succeeded, - }), - ); - } - } - - { - let s = &mut local.block_mut(reserve_failed).instrs; + let s = &mut local.block_mut(seq_id).instrs; + push_instr(s, ptr_const(ptr_ty, frame_size as i64)); push_instr( s, Instr::Const(Const { - value: Value::I32(1), - }), - ); - push_instr( - s, - Instr::LocalSet(LocalSet { - local: abort.live_frame, - }), - ); - // Select the module-owned abort scratch frame. Linked chunks begin - // after the descriptor's larger fixed prefix, so this header-sized - // area can carry the live activation's call index without touching a - // committed node. - push_instr( - s, - Instr::GlobalGet(GlobalGet { - global: runtime.buf_global, - }), - ); - push_instr( - s, - Instr::GlobalGet(GlobalGet { - global: runtime.buf_global, + value: Value::I32(call_idx as i32), }), ); - push_instr(s, ptr_const(ptr_ty, runtime.frames_start_offset as i64)); push_instr( s, - Instr::Binop(Binop { - op: ptr_add(ptr_ty), + Instr::Call(Call { + func: abort.frame_select, }), ); - push_instr(s, store_ptr(memory, ptr_ty, 0)); - push_current_frame_ptr(s, runtime, memory, ptr_ty); push_instr( s, - Instr::Const(Const { - value: Value::I32(call_idx as i32), + Instr::IfElse(IfElse { + consequent: reserve_succeeded, + alternative: reserve_failed, }), ); - push_instr(s, store_i32(memory, CALL_INDEX_OFFSET)); + } + + { + let s = &mut local.block_mut(reserve_failed).instrs; push_instr( s, Instr::Br(Br { @@ -3016,54 +3608,251 @@ fn emit_call_index_store_and_unwind_branch( ); } - emit_phase_6e_writes(local, reserve_succeeded, catch_handlers, catch_state_locals); { let s = &mut local.block_mut(reserve_succeeded).instrs; - push_current_frame_ptr(s, runtime, memory, ptr_ty); - push_instr( - s, - Instr::Const(Const { - value: Value::I32(call_idx as i32), - }), - ); - push_instr(s, store_i32(memory, CALL_INDEX_OFFSET)); push_instr(s, Instr::Br(Br { block: unwind_save })); } +} + +// ---------------------------------------------------------------------- +// Preamble / postamble +// ---------------------------------------------------------------------- - emit_phase_6e_writes(local, normal_else, catch_handlers, catch_state_locals); +fn reference_plan_runs( + plan: &ReferenceFramePlan, +) -> Vec<(usize, usize, Vec, Vec<(LocalId, RefType)>)> { + let mut runs = Vec::new(); + let mut start = 0usize; + while start < plan.slots_by_call.len() { + let slots = &plan.slots_by_call[start]; + let nulls = &plan.null_locals_by_call[start]; + let mut end = start; + while end + 1 < plan.slots_by_call.len() + && plan.slots_by_call[end + 1] == *slots + && plan.null_locals_by_call[end + 1] == *nulls + { + end += 1; + } + if !slots.is_empty() || !nulls.is_empty() { + runs.push((start, end, slots.clone(), nulls.clone())); + } + start = end + 1; + } + runs +} - let s = &mut local.block_mut(seq_id).instrs; +fn push_call_index_in_range( + out: &mut Vec<(Instr, InstrLocId)>, + runtime: &Runtime, + memory: MemoryId, + ptr_ty: ValType, + first: usize, + last: usize, +) { + push_current_call_index(out, runtime, memory, ptr_ty); push_instr( - s, - Instr::GlobalGet(GlobalGet { - global: runtime.state_global, + out, + Instr::Const(Const { + value: Value::I32(first as i32), }), ); + if first == last { + push_instr( + out, + Instr::Binop(Binop { + op: BinaryOp::I32Eq, + }), + ); + return; + } push_instr( - s, + out, + Instr::Binop(Binop { + op: BinaryOp::I32GeU, + }), + ); + push_current_call_index(out, runtime, memory, ptr_ty); + push_instr( + out, Instr::Const(Const { - value: Value::I32(runtime::STATE_UNWINDING), + value: Value::I32(last as i32), }), ); push_instr( - s, + out, Instr::Binop(Binop { - op: BinaryOp::I32Eq, + op: BinaryOp::I32LeU, }), ); push_instr( - s, - Instr::IfElse(IfElse { - consequent: unwind_then, - alternative: normal_else, + out, + Instr::Binop(Binop { + op: BinaryOp::I32And, }), ); } -// ---------------------------------------------------------------------- -// Preamble / postamble -// ---------------------------------------------------------------------- - +#[allow(clippy::too_many_arguments)] +fn emit_reference_restore_dispatch( + local: &mut LocalFunction, + seq: InstrSeqId, + runtime: &Runtime, + memory: MemoryId, + ptr_ty: ValType, + plan: &ReferenceFramePlan, +) { + let codecs = runtime + .reference_codecs + .expect("linked fork reference plan requires typed host codecs"); + let vector_get = runtime + .reference_vector_get + .expect("linked fork reference plan requires recipe-vector lookup"); + for (first, last, slots, nulls) in reference_plan_runs(plan) { + let then_seq = local + .builder_mut() + .dangling_instr_seq(InstrSeqType::Simple(None)) + .id(); + let else_seq = local + .builder_mut() + .dangling_instr_seq(InstrSeqType::Simple(None)) + .id(); + { + let out = &mut local.block_mut(then_seq).instrs; + for (position, slot_idx) in slots.into_iter().enumerate() { + let slot = plan.slots[slot_idx]; + let class = slot.class; + push_current_frame_ptr(out, runtime, memory, ptr_ty); + push_instr(out, load_i32(memory, REFERENCE_VECTOR_OFFSET)); + push_instr( + out, + Instr::Const(Const { + value: Value::I32(position as i32), + }), + ); + push_instr(out, Instr::Call(Call { func: vector_get })); + push_instr( + out, + Instr::Call(Call { + func: class.decoder(codecs), + }), + ); + push_decoded_reference_narrowing(out, class, slot.ty); + push_instr(out, Instr::LocalSet(LocalSet { local: slot.local })); + } + for (local, ty) in nulls { + push_instr(out, Instr::RefNull(RefNull { ty })); + push_instr(out, Instr::LocalSet(LocalSet { local })); + } + } + let out = &mut local.block_mut(seq).instrs; + push_call_index_in_range(out, runtime, memory, ptr_ty, first, last); + push_instr( + out, + Instr::IfElse(IfElse { + consequent: then_seq, + alternative: else_seq, + }), + ); + } +} + +#[allow(clippy::too_many_arguments)] +fn build_reference_save_dispatch( + local: &mut LocalFunction, + runtime: &Runtime, + memory: MemoryId, + ptr_ty: ValType, + plan: &ReferenceFramePlan, +) -> Option { + if plan.slots_by_call.iter().all(Vec::is_empty) { + return None; + } + let codecs = runtime + .reference_codecs + .expect("linked fork reference plan requires typed host codecs"); + let vector_begin = runtime + .reference_vector_begin + .expect("linked fork reference plan requires recipe-vector allocation"); + let vector_append = runtime + .reference_vector_append + .expect("linked fork reference plan requires recipe-vector append"); + let vector_finish = runtime + .reference_vector_finish + .expect("linked fork reference plan requires recipe-vector finish"); + let root = local + .builder_mut() + .dangling_instr_seq(InstrSeqType::Simple(None)) + .id(); + for (first, last, slots, _nulls) in reference_plan_runs(plan) { + if slots.is_empty() { + continue; + } + let then_seq = local + .builder_mut() + .dangling_instr_seq(InstrSeqType::Simple(None)) + .id(); + let else_seq = local + .builder_mut() + .dangling_instr_seq(InstrSeqType::Simple(None)) + .id(); + { + let out = &mut local.block_mut(then_seq).instrs; + push_current_frame_ptr(out, runtime, memory, ptr_ty); + push_instr( + out, + Instr::Const(Const { + value: Value::I32(slots.len() as i32), + }), + ); + push_instr(out, Instr::Call(Call { func: vector_begin })); + push_instr(out, store_i32(memory, REFERENCE_VECTOR_OFFSET)); + for slot_idx in slots { + let slot = plan.slots[slot_idx]; + let class = slot.class; + push_current_frame_ptr(out, runtime, memory, ptr_ty); + push_instr(out, load_i32(memory, REFERENCE_VECTOR_OFFSET)); + push_typed_local_get(out, slot.local, ValType::Ref(slot.ty)); + push_instr( + out, + Instr::Call(Call { + func: class.encoder(codecs), + }), + ); + push_instr( + out, + Instr::Call(Call { + func: vector_append, + }), + ); + } + // WHY: the frame must hold a durable canonical ordinal, never the + // transaction-local builder handle returned by vector_begin. This + // also interns identical vectors across recursive activations + // without adding a source-function local or frame byte. + push_current_frame_ptr(out, runtime, memory, ptr_ty); + push_current_frame_ptr(out, runtime, memory, ptr_ty); + push_instr(out, load_i32(memory, REFERENCE_VECTOR_OFFSET)); + push_instr( + out, + Instr::Call(Call { + func: vector_finish, + }), + ); + push_instr(out, store_i32(memory, REFERENCE_VECTOR_OFFSET)); + } + let out = &mut local.block_mut(root).instrs; + push_call_index_in_range(out, runtime, memory, ptr_ty, first, last); + push_instr( + out, + Instr::IfElse(IfElse { + consequent: then_seq, + alternative: else_seq, + }), + ); + } + Some(root) +} + #[allow(clippy::too_many_arguments)] fn populate_preamble_then( local: &mut LocalFunction, @@ -3073,15 +3862,14 @@ fn populate_preamble_then( ptr_ty: ValType, catch_state_locals: Option, locals_with_offsets: &[(LocalId, ValType, u32)], - ref_plan: &[RefLocalSlot], - aux_tables: &AuxTables, + catch_scalar_restore_dispatch: Option, + reference_plan: &ReferenceFramePlan, frame_size: u32, ) { - let s = &mut local.block_mut(preamble_then).instrs; - // Store the frame selected for replay in *(buf + 0). The linked format // asks the host-managed chain for the next committed frame; the legacy // format walks its contiguous buffer backward. + let s = &mut local.block_mut(preamble_then).instrs; push_instr( s, Instr::GlobalGet(GlobalGet { @@ -3110,22 +3898,14 @@ fn populate_preamble_then( push_instr(s, store_ptr(memory, ptr_ty, 0)); if let Some(catch_state) = catch_state_locals { - // catch_region_id_local / exnref_slot_local - push_current_frame_ptr(s, runtime, memory, ptr_ty); - push_instr(s, load_i32(memory, CATCH_REGION_OFFSET)); - push_instr( - s, - Instr::LocalSet(LocalSet { - local: catch_state.catch_region_id, - }), - ); - + // Frame word +8 owns the exact `(region, arm)` selector. Scalar arm + // payloads are restored separately from their overlaid union. push_current_frame_ptr(s, runtime, memory, ptr_ty); - push_instr(s, load_i32(memory, EXNREF_SLOT_OFFSET)); + push_instr(s, load_i32(memory, CATCH_SELECTOR_OFFSET)); push_instr( s, Instr::LocalSet(LocalSet { - local: catch_state.exnref_slot, + local: catch_state.catch_selector, }), ); } @@ -3136,21 +3916,17 @@ fn populate_preamble_then( push_instr(s, load_scalar(memory, ty, off as u64)); push_instr(s, Instr::LocalSet(LocalSet { local: lid })); } - - // Restore ref-typed user locals from aux tables. - for slot in ref_plan { - let table = aux_tables - .table_for(slot.class) - .expect("aux table for this ref class must be injected"); - push_instr( - s, - Instr::Const(Const { - value: Value::I32(slot.slot as i32), - }), - ); - push_instr(s, Instr::TableGet(TableGet { table })); - push_instr(s, Instr::LocalSet(LocalSet { local: slot.local })); + if let Some(dispatch) = catch_scalar_restore_dispatch { + push_instr(s, Instr::Block(Block { seq: dispatch })); } + emit_reference_restore_dispatch( + local, + preamble_then, + runtime, + memory, + ptr_ty, + reference_plan, + ); } #[allow(clippy::too_many_arguments)] @@ -3161,11 +3937,10 @@ fn populate_postamble( ptr_ty: ValType, catch_state_locals: Option, locals_with_offsets: &[(LocalId, ValType, u32)], - ref_plan: &[RefLocalSlot], - aux_tables: &AuxTables, + catch_scalar_save_dispatch: Option, + reference_save_dispatch: Option, frame_size: u32, func_ordinal: u32, - result_types: &[ValType], ) { // frame[0] = func_ordinal push_current_frame_ptr(out, runtime, memory, ptr_ty); @@ -3178,36 +3953,37 @@ fn populate_postamble( push_instr(out, store_i32(memory, FUNC_INDEX_OFFSET)); if let Some(catch_state) = catch_state_locals { - // frame[8] = dynamic catch_region_id for catch-capable functions. - push_current_frame_ptr(out, runtime, memory, ptr_ty); - push_instr( - out, - Instr::LocalGet(LocalGet { - local: catch_state.catch_region_id, - }), - ); - push_instr(out, store_i32(memory, CATCH_REGION_OFFSET)); - - // frame[12] = dynamic exnref_slot for catch-capable functions. + // frame[8] = exact non-zero `(region, arm)` selector in a catch. push_current_frame_ptr(out, runtime, memory, ptr_ty); push_instr( out, Instr::LocalGet(LocalGet { - local: catch_state.exnref_slot, + local: catch_state.catch_selector, }), ); - push_instr(out, store_i32(memory, EXNREF_SLOT_OFFSET)); + push_instr(out, store_i32(memory, CATCH_SELECTOR_OFFSET)); } else { - // frame[8..16] = zero catch_region_id + exnref_slot. + // frame[8] = no active catch region. push_current_frame_ptr(out, runtime, memory, ptr_ty); push_instr( out, Instr::Const(Const { - value: Value::I64(0), + value: Value::I32(0), }), ); - push_instr(out, store_scalar(memory, ValType::I64, CATCH_REGION_OFFSET)); + push_instr(out, store_i32(memory, CATCH_SELECTOR_OFFSET)); } + // frame[12] starts as the canonical empty reference-vector ordinal. The + // call-specific save dispatch replaces it only when this landing owns + // non-null recipe values. + push_current_frame_ptr(out, runtime, memory, ptr_ty); + push_instr( + out, + Instr::Const(Const { + value: Value::I32(0), + }), + ); + push_instr(out, store_i32(memory, REFERENCE_VECTOR_OFFSET)); // Save scalar user + arg-spill locals for &(lid, ty, off) in locals_with_offsets { @@ -3215,24 +3991,18 @@ fn populate_postamble( push_instr(out, Instr::LocalGet(LocalGet { local: lid })); push_instr(out, store_scalar(memory, ty, off as u64)); } + if let Some(dispatch) = catch_scalar_save_dispatch { + push_instr(out, Instr::Block(Block { seq: dispatch })); + } - // Spill ref-typed user locals to aux tables. - for slot in ref_plan { - let table = aux_tables - .table_for(slot.class) - .expect("aux table for this ref class must be injected"); - push_instr( - out, - Instr::Const(Const { - value: Value::I32(slot.slot as i32), - }), - ); - push_instr(out, Instr::LocalGet(LocalGet { local: slot.local })); - push_instr(out, Instr::TableSet(TableSet { table })); + if let Some(dispatch) = reference_save_dispatch { + // The call selector was written before entering this common postamble. + // Each case encodes only values live for that original call landing. + push_instr(out, Instr::Block(Block { seq: dispatch })); } if let Some(frame_commit) = runtime.frame_commit { - // Publish only after the complete payload and reference stashes exist. + // Publish only after the complete activation-owned payload exists. push_current_frame_ptr(out, runtime, memory, ptr_ty); push_instr(out, Instr::Call(Call { func: frame_commit })); } else { @@ -3254,33 +4024,289 @@ fn populate_postamble( push_instr(out, store_ptr(memory, ptr_ty, 0)); } - // Push defaults for the function's result types, or `unreachable` - // if any result is a non-nullable ref. - let mut fallback_unreachable = false; - for &ty in result_types { - match default_for_type(ty) { - Some(instr) => push_instr(out, instr), - None => { - fallback_unreachable = true; - break; - } - } + // WHY: a synthesized default is not a value owned by this activation, + // and non-nullable reference results do not have a valid default at all. + // The process-owned tag is independent of the function's result type and + // therefore transports unwind through every Wasm signature truthfully. + let unwind_tag = runtime + .unwind_tag + .expect("fork-path instrumentation requires the linked unwind tag"); + push_instr(out, Instr::Throw(Throw { tag: unwind_tag })); +} + +/// Post-call sequence for call site K, appended to sequence `seq_id`. +/// +/// The one-based call selector is installed before entering the callee. Every +/// fork boundary either is an instrumented local function, whose postamble +/// throws the private unwind tag, or a generated transport helper which +/// converts a normal `STATE_UNWINDING` return to that tag before exposing its +/// results. The function-level catch therefore owns all frame reservation and +/// no source result remains on the operand stack across a state probe here. +fn populate_lexical_call( + local: &mut LocalFunction, + sequence: InstrSeqId, + target: CallTarget, + sig_ty: TypeId, + location: InstrLocId, + arguments: &CallArgMaterialization, +) { + let out = &mut local.block_mut(sequence).instrs; + emit_materialized_call_args(out, arguments); + if matches!(target, CallTarget::Ref) { + push_instr( + out, + Instr::RefCast(walrus::ir::RefCast { + nullable: false, + heap_type: HeapType::Concrete(sig_ty), + }), + ); + } + let instruction = match target { + CallTarget::Direct(func) => Instr::Call(Call { func }), + CallTarget::Indirect { table } => Instr::CallIndirect(CallIndirect { ty: sig_ty, table }), + CallTarget::Ref => Instr::CallRef(walrus::ir::CallRef { ty: sig_ty }), + }; + out.push((instruction, location)); +} + +#[allow(clippy::too_many_arguments)] +fn emit_resume_selected_call( + local: &mut LocalFunction, + sequence: InstrSeqId, + target: CallTarget, + sig_ty: TypeId, + resume_ty: TypeId, + location: InstrLocId, + arguments: &CallArgMaterialization, + runtime: &Runtime, + diagnostic_type: i32, +) { + let resume_peek = runtime + .resume_peek + .expect("replay-routed call requires process resume peek"); + let resume_table = runtime + .resume_table + .expect("replay-routed call requires process resume table"); + let branch_ty = InstrSeqType::MultiValue(resume_ty); + let lexical_sentinel = local.builder_mut().dangling_instr_seq(branch_ty).id(); + let dispatch = local.builder_mut().dangling_instr_seq(branch_ty).id(); + + populate_lexical_call(local, lexical_sentinel, target, sig_ty, location, arguments); + { + let out = &mut local.block_mut(dispatch).instrs; + // `resume_peek` is non-consuming and the journal pins its selection + // until frame_next. Calling it again on this replay-only branch avoids + // adding one live i32 local to every ordinary function activation. + push_instr( + out, + Instr::Const(Const { + value: Value::I32(diagnostic_type), + }), + ); + push_instr(out, Instr::Call(Call { func: resume_peek })); + push_instr( + out, + Instr::CallIndirect(CallIndirect { + ty: resume_ty, + table: resume_table, + }), + ); + } + { + let out = &mut local.block_mut(sequence).instrs; + // The ordinal is diagnostic only. Exact template/event identity picks + // the target; Wasm call_indirect is the authoritative recursive-type + // compatibility check and leaves the event unconsumed on mismatch. + push_instr( + out, + Instr::Const(Const { + value: Value::I32(diagnostic_type), + }), + ); + push_instr(out, Instr::Call(Call { func: resume_peek })); + push_instr( + out, + Instr::Unop(walrus::ir::Unop { + op: UnaryOp::I32Eqz, + }), + ); + push_instr( + out, + Instr::IfElse(IfElse { + consequent: lexical_sentinel, + alternative: dispatch, + }), + ); } - if fallback_unreachable { - push_instr(out, Instr::Unreachable(walrus::ir::Unreachable {})); +} + +#[allow(clippy::too_many_arguments)] +fn emit_replay_routed_call( + local: &mut LocalFunction, + sequence: InstrSeqId, + target: CallTarget, + direct_activation: bool, + sig_ty: TypeId, + resume_ty: TypeId, + location: InstrLocId, + arguments: &CallArgMaterialization, + runtime: &Runtime, +) { + let branch_ty = InstrSeqType::MultiValue(resume_ty); + let normal = local.builder_mut().dangling_instr_seq(branch_ty).id(); + let replay = local.builder_mut().dangling_instr_seq(branch_ty).id(); + populate_lexical_call(local, normal, target, sig_ty, location, arguments); + if direct_activation { + // WHY: adding a no-argument resume thunk in front of every ordinary + // recursive activation doubles native rewind depth. A materialized + // direct callee already owns the selected event; its preamble + // validates activation/function identity through frame_next before + // consuming it. Tail-transparent, indirect, and reference calls still + // require the process router because their lexical target need not be + // the next materialized activation. + debug_assert!(matches!(target, CallTarget::Direct(_))); + populate_lexical_call(local, replay, target, sig_ty, location, arguments); + } else { + emit_resume_selected_call( + local, + replay, + target, + sig_ty, + resume_ty, + location, + arguments, + runtime, + sig_ty.index() as i32, + ); } + let out = &mut local.block_mut(sequence).instrs; + push_instr( + out, + Instr::GlobalGet(GlobalGet { + global: runtime.state_global, + }), + ); + push_instr( + out, + Instr::Const(Const { + value: Value::I32(runtime::STATE_REWINDING), + }), + ); + push_instr( + out, + Instr::Binop(Binop { + op: BinaryOp::I32GeU, + }), + ); + push_instr( + out, + Instr::IfElse(IfElse { + consequent: replay, + alternative: normal, + }), + ); } -/// Post-call sequence for call site K, appended to sequence `seq_id`: -/// - reload spilled args -/// - emit the call instruction -/// - Phase 6e writes (compute catch_region_id / exnref_slot from active -/// in_catch flags) -/// - if state == UNWINDING, write K to frame.call_index and branch to -/// `$unwind_save` +/// Emit one result-typed private-tag boundary around a fork-reaching call. /// -/// Takes `&mut LocalFunction` so Phase 6e can allocate dangling -/// IfElse branches for each handler check. +/// Values carried below the call remain below `result_boundary`; a normal +/// call branches out with only its declared results. A private unwind lands +/// after `catch_boundary`, where the statically known call index selects the +/// frame or live-abort restart without any source-function selector local. +#[allow(clippy::too_many_arguments)] +fn emit_replay_routed_call_with_unwind_boundary( + local: &mut LocalFunction, + sequence: InstrSeqId, + target: CallTarget, + direct_activation: bool, + sig_ty: TypeId, + resume_ty: TypeId, + location: InstrLocId, + arguments: &CallArgMaterialization, + call_idx: u32, + runtime: &Runtime, + _memory: MemoryId, + ptr_ty: ValType, + frame_size: u32, + unwind_save: InstrSeqId, + abort: AbortDispatch, +) { + let result_ty = InstrSeqType::MultiValue(resume_ty); + let result_boundary = local.builder_mut().dangling_instr_seq(result_ty).id(); + let catch_boundary = local + .builder_mut() + .dangling_instr_seq(InstrSeqType::Simple(None)) + .id(); + let call_body = local.builder_mut().dangling_instr_seq(result_ty).id(); + + emit_replay_routed_call( + local, + call_body, + target, + direct_activation, + sig_ty, + resume_ty, + location, + arguments, + runtime, + ); + { + let out = &mut local.block_mut(catch_boundary).instrs; + push_instr( + out, + Instr::TryTable(TryTable { + seq: call_body, + catches: vec![TryTableCatch::Catch { + tag: runtime + .unwind_tag + .expect("fork call boundary requires private unwind tag"), + label: catch_boundary, + }], + }), + ); + // On the normal edge the call's results satisfy the result boundary. + // The catch edge branches to the end of this simple block and enters + // the static unwind handler below with no fabricated result values. + push_instr( + out, + Instr::Br(Br { + block: result_boundary, + }), + ); + } + { + let out = &mut local.block_mut(result_boundary).instrs; + push_instr( + out, + Instr::Block(Block { + seq: catch_boundary, + }), + ); + } + emit_static_call_unwind_handler( + local, + result_boundary, + ptr_ty, + frame_size, + call_idx, + unwind_save, + abort, + ); + // Both handler arms branch away, but make that fact explicit to the + // validator: the result-typed boundary has no fallthrough value on the + // caught edge. + push_instr( + &mut local.block_mut(result_boundary).instrs, + Instr::Unreachable(Unreachable {}), + ); + push_instr( + &mut local.block_mut(sequence).instrs, + Instr::Block(Block { + seq: result_boundary, + }), + ); +} + #[allow(clippy::too_many_arguments)] fn emit_post_call_via_local( local: &mut LocalFunction, @@ -3288,13 +4314,13 @@ fn emit_post_call_via_local( call: &CallSiteInfo, call_idx: usize, arg_materialization: &CallArgMaterialization, - carryovers: &[LocalId], - catch_handlers: &[CatchHandlerInfo], + carryovers: &[TypedSpillLocal], + _catch_handlers: &[CatchHandlerInfo], runtime: &Runtime, memory: MemoryId, ptr_ty: ValType, frame_size: u32, - catch_state_locals: Option, + _catch_state_locals: Option, unwind_save: InstrSeqId, abort: AbortDispatch, ) { @@ -3303,33 +4329,254 @@ fn emit_post_call_via_local( // the stack — matching the original code's expected shape. { let s = &mut local.block_mut(seq_id).instrs; - for &l in carryovers.iter() { - push_instr(s, Instr::LocalGet(LocalGet { local: l })); - } - emit_materialized_call_args(s, arg_materialization); - let call_instr = match call.target { - CallTarget::Direct(func) => Instr::Call(Call { func }), - CallTarget::Indirect { table } => Instr::CallIndirect(CallIndirect { - ty: call.sig_ty, - table, - }), + for &(local, ty) in carryovers { + push_typed_local_get(s, local, ty); + } + } + emit_replay_routed_call_with_unwind_boundary( + local, + seq_id, + call.target, + call.direct_activation, + call.sig_ty, + call.resume_ty + .expect("call site resume type was not assigned"), + call.loc, + arg_materialization, + call_idx as u32, + runtime, + memory, + ptr_ty, + frame_size, + unwind_save, + abort, + ); +} + +#[allow(clippy::too_many_arguments)] +fn emit_resume_thunk( + module: &mut Module, + resumed_function: FunctionId, + runtime: &Runtime, + memory: MemoryId, + ptr_ty: ValType, + frame_size: u32, + scalar_offsets: &[(LocalId, ValType, u32)], + references: &ReferenceFramePlan, + func_ordinal: u32, +) -> FunctionId { + let frame_peek = runtime + .frame_peek + .expect("activation resume thunk requires linked frame peek"); + let codecs = runtime + .reference_codecs + .expect("activation resume thunk requires reference codecs"); + let vector_get = runtime + .reference_vector_get + .expect("activation resume thunk requires recipe-vector lookup"); + let (arguments, results) = { + let function = module.funcs.get(resumed_function); + let FunctionKind::Local(local) = &function.kind else { + unreachable!("resume thunk target must be a local function"); }; - s.push((call_instr, call.loc)); + ( + local.args.clone(), + module.types.get(function.ty()).results().to_vec(), + ) + }; + let scalar_offsets: HashMap = scalar_offsets + .iter() + .map(|&(local, ty, offset)| (local, (ty, offset))) + .collect(); + let reference_slots: HashMap = references + .slots + .iter() + .copied() + .map(|slot| (slot.local, slot)) + .collect(); + let frame = module.locals.add(ptr_ty); + let mut builder = FunctionBuilder::new(&mut module.types, &[], &results); + builder.name(format!("__wpk_fork_resume_{func_ordinal}")); + { + let mut body = builder.func_body(); + let out = body.instrs_mut(); + push_instr(out, ptr_const(ptr_ty, frame_size as i64)); + push_instr(out, Instr::Call(Call { func: frame_peek })); + push_instr(out, Instr::LocalSet(LocalSet { local: frame })); + + for argument in arguments { + let ty = module.locals.get(argument).ty(); + match ty { + ValType::Ref(reference) => { + let slot = reference_slots.get(&argument).unwrap_or_else(|| { + panic!("resume thunk parameter {argument:?} has no activation-owned recipe") + }); + let position = slot.universal_position.unwrap_or_else(|| { + panic!( + "resume thunk parameter {argument:?} does not have a stable recipe-vector position" + ) + }); + push_instr(out, Instr::LocalGet(LocalGet { local: frame })); + push_instr(out, load_i32(memory, REFERENCE_VECTOR_OFFSET)); + push_instr( + out, + Instr::Const(Const { + value: Value::I32(position as i32), + }), + ); + push_instr(out, Instr::Call(Call { func: vector_get })); + push_instr( + out, + Instr::Call(Call { + func: slot.class.decoder(codecs), + }), + ); + push_decoded_reference_narrowing(out, slot.class, reference); + } + scalar => { + let &(saved_ty, offset) = scalar_offsets.get(&argument).unwrap_or_else(|| { + panic!("resume thunk scalar parameter {argument:?} has no frame offset") + }); + debug_assert_eq!(scalar, saved_ty); + push_instr(out, Instr::LocalGet(LocalGet { local: frame })); + push_instr(out, load_scalar(memory, scalar, offset as u64)); + } + } + } + push_instr( + out, + Instr::Call(Call { + func: resumed_function, + }), + ); + } + builder.finish(Vec::new(), &mut module.funcs) +} + +fn emit_resume_catalog(module: &mut Module, thunks: &[ResumeThunk]) { + let size = thunks.len() as u64; + let table = module + .tables + .add_local(false, size, Some(size), RefType::FUNCREF); + module.tables.get_mut(table).name = Some(RESUME_CATALOG_EXPORT.into()); + if !thunks.is_empty() { + module.elements.add( + ElementKind::Active { + table, + offset: walrus::ConstExpr::Value(Value::I32(0)), + }, + ElementItems::Functions(thunks.iter().map(|thunk| thunk.function).collect()), + ); + } + module.exports.add(RESUME_CATALOG_EXPORT, table); + + // The host already validates the exact module template and event target. + // Function type equivalence remains the engine's job at the generated + // call_indirect site, avoiding a second recursive-type implementation. + let mut data = Vec::with_capacity(usize::from(RESUME_CATALOG_HEADER_SIZE) + thunks.len() * 8); + data.extend_from_slice(&RESUME_CATALOG_MAGIC); + data.extend_from_slice(&RESUME_CATALOG_VERSION.to_le_bytes()); + data.extend_from_slice(&RESUME_CATALOG_HEADER_SIZE.to_le_bytes()); + data.extend_from_slice(&(thunks.len() as u32).to_le_bytes()); + for (slot, thunk) in thunks.iter().enumerate() { + debug_assert_eq!(thunk.func_ordinal, slot as u32); + data.extend_from_slice(&thunk.func_ordinal.to_le_bytes()); + data.extend_from_slice(&(slot as u32).to_le_bytes()); + } + module.customs.add(RawCustomSection { + name: RESUME_CATALOG_SECTION.into(), + data, + }); +} + +fn exported_function(module: &Module, name: &str) -> Option { + module.exports.iter().find_map(|export| { + if export.name != name { + return None; + } + match export.item { + ExportItem::Function(function) => Some(function), + _ => None, + } + }) +} + +fn exported_table(module: &Module, name: &str) -> Option { + module.exports.iter().find_map(|export| { + if export.name != name { + return None; + } + match export.item { + ExportItem::Table(table) => Some(table), + _ => None, + } + }) +} + +fn emit_fixed_resume_boundaries(module: &mut Module, runtime: &Runtime) { + if runtime.resume_peek.is_none() || runtime.resume_table.is_none() { + return; } - emit_call_index_store_and_unwind_branch( - local, - seq_id, - runtime, - memory, - ptr_ty, - frame_size, - call_idx as u32, - unwind_save, - catch_handlers, - catch_state_locals, - abort, - ); + if let Some(start) = exported_function(module, "_start") { + let start_ty = module.funcs.get(start).ty(); + let signature = module.types.get(start_ty); + if signature.params().is_empty() && signature.results().is_empty() { + let resume_ty = module.types.add(&[], &[]); + let mut builder = FunctionBuilder::new(&mut module.types, &[], &[]); + builder.name(RESUME_START_EXPORT.into()); + let wrapper = builder.finish(Vec::new(), &mut module.funcs); + let entry = local_mut(module, wrapper).entry_block(); + emit_resume_selected_call( + local_mut(module, wrapper), + entry, + CallTarget::Direct(start), + start_ty, + resume_ty, + InstrLocId::default(), + &CallArgMaterialization::Spill { + locals: Vec::new(), + types: Vec::new(), + }, + runtime, + 0, + ); + module.exports.add(RESUME_START_EXPORT, wrapper); + } + } + + if let Some(function_table) = exported_table(module, "__indirect_function_table") { + let ptr_ty = runtime.buf_type; + let thread_ty = module.types.add(&[ptr_ty], &[ptr_ty]); + let resume_ty = module.types.add(&[], &[ptr_ty]); + let table_index = module.locals.add(ValType::I32); + let argument = module.locals.add(ptr_ty); + let mut builder = + FunctionBuilder::new(&mut module.types, &[ValType::I32, ptr_ty], &[ptr_ty]); + builder.name(RESUME_THREAD_EXPORT.into()); + let wrapper = builder.finish(vec![table_index, argument], &mut module.funcs); + let entry = local_mut(module, wrapper).entry_block(); + emit_resume_selected_call( + local_mut(module, wrapper), + entry, + CallTarget::Indirect { + table: function_table, + }, + thread_ty, + resume_ty, + InstrLocId::default(), + &CallArgMaterialization::Spill { + // call_indirect consumes function parameters first and its + // table index last; the public wrapper keeps the ergonomic + // host ABI `(table_index, arg)`. + locals: vec![argument, table_index], + types: vec![ptr_ty, ValType::I32], + }, + runtime, + 0, + ); + module.exports.add(RESUME_THREAD_EXPORT, wrapper); + } } // ---------------------------------------------------------------------- @@ -3405,6 +4652,30 @@ fn collect_user_locals(module: &Module, func_id: FunctionId) -> Vec<(LocalId, Va .collect() } +fn append_resume_parameter_references( + module: &Module, + func_id: FunctionId, + per_call_references: &mut [Vec<(LocalId, RefType)>], +) { + let FunctionKind::Local(local) = &module.funcs.get(func_id).kind else { + return; + }; + let params = module.types.get(module.funcs.get(func_id).ty()).params(); + debug_assert_eq!(local.args.len(), params.len()); + for (&argument, &ty) in local.args.iter().zip(params) { + let Some(reference) = supported_reference(ty) else { + continue; + }; + // WHY: a resume thunk has no parameters so callers with a different + // lexical signature can bypass eliminated tail frames. Even a dead + // non-nullable parameter needs a valid value to enter the original + // function, whose preamble then consumes and restores this frame. + for references in per_call_references.iter_mut() { + references.push((argument, reference)); + } + } +} + // ---------------------------------------------------------------------- // Nested-seq traversal // ---------------------------------------------------------------------- @@ -3442,6 +4713,152 @@ fn is_scalar(ty: ValType) -> bool { !matches!(ty, ValType::Ref(_)) } +fn supported_reference(ty: ValType) -> Option { + match ty { + ValType::Ref(reference) => Some(reference), + _ => None, + } +} + +fn spill_storage_type(ty: ValType) -> ValType { + match supported_reference(ty) { + Some(reference) if !reference.nullable => { + let mut storage = reference; + storage.nullable = true; + ValType::Ref(storage) + } + _ => ty, + } +} + +fn push_typed_local_get(out: &mut Vec<(Instr, InstrLocId)>, local: LocalId, expected: ValType) { + push_instr(out, Instr::LocalGet(LocalGet { local })); + if matches!(expected, ValType::Ref(reference) if !reference.nullable) { + push_instr(out, Instr::RefAsNonNull(RefAsNonNull {})); + } +} + +fn push_decoded_reference_narrowing( + out: &mut Vec<(Instr, InstrLocId)>, + class: RefClass, + expected: RefType, +) { + let broad = class.nullable_type(); + if expected.heap_type != broad.heap_type { + push_instr( + out, + Instr::RefCast(walrus::ir::RefCast { + nullable: expected.nullable, + heap_type: expected.heap_type, + }), + ); + } else if !expected.nullable { + push_instr(out, Instr::RefAsNonNull(RefAsNonNull {})); + } +} + +fn plan_reference_frame( + module: &Module, + analysis: &FunctionReferenceAnalysis, + mut per_call_synthetic: Vec>, +) -> ReferenceFramePlan { + debug_assert_eq!(analysis.call_sites.len(), per_call_synthetic.len()); + let call_count = analysis.call_sites.len(); + let mut per_call_refs: Vec> = vec![BTreeMap::new(); call_count]; + let mut null_locals_by_call = vec![Vec::new(); call_count]; + + for (call_idx, site) in analysis.call_sites.iter().enumerate() { + if site.reachable { + // The replayed callee can still throw after the child-side fork + // return. Preserve references used by either normal continuation + // or an exceptional successor; definitely-null cleanup locals + // remain recipe-free below. + for &local in &site.live_ref_locals_on_any_successor { + let ty = analysis.reference_locals[&local]; + match site + .local_nullability_before_call + .get(&local) + .copied() + .unwrap_or(ReferenceNullability::MaybeNonNull) + { + ReferenceNullability::DefinitelyNull if ty.nullable => { + null_locals_by_call[call_idx].push((local, ty)); + } + ReferenceNullability::DefinitelyNull | ReferenceNullability::MaybeNonNull => { + per_call_refs[call_idx].insert(local, ty); + } + } + } + } + for (local, ty) in per_call_synthetic[call_idx].drain(..) { + per_call_refs[call_idx] + .entry(local) + .and_modify(|existing| { + debug_assert_eq!(RefClass::of(module, *existing), RefClass::of(module, ty)); + existing.nullable &= ty.nullable; + }) + .or_insert(ty); + } + } + + let mut union = BTreeMap::::new(); + let mut occurrence_count = BTreeMap::::new(); + for refs in &per_call_refs { + for (&local, &ty) in refs { + union + .entry(local) + .and_modify(|existing| existing.nullable &= ty.nullable) + .or_insert(ty); + *occurrence_count.entry(local).or_default() += 1; + } + } + + let mut slots = Vec::with_capacity(union.len()); + let mut slot_by_local = BTreeMap::::new(); + // Values present at every landing form a stable vector prefix. Function + // reference parameters are deliberately added to every landing, allowing + // a no-parameter resume thunk to decode them without a synthetic local or + // a call-index dispatch. + let mut ordered: Vec<_> = union.into_iter().collect(); + ordered.sort_by_key(|(local, _)| { + ( + occurrence_count.get(local).copied().unwrap_or(0) != call_count, + *local, + ) + }); + let universal_count = ordered + .iter() + .take_while(|(local, _)| occurrence_count.get(local).copied().unwrap_or(0) == call_count) + .count(); + for (position, (local, ty)) in ordered.into_iter().enumerate() { + let index = slots.len(); + slots.push(ReferenceFrameSlot { + local, + ty, + class: RefClass::of(module, ty), + universal_position: (position < universal_count).then_some(position as u32), + }); + slot_by_local.insert(local, index); + } + let slots_by_call = per_call_refs + .into_iter() + .map(|refs| { + let mut slots: Vec<_> = refs + .into_keys() + .map(|local| slot_by_local[&local]) + .collect(); + slots.sort_unstable(); + slots + }) + .collect(); + + ReferenceFramePlan { + slots, + slots_by_call, + null_locals_by_call, + } +} + fn scalar_size(ty: ValType) -> u32 { match ty { ValType::I32 | ValType::F32 => 4, @@ -3455,28 +4872,6 @@ fn natural_align(ty: ValType) -> u32 { scalar_size(ty) } -fn default_for_type(ty: ValType) -> Option { - Some(match ty { - ValType::I32 => Instr::Const(Const { - value: Value::I32(0), - }), - ValType::I64 => Instr::Const(Const { - value: Value::I64(0), - }), - ValType::F32 => Instr::Const(Const { - value: Value::F32(0.0), - }), - ValType::F64 => Instr::Const(Const { - value: Value::F64(0.0), - }), - ValType::V128 => Instr::Const(Const { - value: Value::V128(0), - }), - ValType::Ref(rt) if rt.nullable => Instr::RefNull(RefNull { ty: rt }), - ValType::Ref(_) => return None, - }) -} - fn load_i32(memory: MemoryId, offset: u64) -> Instr { Instr::Load(walrus::ir::Load { memory, @@ -3631,183 +5026,34 @@ fn push_current_call_index( } // ---------------------------------------------------------------------- -// Phase 4f — ref-typed local spilling via aux tables +// Tagged-catch region planning // ---------------------------------------------------------------------- -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub enum RefClass { - Funcref, - Externref, - Exnref, -} - -#[derive(Debug, Clone, Copy)] -pub struct RefLocalSlot { - pub local: LocalId, - pub class: RefClass, - pub slot: u32, -} - -#[derive(Debug, Clone, Copy, Default)] -pub struct AuxTables { - pub funcref: Option, - pub externref: Option, - pub exnref: Option, -} - -impl AuxTables { - pub fn table_for(&self, class: RefClass) -> Option { - match class { - RefClass::Funcref => self.funcref, - RefClass::Externref => self.externref, - RefClass::Exnref => self.exnref, - } - } -} - -fn classify_ref(rt: RefType) -> Option { - if !rt.nullable { - return None; - } - match rt.heap_type { - HeapType::Abstract(AbstractHeapType::Func) => Some(RefClass::Funcref), - HeapType::Abstract(AbstractHeapType::NoFunc) => Some(RefClass::Funcref), - HeapType::Abstract(AbstractHeapType::Extern) => Some(RefClass::Externref), - HeapType::Abstract(AbstractHeapType::NoExtern) => Some(RefClass::Externref), - HeapType::Abstract(AbstractHeapType::Exn) => Some(RefClass::Exnref), - HeapType::Abstract(AbstractHeapType::NoExn) => Some(RefClass::Exnref), - _ => None, - } -} - -fn plan_and_inject_aux_tables( - module: &mut Module, +fn plan_catch_regions( + module: &Module, targets: &[FunctionId], -) -> ( - AuxTables, - HashMap>, - HashMap>, -) { - let mut funcref_cursor: u32 = 0; - let mut externref_cursor: u32 = 0; - let mut exnref_cursor: u32 = 0; - - let mut plan: HashMap> = HashMap::new(); - - for &id in targets { - let mut per_func: Vec = Vec::new(); - for (local, ty) in collect_user_locals(module, id) { - let rt = match ty { - ValType::Ref(rt) => rt, - _ => continue, - }; - let class = classify_ref(rt).unwrap_or_else(|| { - let name = module.funcs.get(id).name.as_deref().unwrap_or(""); - panic!( - "fork-instrument 4f: function `{name}` has a ref-typed local of \ - type {rt:?} which is not yet supported (non-nullable or non-abstract \ - ref).", - ) - }); - let slot = match class { - RefClass::Funcref => { - let s = funcref_cursor; - funcref_cursor += 1; - s - } - RefClass::Externref => { - let s = externref_cursor; - externref_cursor += 1; - s - } - RefClass::Exnref => { - let s = exnref_cursor; - exnref_cursor += 1; - s - } - }; - per_func.push(RefLocalSlot { local, class, slot }); - } - if !per_func.is_empty() { - plan.insert(id, per_func); - } - } - +) -> HashMap> { let mut catch_plans: HashMap> = HashMap::new(); for &id in targets { let bodies = discover_try_table_bodies(module, id); let mut per_func: Vec = Vec::with_capacity(bodies.len()); for (lex_idx, body_seq) in bodies.into_iter().enumerate() { - let slot = exnref_cursor; - exnref_cursor += 1; per_func.push(CatchRegionPlan { body_seq, catch_region_id: (lex_idx as u32) + 1, - exnref_slot: slot, }); } if !per_func.is_empty() { catch_plans.insert(id, per_func); } } - - let funcref = if funcref_cursor > 0 { - let id = module.tables.add_local( - false, - funcref_cursor as u64, - Some(funcref_cursor as u64), - RefType::FUNCREF, - ); - module.tables.get_mut(id).name = Some("_wpk_fork_funcref_stash".into()); - Some(id) - } else { - None - }; - let externref = if externref_cursor > 0 { - let id = module.tables.add_local( - false, - externref_cursor as u64, - Some(externref_cursor as u64), - RefType::EXTERNREF, - ); - module.tables.get_mut(id).name = Some("_wpk_fork_externref_stash".into()); - Some(id) - } else { - None - }; - let exnref = if exnref_cursor > 0 { - let exn_rt = RefType { - nullable: true, - heap_type: HeapType::Abstract(AbstractHeapType::Exn), - }; - let id = module.tables.add_local( - false, - exnref_cursor as u64, - Some(exnref_cursor as u64), - exn_rt, - ); - module.tables.get_mut(id).name = Some("_wpk_fork_exnref_stash".into()); - Some(id) - } else { - None - }; - - ( - AuxTables { - funcref, - externref, - exnref, - }, - plan, - catch_plans, - ) + catch_plans } #[derive(Debug, Clone, Copy)] pub struct CatchRegionPlan { pub body_seq: InstrSeqId, pub catch_region_id: u32, - pub exnref_slot: u32, } fn discover_try_table_bodies(module: &Module, func_id: FunctionId) -> Vec { @@ -3831,33 +5077,58 @@ fn visit_try_tables(f: &LocalFunction, seq: InstrSeqId, out: &mut Vec bool { + matches!(self, Self::Plain | Self::AllPlain) + } + + fn is_ref(self) -> bool { + matches!(self, Self::Ref | Self::AllRef) + } +} + +/// Describes one catch clause in a fork-path try_table. #[derive(Debug, Clone)] pub struct PlainCatchArm { - /// Index of this arm within its try_table's `catches` list. Stage 2 - /// writes this value to the region's frame-backed `active_arm` - /// local; the rewind path reads it to select which - /// `throw $tag (operands)` to emit. Combined with the function's - /// `catch_region_id` (tracked by `CatchRegionPlan`), the pair is - /// unique within the function — no module-wide arm_id is needed. + /// Index of this arm within its try_table's `catches` list. The emitted + /// state assigns the `(catch_region_id, arm_idx)` pair one non-zero + /// function-local selector stored directly in frame word +8. pub arm_idx: u32, - /// Tag this arm catches. - pub tag: TagId, + /// Whether normal handler entry receives only the tag payload or the tag + /// payload followed by an instance-local exnref. + pub kind: TaggedCatchKind, + /// Tag this arm catches, or `None` for CatchAll/CatchAllRef. + pub tag: Option, /// Label the arm branches to on catch (target block id). pub label: InstrSeqId, /// Tag's operand types (matches the params of the type that /// `module.tags.get(tag).ty()` references). Cached at discovery /// time so we don't re-look-up on emission. pub operand_tys: Vec, + /// JavaScript cannot inspect `v128`, `exnref`, or GC/reference payloads. + /// Capture the entire exception as one exnref recipe and replay it with + /// `throw_ref` instead of serializing those operands independently. + pub uses_exception_recipe: bool, } /// Stage 1 (B1) — for each try_table in `func_id`, returns -/// `(body_seq, plain_catch_arms)` where `plain_catch_arms` lists -/// every plain `Catch { tag, label }` clause. Following Phase 6's -/// pattern: catch_ref / catch_all_ref clauses are skipped (Phase 6 -/// territory); plain catch is enumerated unfiltered. +/// `(body_seq, catch_arms)` where every tagged and catch-all arm is represented. +/// CatchAll is retargeted through CatchAllRef so an arbitrary Wasm/JSTag/raw +/// exception has the same positive broker/recipe path as an explicit +/// CatchAllRef. Tagged payloads that JavaScript cannot inspect are likewise +/// captured through CatchRef and owned by one exnref recipe. /// /// Function-level filtering happens at the call site (caller passes /// only fork-path `FunctionId`s, mirroring `discover_try_table_bodies`). @@ -3887,22 +5158,36 @@ fn visit_for_plain_catch( if let Instr::TryTable(tt) = instr { let mut arms: Vec = Vec::new(); for (i, c) in tt.catches.iter().enumerate() { - let (tag, label) = match c { - TryTableCatch::Catch { tag, label } => (*tag, *label), - _ => continue, // CatchRef / CatchAllRef: handled by Phase 6. - // CatchAll: unsupported today; not in B1 scope - // (no tag → no operand_tys to save). + let (kind, tag, label) = match c { + TryTableCatch::Catch { tag, label } => { + (TaggedCatchKind::Plain, Some(*tag), *label) + } + TryTableCatch::CatchRef { tag, label } => { + (TaggedCatchKind::Ref, Some(*tag), *label) + } + TryTableCatch::CatchAll { label } => (TaggedCatchKind::AllPlain, None, *label), + TryTableCatch::CatchAllRef { label } => (TaggedCatchKind::AllRef, None, *label), }; - let operand_tys: Vec = module - .types - .get(module.tags.get(tag).ty()) - .params() - .to_vec(); + let operand_tys: Vec = tag + .map(|tag| { + module + .types + .get(module.tags.get(tag).ty()) + .params() + .to_vec() + }) + .unwrap_or_default(); + let uses_exception_recipe = tag.is_none() + || operand_tys + .iter() + .any(|ty| matches!(ty, ValType::Ref(_) | ValType::V128)); arms.push(PlainCatchArm { arm_idx: i as u32, + kind, tag, label, operand_tys, + uses_exception_recipe, }); } if !arms.is_empty() { @@ -3915,45 +5200,20 @@ fn visit_for_plain_catch( } } -/// Module-wide static plain-catch plan. -/// -/// Stage 2 (Task 2.1) adds `b2_carveout`: functions whose plain-catch -/// arms include unsupported operand types (e.g., ref-typed) land here -/// instead of `per_function`. Stage 2 emission tasks check this set -/// and skip plain-catch instrumentation for carved-out functions — -/// falling back to today's behavior (Phase 6 catch_ref still works, -/// plain-catch fork remains unsupported for those specific shapes). +/// Module-wide static tagged-catch plan. #[derive(Debug, Clone, Default)] pub struct PlainCatchPlan { /// Per-function per-region arm metadata. Outer Vec /// parallels `discover_plain_catch_arms`'s return shape (one /// entry per try_table that has at least one plain-catch arm). pub per_function: std::collections::HashMap)>>, - /// Stage 2 (B1): functions whose plain-catch arms include - /// unsupported operand types (e.g., ref-typed). For these - /// functions, B1 emission tasks fall back to today's behavior - /// (Phase 6 doesn't intercept plain-catch arms — the function - /// works for catch_ref but is unsupported for plain-catch fork). - pub b2_carveout: std::collections::HashSet, } -/// Discover supported plain-catch arms across all fork-path functions. -/// -/// Operand types are restricted to scalars (i32/i64/f32/f64/v128). -/// Ref-typed operands (externref/funcref/exnref/GC refs) require -/// auxiliary-table spilling and remain a conservative carve-out. -/// -/// Stage 2 (Task 2.1) detects ref-typed payloads here and routes the -/// affected function to `PlainCatchPlan.b2_carveout` instead of -/// `per_function`. Stage 2 emission tasks check the carve-out set -/// and skip plain-catch instrumentation for those functions. -/// -/// The carve-out is whole-function: if any arm in any region of a -/// function has a ref-typed operand, the entire function's -/// plain-catch instrumentation is skipped. We don't selectively drop -/// arms because Task 2.3's rewind dispatcher needs the whole -/// region's arm set or none. +/// Discover every tagged and catch-all clause across fork-path functions. /// +/// Every arm receives either exact scalar payload ownership or one complete +/// exception recipe. Silently omitting an arm would be an instrumenter bug +/// that surfaced only in a fresh child. pub fn plan_plain_catches(module: &Module, targets: &[FunctionId]) -> PlainCatchPlan { let mut plan = PlainCatchPlan::default(); for &fid in targets { @@ -3961,34 +5221,6 @@ pub fn plan_plain_catches(module: &Module, targets: &[FunctionId]) -> PlainCatch if arms_per_region.is_empty() { continue; } - // Stage 2 (B1): detect unsupported operand types and carve out - // the entire function. We can't selectively drop just the bad - // arms because replay needs a complete region-wide arm set. - let has_unsupported = arms_per_region.iter().any(|(_, arms)| { - arms.iter() - .any(|arm| arm.operand_tys.iter().any(|t| matches!(t, ValType::Ref(_)))) - }); - // Stage 2 (B1) Task 2.4: multi-target plain-catch guard. - // A try_table whose plain-catch arms branch to *different* - // labels has not been verified end-to-end. Per-arm capture - // blocks each branch to their own original target label, and - // the rewind dispatcher's re-throw routes through the - // try_table's catch clauses to reach those captures, so in - // principle multi-target should work — but until a real port - // exercises it, conservatively treat such functions as - // b2_carveout. Single-target multi-arm (multiple catches all - // pointing at the same label) remains supported. - let has_multi_target = arms_per_region.iter().any(|(_, arms)| { - if arms.len() <= 1 { - return false; - } - let first = arms[0].label; - arms.iter().any(|arm| arm.label != first) - }); - if has_unsupported || has_multi_target { - plan.b2_carveout.insert(fid); - continue; - } plan.per_function.insert(fid, arms_per_region); } plan @@ -3998,7 +5230,21 @@ pub fn plan_plain_catches(module: &Module, targets: &[FunctionId]) -> PlainCatch #[derive(Debug, Clone)] struct PlainCatchArmState { arm: PlainCatchArm, + /// Non-zero function-local identity for this exact `(region, arm)` pair. + /// + /// WHY: frame word +8 is copied to a fresh child. Keeping the selector in + /// that existing header word avoids both a frame-backed `active_arm` local + /// and any module-instance auxiliary state. + selector: u32, + /// Typed per-region union used to forward the original tag payload and, + /// for scalar arms, back the selector-overlaid frame bytes. Recipe-backed + /// arms do not serialize these values independently because the exception + /// codec owns their payload atomically. operand_locals: Vec, + /// Function-shared CatchRef forwarding scratch for scalar arms, or + /// region-shared activation state for recipe-backed exceptions. Only the + /// latter is added to the linked frame's reference plan. + captured_exnref: Option, } /// Activation-owned state for one try_table with plain catches. @@ -4008,52 +5254,351 @@ struct PlainCatchArmState { #[derive(Debug, Clone)] struct PlainCatchRegionState { body_seq: InstrSeqId, - active_arm: LocalId, + /// Function-wide retained exception selected by frame word +8. + /// + /// WHY: only one catch selector is activation-live at a time. A retained + /// complete-exception recipe that cannot be named by that selector is not + /// replay state; user-visible exceptions that remain live have separate + /// typed local/operand ownership. Sharing this slot across regions keeps + /// static catch-region count out of the native activation footprint and + /// out of the process reference vector. + retained_recipe_exnref: Option, + /// Arms in a region receive one contiguous selector interval, allowing + /// the rewind guard to recognize the region with two unsigned compares. + first_selector: u32, + last_selector: u32, arms: Vec, } -fn allocate_plain_catch_state( - module: &mut Module, - plain_catches: &[(InstrSeqId, Vec)], -) -> Vec { - plain_catches - .iter() - .map(|(body_seq, arms)| PlainCatchRegionState { - body_seq: *body_seq, - active_arm: module.locals.add(ValType::I32), - arms: arms - .iter() - .cloned() - .map(|arm| { - let operand_locals = arm - .operand_tys - .iter() - .map(|&ty| module.locals.add(ty)) - .collect(); - PlainCatchArmState { - arm, - operand_locals, +fn allocate_plain_catch_state( + module: &mut Module, + plain_catches: &[(InstrSeqId, Vec)], +) -> Vec { + let forwarding_exnref_scratch = plain_catches + .iter() + .flat_map(|(_, arms)| arms) + .any(|arm| arm.kind.is_ref() && !arm.uses_exception_recipe) + .then(|| { + module.locals.add(ValType::Ref(RefType { + nullable: true, + heap_type: HeapType::Abstract(AbstractHeapType::Exn), + })) + }); + let retained_recipe_exnref = plain_catches + .iter() + .flat_map(|(_, arms)| arms) + .any(|arm| arm.uses_exception_recipe) + .then(|| { + module.locals.add(ValType::Ref(RefType { + nullable: true, + heap_type: HeapType::Abstract(AbstractHeapType::Exn), + })) + }); + let mut next_selector = 1u32; + let mut regions = Vec::with_capacity(plain_catches.len()); + let mut operand_pools: Vec<(ValType, Vec)> = Vec::new(); + for (body_seq, arms) in plain_catches { + debug_assert!(!arms.is_empty()); + let first_selector = next_selector; + let mut arm_states = Vec::with_capacity(arms.len()); + for arm in arms.iter().cloned() { + let selector = next_selector; + next_selector = next_selector + .checked_add(1) + .expect("a Wasm function cannot contain 2^32 catch arms"); + let mut uses_by_type: Vec<(ValType, usize)> = Vec::new(); + let mut operand_locals = Vec::with_capacity(arm.operand_tys.len()); + for &ty in &arm.operand_tys { + let storage_ty = spill_storage_type(ty); + let ordinal = match uses_by_type + .iter_mut() + .find(|(candidate, _)| *candidate == storage_ty) + { + Some((_, next)) => { + let ordinal = *next; + *next += 1; + ordinal + } + None => { + uses_by_type.push((storage_ty, 1)); + 0 + } + }; + let pool_index = operand_pools + .iter() + .position(|(candidate, _)| *candidate == storage_ty) + .unwrap_or_else(|| { + operand_pools.push((storage_ty, Vec::new())); + operand_pools.len() - 1 + }); + let pool = &mut operand_pools[pool_index].1; + if pool.len() == ordinal { + pool.push(module.locals.add(storage_ty)); + } + // WHY: catch capture publishes one dynamically latest selector + // per activation, and the capture tail contains no call or + // throw between overwriting this typed scratch and publishing + // that selector. A function-wide typed union therefore cannot + // be observed half-updated by fork. Guest-visible values from + // earlier handlers have ordinary local/operand liveness + // ownership; this scratch exists only to rethrow the selected + // catch. Recursive activations still receive distinct native + // local tuples. + operand_locals.push(pool[ordinal]); + } + let captured_exnref = if arm.uses_exception_recipe { + // The single latest-catch selector owns this value. An older + // synthetic recipe cannot be replayed after another catch + // supersedes its selector; any guest-visible exception that + // remains live is captured independently by typed liveness. + retained_recipe_exnref + } else if arm.kind.is_ref() { + // WHY: a scalar CatchRef needs this local only to move the + // non-null exception past its scalar payload. The generated + // capture tail contains no call or throw and clears the local + // before entering user code, so mutually exclusive arms can + // share one function-local scratch without retaining a GC + // root or increasing every activation by one exnref per arm. + forwarding_exnref_scratch + } else { + None + }; + arm_states.push(PlainCatchArmState { + arm, + selector, + operand_locals, + captured_exnref, + }); + } + regions.push(PlainCatchRegionState { + body_seq: *body_seq, + retained_recipe_exnref, + first_selector, + last_selector: next_selector - 1, + arms: arm_states, + }); + } + regions +} + +#[derive(Debug, Clone)] +struct PlainCatchScalarArmFrame { + selector: u32, + fields: Vec<(LocalId, ValType, u32)>, +} + +/// One overlaid scalar payload range shared by every catch arm in a function. +/// +/// Only one `(region, arm)` selector can own a continuation landing. Giving +/// each arm offsets relative to the same `start` therefore preserves the +/// selected payload in `max(arm_size)` bytes instead of summing all static +/// arms. Save/restore dispatch below makes the aliasing explicit and prevents +/// inactive locals from overwriting the active arm. +#[derive(Debug, Clone, Default)] +struct PlainCatchScalarFrame { + arms: Vec, + byte_len: u32, +} + +impl PlainCatchScalarFrame { + fn frame_end(&self, start: u32) -> u32 { + start + .checked_add(self.byte_len) + .expect("catch payload frame exceeds the 32-bit continuation format") + } +} + +fn plan_plain_catch_scalar_frame( + regions: &[PlainCatchRegionState], + start: u32, +) -> PlainCatchScalarFrame { + let mut plan = PlainCatchScalarFrame::default(); + for region in regions { + for arm in ®ion.arms { + let mut relative = 0u32; + let mut fields = Vec::new(); + if !arm.arm.uses_exception_recipe { + for (&local, &ty) in arm.operand_locals.iter().zip(&arm.arm.operand_tys) { + fields.push(( + local, + ty, + start + .checked_add(relative) + .expect("catch payload offset exceeds the frame format"), + )); + relative = relative + .checked_add(scalar_size(ty)) + .expect("catch payload exceeds the frame format"); + } + } + plan.byte_len = plan.byte_len.max(relative); + plan.arms.push(PlainCatchScalarArmFrame { + selector: arm.selector, + fields, + }); + } + } + plan +} + +#[derive(Debug, Clone, Copy)] +enum PlainCatchScalarIo { + Save, + Restore, +} + +/// Build a selector-guarded frame I/O tree for the overlaid catch payload. +/// +/// Inactive arms intentionally perform no memory access. A non-zero selector +/// not present in the static function plan is corrupt continuation state and +/// traps before replay can branch into user code. +fn build_plain_catch_scalar_dispatch( + local: &mut LocalFunction, + runtime: &Runtime, + memory: MemoryId, + ptr_ty: ValType, + catch_selector: LocalId, + plan: &PlainCatchScalarFrame, + io: PlainCatchScalarIo, +) -> Option { + if plan.arms.is_empty() { + return None; + } + + let empty = local + .builder_mut() + .dangling_instr_seq(InstrSeqType::Simple(None)) + .id(); + let invalid = local + .builder_mut() + .dangling_instr_seq(InstrSeqType::Simple(None)) + .id(); + push_instr( + &mut local.block_mut(invalid).instrs, + Instr::Unreachable(Unreachable {}), + ); + + // Selector zero is the common path outside a catch. Every other value + // must match an exact static arm below. + let mut chain = local + .builder_mut() + .dangling_instr_seq(InstrSeqType::Simple(None)) + .id(); + { + let out = &mut local.block_mut(chain).instrs; + push_instr( + out, + Instr::LocalGet(LocalGet { + local: catch_selector, + }), + ); + push_instr( + out, + Instr::Unop(walrus::ir::Unop { + op: UnaryOp::I32Eqz, + }), + ); + push_instr( + out, + Instr::IfElse(IfElse { + consequent: empty, + alternative: invalid, + }), + ); + } + + for arm in plan.arms.iter().rev() { + let action = local + .builder_mut() + .dangling_instr_seq(InstrSeqType::Simple(None)) + .id(); + { + let out = &mut local.block_mut(action).instrs; + for &(field, ty, offset) in &arm.fields { + match io { + PlainCatchScalarIo::Save => { + push_current_frame_ptr(out, runtime, memory, ptr_ty); + push_instr(out, Instr::LocalGet(LocalGet { local: field })); + push_instr(out, store_scalar(memory, ty, offset as u64)); } - }) - .collect(), - }) - .collect() + PlainCatchScalarIo::Restore => { + push_current_frame_ptr(out, runtime, memory, ptr_ty); + push_instr(out, load_scalar(memory, ty, offset as u64)); + push_instr(out, Instr::LocalSet(LocalSet { local: field })); + } + } + } + } + + let select = local + .builder_mut() + .dangling_instr_seq(InstrSeqType::Simple(None)) + .id(); + { + let out = &mut local.block_mut(select).instrs; + push_instr( + out, + Instr::LocalGet(LocalGet { + local: catch_selector, + }), + ); + push_instr( + out, + Instr::Const(Const { + value: Value::I32(arm.selector as i32), + }), + ); + push_instr( + out, + Instr::Binop(Binop { + op: BinaryOp::I32Eq, + }), + ); + push_instr( + out, + Instr::IfElse(IfElse { + consequent: action, + alternative: chain, + }), + ); + } + chain = select; + } + Some(chain) } -fn append_plain_catch_frame_scalars( - frame_scalars: &mut Vec<(LocalId, ValType)>, +fn append_plain_catch_frame_references( + per_call_references: &mut [Vec<(LocalId, RefType)>], regions: &[PlainCatchRegionState], ) { - for region in regions { - frame_scalars.push((region.active_arm, ValType::I32)); - for arm in ®ion.arms { - frame_scalars.extend( - arm.operand_locals - .iter() - .copied() - .zip(arm.arm.operand_tys.iter().copied()), - ); - } + let exnref = RefType { + nullable: true, + heap_type: HeapType::Abstract(AbstractHeapType::Exn), + }; + let Some(exception) = regions + .iter() + .find_map(|region| region.retained_recipe_exnref) + else { + return; + }; + debug_assert!( + regions + .iter() + .all(|region| region.retained_recipe_exnref == Some(exception)) + ); + debug_assert!( + regions + .iter() + .flat_map(|region| ®ion.arms) + .filter(|arm| arm.arm.uses_exception_recipe) + .all(|arm| arm.captured_exnref == Some(exception)) + ); + // Any call can be reached while a handler activation is live. Encoding + // null when no recipe catch is selected is the deterministic zero-recipe + // fast path; the selected arm's non-null exception is activation state and + // appears exactly once in each call-specific reference vector. + for references in per_call_references.iter_mut() { + references.push((exception, exnref)); } } @@ -4065,32 +5610,21 @@ fn append_plain_catch_frame_scalars( /// Phase 6c (extended by B1 Stage 2 Task 2.3) — prepend a rewind-throw /// stub at the top of each fork-path try_table body. /// -/// On REWIND, when `catch_region_id_local == K`, the stub re-enters the -/// try_table's catch dispatch so the original handler observes the same -/// exception that was caught pre-fork. The shape depends on what kind -/// of catch was originally taken: +/// On REWIND, when `catch_selector_local` falls in this region's selector +/// interval, the stub re-enters the try_table's catch dispatch so the original +/// handler observes the same exception that was caught pre-fork. /// -/// Plain catches restore a frame-backed active-arm local and operand locals, -/// then throw the matching tag. A catch_ref capture writes active-arm `-1`, -/// so a mixed region falls through to the exnref `throw_ref` path without -/// treating stale auxiliary-table contents as mode state. +/// Both Catch and CatchRef restore the exact selector from frame word +8 and +/// scalar tag operands from the overlaid payload range, then throw the matching +/// tag. The original CatchRef clause creates a fresh exnref in the child. fn inject_rewind_throw_stubs( module: &mut Module, func_id: FunctionId, runtime: &Runtime, - catch_region_id_local: LocalId, - aux_tables: &AuxTables, + catch_selector_local: LocalId, catch_plan: &[CatchRegionPlan], plain_catches: &[PlainCatchRegionState], ) { - let exnref_table = match aux_tables.exnref { - Some(t) => t, - None => { - debug_assert!(catch_plan.is_empty()); - return; - } - }; - let plain_lookup: HashMap = plain_catches .iter() .map(|region| (region.body_seq, region)) @@ -4098,40 +5632,26 @@ fn inject_rewind_throw_stubs( for plan in catch_plan { let body_seq_id = plan.body_seq; - let region_id = plan.catch_region_id; - let slot = plan.exnref_slot; - let plain_region = plain_lookup.get(&body_seq_id).copied(); - - // Build the inner "catch_ref path" sequence (Phase 6's existing - // logic). Always emitted — used either as the only path or as - // the fallback when no exact plain arm is active. - let throw_ref_seq_id = { + let Some(region) = plain_lookup.get(&body_seq_id).copied() else { + continue; + }; + + // An unknown arm means the continuation is corrupt or from an + // incompatible artifact. There is no module-instance reference + // fallback in ABI 43. + let invalid_arm = { let local = local_mut(module, func_id); let s = local .builder_mut() .dangling_instr_seq(InstrSeqType::Simple(None)) .id(); let block = &mut local.block_mut(s).instrs; - push_instr( - block, - Instr::Const(Const { - value: Value::I32(slot as i32), - }), - ); - push_instr( - block, - Instr::TableGet(TableGet { - table: exnref_table, - }), - ); - push_instr(block, Instr::RefAsNonNull(RefAsNonNull {})); - push_instr(block, Instr::ThrowRef(ThrowRef {})); + push_instr(block, Instr::Unreachable(Unreachable {})); s }; - let dispatch_seq_id = plain_region.map_or(throw_ref_seq_id, |region| { - build_plain_catch_dispatch(module, func_id, region, throw_ref_seq_id) - }); + let dispatch_seq_id = + build_plain_catch_dispatch(module, func_id, region, catch_selector_local, invalid_arm); // Build the empty else for the outer REWIND-match guard. let else_id = { @@ -4142,8 +5662,9 @@ fn inject_rewind_throw_stubs( .id() }; - // Prepend the outer guard `if state>=REWINDING && cri == K` - // to the try_table body. + // Prepend the outer guard. Selectors are allocated contiguously per + // region, so two unsigned comparisons recognize this exact lexical + // try_table without another activation-local word. let local = local_mut(module, func_id); let original: Vec<(Instr, InstrLocId)> = std::mem::take(&mut local.block_mut(body_seq_id).instrs); @@ -4170,19 +5691,43 @@ fn inject_rewind_throw_stubs( push_instr( body, Instr::LocalGet(LocalGet { - local: catch_region_id_local, + local: catch_selector_local, }), ); push_instr( body, Instr::Const(Const { - value: Value::I32(region_id as i32), + value: Value::I32(region.first_selector as i32), }), ); push_instr( body, Instr::Binop(Binop { - op: BinaryOp::I32Eq, + op: BinaryOp::I32GeU, + }), + ); + push_instr( + body, + Instr::Binop(Binop { + op: BinaryOp::I32And, + }), + ); + push_instr( + body, + Instr::LocalGet(LocalGet { + local: catch_selector_local, + }), + ); + push_instr( + body, + Instr::Const(Const { + value: Value::I32(region.last_selector as i32), + }), + ); + push_instr( + body, + Instr::Binop(Binop { + op: BinaryOp::I32LeU, }), ); push_instr( @@ -4203,20 +5748,17 @@ fn inject_rewind_throw_stubs( } } -/// Build a dangling sequence that rethrows one frame-restored plain catch. -/// -/// Exact nonnegative arm IDs select plain catches. `-1` deliberately falls -/// through to `throw_ref_fallback`, which is what a catch_ref capture records -/// for a mixed region. +/// Build a dangling sequence that rethrows one frame-restored tagged catch. fn build_plain_catch_dispatch( module: &mut Module, func_id: FunctionId, region: &PlainCatchRegionState, - throw_ref_fallback: InstrSeqId, + catch_selector_local: LocalId, + invalid_arm: InstrSeqId, ) -> InstrSeqId { debug_assert!(!region.arms.is_empty()); - let mut chain = throw_ref_fallback; + let mut chain = invalid_arm; for arm in region.arms.iter().rev() { let throw_id = { let local = local_mut(module, func_id); @@ -4227,333 +5769,356 @@ fn build_plain_catch_dispatch( }; let local = local_mut(module, func_id); let s = &mut local.block_mut(throw_id).instrs; - for &operand in &arm.operand_locals { - push_instr(s, Instr::LocalGet(LocalGet { local: operand })); - } - push_instr(s, Instr::Throw(Throw { tag: arm.arm.tag })); - - let outer_id = { - let local = local_mut(module, func_id); - local - .builder_mut() - .dangling_instr_seq(InstrSeqType::Simple(None)) - .id() - }; - { - let local = local_mut(module, func_id); - let s = &mut local.block_mut(outer_id).instrs; - push_instr( - s, - Instr::LocalGet(LocalGet { - local: region.active_arm, - }), - ); - push_instr( - s, - Instr::Const(Const { - value: Value::I32(arm.arm.arm_idx as i32), - }), - ); - push_instr( - s, - Instr::Binop(Binop { - op: BinaryOp::I32Eq, - }), - ); - push_instr( - s, - Instr::IfElse(IfElse { - consequent: throw_id, - alternative: chain, - }), - ); - } - chain = outer_id; - } - - chain -} - -// ---------------------------------------------------------------------- -// Phase 6d — catch-handler entry capture -// ---------------------------------------------------------------------- - -#[derive(Debug, Clone, Copy)] -struct CatchHandlerInfo { - catch_region_id: u32, - exnref_slot: u32, - body_seq: InstrSeqId, - target_label: InstrSeqId, - in_catch_local: LocalId, - captured_exnref_local: LocalId, - plain_active_arm: Option, -} - -fn plan_catch_ref_handlers( - module: &mut Module, - func_id: FunctionId, - catch_plan: &[CatchRegionPlan], - aux_tables: &AuxTables, - plain_catches: &[PlainCatchRegionState], -) -> Vec { - let mut infos = Vec::new(); - if aux_tables.exnref.is_none() { - return infos; - } - let exnref_ty = RefType { - nullable: true, - heap_type: HeapType::Abstract(AbstractHeapType::Exn), - }; - - for plan in catch_plan { - let target_label_opt = { - let local = match &module.funcs.get(func_id).kind { - FunctionKind::Local(l) => l, - _ => continue, - }; - let (_, tt) = match find_try_table_parent_seq(local, local.entry_block(), plan.body_seq) - { - Some(v) => v, - None => continue, - }; - - let mut ref_targets: HashSet = HashSet::new(); - for c in &tt.catches { - match c { - TryTableCatch::CatchRef { label, .. } - | TryTableCatch::CatchAllRef { label } => { - ref_targets.insert(*label); - } - _ => {} - } - } - if ref_targets.len() != 1 { - None - } else { - Some(*ref_targets.iter().next().unwrap()) + if arm.arm.uses_exception_recipe { + let exception = arm + .captured_exnref + .expect("recipe-backed catch must own an exnref local"); + push_instr(s, Instr::LocalGet(LocalGet { local: exception })); + push_instr(s, Instr::RefAsNonNull(RefAsNonNull {})); + push_instr(s, Instr::ThrowRef(walrus::ir::ThrowRef {})); + } else { + for (&operand, &ty) in arm.operand_locals.iter().zip(&arm.arm.operand_tys) { + push_typed_local_get(s, operand, ty); } - }; - let target_label = match target_label_opt { - Some(t) => t, - None => continue, - }; - - let in_catch_local = module.locals.add(ValType::I32); - let captured_exnref_local = module.locals.add(ValType::Ref(exnref_ty)); - - infos.push(CatchHandlerInfo { - catch_region_id: plan.catch_region_id, - exnref_slot: plan.exnref_slot, - body_seq: plan.body_seq, - target_label, - in_catch_local, - captured_exnref_local, - plain_active_arm: plain_catches - .iter() - .find(|region| region.body_seq == plan.body_seq) - .map(|region| region.active_arm), - }); - } - - infos -} - -fn apply_catch_ref_handlers( - module: &mut Module, - func_id: FunctionId, - handlers: &[CatchHandlerInfo], - aux_tables: &AuxTables, -) { - let exnref_table = match aux_tables.exnref { - Some(t) => t, - None => return, - }; - - for info in handlers { - let (parent_seq, original_catches, try_table_type, catch_sig_type) = { - let local = match &module.funcs.get(func_id).kind { - FunctionKind::Local(l) => l, - _ => continue, - }; - let (parent, tt) = - match find_try_table_parent_seq(local, local.entry_block(), info.body_seq) { - Some(v) => v, - None => continue, - }; - let catches = tt.catches.clone(); - let try_sig = local.block(info.body_seq).ty; - let catch_sig = local.block(info.target_label).ty; - (parent, catches, try_sig, catch_sig) - }; - - let (outer_seq_id, capture_seq_id) = { - let local = local_mut(module, func_id); - let cap = local.builder_mut().dangling_instr_seq(catch_sig_type).id(); - let out = local.builder_mut().dangling_instr_seq(try_table_type).id(); - (out, cap) - }; - - let new_catches: Vec = original_catches - .iter() - .map(|c| match c { - TryTableCatch::CatchRef { tag, .. } => TryTableCatch::CatchRef { - tag: *tag, - label: capture_seq_id, - }, - TryTableCatch::CatchAllRef { .. } => TryTableCatch::CatchAllRef { - label: capture_seq_id, - }, - TryTableCatch::Catch { tag, label } => TryTableCatch::Catch { - tag: *tag, - label: *label, - }, - TryTableCatch::CatchAll { label } => TryTableCatch::CatchAll { label: *label }, - }) - .collect(); - - { - let local = local_mut(module, func_id); - let s = &mut local.block_mut(capture_seq_id).instrs; - push_instr( - s, - Instr::TryTable(TryTable { - seq: info.body_seq, - catches: new_catches, - }), - ); push_instr( s, - Instr::Br(Br { - block: outer_seq_id, + Instr::Throw(Throw { + tag: arm + .arm + .tag + .expect("scalar tagged catch dispatch must have a tag"), }), ); } - { + let outer_id = { let local = local_mut(module, func_id); - let s = &mut local.block_mut(outer_seq_id).instrs; - push_instr( - s, - Instr::Block(Block { - seq: capture_seq_id, - }), - ); - push_instr( - s, - Instr::LocalTee(LocalTee { - local: info.captured_exnref_local, - }), - ); - if let Some(active_arm) = info.plain_active_arm { - // WHY: mixed catch regions must restore their capture kind - // from frame-owned state. A negative arm cannot match any - // plain catch, so replay falls through to throw_ref without - // consulting possibly stale exnref-table nullness. - push_instr( - s, - Instr::Const(Const { - value: Value::I32(-1), - }), - ); - push_instr(s, Instr::LocalSet(LocalSet { local: active_arm })); - } - push_instr( - s, - Instr::Const(Const { - value: Value::I32(1), - }), - ); + local + .builder_mut() + .dangling_instr_seq(InstrSeqType::Simple(None)) + .id() + }; + { + let local = local_mut(module, func_id); + let s = &mut local.block_mut(outer_id).instrs; push_instr( s, - Instr::LocalSet(LocalSet { - local: info.in_catch_local, + Instr::LocalGet(LocalGet { + local: catch_selector_local, }), ); push_instr( s, Instr::Const(Const { - value: Value::I32(info.exnref_slot as i32), + value: Value::I32(arm.selector as i32), }), ); push_instr( s, - Instr::LocalGet(LocalGet { - local: info.captured_exnref_local, - }), - ); - push_instr( - s, - Instr::TableSet(TableSet { - table: exnref_table, + Instr::Binop(Binop { + op: BinaryOp::I32Eq, }), ); push_instr( s, - Instr::Br(Br { - block: info.target_label, + Instr::IfElse(IfElse { + consequent: throw_id, + alternative: chain, }), ); } + chain = outer_id; + } - { - let local = local_mut(module, func_id); - let parent_instrs = &mut local.block_mut(parent_seq).instrs; - let tt_idx = parent_instrs + chain +} + +// ---------------------------------------------------------------------- +// Phase 6d — catch-handler entry capture +// ---------------------------------------------------------------------- + +#[derive(Debug, Clone, Copy)] +struct CatchHandlerInfo { + body_seq: InstrSeqId, +} + +fn plan_catch_handlers( + catch_plan: &[CatchRegionPlan], + plain_catches: &[PlainCatchRegionState], +) -> Vec { + plain_catches + .iter() + .filter_map(|region| { + catch_plan .iter() - .position(|(i, _)| matches!(i, Instr::TryTable(tt) if tt.seq == info.body_seq)) - .expect("try_table not found in its parent"); - parent_instrs[tt_idx].0 = Instr::Block(Block { seq: outer_seq_id }); - } - } + .find(|plan| plan.body_seq == region.body_seq) + .map(|plan| CatchHandlerInfo { + body_seq: plan.body_seq, + }) + }) + .collect() } // ---------------------------------------------------------------------- -// Stage 2 (B1) — per-arm capture-block emission for plain catch +// Per-arm capture-block emission for tagged catches // ---------------------------------------------------------------------- -/// Stage 2 (B1) — emit per-arm capture blocks that intercept plain -/// catch dispatch. +/// Ensure a user `catch_all`/`catch_all_ref` can never consume the +/// process-owned unwind transport. /// -/// For each fork-path try_table that has at least one plain-catch arm -/// (and whose function is NOT in `b2_carveout`), this rewrites: +/// A modern `try_table` catch transfers directly to an enclosing label, so a +/// zero-result shield block is inserted around the original instruction: /// -/// ```wat -/// (try_table (catch $tag $h) ... body ...) ;; original -/// ``` -/// into: -/// ```wat -/// (block $b1_outer -/// (block $cap_arm_0 -/// ... -/// (block $cap_arm_N-1 -/// (try_table (catch $tag0 $cap_arm_0) ... (catch $tagN-1 $cap_arm_N-1) body) -/// br $b1_outer) -/// ;; cap_arm_N-1 body: tagN-1.params on stack — save, set flags, br $hN-1 -/// ... -/// ;; cap_arm_0 body: tag0.params on stack — save, set flags, br $h0 +/// ```text +/// block $outer (param P) (result R) +/// block $private_shield (param P) +/// try_table (param P) (result R) +/// (catch $__wpk_fork_unwind $private_shield) +/// ...original catches... +/// br $outer +/// end +/// throw $__wpk_fork_unwind +/// end /// ``` /// -/// Inside each cap_arm_J body the operands are: -/// 1. spilled to activation-local, frame-backed locals. -/// 2. used to set the frame-backed active arm plus -/// `in_catch_local = 1` and `catch_region_id_local = -/// region_id`. -/// 3. re-pushed (in declaration order) and `br $hJ` executes. -/// -/// CatchAll/CatchRef/CatchAllRef clauses are preserved verbatim. If -/// Phase 6 already retargeted CatchRef/CatchAllRef clauses, those -/// retargets are passed through unchanged. -/// -/// `catch_handlers` is Phase 6's per-region info; the `in_catch_local` -/// is reused for any region that overlaps with B1's emission. For -/// plain-catch-only regions, a fresh `in_catch_local` is allocated. +/// Typed block parameters preserve the original operand stack without +/// allocating reference temporaries (which would themselves become stale GC +/// roots). Legacy EH has explicit handler sequences, so it only needs a typed +/// private handler inserted immediately before its catch-all. +fn shield_private_unwind_from_user_catches( + module: &mut Module, + func_id: FunctionId, + runtime: &Runtime, +) { + #[derive(Clone, Copy)] + enum Site { + TryTable { body: InstrSeqId, depth: u32 }, + LegacyTry { body: InstrSeqId, depth: u32 }, + } + + fn collect( + local: &LocalFunction, + seq: InstrSeqId, + depth: u32, + seen: &mut HashSet, + out: &mut Vec, + ) { + if !seen.insert(seq) { + return; + } + for (instr, _) in &local.block(seq).instrs { + match instr { + Instr::TryTable(tt) + if tt.catches.iter().any(|catch| { + matches!( + catch, + TryTableCatch::CatchAll { .. } | TryTableCatch::CatchAllRef { .. } + ) + }) => + { + out.push(Site::TryTable { + body: tt.seq, + depth, + }); + } + Instr::Try(legacy) + if legacy + .catches + .iter() + .any(|catch| matches!(catch, LegacyCatch::CatchAll { .. })) => + { + out.push(Site::LegacyTry { + body: legacy.seq, + depth, + }); + } + _ => {} + } + for child in nested_seqs(instr) { + collect(local, child, depth + 1, seen, out); + } + } + } + + let unwind_tag = runtime + .unwind_tag + .expect("fork-path instrumentation requires the linked unwind tag"); + let mut sites = Vec::new(); + { + let local = match &module.funcs.get(func_id).kind { + FunctionKind::Local(local) => local, + _ => return, + }; + collect( + local, + local.entry_block(), + 0, + &mut HashSet::new(), + &mut sites, + ); + } + sites.sort_by_key(|site| match site { + Site::TryTable { depth, .. } | Site::LegacyTry { depth, .. } => std::cmp::Reverse(*depth), + }); + + for site in sites { + match site { + Site::TryTable { body, .. } => { + let Some((parent, index, loc, mut table, body_ty)) = ({ + let local = match &module.funcs.get(func_id).kind { + FunctionKind::Local(local) => local, + _ => return, + }; + find_try_table_instr_site(local, local.entry_block(), body) + }) else { + continue; + }; + + let params = match body_ty { + InstrSeqType::Simple(_) => Vec::new(), + InstrSeqType::MultiValue(ty) => module.types.get(ty).params().to_vec(), + }; + let shield_ty = InstrSeqType::new(&mut module.types, ¶ms, &[]); + let (outer, shield) = { + let local = local_mut(module, func_id); + let outer = local.builder_mut().dangling_instr_seq(body_ty).id(); + let shield = local.builder_mut().dangling_instr_seq(shield_ty).id(); + (outer, shield) + }; + + let catch_all_index = table + .catches + .iter() + .position(|catch| { + matches!( + catch, + TryTableCatch::CatchAll { .. } | TryTableCatch::CatchAllRef { .. } + ) + }) + .expect("collected try_table still has a catch-all"); + table.catches.insert( + catch_all_index, + TryTableCatch::Catch { + tag: unwind_tag, + label: shield, + }, + ); + + { + let local = local_mut(module, func_id); + let s = &mut local.block_mut(shield).instrs; + push_instr(s, Instr::TryTable(table)); + push_instr(s, Instr::Br(Br { block: outer })); + } + { + let local = local_mut(module, func_id); + let s = &mut local.block_mut(outer).instrs; + push_instr(s, Instr::Block(Block { seq: shield })); + push_instr(s, Instr::Throw(Throw { tag: unwind_tag })); + } + local_mut(module, func_id).block_mut(parent).instrs[index] = + (Instr::Block(Block { seq: outer }), loc); + } + Site::LegacyTry { body, .. } => { + let Some((parent, index, body_ty)) = ({ + let local = match &module.funcs.get(func_id).kind { + FunctionKind::Local(local) => local, + _ => return, + }; + find_legacy_try_instr_site(local, local.entry_block(), body) + }) else { + continue; + }; + let results = match body_ty { + InstrSeqType::Simple(None) => Vec::new(), + InstrSeqType::Simple(Some(result)) => vec![result], + InstrSeqType::MultiValue(ty) => module.types.get(ty).results().to_vec(), + }; + let handler_ty = InstrSeqType::new(&mut module.types, &[], &results); + let handler = { + let local = local_mut(module, func_id); + local.builder_mut().dangling_instr_seq(handler_ty).id() + }; + { + let local = local_mut(module, func_id); + push_instr( + &mut local.block_mut(handler).instrs, + Instr::Throw(Throw { tag: unwind_tag }), + ); + let Instr::Try(legacy) = &mut local.block_mut(parent).instrs[index].0 else { + unreachable!("legacy try site changed during shielding"); + }; + let catch_all_index = legacy + .catches + .iter() + .position(|catch| matches!(catch, LegacyCatch::CatchAll { .. })) + .expect("collected legacy try still has a catch-all"); + legacy.catches.insert( + catch_all_index, + LegacyCatch::Catch { + tag: unwind_tag, + handler, + }, + ); + } + } + } + } +} + +fn find_try_table_instr_site( + local: &LocalFunction, + seq: InstrSeqId, + body: InstrSeqId, +) -> Option<(InstrSeqId, usize, InstrLocId, TryTable, InstrSeqType)> { + for (index, (instr, loc)) in local.block(seq).instrs.iter().enumerate() { + if let Instr::TryTable(table) = instr { + if table.seq == body { + return Some((seq, index, *loc, table.clone(), local.block(body).ty)); + } + } + for child in nested_seqs(instr) { + if let Some(site) = find_try_table_instr_site(local, child, body) { + return Some(site); + } + } + } + None +} + +fn find_legacy_try_instr_site( + local: &LocalFunction, + seq: InstrSeqId, + body: InstrSeqId, +) -> Option<(InstrSeqId, usize, InstrSeqType)> { + for (index, (instr, _)) in local.block(seq).instrs.iter().enumerate() { + if let Instr::Try(legacy) = instr { + if legacy.seq == body { + return Some((seq, index, local.block(body).ty)); + } + } + for child in nested_seqs(instr) { + if let Some(site) = find_legacy_try_instr_site(local, child, body) { + return Some(site); + } + } + } + None +} + +/// Emit per-arm capture blocks that intercept tagged Catch and CatchRef +/// dispatch. /// +/// Each capture spills its scalar operands to activation-owned frame locals, +/// records the active region/arm, re-pushes the original operands, and branches +/// to the user's handler. CatchRef captures only the scalar tag payload; its +/// instance-local exnref is forwarded and then cleared from the synthetic +/// local so it does not survive as a stale GC root. fn apply_plain_catch_handlers( module: &mut Module, func_id: FunctionId, - catch_region_id_local: LocalId, + catch_selector_local: LocalId, plain_catches: &[PlainCatchRegionState], - catch_plan: &[CatchRegionPlan], catch_handlers: &[CatchHandlerInfo], ) { if plain_catches.is_empty() { @@ -4567,10 +6132,6 @@ fn apply_plain_catch_handlers( continue; } - // Phase 6's `apply_catch_ref_handlers` may have moved this - // try_table inside its own capture block. `find_try_table_parent_seq` - // walks recursively from the entry block, so the new parent - // is discovered automatically. let (parent_seq, original_catches, try_table_type) = { let local = match &module.funcs.get(func_id).kind { FunctionKind::Local(l) => l, @@ -4584,23 +6145,7 @@ fn apply_plain_catch_handlers( (parent, tt.catches.clone(), local.block(body_seq).ty) }; - // Look up region_id from catch_plan (every fork-path try_table - // gets a catch_region_id assigned in plan_and_inject_aux_tables). - let catch_region_id = catch_plan - .iter() - .find(|p| p.body_seq == body_seq) - .map(|p| p.catch_region_id) - .unwrap_or(0); - - // Reuse Phase 6's in_catch_local if the region overlaps; else - // allocate a fresh one. (Mixed catch_ref+plain regions share - // the same flag so post-call dispatch sees a single signal per - // region.) - let in_catch_local = catch_handlers - .iter() - .find(|h| h.body_seq == body_seq) - .map(|h| h.in_catch_local) - .unwrap_or_else(|| module.locals.add(ValType::I32)); + debug_assert!(catch_handlers.iter().any(|h| h.body_seq == body_seq)); // ---------------------------------------------------------- // Build dangling sequences: outer + N caps. @@ -4614,40 +6159,100 @@ fn apply_plain_catch_handlers( local.builder_mut().dangling_instr_seq(try_table_type).id() }; - // Build per-arm InstrSeqType up-front (mutates module.types) - // before any &mut LocalFunction borrow is needed. - let cap_types: Vec = arm_states - .iter() - .map(|state| InstrSeqType::new(&mut module.types, &[], &state.arm.operand_tys)) - .collect(); - + // The original target label already carries the exact catch branch + // type. For CatchRef that is tag.params followed by a non-null exnref. + // Reusing it avoids weakening concrete EH reference types. let mut cap_seq_ids: Vec = Vec::with_capacity(arm_states.len()); - for cap_ty in &cap_types { + for state in arm_states { + let original_ty = { + let local = match &module.funcs.get(func_id).kind { + FunctionKind::Local(local) => local, + _ => continue, + }; + local.block(state.arm.label).ty + }; + let cap_ty = if state.arm.kind.is_plain() && state.arm.uses_exception_recipe { + let mut results = match original_ty { + InstrSeqType::Simple(None) => Vec::new(), + InstrSeqType::Simple(Some(result)) => vec![result], + InstrSeqType::MultiValue(ty) => module.types.get(ty).results().to_vec(), + }; + // Retarget a plain catch to CatchRef so the capture owns + // the complete exception. The extra non-null exnref is + // consumed by the synthetic tail and never reaches the + // original plain handler. + results.push(ValType::Ref(RefType { + nullable: false, + heap_type: HeapType::Abstract(AbstractHeapType::Exn), + })); + InstrSeqType::new(&mut module.types, &[], &results) + } else { + original_ty + }; let local = local_mut(module, func_id); - cap_seq_ids.push(local.builder_mut().dangling_instr_seq(*cap_ty).id()); + cap_seq_ids.push(local.builder_mut().dangling_instr_seq(cap_ty).id()); } // ---------------------------------------------------------- - // Rewrite the inner try_table's catches: each plain Catch - // arm now points at its capture block; everything else (incl. - // catch_ref/catch_all_ref already retargeted by Phase 6) is - // preserved verbatim. + // Rewrite every planned tagged arm to its activation capture. // // We map by arm position within `arm_states` -- each entry's // `arm.arm_idx` is the arm's index in the original try_table's - // catches list. We walk `original_catches` and substitute each - // matching plain Catch with its capture target. + // catches list before private-unwind shielding. Locate the live clause + // by its original label/tag instead of indexing directly: shielding + // inserts a private catch immediately before a user catch-all. // ---------------------------------------------------------- let mut new_catches: Vec = original_catches.clone(); for (j, state) in arm_states.iter().enumerate() { - let arm_idx = state.arm.arm_idx as usize; + let arm_idx = new_catches + .iter() + .position(|catch| match (state.arm.kind, catch) { + (TaggedCatchKind::Plain, TryTableCatch::Catch { tag, label }) + | (TaggedCatchKind::Ref, TryTableCatch::CatchRef { tag, label }) => { + Some(*tag) == state.arm.tag && *label == state.arm.label + } + (TaggedCatchKind::AllPlain, TryTableCatch::CatchAll { label }) + | (TaggedCatchKind::AllRef, TryTableCatch::CatchAllRef { label }) => { + *label == state.arm.label + } + _ => false, + }) + .expect("planned user catch no longer exists after private shielding"); if let Some(c) = new_catches.get_mut(arm_idx) { - if let TryTableCatch::Catch { tag, .. } = c { - *c = TryTableCatch::Catch { - tag: *tag, - label: cap_seq_ids[j], - }; - } + let replacement = match (state.arm.kind, &*c) { + (TaggedCatchKind::Plain, TryTableCatch::Catch { tag, .. }) + if state.arm.uses_exception_recipe => + { + TryTableCatch::CatchRef { + tag: *tag, + label: cap_seq_ids[j], + } + } + (TaggedCatchKind::Plain, TryTableCatch::Catch { tag, .. }) => { + TryTableCatch::Catch { + tag: *tag, + label: cap_seq_ids[j], + } + } + (TaggedCatchKind::Ref, TryTableCatch::CatchRef { tag, .. }) => { + TryTableCatch::CatchRef { + tag: *tag, + label: cap_seq_ids[j], + } + } + (TaggedCatchKind::AllPlain, TryTableCatch::CatchAll { .. }) => { + TryTableCatch::CatchAllRef { + label: cap_seq_ids[j], + } + } + (TaggedCatchKind::AllRef, TryTableCatch::CatchAllRef { .. }) => { + TryTableCatch::CatchAllRef { + label: cap_seq_ids[j], + } + } + _ => unreachable!("validated catch plan no longer matches try_table"), + }; + *c = replacement; } } @@ -4724,11 +6329,9 @@ fn apply_plain_catch_handlers( module, func_id, cap_seq_ids[j], - region.active_arm, &arm_states[j + 1], - in_catch_local, - catch_region_id_local, - catch_region_id, + catch_selector_local, + region.retained_recipe_exnref, ); } @@ -4763,11 +6366,9 @@ fn apply_plain_catch_handlers( module, func_id, outer_seq_id, - region.active_arm, &arm_states[0], - in_catch_local, - catch_region_id_local, - catch_region_id, + catch_selector_local, + region.retained_recipe_exnref, ); // ---------------------------------------------------------- @@ -4787,28 +6388,38 @@ fn apply_plain_catch_handlers( } } -/// Emit the capture-block "tail" for a single plain-catch arm: at the -/// point where this is invoked, `tag.params()` are on the operand -/// stack. The emitted sequence: +/// Emit the capture-block tail for one tagged catch. The operand stack holds +/// `tag.params()` and, for CatchRef, a final exnref. /// -/// 1. Spill operands to per-arm frame locals (top-of-stack first). -/// 2. Set the region's frame-backed active arm. -/// 3. Set `in_catch_local = 1`, `catch_region_id_local = region_id`. -/// 4. Re-push operands (declaration order). +/// 1. Temporarily pop CatchRef's exnref, then spill scalar operands. +/// 2. Replace this activation's latest-catch selector with this exact arm. +/// 4. Re-push operands and, for the user's CatchRef, the exnref. +/// Clear capture-only reference locals; retain a recipe-owned exception +/// until this activation no longer needs catch replay. /// 5. `br arm.label` (original handler). fn emit_capture_save_and_branch( module: &mut Module, func_id: FunctionId, cap_seq_id: InstrSeqId, - active_arm: LocalId, arm: &PlainCatchArmState, - in_catch_local: LocalId, - catch_region_id_local: LocalId, - catch_region_id: u32, + catch_selector_local: LocalId, + retained_recipe_exnref: Option, ) { let local = local_mut(module, func_id); let s = &mut local.block_mut(cap_seq_id).instrs; + // CatchRef appends exnref after the tag payload, so it is first off the + // stack. A converted plain Catch also arrives here through CatchRef when + // the typed Wasm codec must own a reference/v128-bearing payload. + if let Some(captured_exnref) = arm.captured_exnref { + push_instr( + s, + Instr::LocalSet(LocalSet { + local: captured_exnref, + }), + ); + } + // 1. Spill operands. Operands were declared L-to-R but appear on // the stack with the LAST one on top — so we spill in reverse // declaration order: spills[M-1] first, then [M-2], ..., [0]. @@ -4821,44 +6432,99 @@ fn emit_capture_save_and_branch( ); } - // 2. Record which arm owns the operand locals for this activation. - push_instr( - s, - Instr::Const(Const { - value: Value::I32(arm.arm.arm_idx as i32), - }), - ); - push_instr(s, Instr::LocalSet(LocalSet { local: active_arm })); - - // 3. Set flags. - push_instr( - s, - Instr::Const(Const { - value: Value::I32(1), - }), - ); - push_instr( - s, - Instr::LocalSet(LocalSet { - local: in_catch_local, - }), - ); + // 2. Record the dynamically latest catch for this activation. + // + // WHY: replay needs the most recently taken exception edge, not the + // lexically last try_table that ever caught. A loop can execute region B + // and later region A, and nested handlers can likewise supersede one + // another. One activation-local selector naturally follows that dynamic + // order and avoids both stale per-region markers and one native i32 local + // per static try_table. push_instr( s, Instr::Const(Const { - value: Value::I32(catch_region_id as i32), + value: Value::I32(arm.selector as i32), }), ); push_instr( s, Instr::LocalSet(LocalSet { - local: catch_region_id_local, + local: catch_selector_local, }), ); + if !arm.arm.uses_exception_recipe { + if let Some(retained_recipe_exnref) = retained_recipe_exnref { + // WHY: a scalar catch has just superseded the only selector that + // could name the previous complete-exception recipe. Clear the + // function-wide slot before entering user code so the obsolete + // exception is neither serialized nor retained as a hidden GC + // root. Guest-visible references have independent typed owners. + push_instr( + s, + Instr::RefNull(RefNull { + ty: RefType { + nullable: true, + heap_type: HeapType::Abstract(AbstractHeapType::Exn), + }, + }), + ); + push_instr( + s, + Instr::LocalSet(LocalSet { + local: retained_recipe_exnref, + }), + ); + } + } // 4. Re-push operands in declaration order. - for &operand in &arm.operand_locals { - push_instr(s, Instr::LocalGet(LocalGet { local: operand })); + for (&operand, &ty) in arm.operand_locals.iter().zip(&arm.arm.operand_tys) { + push_typed_local_get(s, operand, ty); + } + // Capture-only payload locals must not retain reference values as hidden + // GC roots after the branch. The already-pushed handler operands remain on + // the operand stack while these nullable storage locals are cleared. + for (&operand, &ty) in arm.operand_locals.iter().zip(&arm.arm.operand_tys) { + let ValType::Ref(mut reference) = ty else { + continue; + }; + reference.nullable = true; + push_instr(s, Instr::RefNull(RefNull { ty: reference })); + push_instr(s, Instr::LocalSet(LocalSet { local: operand })); + } + if arm.arm.kind.is_ref() { + let captured_exnref = arm + .captured_exnref + .expect("CatchRef capture must own an exnref local"); + push_instr( + s, + Instr::LocalGet(LocalGet { + local: captured_exnref, + }), + ); + push_instr(s, Instr::RefAsNonNull(RefAsNonNull {})); + } + if let Some(captured_exnref) = arm + .captured_exnref + .filter(|_| !arm.arm.uses_exception_recipe) + { + // Scalar CatchRef replay reconstructs from the tag/payload, so its + // forwarding local is scratch and must not retain a stale exception. + push_instr( + s, + Instr::RefNull(RefNull { + ty: RefType { + nullable: true, + heap_type: HeapType::Abstract(AbstractHeapType::Exn), + }, + }), + ); + push_instr( + s, + Instr::LocalSet(LocalSet { + local: captured_exnref, + }), + ); } // 5. Branch to original handler. @@ -4895,26 +6561,18 @@ fn find_try_table_parent_seq<'a>( // (docs/plans/2026-05-13-fork-instrument-megaPR-eliminate-guard-dispatch-and-modern-EH-plan.md) // ===================================================================== // -// Note: guard-dispatch was deleted in commit 4 (2026-05-14) after the -// 2.5/2.6 sub-commits absorbed UnsupportedCarryover/MultiValueParams -// into nested switch-dispatch, commit 9's modern-EH SDK flip -// eliminated UnsupportedLegacyTry from shipping wasm, and commit 3 -// replaced the two `instrument_one_function_guard_dispatch` callers -// with panics. The trampoline scaffolding below remains UNWIRED in -// shipping fork-instrument runs; it's preserved for future use if a -// new "genuinely impossible for switch-dispatch" case ever emerges. +// Guard-dispatch was deleted after nested switch-dispatch absorbed structured +// carryovers and multi-value parameters. Fork-reachable legacy handlers are +// now normalized to activation-owned modern EH before this file runs. The +// trampoline scaffolding below remains unwired historical implementation. // -// Replaces guard-dispatch as the fallback for the three classes -// switch-dispatch can't handle today: +// It was originally intended for three historical classes: // (a) Nested fork-path call inside a Loop/IfElse/TryTable body that -// `classify_nested_pattern` rejects (UnsupportedLegacyTry, -// UnsupportedMultiValueParams, UnsupportedCarryover). +// `classify_nested_pattern` could not type. // (b) Top-level fork-path call with operand-stack carryover. // (c) Nested call_indirect to a fork-path callee, in combination -// with another unsupported pattern. (Simple nested call_indirect -// in a loop is empirically already handled by nested switch- -// dispatch — see crates/fork-instrument/tests/trampoline.rs's -// `today_nested_call_indirect_uses_nested_switch_dispatch`.) +// with a carryover shape absent from the old typed model. +// Nested switch-dispatch now owns all three. // // Per-function dispatch table (open Q #3, resolved 2026-05-13): // each instrumented fork-path function emits its own @@ -4924,10 +6582,9 @@ fn find_try_table_parent_seq<'a>( // // State after sub-commit 2.2 (this commit): the function below is // defined but UNREACHABLE — no caller exists. The body emission -// lands in 2.3; sub-commits 2.4 (carryover), 2.5 (call_indirect + -// pattern), 2.6 (nested unsupported) wire callers one class at a -// time. Once 2.6 ships, guard-dispatch is unreachable; commits 3-4 -// verify and delete it. +// landed in 2.3; later sub-commits wired carryovers, call_indirect, and +// nested typed state one class at a time. Guard-dispatch has since been +// deleted. /// Emit a per-function funcref dispatch table populated with the /// extracted post-call functions for one fork-path function. @@ -5100,8 +6757,6 @@ fn instrument_one_function_trampoline_dispatch( _runtime: &Runtime, _fork_path: &HashSet, _func_ordinal: u32, - _aux_tables: &AuxTables, - _ref_plan: &[RefLocalSlot], _catch_plan: &[CatchRegionPlan], _plain_catches: &[(InstrSeqId, Vec)], ) { @@ -5142,21 +6797,13 @@ fn instrument_one_function_trampoline_dispatch( // back via `local.get` in the post-call sequence. // // MVP supported nesting: `Block` (any result type), `IfElse`, -// `Loop`, `TryTable` body. Unsupported (routes to guard-dispatch): -// legacy `Try`, multi-value-params blocks, sub-region landings whose -// preceding chunk has a stack carryover. +// `Loop`, `TryTable`, or normalized legacy-handler body. #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum NestedSupportStatus { Supported, - UnsupportedLegacyTry, - /// Sub-commit 2.6c: no longer produced — multi-value-params - /// SubRegions now route to nested switch-dispatch via the body- - /// param prespill + reload mechanism in `transform_region_seq`. - /// Kept as a documented enum variant for future defensive use - /// (e.g., if a shape regression appears). - #[allow(dead_code)] - UnsupportedMultiValueParams, - UnsupportedCarryover, + /// Exhaustive stack effects disagreed with already-validated Wasm IR. + /// This is an instrumenter invariant failure, never a source-shape policy. + AnalysisInvariantFailed, } impl NestedSupportStatus { @@ -5165,18 +6812,9 @@ impl NestedSupportStatus { } } -/// Classify a function's nesting pattern. MVP scope: returns -/// `Supported` iff every fork-path call in the function lives inside a -/// chain of `Block` bodies only (any depth), no enclosing `IfElse` / -/// `Loop` / `TryTable` / legacy `Try`, no multi-value-params blocks, -/// no nested-seq stack carryovers. -/// -/// This narrow scope handles the popen-class regression — popen's -/// `__fork` and `posix_spawn` reach `kernel_fork` through `block` -/// nesting (no IfElse around the fork-path call). Functions with -/// IfElse-around-fork-call still fall back to guard-dispatch (today's -/// behavior); that's a known divergence-bug exposure but is not the -/// pattern that hangs popen on this branch. +/// Verify that nested switch-dispatch can statically type every activation +/// carryover. Structured control, multi-value parameters, modern EH, and +/// normalized legacy handlers all use this path. fn classify_nested_pattern( module: &Module, func_id: FunctionId, @@ -5187,256 +6825,70 @@ fn classify_nested_pattern( _ => return NestedSupportStatus::Supported, }; let status = classify_seq(module, local, local.entry_block(), fork_path); - if !status.is_supported() { - return status; - } - - // Sub-commit 2.5c (2026-05-14): direct fork-path call landings - // with operand-stack carryovers are absorbed by nested switch- - // dispatch via the per-call carryover-spilling extension wired in - // 2.5b. The carryover types must be statically determinable — - // fall back to guard-dispatch when the analyser can't type them, - // mirroring the policy used by top-level switch-dispatch's 2.4c - // gate in `instrument_one_function`. The seq-level check in - // `seq_has_unsupported_carryover` no longer rejects these; - // function-level here is the appropriate granularity (the - // analyser needs the whole function's call_idx assignment to - // produce its result). - if compute_nested_carryover_types(module, func_id, fork_path).is_none() { - return NestedSupportStatus::UnsupportedCarryover; - } - - NestedSupportStatus::Supported -} - -fn classify_seq( - module: &Module, - f: &LocalFunction, - seq: InstrSeqId, - fork_path: &HashSet, -) -> NestedSupportStatus { - let carryover = seq_has_unsupported_carryover(module, f, seq, fork_path); - if carryover { - return NestedSupportStatus::UnsupportedCarryover; - } - - for (instr, _) in &f.block(seq).instrs { - match instr { - Instr::Loop(_) | Instr::Block(_) | Instr::IfElse(_) | Instr::TryTable(_) => { - // Allowed. Loops, blocks, ifs, and try_tables are - // handled by per-block dispatch inside their body. - // For IfElse, the cond rewrite via `select` selects - // between original cond (NORMAL) and force-flag - // (REWIND). Try_table catches branch to outer - // labels — fork-path calls reachable only via a - // catch (fork-from-catch) are still unsupported, but - // are detected separately as "carryover" / "unknown - // stack-effect" patterns and routed to guard-dispatch. - } - Instr::Try(t) => { - // Legacy try bodies follow the same nested-switch route - // as block/loop/try_table bodies. Legacy catch handlers - // remain unsupported because REWIND cannot re-enter a - // handler without reconstructing the exception path. - for c in &t.catches { - let handler = match c { - LegacyCatch::Catch { handler, .. } => Some(*handler), - LegacyCatch::CatchAll { handler } => Some(*handler), - LegacyCatch::Delegate { .. } => None, - }; - if let Some(h) = handler { - if subtree_contains_fork_call(f, h, fork_path) { - return NestedSupportStatus::UnsupportedLegacyTry; - } - } - } - } - _ => {} - } - for child in nested_seqs(instr) { - // Sub-commit 2.6c (2026-05-14): multi-value-params - // SubRegions are now absorbed by nested switch-dispatch - // via the typed `CarryoverPlan::spill_locals` machinery - // (2.6a/2.6b). The Block's declared type-params are - // spilled at the chunk tail (like any other carryover) - // and pushed back BEFORE the SubRegion runs at - // emit_post_landing. The function-level - // `UnsupportedMultiValueParams` rejection here is gone. - let status = classify_seq(module, f, child, fork_path); - if !status.is_supported() { - return status; - } - } + if !status.is_supported() { + return status; } - NestedSupportStatus::Supported -} -fn subtree_contains_fork_call( - f: &LocalFunction, - seq: InstrSeqId, - fork_path: &HashSet, -) -> bool { - for (instr, _) in &f.block(seq).instrs { - match instr { - Instr::Call(c) if fork_path.contains(&c.func) => return true, - Instr::CallIndirect(_) => return true, - _ => {} - } - for child in nested_seqs(instr) { - if subtree_contains_fork_call(f, child, fork_path) { - return true; - } - } + // Direct fork-path call landings with operand-stack carryovers are + // absorbed by per-call typed spill locals. Failure here means the + // exhaustive stack model disagreed with validated IR. + if compute_nested_carryover_types(module, func_id, fork_path).is_none() { + return NestedSupportStatus::AnalysisInvariantFailed; } - false + + NestedSupportStatus::Supported } -/// Returns true iff `seq` contains a fork-path landing (direct call -/// or sub-region whose nested seq is fork-bearing) whose pre-landing -/// stack has a carryover — extra values left on the stack from before -/// the chunk's start that aren't part of the landing's required -/// inputs. Per-block dispatch's `POST_K` blocks are typed `Simple(None)` -/// (0 → 0), so carryovers can't be expressed. -/// -/// "Required inputs" per landing: -/// - Direct fork-path Call: the call's params count. -/// - CallIndirect: params + 1 (the table index). -/// - Block / Loop / TryTable: 0 (we already reject multi-value -/// params blocks elsewhere in classify_nested_pattern). -/// - IfElse: 1 (the cond). -/// -/// ## Known unfixed case: `tests/sortix/os-test/basic/spawn/posix_spawnattr_setpgroup` -O2 -/// -/// LLVM-O2 inlines `posix_spawn` into `main` and emits a sub-region -/// carryover at the `kernel_fork`-bearing block: -/// -/// ```text -/// local.get 0 ;; push __errno_location() — carryover -/// block (result i32) ;; the block contains kernel_fork -/// ... kernel_fork wrap ... -/// end -/// local.tee 1 ;; save posix_spawn return value -/// i32.store ;; *errno_location = posix_spawn_rc — consumes both -/// ``` -/// -/// We currently route this to **guard-dispatch** (because -/// switch-dispatch's `POST_K` blocks are 0 → 0 and can't express the -/// carryover). On `-O0`/`-O1` the function passes; on `-O2` it fails -/// with `waitpid: ECHILD` because the parent's local `pid` ends up at -/// 0 instead of the child's pid. The `pid` write happens through the -/// `&pid` pointer inside the inlined `posix_spawn` body, after the -/// kernel_fork rewind handshake — but some divergence specific to the -/// `-O2` shape causes that write not to take effect on the parent's -/// REWIND path. -/// -/// We **expected guard-dispatch + the c01554940 non-fork-path-call -/// gate + the LocalTee identity-passthrough fix to handle this case**, -/// but multiple sessions of debugging haven't pinned down the exact -/// divergence; the bug is highly LLVM-codegen-sensitive (adding a -/// single `fprintf`/`fflush` at the right spot makes it pass). The -/// current best understanding is that some pre-call op leaves the -/// stack or shadow-stack in a state that REWIND replay diverges from -/// NORMAL, despite all the targeted fixes. -/// -/// **Next step:** the proper fix is to extend per-block switch-dispatch -/// to handle carryovers at sub-region landings via local-spilling -/// (allocate spill locals, push values from carryover into them -/// before the enclosing instruction, reload after) — that takes the -/// function off the guard-dispatch path entirely and avoids the -/// divergence. Implementation begins immediately below; see -/// `partition_region_instrs` and `emit_chunk_tail_for_landing`. -/// -/// **It remains worth revisiting whether guard-dispatch could be -/// fixed for this case in the future** — a successful guard-dispatch -/// solution would cover any other LLVM-codegen-sensitive carryover -/// shape we discover later, not just the ones that fit the -/// switch-dispatch carryover-spilling extension. -#[allow(dead_code)] -fn seq_has_direct_fork_carryover( +fn classify_seq( module: &Module, f: &LocalFunction, seq: InstrSeqId, fork_path: &HashSet, -) -> bool { - let mut depth: usize = 0; +) -> NestedSupportStatus { + if seq_stack_analysis_invariant_failed(module, f, seq) { + return NestedSupportStatus::AnalysisInvariantFailed; + } + for (instr, _) in &f.block(seq).instrs { - // Check direct fork-path call landings. - let direct_expected: Option = match instr { - Instr::Call(c) if fork_path.contains(&c.func) => Some( - module - .types - .get(module.funcs.get(c.func).ty()) - .params() - .len(), - ), - Instr::CallIndirect(ci) => Some(module.types.get(ci.ty).params().len() + 1), - _ => None, - }; - if let Some(expected) = direct_expected { - if depth > expected { - return true; + match instr { + Instr::Loop(_) | Instr::Block(_) | Instr::IfElse(_) | Instr::TryTable(_) => { + // Loops, blocks, ifs, and try_tables are handled by + // per-block dispatch. For IfElse, the condition rewrite + // selects between the original condition (NORMAL) and the + // replay force flag. Catch handlers use activation-owned + // selector/payload or complete-exception recipes. } - } - - // Check sub-region landings: any enclosing instruction whose - // nested seq's subtree contains a fork-path call (so the - // partition would emit a SubRegion landing for it). - let is_subregion_landing = match instr { - Instr::Block(_) - | Instr::Loop(_) - | Instr::TryTable(_) - | Instr::Try(_) - | Instr::IfElse(_) => nested_seqs(instr) - .iter() - .any(|s| subtree_contains_fork_call(f, *s, fork_path)), - _ => false, - }; - if is_subregion_landing { - let subregion_expected = match instr { - Instr::IfElse(_) => 1, - _ => 0, - }; - if depth > subregion_expected { - return true; + Instr::Try(_) => { + // Fork-reachable legacy handlers were converted to modern + // try_table/catch_ref before this classifier runs. A surviving + // legacy try can therefore only be a handler-free delegate, + // whose body follows the ordinary nested-switch route. } + _ => {} } - - match top_level_stack_effect(module, f, instr) { - StackEffect::Delta { pops, pushes } => { - if depth < pops { - return true; - } - depth = depth - pops + pushes; + for child in nested_seqs(instr) { + // Multi-value SubRegion parameters use the same typed + // `CarryoverPlan::spill_locals` machinery: spill at the chunk + // tail and restore before entering the SubRegion. + let status = classify_seq(module, f, child, fork_path); + if !status.is_supported() { + return status; } - StackEffect::Terminator => return false, - StackEffect::Unknown => return true, } } - false + NestedSupportStatus::Supported } -/// Like `seq_has_direct_fork_carryover` but only flags carryovers -/// that the per-block switch-dispatch transform can NOT handle via -/// local-spilling. Currently: every carryover except a 1-i32 stack -/// item at a SubRegion (non-IfElse) landing whose enclosing -/// instruction produces 0 or 1 i32 result. +/// Check that the depth-only structured walk agrees with validated Wasm IR. /// -/// MVP rationale: in C-emitted wasm at -O2, the most common carryover -/// pattern is `local.get $ptr; block (result i32) { ... fork ... }; -/// local.tee; i32.store` — a single i32 (typically a pointer) pushed -/// before a fork-bearing block and consumed after. We spill the i32 -/// via `local.set $carryover_local` at the chunk tail, then reload it -/// after the block runs (juggling with a `tmp_result_local` if the -/// block produces an i32 result). -/// -/// Wider carryover patterns (multi-value, non-i32 types, carryovers at -/// DirectCall landings) still reject; extending support is -/// straightforward but not needed for the cases we've seen so far. -fn seq_has_unsupported_carryover( +/// All carryover types—including references, GC values, EH references, and +/// multi-value parameters/results—are handled by the typed spill planners. +/// A `true` result here therefore signals an analyzer invariant failure, not a +/// source shape the instrumenter intentionally excludes. +fn seq_stack_analysis_invariant_failed( module: &Module, f: &LocalFunction, seq: InstrSeqId, - fork_path: &HashSet, ) -> bool { // Sub-commit 2.6c: a Block/Loop/TryTable body with declared // type-params enters with those values already on the seq's @@ -5448,46 +6900,6 @@ fn seq_has_unsupported_carryover( _ => 0, }; for (instr, _) in &f.block(seq).instrs { - // Sub-commit 2.5c (2026-05-14): direct fork-path call landings - // with operand-stack carryovers are now absorbed by nested - // switch-dispatch via the carryover-spilling extension (see - // `compute_nested_carryover_types` + the per-call - // `carryover_spills` wiring in `instrument_one_function_nested_switch`). - // The function-level fallback for shapes the analyser can't - // statically type lives at `classify_nested_pattern`. No - // per-seq direct-call rejection here. - - // Sub-region landings. - let is_subregion = match instr { - Instr::Block(_) - | Instr::Loop(_) - | Instr::TryTable(_) - | Instr::Try(_) - | Instr::IfElse(_) => nested_seqs(instr) - .iter() - .any(|s| subtree_contains_fork_call(f, *s, fork_path)), - _ => false, - }; - if is_subregion { - let is_ifelse = matches!(instr, Instr::IfElse(_)); - // Sub-commit 2.6b: a SubRegion's `expected_input` includes - // both the cond (for IfElse) AND any declared type-params - // (for multi-value-params Block/Loop/TryTable). Values - // beyond that are real "extra carryover" above the params. - // The 2.6a analyser spills both bands uniformly into - // `CarryoverPlan::spill_locals`, so multi-value-params - // SubRegions are no longer rejected — their params are - // just one source of spill values. - let subregion_params = subregion_input_param_count(module, f, instr); - let expected_input: usize = if is_ifelse { 1 } else { subregion_params }; - let carryover_depth = depth.saturating_sub(expected_input); - if carryover_depth > 0 { - // Otherwise: this is a supported extra-carryover - // (possibly combined with multi-value params, both - // spilled together by 2.6a's analyser). - } - } - match top_level_stack_effect(module, f, instr) { StackEffect::Delta { pops, pushes } => { if depth < pops { @@ -5496,32 +6908,11 @@ fn seq_has_unsupported_carryover( depth = depth - pops + pushes; } StackEffect::Terminator => return false, - StackEffect::Unknown => return true, } } false } -/// Sub-commit 2.6b: count the declared type-params of a SubRegion -/// (Block/Loop/TryTable). Returns 0 for simple (non-multi-value) -/// signatures, and 0 for non-SubRegion instructions. -fn subregion_input_param_count(module: &Module, f: &LocalFunction, instr: &Instr) -> usize { - let body_seq = match instr { - Instr::Block(b) => Some(b.seq), - Instr::Loop(l) => Some(l.seq), - Instr::TryTable(t) => Some(t.seq), - Instr::Try(t) => Some(t.seq), - _ => None, - }; - let Some(seq) = body_seq else { - return 0; - }; - match f.block(seq).ty { - InstrSeqType::MultiValue(ty_id) => module.types.get(ty_id).params().len(), - _ => 0, - } -} - /// For each non-IfElse SubRegion landing in a fork-bearing seq, /// returns the full Vec of values to spill at that landing — /// covering BOTH the SubRegion's type-params (consumed on entry) AND @@ -5547,10 +6938,9 @@ fn subregion_input_param_count(module: &Module, f: &LocalFunction, instr: &Instr /// value is the original condition and preceding values are restored /// as carryovers below the condition. /// -/// Returns `None` if any producer in this seq pushes a value whose -/// type can't be statically determined AND that value ends up in a -/// SubRegion's spill list. Producers that push unknown-type values -/// consumed before any SubRegion landing are harmless. +/// Returns `None` only if the exhaustive producer model disagrees with +/// validated IR. Reference, GC, EH, and multi-value spill types are all +/// preserved exactly. fn analyze_subregion_spill_types( module: &Module, f: &LocalFunction, @@ -5586,6 +6976,7 @@ fn analyze_subregion_spill_types( let is_fork_landing = match instr { Instr::Call(c) => fork_path.contains(&c.func), Instr::CallIndirect(_) => true, + Instr::CallRef(_) => true, _ => false, }; if is_fork_landing && direct_cursor < direct_idxs_at_this_seq.len() { @@ -5625,10 +7016,8 @@ fn analyze_subregion_spill_types( } } - // Advance the typed stack. Same logic as - // `walk_seq_for_carryovers` — known producers push `Some(ty)`, - // unknown producers push `None`. Fork-path Call/CallIndirect - // pops args and pushes typed results. + // Advance the exact typed stack. Fork-path Call/CallIndirect/CallRef + // pops args and pushes its declared result types. match instr { Instr::Call(c) if fork_path.contains(&c.func) => { let sig = module.types.get(module.funcs.get(c.func).ty()); @@ -5654,6 +7043,18 @@ fn analyze_subregion_spill_types( } continue; } + Instr::CallRef(call) => { + let sig = module.types.get(call.ty); + let n_args = sig.params().len() + 1; + if stack.len() < n_args { + return None; + } + stack.truncate(stack.len() - n_args); + for &ty in sig.results() { + stack.push(Some(ty)); + } + continue; + } _ => {} } @@ -5667,105 +7068,26 @@ fn analyze_subregion_spill_types( if pushes == 0 { continue; } - match instr { - Instr::Call(c) => { - let sig = module.types.get(module.funcs.get(c.func).ty()); - for &ty in sig.results() { - stack.push(Some(ty)); - } - continue; - } - Instr::CallRef(cr) => { - let sig = module.types.get(cr.ty); - for _ in sig.results() { - stack.push(None); - } - continue; - } - Instr::Block(b) => { - push_structured_results(&mut stack, module, f, b.seq, pushes); - continue; - } - Instr::Loop(l) => { - push_structured_results(&mut stack, module, f, l.seq, pushes); - continue; - } - Instr::IfElse(ie) => { - push_structured_results(&mut stack, module, f, ie.consequent, pushes); - continue; - } - Instr::TryTable(t) => { - push_structured_results(&mut stack, module, f, t.seq, pushes); - continue; - } - Instr::Try(t) => { - push_structured_results(&mut stack, module, f, t.seq, pushes); - continue; - } - _ => {} + let produced = typed_instruction_pushes(module, f, instr, &pre_stack)?; + if produced.len() != pushes { + return None; } - debug_assert_eq!(pushes, 1); - stack.push(typed_single_push(module, instr, &pre_stack)); + stack.extend(produced.into_iter().map(Some)); } StackEffect::Terminator => return Some(out), - StackEffect::Unknown => return None, } } Some(out) } -fn has_fork_call_in_catch_handler( - module: &Module, - func_id: FunctionId, - fork_path: &HashSet, -) -> bool { - let local = match &module.funcs.get(func_id).kind { - FunctionKind::Local(l) => l, - _ => return false, - }; - fn walk(f: &LocalFunction, seq: InstrSeqId, fork_path: &HashSet) -> bool { - for (instr, _) in &f.block(seq).instrs { - if let Instr::TryTable(tt) = instr { - for c in &tt.catches { - let handler = match c { - TryTableCatch::Catch { label, .. } - | TryTableCatch::CatchAll { label } - | TryTableCatch::CatchRef { label, .. } - | TryTableCatch::CatchAllRef { label } => *label, - }; - // try_table catch labels target an enclosing block, - // so a fork call in the body of THAT block is a - // fork-from-catch candidate. We approximate: if the - // handler label is reachable from a fork-path call - // in the function. Since walrus IR doesn't easily - // give us "code AT label X", we rely on the simpler - // existing detection in classify_nested_pattern - // which flags TryTable bodies; if you got here via - // the fall-through guard-dispatch path, B1 status - // is already known. Here we conservatively return - // false — let guard-dispatch handle (today's - // behavior). - let _ = handler; - } - } - for child in nested_seqs(instr) { - if walk(f, child, fork_path) { - return true; - } - } - } - false - } - walk(local, local.entry_block(), fork_path) -} - // --- Discovery: walk the function in DFS order, assigning call_idx -- #[derive(Debug, Clone, Copy)] enum NestedTarget { Direct(FunctionId), Indirect { table: TableId }, + Ref, } #[derive(Debug, Clone)] @@ -5773,7 +7095,9 @@ struct NestedCallSite { call_idx: u32, seq_id: InstrSeqId, target: NestedTarget, + direct_activation: bool, sig_ty: TypeId, + resume_ty: Option, loc: InstrLocId, } @@ -5846,7 +7170,9 @@ fn walk_discover( call_idx: idx, seq_id: seq, target: NestedTarget::Direct(c.func), + direct_activation: false, sig_ty: module.funcs.get(c.func).ty(), + resume_ty: None, loc: *loc, }); my_idxs.push(idx); @@ -5858,7 +7184,23 @@ fn walk_discover( call_idx: idx, seq_id: seq, target: NestedTarget::Indirect { table: ci.table }, + direct_activation: false, sig_ty: ci.ty, + resume_ty: None, + loc: *loc, + }); + my_idxs.push(idx); + } + Instr::CallRef(call) => { + let idx = *next_idx; + *next_idx += 1; + sites.push(NestedCallSite { + call_idx: idx, + seq_id: seq, + target: NestedTarget::Ref, + direct_activation: false, + sig_ty: call.ty, + resume_ty: None, loc: *loc, }); my_idxs.push(idx); @@ -5891,6 +7233,40 @@ fn walk_discover( } } +fn assert_nested_reference_call_alignment( + analysis: &FunctionReferenceAnalysis, + sites: &[NestedCallSite], +) { + assert_eq!( + analysis.call_sites.len(), + sites.len(), + "original reference analysis and nested transform discovered different call counts" + ); + for (reference, site) in analysis.call_sites.iter().zip(sites) { + let aligned = match (reference.kind, site.target) { + (OriginalCallKind::Direct(expected), NestedTarget::Direct(actual)) => { + expected == actual + } + ( + OriginalCallKind::Indirect { + table: expected_table, + ty: expected_ty, + }, + NestedTarget::Indirect { + table: actual_table, + }, + ) => expected_table == actual_table && expected_ty == site.sig_ty, + (OriginalCallKind::Ref { ty }, NestedTarget::Ref) => ty == site.sig_ty, + _ => false, + }; + assert!( + aligned, + "reference analysis call {:?} does not align with nested target {:?}", + reference.kind, site.target + ); + } +} + // --- The main transform ---------------------------------------------- #[allow(clippy::too_many_arguments)] @@ -5898,13 +7274,14 @@ fn instrument_one_function_nested_switch( module: &mut Module, func_id: FunctionId, runtime: &Runtime, + activations: &HashSet, fork_path: &HashSet, func_ordinal: u32, - aux_tables: &AuxTables, - ref_plan: &[RefLocalSlot], catch_plan: &[CatchRegionPlan], plain_catches: &[(InstrSeqId, Vec)], -) { + reference_analysis: &FunctionReferenceAnalysis, + unwind_frame_select: FunctionId, +) -> ResumeThunk { // Pre-existing user locals. let all_user_locals = collect_user_locals(module, func_id); let user_scalar_locals: Vec<(LocalId, ValType)> = all_user_locals @@ -5915,24 +7292,33 @@ fn instrument_one_function_nested_switch( // Discover all fork-path call sites (with assigned call_idxs in // DFS order) and the per-seq region info. - let (sites, regions) = discover_calls_and_regions(module, func_id, fork_path); + let (mut sites, regions) = discover_calls_and_regions(module, func_id, fork_path); + for site in &mut sites { + site.direct_activation = matches!( + site.target, + NestedTarget::Direct(target) if activations.contains(&target) + ); + let results = module.types.get(site.sig_ty).results().to_vec(); + site.resume_ty = Some(module.types.add(&[], &results)); + } + assert_nested_reference_call_alignment(reference_analysis, &sites); let n_calls = sites.len(); if n_calls == 0 { // Defensive: function should have at least one fork-path call // by virtue of being in fork_path. Bail out to existing // top-level switch-dispatch (which handles n_calls==0 cleanly). - instrument_one_function_switch( + return instrument_one_function_switch( module, func_id, runtime, + activations, fork_path, func_ordinal, - aux_tables, - ref_plan, catch_plan, plain_catches, + reference_analysis, + unwind_frame_select, ); - return; } // `HashMap` deliberately randomizes its iteration order. Keep one stable @@ -5955,7 +7341,7 @@ fn instrument_one_function_nested_switch( }; // Plan per-call argument materialization before allocating the - // frame. Pure scalar argument tails are replayed after POST_K; + // frame. Side-effect-free argument tails are replayed after POST_K; // all other shapes keep the existing frame-backed spill locals. let mut pending_arg_materializations: HashMap = HashMap::new(); @@ -5980,16 +7366,6 @@ fn instrument_one_function_nested_switch( .find(|site| site.call_idx == call_idx) .expect("call_idx must have a discovered site"); let arg_types = nested_call_arg_types(module, site); - for &ty in &arg_types { - if !is_scalar(ty) { - let name = func_name(module, func_id); - panic!( - "fork-instrument: function `{name}` has a nested fork-path call \ - with a ref-typed argument ({ty:?}). Aux-table arg spilling \ - is not yet supported in the nested per-block transform." - ); - } - } pending_arg_materializations.insert( call_idx, plan_call_arg_materialization(module, &chunks[landing_idx], arg_types), @@ -6018,21 +7394,23 @@ fn instrument_one_function_nested_switch( // round-trip through the fork frame so REWIND can reload them // beneath the call's result. // - // `compute_nested_carryover_types` may return `None` for shapes - // it can't statically type. Until sub-commit 2.5c flips the - // rejection in `seq_has_unsupported_carryover`, the only seqs - // that actually reach this point have already passed that check, - // so `None` here is unexpected — but we treat it identically to - // "no carryovers at any call" for safety (matches 2.4c behavior). + // Classification already proved the exact typed stack model. Do not turn + // a later analyzer disagreement into empty carryovers: that would silently + // lose activation state. let nested_carryover_types: HashMap> = - compute_nested_carryover_types(module, func_id, fork_path).unwrap_or_default(); - let mut carryover_spills: HashMap> = HashMap::new(); + compute_nested_carryover_types(module, func_id, fork_path).unwrap_or_else(|| { + panic!("typed nested carryover analysis changed after classification") + }); + let mut carryover_spills: HashMap> = HashMap::new(); for site in &sites { let cr_types: &[ValType] = nested_carryover_types .get(&site.call_idx) .map(Vec::as_slice) - .unwrap_or(&[]); - let spills: Vec = cr_types.iter().map(|&ty| module.locals.add(ty)).collect(); + .unwrap_or_else(|| panic!("typed carryover plan omitted call {}", site.call_idx)); + let spills: Vec = cr_types + .iter() + .map(|&ty| (module.locals.add(spill_storage_type(ty)), ty)) + .collect(); carryover_spills.insert(site.call_idx, spills); } @@ -6050,35 +7428,30 @@ fn instrument_one_function_nested_switch( .iter() .zip(arg_types.iter()) { - frame_scalars.push((lid, ty)); + if is_scalar(ty) { + frame_scalars.push((lid, ty)); + } } } for site in &sites { - let cr_types: &[ValType] = nested_carryover_types - .get(&site.call_idx) - .map(Vec::as_slice) - .unwrap_or(&[]); - for (&lid, &ty) in carryover_spills[&site.call_idx].iter().zip(cr_types.iter()) { - frame_scalars.push((lid, ty)); + for &(lid, ty) in &carryover_spills[&site.call_idx] { + if is_scalar(ty) { + frame_scalars.push((lid, ty)); + } } } - append_plain_catch_frame_scalars(&mut frame_scalars, &plain_catch_state); - // Synthetic locals. let catch_state_locals = if catch_plan.is_empty() && plain_catches.is_empty() { None } else { Some(CatchStateLocals { - catch_region_id: module.locals.add(ValType::I32), - exnref_slot: module.locals.add(ValType::I32), + catch_selector: module.locals.add(ValType::I32), }) }; // Tmp i32 used by the IfElse cond rewrite to swap stack order // (preserve original cond while computing force_flag and // is_rewind without touching the operand stack). let cond_swap_local = module.locals.add(ValType::I32); - let abort_live_frame = module.locals.add(ValType::I32); - // Pre-pass: walk each fork-bearing seq, identify its // SubRegion-with-1-i32-carryover landings, and pre-allocate spill // locals (+ tmp_result_local for blocks producing 1 i32). The @@ -6102,24 +7475,25 @@ fn instrument_one_function_nested_switch( let mut pending_plans: Vec<(InstrSeqId, usize, PendingCarryoverPlan)> = Vec::new(); for &seq_id in ®ion_ids { let direct = direct_idxs_per_seq.get(&seq_id).unwrap_or(&empty_idxs); - // Sub-commit 2.6a: typed analyser captures per-landing - // spill ValTypes (covering both SubRegion type-params and - // any extra carryover above them). None on unanalyzable - // shapes — `classify_nested_pattern` already gated on a - // best-effort version of this, so None here is the same - // conservative fallback (function routes to guard-dispatch - // unless 2.5c/2.6a's combined gates accepted it). + // The typed analyser captures both SubRegion parameters and extra + // carryovers. An analysis failure here is an internal invariant; + // defaulting to no spills would silently lose activation state. let spill_types = analyze_subregion_spill_types( module, local_ro, seq_id, fork_path, direct, ®ions, ) - .unwrap_or_default(); + .unwrap_or_else(|| { + panic!("typed SubRegion carryover analysis failed after classification") + }); let original = &local_ro.block(seq_id).instrs; let (chunks, landings) = partition_region_instrs(local_ro, original, direct, ®ions, fork_path); + assert_eq!( + spill_types.len(), + landings.len(), + "typed SubRegion carryover analysis and landing partition disagree" + ); for (landing_idx, landing) in landings.iter().enumerate() { - let Some(types) = spill_types.get(landing_idx) else { - continue; - }; + let types = &spill_types[landing_idx]; if types.is_empty() { continue; } @@ -6130,7 +7504,7 @@ fn instrument_one_function_nested_switch( }; if pure_allowed { if let Some((tail_len, tail)) = - split_pure_scalar_tail(module, &chunks[landing_idx], types) + split_pure_replay_tail(module, &chunks[landing_idx], types) { pending_plans.push(( seq_id, @@ -6164,8 +7538,10 @@ fn instrument_one_function_nested_switch( PendingCarryoverPlan::Spill { types } => { let mut spill_locals: Vec<(LocalId, ValType)> = Vec::with_capacity(types.len()); for &ty in &types { - let lid = module.locals.add(ty); - frame_scalars.push((lid, ty)); + let lid = module.locals.add(spill_storage_type(ty)); + if is_scalar(ty) { + frame_scalars.push((lid, ty)); + } spill_locals.push((lid, ty)); } CarryoverPlan::Spill { spill_locals } @@ -6218,7 +7594,7 @@ fn instrument_one_function_nested_switch( for (seq_id, types) in to_allocate { let mut locals: Vec<(LocalId, ValType)> = Vec::with_capacity(types.len()); for &ty in &types { - let lid = module.locals.add(ty); + let lid = module.locals.add(spill_storage_type(ty)); locals.push((lid, ty)); } body_param_locals.insert(seq_id, locals); @@ -6226,7 +7602,59 @@ fn instrument_one_function_nested_switch( } let locals_with_offsets = assign_local_offsets(&frame_scalars, LOCALS_START_OFFSET); - let frame_size = HEADER_SIZE + user_locals_size(&frame_scalars); + let ordinary_scalar_end = HEADER_SIZE + user_locals_size(&frame_scalars); + let catch_scalar_frame = plan_plain_catch_scalar_frame(&plain_catch_state, ordinary_scalar_end); + let scalar_end = catch_scalar_frame.frame_end(ordinary_scalar_end); + let mut per_call_references = vec![Vec::new(); n_calls]; + for site in &sites { + let call_idx = site.call_idx as usize; + arg_materializations[&site.call_idx] + .append_reference_inputs(module, &mut per_call_references[call_idx]); + for &(local, ty) in &carryover_spills[&site.call_idx] { + if let Some(reference) = supported_reference(ty) { + per_call_references[call_idx].push((local, reference)); + } + } + } + { + let local_ro = match &module.funcs.get(func_id).kind { + FunctionKind::Local(local) => local, + _ => unreachable!(), + }; + let empty_idxs = Vec::new(); + for &seq_id in ®ion_ids { + let direct = direct_idxs_per_seq.get(&seq_id).unwrap_or(&empty_idxs); + let original = &local_ro.block(seq_id).instrs; + let (_, landings) = + partition_region_instrs(local_ro, original, direct, ®ions, fork_path); + for (landing_idx, landing) in landings.iter().enumerate() { + let Some(CarryoverPlan::Spill { spill_locals }) = + carryover_plans.get(&(seq_id, landing_idx)) + else { + continue; + }; + let (first, last) = match landing.kind { + LandingKind::SubRegion { range_lo, range_hi } + | LandingKind::SubRegionIfElse { + range_lo, range_hi, .. + } => (range_lo, range_hi), + LandingKind::DirectCall { .. } => continue, + }; + for &(local, ty) in spill_locals { + let Some(reference) = supported_reference(ty) else { + continue; + }; + for call_idx in first..=last { + per_call_references[call_idx as usize].push((local, reference)); + } + } + } + } + } + append_resume_parameter_references(module, func_id, &mut per_call_references); + append_plain_catch_frame_references(&mut per_call_references, &plain_catch_state); + let reference_frame = plan_reference_frame(module, reference_analysis, per_call_references); + let frame_size = reference_frame.frame_end(scalar_end); let result_types: Vec = { let ty_id = module.funcs.get(func_id).ty(); @@ -6234,28 +7662,19 @@ fn instrument_one_function_nested_switch( }; let restart_loop_ty = InstrSeqType::new(&mut module.types, &[], &result_types); - // Plan catch handlers (Phase 6d). These remain dead code for the - // nested transform's MVP (no fork-from-catch), but the plumbing is - // preserved for ref-typed exnref locals that still round-trip. - let catch_handlers = - plan_catch_ref_handlers(module, func_id, catch_plan, aux_tables, &plain_catch_state); + let catch_handlers = plan_catch_handlers(catch_plan, &plain_catch_state); let memory = first_memory(module); let ptr_ty = runtime.buf_type; - // Phase 6c rewind-throw stubs (still emitted for try_table bodies - // without fork-path calls — preserves the exnref serialization - // path). Extended by B1 Stage 2 Task 2.3 with plain-catch arm - // dispatch when `plain_catches` lists arms for the region. - if !catch_plan.is_empty() && aux_tables.exnref.is_some() { + if !plain_catch_state.is_empty() { let catch_state = - catch_state_locals.expect("exnref catch plan requires catch-state locals"); + catch_state_locals.expect("tagged catch plan requires catch-state locals"); inject_rewind_throw_stubs( module, func_id, runtime, - catch_state.catch_region_id, - aux_tables, + catch_state.catch_selector, catch_plan, &plain_catch_state, ); @@ -6277,9 +7696,20 @@ fn instrument_one_function_nested_switch( .id(); let restart_loop = local.builder_mut().dangling_instr_seq(restart_loop_ty).id(); let abort = AbortDispatch { - live_frame: abort_live_frame, restart_loop, + frame_select: unwind_frame_select, }; + let catch_scalar_restore_dispatch = catch_state_locals.and_then(|catch_state| { + build_plain_catch_scalar_dispatch( + local, + runtime, + memory, + ptr_ty, + catch_state.catch_selector, + &catch_scalar_frame, + PlainCatchScalarIo::Restore, + ) + }); populate_preamble_then( local, @@ -6289,8 +7719,8 @@ fn instrument_one_function_nested_switch( ptr_ty, catch_state_locals, &locals_with_offsets, - ref_plan, - aux_tables, + catch_scalar_restore_dispatch, + &reference_frame, frame_size, ); @@ -6390,6 +7820,19 @@ fn instrument_one_function_nested_switch( // Build postamble — same as switch-dispatch. let mut postamble: Vec<(Instr, InstrLocId)> = Vec::new(); + let catch_scalar_save_dispatch = catch_state_locals.and_then(|catch_state| { + build_plain_catch_scalar_dispatch( + local, + runtime, + memory, + ptr_ty, + catch_state.catch_selector, + &catch_scalar_frame, + PlainCatchScalarIo::Save, + ) + }); + let reference_save_dispatch = + build_reference_save_dispatch(local, runtime, memory, ptr_ty, &reference_frame); populate_postamble( &mut postamble, runtime, @@ -6397,14 +7840,13 @@ fn instrument_one_function_nested_switch( ptr_ty, catch_state_locals, &locals_with_offsets, - ref_plan, - aux_tables, + catch_scalar_save_dispatch, + reference_save_dispatch, frame_size, func_ordinal, - &result_types, ); - // Wrap entry block with [preamble-if-else, Block(unwind_save), postamble]. + // Wrap entry block with [preamble-if-else, live-restart loop]. // The entry block's instrs (set by transform_entry_region) become // the body of `unwind_save`. We pull them out and place them inside // unwind_save here, then install the wrapper structure in entry. @@ -6414,8 +7856,7 @@ fn instrument_one_function_nested_switch( let s = &mut local.block_mut(unwind_save).instrs; s.extend(entry_body); } - - let entry_seq = &mut local.block_mut(restart_loop).instrs; + let entry_seq = &mut local.block_mut(entry_id).instrs; push_instr( entry_seq, Instr::GlobalGet(GlobalGet { @@ -6434,24 +7875,6 @@ fn instrument_one_function_nested_switch( op: BinaryOp::I32GeU, }), ); - push_instr( - entry_seq, - Instr::LocalGet(LocalGet { - local: abort_live_frame, - }), - ); - push_instr( - entry_seq, - Instr::Unop(walrus::ir::Unop { - op: UnaryOp::I32Eqz, - }), - ); - push_instr( - entry_seq, - Instr::Binop(Binop { - op: BinaryOp::I32And, - }), - ); push_instr( entry_seq, Instr::IfElse(IfElse { @@ -6459,27 +7882,39 @@ fn instrument_one_function_nested_switch( alternative: preamble_else, }), ); - push_instr(entry_seq, Instr::Block(Block { seq: unwind_save })); - entry_seq.extend(postamble); - let entry_seq = &mut local.block_mut(entry_id).instrs; - entry_seq.clear(); push_instr(entry_seq, Instr::Loop(Loop { seq: restart_loop })); + let restart_seq = &mut local.block_mut(restart_loop).instrs; + push_instr(restart_seq, Instr::Block(Block { seq: unwind_save })); + restart_seq.extend(postamble); - apply_catch_ref_handlers(module, func_id, &catch_handlers, aux_tables); - - // Stage 2 (B1) plain-catch capture-block emission. Runs AFTER - // Phase 6 so it sees post-Phase-6 try_table locations. + // Tagged-catch capture emission runs after the nested body rebuild. if let Some(catch_state) = catch_state_locals { + shield_private_unwind_from_user_catches(module, func_id, runtime); apply_plain_catch_handlers( module, func_id, - catch_state.catch_region_id, + catch_state.catch_selector, &plain_catch_state, - catch_plan, &catch_handlers, ); } else { debug_assert!(plain_catches.is_empty()); + shield_private_unwind_from_user_catches(module, func_id, runtime); + } + + ResumeThunk { + func_ordinal, + function: emit_resume_thunk( + module, + func_id, + runtime, + memory, + ptr_ty, + frame_size, + &locals_with_offsets, + &reference_frame, + func_ordinal, + ), } } @@ -6498,7 +7933,7 @@ fn emit_chunk_tail_for_landing( out: &mut Vec<(Instr, InstrLocId)>, landing: &LandingInfo, arg_materializations: &HashMap, - carryover_spills: &HashMap>, + carryover_spills: &HashMap>, cond_swap_local: LocalId, ) { match &landing.kind { @@ -6510,7 +7945,7 @@ fn emit_chunk_tail_for_landing( // matching top-level switch-dispatch's 2.4c behavior). // `carryover_spills` is keyed by call_idx; an absent entry // is treated as no-carryover. - let empty: Vec = Vec::new(); + let empty: Vec = Vec::new(); let cr = carryover_spills.get(call_idx).unwrap_or(&empty); emit_spill_call_tail(out, &arg_materializations[call_idx], cr); } @@ -6551,8 +7986,10 @@ fn emit_chunk_tail_for_landing( fn nested_call_arg_types(module: &Module, site: &NestedCallSite) -> Vec { let mut arg_types: Vec = module.types.get(site.sig_ty).params().to_vec(); - if matches!(site.target, NestedTarget::Indirect { .. }) { - arg_types.push(ValType::I32); + match site.target { + NestedTarget::Indirect { .. } => arg_types.push(ValType::I32), + NestedTarget::Ref => arg_types.push(ValType::Ref(RefType::FUNCREF)), + NestedTarget::Direct(_) => {} } arg_types } @@ -6585,7 +8022,7 @@ fn transform_region_seq( sites: &[NestedCallSite], fork_path: &HashSet, arg_materializations: &HashMap, - carryover_spills: &HashMap>, + carryover_spills: &HashMap>, carryover_plans: &HashMap<(InstrSeqId, usize), CarryoverPlan>, catch_handlers: &[CatchHandlerInfo], runtime: &Runtime, @@ -6619,11 +8056,8 @@ fn transform_region_seq( // Ordered deepest-first to match the original parent-stack layout. if !body_param_locals.is_empty() && !chunks.is_empty() { let mut prefix: Vec<(Instr, InstrLocId)> = Vec::with_capacity(body_param_locals.len()); - for (lid, _ty) in body_param_locals.iter() { - prefix.push(( - Instr::LocalGet(LocalGet { local: *lid }), - InstrLocId::default(), - )); + for &(local, ty) in body_param_locals { + push_typed_local_get(&mut prefix, local, ty); } prefix.extend(std::mem::take(&mut chunks[0])); chunks[0] = prefix; @@ -6724,7 +8158,7 @@ fn transform_entry_region( sites: &[NestedCallSite], fork_path: &HashSet, arg_materializations: &HashMap, - carryover_spills: &HashMap>, + carryover_spills: &HashMap>, carryover_plans: &HashMap<(InstrSeqId, usize), CarryoverPlan>, catch_handlers: &[CatchHandlerInfo], runtime: &Runtime, @@ -6956,6 +8390,7 @@ fn partition_region_instrs( let is_fork_landing = match instr { Instr::Call(c) => fork_path.contains(&c.func), Instr::CallIndirect(_) => true, + Instr::CallRef(_) => true, _ => false, }; if is_fork_landing && direct_cursor < direct_idxs_at_this_seq.len() { @@ -7160,7 +8595,7 @@ fn populate_region_dispatch_structure( landings: &[LandingInfo], sites: &[NestedCallSite], arg_materializations: &HashMap, - carryover_spills: &HashMap>, + carryover_spills: &HashMap>, catch_handlers: &[CatchHandlerInfo], runtime: &Runtime, memory: MemoryId, @@ -7295,14 +8730,14 @@ fn emit_post_landing( landing: &LandingInfo, sites: &[NestedCallSite], arg_materializations: &HashMap, - carryover_spills: &HashMap>, - catch_handlers: &[CatchHandlerInfo], + carryover_spills: &HashMap>, + _catch_handlers: &[CatchHandlerInfo], runtime: &Runtime, memory: MemoryId, ptr_ty: ValType, frame_size: u32, cond_swap_local: LocalId, - catch_state_locals: Option, + _catch_state_locals: Option, unwind_save: InstrSeqId, abort: AbortDispatch, ) { @@ -7317,39 +8752,39 @@ fn emit_post_landing( // the carryovers + result on the stack — matching the // original code's expected shape, same as top-level // switch-dispatch's `emit_post_call_via_local`. - let empty: Vec = Vec::new(); + let empty: Vec = Vec::new(); let carryovers = carryover_spills.get(call_idx).unwrap_or(&empty); { let s = &mut local.block_mut(seq_id).instrs; - for &l in carryovers.iter() { - push_instr(s, Instr::LocalGet(LocalGet { local: l })); + for &(local, ty) in carryovers { + push_typed_local_get(s, local, ty); } - emit_materialized_call_args(s, &arg_materializations[call_idx]); - let call_instr = match site.target { - NestedTarget::Direct(func) => Instr::Call(Call { func }), - NestedTarget::Indirect { table } => Instr::CallIndirect(CallIndirect { - ty: site.sig_ty, - table, - }), - }; - s.push((call_instr, site.loc)); } - // Phase 6e + call_idx frame write + UNWIND branch. Phase 6e is - // delayed until after frame reservation succeeds so an abort can - // replay the still-live activation without publishing state. - emit_call_index_store_and_unwind_branch( + let target = match site.target { + NestedTarget::Direct(func) => CallTarget::Direct(func), + NestedTarget::Indirect { table } => CallTarget::Indirect { table }, + NestedTarget::Ref => CallTarget::Ref, + }; + emit_replay_routed_call_with_unwind_boundary( local, seq_id, + target, + site.direct_activation, + site.sig_ty, + site.resume_ty + .expect("nested call site resume type was not assigned"), + site.loc, + &arg_materializations[call_idx], + *call_idx, runtime, memory, ptr_ty, frame_size, - *call_idx, unwind_save, - catch_handlers, - catch_state_locals, abort, ); + // The statically scoped private-tag boundary records this exact + // call before any result becomes visible to the continuation. } LandingKind::SubRegion { .. } => { // Block/Loop/TryTable: preserve the enclosing instr @@ -7382,8 +8817,8 @@ fn emit_post_landing( if let Some(plan) = &landing.carryover { match plan { CarryoverPlan::Spill { spill_locals } => { - for (l, _ty) in spill_locals.iter() { - push_instr(s, Instr::LocalGet(LocalGet { local: *l })); + for &(local, ty) in spill_locals { + push_typed_local_get(s, local, ty); } } CarryoverPlan::PureTail { tail, .. } => { @@ -7425,8 +8860,8 @@ fn emit_post_landing( .last() .copied() .expect("IfElse spill plan must include the condition"); - for (l, _ty) in spill_locals.iter().take(spill_locals.len() - 1) { - push_instr(s, Instr::LocalGet(LocalGet { local: *l })); + for &(local, ty) in spill_locals.iter().take(spill_locals.len() - 1) { + push_typed_local_get(s, local, ty); } IfElseCondSource::Local(cond_local) } @@ -7965,10 +9400,10 @@ mod trampoline_tests { } #[test] - fn compute_carryover_types_unknown_producer_in_carryover_returns_none() { - // Contrast case: a ref-typed producer's value IS the carryover - // at a fork-path call. The switch-dispatch spill path only - // supports scalar ValTypes, so the analyser correctly fails. + fn compute_carryover_types_preserves_reference_for_validation() { + // A reference producer's value is the carryover at a fork-path call. + // Preserve the exact type so the activation recipe planner can assign + // its codec class and call-specific vector ownership. let wat = r#" (module (import "kernel" "kernel_fork" (func $fork (result i32))) @@ -7987,9 +9422,7 @@ mod trampoline_tests { let main = find_func_id(&module, "main"); let fork_path = build_fork_path(&module, &["fork", "main"]); let result = compute_carryover_types(&module, main, &fork_path); - // Carryover would be [None] (the ref-typed result). Analyser - // refuses -> None. - assert_eq!(result, None); + assert_eq!(result, Some(vec![vec![ValType::Ref(RefType::EXTERNREF)]])); } // Sub-commit 2.5a: nested-aware carryover analyser. The analyser diff --git a/crates/fork-instrument/src/legacy_dlopen.rs b/crates/fork-instrument/src/legacy_dlopen.rs new file mode 100644 index 0000000000..08ea0aee0b --- /dev/null +++ b/crates/fork-instrument/src/legacy_dlopen.rs @@ -0,0 +1,385 @@ +//! Lower the historical monolithic dynamic-loader import to ABI 43's staged +//! non-reentrant protocol. +//! +//! `env.__wasm_dlopen` used to compile, instantiate, and synchronously call +//! side-module initialization code before its host import returned. A fork +//! below that callback leaves a JavaScript activation in the middle of the +//! Wasm stack, and that activation has no deterministic fresh-child recipe. +//! The staged imports return one initializer table entry at a time so the +//! initializer instead runs as an ordinary Wasm-to-Wasm call. + +use anyhow::{Result, ensure}; +use walrus::ir::{ + BinaryOp, Binop, Br, Call, CallIndirect, Const, LocalGet, LocalSet, Return, ReturnCall, + UnaryOp, Unop, Unreachable, Value, +}; +use walrus::{ + ExportItem, FunctionBuilder, FunctionId, FunctionKind, ImportKind, LocalId, Module, RefType, + TableId, TypeId, ValType, +}; + +const IMPORT_MODULE: &str = "env"; +const LEGACY_IMPORT: &str = "__wasm_dlopen"; +const MAIN_IMPORT: &str = "__wasm_dlopen_main"; +const PREPARE_IMPORT: &str = "__wasm_dlopen_prepare"; +const NEXT_IMPORT: &str = "__wasm_dlopen_next"; +const COMMIT_IMPORT: &str = "__wasm_dlopen_commit"; +const SIGNAL_CHECKPOINT_EXPORT: &str = "__wasm_posix_signal_checkpoint"; +const DEFAULT_RTLD_GLOBAL: i32 = 0x100; + +#[derive(Clone)] +struct LegacyImport { + function: FunctionId, + import: walrus::ImportId, + ty: TypeId, + params: Vec, +} + +/// Replace every canonical legacy loader import with a local staged adapter. +/// +/// The original `FunctionId` is retained. That is important beyond direct +/// calls: exports, active/passive element segments, constant expressions, and +/// `ref.func` instructions all continue to name the now-local adapter without +/// an incomplete graph-wide reference rewrite. +pub fn lower(module: &mut Module) -> Result { + let legacy = collect_legacy_imports(module); + if legacy.is_empty() { + return Ok(0); + } + + for import in &legacy { + validate_signature(module, import)?; + } + + let table = process_function_table(module); + let checkpoint = exported_signal_checkpoint(module); + let main = import_function(module, MAIN_IMPORT, &[], &[ValType::I32]); + let next = import_function( + module, + NEXT_IMPORT, + &[ValType::I32], + &[ValType::I32], + ); + let commit = import_function( + module, + COMMIT_IMPORT, + &[ValType::I32], + &[ValType::I32], + ); + let driver = add_staged_driver(module, table, next, commit, checkpoint); + + for import in &legacy { + let prepare_params = match import.params.as_slice() { + [pointer, length] => vec![*pointer, *length, *pointer, *length, ValType::I32], + [_, _, _, _] => { + let mut params = import.params.clone(); + params.push(ValType::I32); + params + } + _ => import.params.clone(), + }; + let prepare = import_function( + module, + PREPARE_IMPORT, + &prepare_params, + &[ValType::I32], + ); + replace_import_with_adapter(module, import, main, prepare, driver)?; + } + + Ok(legacy.len()) +} + +fn collect_legacy_imports(module: &Module) -> Vec { + module + .imports + .iter() + .filter_map(|import| { + if import.module != IMPORT_MODULE || import.name != LEGACY_IMPORT { + return None; + } + let ImportKind::Function(function) = import.kind else { + return None; + }; + let ty = module.funcs.get(function).ty(); + Some(LegacyImport { + function, + import: import.id(), + ty, + params: module.types.get(ty).params().to_vec(), + }) + }) + .collect() +} + +fn validate_signature(module: &Module, import: &LegacyImport) -> Result<()> { + let signature = module.types.get(import.ty); + ensure!( + matches!(import.params.len(), 2 | 4 | 5) + && signature.results() == [ValType::I32] + && matches!(import.params[0], ValType::I32 | ValType::I64) + && matches!(import.params[1], ValType::I32 | ValType::I64) + && (import.params.len() == 2 || import.params[2] == import.params[0]) + && (import.params.len() == 2 + || matches!(import.params[3], ValType::I32 | ValType::I64)) + && (import.params.len() != 5 || import.params[4] == ValType::I32), + "fork-instrument: reserved env.__wasm_dlopen import has signature \ + {:?} -> {:?}; expected (pointer, integer[, pointer, integer[, i32]]) -> i32", + signature.params(), + signature.results(), + ); + Ok(()) +} + +fn process_function_table(module: &mut Module) -> TableId { + let exported = module.exports.iter().find_map(|export| { + if export.name != "__indirect_function_table" { + return None; + } + match export.item { + ExportItem::Table(table) if module.tables.get(table).element_ty == RefType::FUNCREF => { + Some(table) + } + _ => None, + } + }); + exported.unwrap_or_else(|| { + // A successful Kandelo side-module load already requires the canonical + // exported table and stack pointer. Keep malformed/dead legacy imports + // valid after lowering without inventing a second observable process + // function-pointer table; `prepare` will report the missing host + // linker contract before this private fallback can be reached. + module.tables.add_local(false, 1, None, RefType::FUNCREF) + }) +} + +fn exported_signal_checkpoint(module: &Module) -> Option { + module.exports.iter().find_map(|export| { + if export.name != SIGNAL_CHECKPOINT_EXPORT { + return None; + } + let ExportItem::Function(function) = export.item else { + return None; + }; + let signature = module.types.get(module.funcs.get(function).ty()); + (signature.params().is_empty() && signature.results().is_empty()).then_some(function) + }) +} + +fn import_function( + module: &mut Module, + name: &str, + params: &[ValType], + results: &[ValType], +) -> FunctionId { + if let Some(function) = module.imports.iter().find_map(|import| { + if import.module != IMPORT_MODULE || import.name != name { + return None; + } + let ImportKind::Function(function) = import.kind else { + return None; + }; + let signature = module.types.get(module.funcs.get(function).ty()); + (signature.params() == params && signature.results() == results).then_some(function) + }) { + return function; + } + let ty = module.types.add(params, results); + module.add_import_func(IMPORT_MODULE, name, ty).0 +} + +fn add_staged_driver( + module: &mut Module, + table: TableId, + next: FunctionId, + commit: FunctionId, + checkpoint: Option, +) -> FunctionId { + let token = module.locals.add(ValType::I32); + let entry = module.locals.add(ValType::I32); + let call_ty = module.types.add(&[], &[]); + let mut builder = + FunctionBuilder::new(&mut module.types, &[ValType::I32], &[ValType::I32]); + builder.name("__wpk_fork_legacy_dlopen_driver".into()); + + let mut loop_body = builder.dangling_instr_seq(None); + let loop_id = loop_body.id(); + + // A prepare call can issue loader-owned channel requests. Checkpoint only + // after the adapter's tail call has removed its dead pointer parameters, + // keeping those values out of a continuation captured by a signal handler. + call_optional(&mut loop_body, checkpoint); + + local_get(&mut loop_body, token); + call(&mut loop_body, next); + local_set(&mut loop_body, entry); + call_optional(&mut loop_body, checkpoint); + + local_get(&mut loop_body, entry); + i32_const(&mut loop_body, 0); + binop(&mut loop_body, BinaryOp::I32LtS); + loop_body.if_else( + None, + |failed| { + i32_const(failed, 0); + ret(failed); + }, + |_| {}, + ); + + local_get(&mut loop_body, entry); + unop(&mut loop_body, UnaryOp::I32Eqz); + loop_body.if_else( + None, + |finished| { + local_get(finished, token); + call(finished, commit); + local_set(finished, entry); + call_optional(finished, checkpoint); + local_get(finished, entry); + ret(finished); + }, + |_| {}, + ); + + local_get(&mut loop_body, entry); + if module.tables.get(table).table64 { + unop(&mut loop_body, UnaryOp::I64ExtendUI32); + } + loop_body.instr(CallIndirect { ty: call_ty, table }); + loop_body.instr(Br { block: loop_id }); + drop(loop_body); + + let mut body = builder.func_body(); + body.instr(walrus::ir::Loop { seq: loop_id }); + body.instr(Unreachable {}); + builder.finish(vec![token], &mut module.funcs) +} + +fn replace_import_with_adapter( + module: &mut Module, + import: &LegacyImport, + main: FunctionId, + prepare: FunctionId, + driver: FunctionId, +) -> Result<()> { + let args: Vec = import + .params + .iter() + .copied() + .map(|ty| module.locals.add(ty)) + .collect(); + let mut builder = + FunctionBuilder::new(&mut module.types, &import.params, &[ValType::I32]); + builder.name("__wpk_fork_legacy_dlopen_adapter".into()); + + let body = &mut builder.func_body(); + if args.len() >= 4 { + local_get(body, args[1]); + integer_eqz(body, import.params[1]); + local_get(body, args[3]); + integer_eqz(body, import.params[3]); + binop(body, BinaryOp::I32And); + body.if_else( + None, + |main_program| { + call(main_program, main); + ret(main_program); + }, + |_| {}, + ); + } + + local_get(body, args[0]); + local_get(body, args[1]); + if args.len() == 2 { + // The original Kandelo loader ABI supplied no pathname. Preserve its + // deterministic `dlopen::` naming rule by passing an + // empty name range; the process Worker derives the historical name + // after validating both ranges. + integer_const_zero(body, import.params[0]); + integer_const_zero(body, import.params[1]); + } else { + local_get(body, args[2]); + local_get(body, args[3]); + } + if args.len() == 5 { + local_get(body, args[4]); + } else { + i32_const(body, DEFAULT_RTLD_GLOBAL); + } + call(body, prepare); + // WHY: no guest code ran during prepare, so this true tail call removes + // the adapter's byte/name pointer parameters before a constructor or + // signal handler can fork. Only the driver's two i32 values can then add + // to the continuation payload, regardless of pointer width. + body.instr(ReturnCall { func: driver }); + + let local = builder.local_func(args); + ensure!( + local.ty() == import.ty, + "fork-instrument: legacy dlopen adapter did not retain its canonical function type", + ); + let function = module.funcs.get_mut(import.function); + function.kind = FunctionKind::Local(local); + function.name = Some("__wpk_fork_legacy_dlopen_adapter".into()); + module.imports.delete(import.import); + Ok(()) +} + +fn integer_eqz(body: &mut walrus::InstrSeqBuilder<'_>, ty: ValType) { + unop( + body, + match ty { + ValType::I32 => UnaryOp::I32Eqz, + ValType::I64 => UnaryOp::I64Eqz, + _ => unreachable!("validated legacy dlopen integer"), + }, + ); +} + +fn integer_const_zero(body: &mut walrus::InstrSeqBuilder<'_>, ty: ValType) { + body.instr(Const { + value: match ty { + ValType::I32 => Value::I32(0), + ValType::I64 => Value::I64(0), + _ => unreachable!("validated legacy dlopen integer"), + }, + }); +} + +fn local_get(body: &mut walrus::InstrSeqBuilder<'_>, local: LocalId) { + body.instr(LocalGet { local }); +} + +fn local_set(body: &mut walrus::InstrSeqBuilder<'_>, local: LocalId) { + body.instr(LocalSet { local }); +} + +fn call(body: &mut walrus::InstrSeqBuilder<'_>, function: FunctionId) { + body.instr(Call { func: function }); +} + +fn call_optional(body: &mut walrus::InstrSeqBuilder<'_>, function: Option) { + if let Some(function) = function { + call(body, function); + } +} + +fn i32_const(body: &mut walrus::InstrSeqBuilder<'_>, value: i32) { + body.instr(Const { + value: Value::I32(value), + }); +} + +fn binop(body: &mut walrus::InstrSeqBuilder<'_>, op: BinaryOp) { + body.instr(Binop { op }); +} + +fn unop(body: &mut walrus::InstrSeqBuilder<'_>, op: UnaryOp) { + body.instr(Unop { op }); +} + +fn ret(body: &mut walrus::InstrSeqBuilder<'_>) { + body.instr(Return {}); +} diff --git a/crates/fork-instrument/src/legacy_eh.rs b/crates/fork-instrument/src/legacy_eh.rs new file mode 100644 index 0000000000..5bdc61092c --- /dev/null +++ b/crates/fork-instrument/src/legacy_eh.rs @@ -0,0 +1,880 @@ +//! Legacy exception-handler normalization for fork-reachable functions. +//! +//! A legacy `catch` enters an implicit engine-owned exception context. That +//! context is exactly the state a continuation cannot recover in a fresh Wasm +//! instance. This pass converts legacy handlers to modern `try_table` +//! `catch_ref`/`catch_all_ref` clauses before continuation planning. The caught +//! exception is held in an ordinary activation local, so the reference recipe +//! analysis gives it the same linked-frame ownership as every other live +//! reference. +//! +//! Legacy `delegate` has no handler activation and therefore needs no +//! conversion. Keeping it native also preserves its relative-depth semantics +//! without inventing an exception round trip. + +use anyhow::{Result, bail, ensure}; +use std::collections::{BTreeMap, HashMap, HashSet}; +use walrus::{ + AbstractHeapType, FunctionId, FunctionKind, HeapType, LocalFunction, LocalId, Module, RefType, + ValType, + ir::{ + Block, Br, Instr, InstrLocId, InstrSeqId, InstrSeqType, LegacyCatch, LocalGet, LocalSet, + RefAsNonNull, RefNull, ThrowRef, Try, TryTable, TryTableCatch, + }, +}; + +#[derive(Clone)] +struct HandlerMeta { + root: InstrSeqId, + exception: LocalId, + /// Outer-to-inner legacy handlers active while this handler executes. + ancestors: Vec, +} + +#[derive(Clone)] +struct ExitShim { + seq: InstrSeqId, + target: InstrSeqId, + clear: Vec, +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum LabelKind { + Loop, + Other, +} + +#[derive(Clone, Copy)] +struct TrySite { + body: InstrSeqId, + depth: u32, +} + +#[derive(Clone)] +struct RethrowRewrite { + seq: InstrSeqId, + index: usize, + loc: InstrLocId, + exception: LocalId, + clear: Vec, +} + +const NULLABLE_EXNREF: RefType = RefType { + nullable: true, + heap_type: HeapType::Abstract(AbstractHeapType::Exn), +}; + +const NON_NULL_EXNREF: RefType = RefType { + nullable: false, + heap_type: HeapType::Abstract(AbstractHeapType::Exn), +}; + +/// Normalize legacy handlers only in functions that can own a fork +/// continuation. References and legacy EH outside the fork closure remain +/// byte-for-byte under Walrus's normal re-emission. +pub fn normalize_fork_path(module: &mut Module, fork_path: &HashSet) -> Result<()> { + let mut functions: Vec<_> = fork_path.iter().copied().collect(); + functions.sort(); + for function in functions { + if !matches!(module.funcs.get(function).kind, FunctionKind::Local(_)) { + continue; + } + normalize_function(module, function)?; + } + Ok(()) +} + +fn normalize_function(module: &mut Module, function: FunctionId) -> Result<()> { + let (entry, handler_layout, label_kinds, try_sites) = { + let local = local(module, function); + let entry = local.entry_block(); + let mut handlers = Vec::new(); + let mut labels = HashMap::from([(entry, LabelKind::Other)]); + let mut tries = Vec::new(); + collect_structure(local, entry, 0, &[], &mut handlers, &mut labels, &mut tries); + (entry, handlers, labels, tries) + }; + + if handler_layout.is_empty() { + return Ok(()); + } + + let mut handlers = HashMap::new(); + for (root, ancestors) in handler_layout { + let exception = module.locals.add(ValType::Ref(NULLABLE_EXNREF)); + handlers.insert( + root, + HandlerMeta { + root, + exception, + ancestors, + }, + ); + } + + rewrite_rethrows(module, function, entry, &handlers)?; + + let full_subtrees: HashMap> = handlers + .keys() + .copied() + .map(|root| { + let mut subtree = HashSet::new(); + collect_subtree(local(module, function), root, &mut subtree); + (root, subtree) + }) + .collect(); + + // Allocate every branch-cleanup label before lowering changes ancestry. + // All branch opcodes, including br_table and br_on_*, can then be retargeted + // without lowering them into slower instruction sequences. + let mut handler_shims: HashMap> = HashMap::new(); + let handler_roots: HashSet<_> = handlers.keys().copied().collect(); + let mut ordered_handlers: Vec<_> = handlers.values().cloned().collect(); + ordered_handlers.sort_by_key(|handler| handler.root); + for handler in ordered_handlers { + let direct_subtree = + collect_direct_handler_subtree(local(module, function), handler.root, &handler_roots); + let mut targets = BTreeMap::>::new(); + collect_exit_targets( + local(module, function), + handler.root, + &direct_subtree, + &handler, + &handlers, + &full_subtrees, + &mut targets, + ); + + let mut shims = Vec::new(); + let (handler_params, _) = + params_results(module, local(module, function).block(handler.root).ty); + for (target, clear) in targets { + let branch_values = branch_value_types(module, function, target, &label_kinds)?; + // The normal catch-entry path carries the legacy tag payload + // through every enclosing cleanup shim. A branch to a shim label + // supplies only its result tuple, so these params do not alter the + // retargeted branch signature. + let ty = InstrSeqType::new(&mut module.types, &handler_params, &branch_values); + let seq = local_mut(module, function) + .builder_mut() + .dangling_instr_seq(ty) + .id(); + shims.push(ExitShim { seq, target, clear }); + } + let replacements: HashMap<_, _> = + shims.iter().map(|shim| (shim.target, shim.seq)).collect(); + retarget_handler_exits( + local_mut(module, function), + handler.root, + &direct_subtree, + &replacements, + ); + handler_shims.insert(handler.root, shims); + } + + let mut sites = try_sites; + sites.sort_by_key(|site| std::cmp::Reverse(site.depth)); + for site in sites { + lower_try( + module, + function, + entry, + site.body, + &handlers, + &handler_shims, + )?; + } + + Ok(()) +} + +fn collect_structure( + local: &LocalFunction, + seq: InstrSeqId, + depth: u32, + active_handlers: &[InstrSeqId], + handlers: &mut Vec<(InstrSeqId, Vec)>, + labels: &mut HashMap, + tries: &mut Vec, +) { + for (instr, _) in &local.block(seq).instrs { + match instr { + Instr::Block(block) => { + labels.insert(block.seq, LabelKind::Other); + collect_structure( + local, + block.seq, + depth + 1, + active_handlers, + handlers, + labels, + tries, + ); + } + Instr::Loop(loop_) => { + labels.insert(loop_.seq, LabelKind::Loop); + collect_structure( + local, + loop_.seq, + depth + 1, + active_handlers, + handlers, + labels, + tries, + ); + } + Instr::IfElse(if_) => { + for child in [if_.consequent, if_.alternative] { + labels.insert(child, LabelKind::Other); + collect_structure( + local, + child, + depth + 1, + active_handlers, + handlers, + labels, + tries, + ); + } + } + Instr::TryTable(table) => { + labels.insert(table.seq, LabelKind::Other); + collect_structure( + local, + table.seq, + depth + 1, + active_handlers, + handlers, + labels, + tries, + ); + } + Instr::Try(try_) => { + labels.insert(try_.seq, LabelKind::Other); + collect_structure( + local, + try_.seq, + depth + 1, + active_handlers, + handlers, + labels, + tries, + ); + if try_ + .catches + .iter() + .any(|catch| !matches!(catch, LegacyCatch::Delegate { .. })) + { + tries.push(TrySite { + body: try_.seq, + depth, + }); + } + for catch in &try_.catches { + let handler = match catch { + LegacyCatch::Catch { handler, .. } | LegacyCatch::CatchAll { handler } => { + *handler + } + LegacyCatch::Delegate { .. } => continue, + }; + labels.insert(handler, LabelKind::Other); + handlers.push((handler, active_handlers.to_vec())); + let mut nested = active_handlers.to_vec(); + nested.push(handler); + collect_structure(local, handler, depth + 1, &nested, handlers, labels, tries); + } + } + _ => {} + } + } +} + +fn rewrite_rethrows( + module: &mut Module, + function: FunctionId, + entry: InstrSeqId, + handlers: &HashMap, +) -> Result<()> { + let rewrites = { + let mut rewrites = Vec::new(); + collect_rethrows( + local(module, function), + entry, + &mut vec![(entry, None)], + handlers, + &mut rewrites, + )?; + rewrites + }; + + let mut by_seq = BTreeMap::>::new(); + for rewrite in rewrites { + by_seq.entry(rewrite.seq).or_default().push(rewrite); + } + for (seq, mut rewrites) in by_seq { + rewrites.sort_by_key(|rewrite| std::cmp::Reverse(rewrite.index)); + let instrs = &mut local_mut(module, function).block_mut(seq).instrs; + for rewrite in rewrites { + let mut replacement = Vec::new(); + replacement.push(( + Instr::LocalGet(LocalGet { + local: rewrite.exception, + }), + rewrite.loc, + )); + replacement.push((Instr::RefAsNonNull(RefAsNonNull {}), rewrite.loc)); + for local in rewrite.clear { + replacement.push(( + Instr::RefNull(RefNull { + ty: NULLABLE_EXNREF, + }), + rewrite.loc, + )); + replacement.push((Instr::LocalSet(LocalSet { local }), rewrite.loc)); + } + replacement.push((Instr::ThrowRef(ThrowRef {}), rewrite.loc)); + instrs.splice(rewrite.index..=rewrite.index, replacement); + } + } + Ok(()) +} + +fn collect_rethrows( + local: &LocalFunction, + seq: InstrSeqId, + stack: &mut Vec<(InstrSeqId, Option)>, + handlers: &HashMap, + out: &mut Vec, +) -> Result<()> { + for (index, (instr, loc)) in local.block(seq).instrs.iter().enumerate() { + if let Instr::Rethrow(rethrow) = instr { + let depth = rethrow.relative_depth as usize; + ensure!( + depth < stack.len(), + "fork-instrument: legacy rethrow depth {} exceeds control depth {}", + depth, + stack.len(), + ); + let target_index = stack.len() - 1 - depth; + let Some(exception) = stack[target_index].1 else { + bail!( + "fork-instrument: legacy rethrow depth {} does not target a catch handler", + depth, + ); + }; + let mut clear = Vec::new(); + for (_, local) in stack[target_index..].iter().rev() { + if let Some(local) = local + && !clear.contains(local) + { + clear.push(*local); + } + } + out.push(RethrowRewrite { + seq, + index, + loc: *loc, + exception, + clear, + }); + } + + match instr { + Instr::Block(block) => { + collect_rethrows_child(local, block.seq, None, stack, handlers, out)? + } + Instr::Loop(loop_) => { + collect_rethrows_child(local, loop_.seq, None, stack, handlers, out)? + } + Instr::IfElse(if_) => { + collect_rethrows_child(local, if_.consequent, None, stack, handlers, out)?; + collect_rethrows_child(local, if_.alternative, None, stack, handlers, out)?; + } + Instr::TryTable(table) => { + collect_rethrows_child(local, table.seq, None, stack, handlers, out)? + } + Instr::Try(try_) => { + collect_rethrows_child(local, try_.seq, None, stack, handlers, out)?; + for catch in &try_.catches { + let handler = match catch { + LegacyCatch::Catch { handler, .. } | LegacyCatch::CatchAll { handler } => { + *handler + } + LegacyCatch::Delegate { .. } => continue, + }; + collect_rethrows_child( + local, + handler, + Some(handlers[&handler].exception), + stack, + handlers, + out, + )?; + } + } + _ => {} + } + } + Ok(()) +} + +fn collect_rethrows_child( + local: &LocalFunction, + child: InstrSeqId, + handler: Option, + stack: &mut Vec<(InstrSeqId, Option)>, + handlers: &HashMap, + out: &mut Vec, +) -> Result<()> { + stack.push((child, handler)); + collect_rethrows(local, child, stack, handlers, out)?; + stack.pop(); + Ok(()) +} + +fn collect_subtree(local: &LocalFunction, seq: InstrSeqId, out: &mut HashSet) { + if !out.insert(seq) { + return; + } + for (instr, _) in &local.block(seq).instrs { + for child in children(instr) { + collect_subtree(local, child, out); + } + } +} + +fn collect_direct_handler_subtree( + local: &LocalFunction, + root: InstrSeqId, + handler_roots: &HashSet, +) -> HashSet { + fn visit( + local: &LocalFunction, + root: InstrSeqId, + seq: InstrSeqId, + handler_roots: &HashSet, + out: &mut HashSet, + ) { + if seq != root && handler_roots.contains(&seq) { + return; + } + if !out.insert(seq) { + return; + } + for (instr, _) in &local.block(seq).instrs { + for child in children(instr) { + visit(local, root, child, handler_roots, out); + } + } + } + + let mut out = HashSet::new(); + visit(local, root, root, handler_roots, &mut out); + out +} + +fn collect_exit_targets( + local: &LocalFunction, + seq: InstrSeqId, + direct_subtree: &HashSet, + current: &HandlerMeta, + handlers: &HashMap, + full_subtrees: &HashMap>, + out: &mut BTreeMap>, +) { + for (instr, _) in &local.block(seq).instrs { + for target in branch_targets(instr) { + if target != current.root && full_subtrees[¤t.root].contains(&target) { + continue; + } + let mut clear = Vec::new(); + for root in current + .ancestors + .iter() + .copied() + .chain(std::iter::once(current.root)) + .rev() + { + if target == root || !full_subtrees[&root].contains(&target) { + clear.push(handlers[&root].exception); + } + } + out.entry(target).or_insert(clear); + } + for child in children(instr) { + if direct_subtree.contains(&child) { + collect_exit_targets( + local, + child, + direct_subtree, + current, + handlers, + full_subtrees, + out, + ); + } + } + } +} + +fn retarget_handler_exits( + local: &mut LocalFunction, + seq: InstrSeqId, + direct_subtree: &HashSet, + replacements: &HashMap, +) { + let children_to_visit: Vec<_> = local + .block(seq) + .instrs + .iter() + .flat_map(|(instr, _)| children(instr)) + .filter(|child| direct_subtree.contains(child)) + .collect(); + for (instr, _) in &mut local.block_mut(seq).instrs { + replace_branch_targets(instr, replacements); + } + for child in children_to_visit { + retarget_handler_exits(local, child, direct_subtree, replacements); + } +} + +fn lower_try( + module: &mut Module, + function: FunctionId, + entry: InstrSeqId, + body: InstrSeqId, + handlers: &HashMap, + handler_shims: &HashMap>, +) -> Result<()> { + let Some((parent, index, loc, try_)) = find_try(local(module, function), entry, body) else { + return Ok(()); + }; + if try_ + .catches + .iter() + .all(|catch| matches!(catch, LegacyCatch::Delegate { .. })) + { + return Ok(()); + } + ensure!( + try_.catches + .iter() + .all(|catch| !matches!(catch, LegacyCatch::Delegate { .. })), + "fork-instrument: malformed legacy try mixes delegate with catch handlers", + ); + + let body_ty = local(module, function).block(body).ty; + let (try_params, _) = params_results(module, body_ty); + let outer = local_mut(module, function) + .builder_mut() + .dangling_instr_seq(body_ty) + .id(); + + let mut caps = Vec::new(); + let mut modern_catches = Vec::new(); + let mut catch_handlers = Vec::new(); + for catch in &try_.catches { + let (handler, tag) = match catch { + LegacyCatch::Catch { tag, handler } => (*handler, Some(*tag)), + LegacyCatch::CatchAll { handler } => (*handler, None), + LegacyCatch::Delegate { .. } => unreachable!(), + }; + let (handler_params, _) = params_results(module, local(module, function).block(handler).ty); + let mut catch_values = handler_params; + catch_values.push(ValType::Ref(NON_NULL_EXNREF)); + let cap_ty = InstrSeqType::new(&mut module.types, &try_params, &catch_values); + let cap = local_mut(module, function) + .builder_mut() + .dangling_instr_seq(cap_ty) + .id(); + let modern = match tag { + Some(tag) => TryTableCatch::CatchRef { tag, label: cap }, + None => TryTableCatch::CatchAllRef { label: cap }, + }; + caps.push(cap); + modern_catches.push(modern); + catch_handlers.push(handler); + } + + let innermost = *caps.last().expect("legacy try has at least one handler"); + { + let instrs = &mut local_mut(module, function).block_mut(innermost).instrs; + push( + instrs, + Instr::TryTable(TryTable { + seq: body, + catches: modern_catches, + }), + ); + push(instrs, Instr::Br(Br { block: outer })); + } + + for index in (0..caps.len() - 1).rev() { + let child = caps[index + 1]; + push( + &mut local_mut(module, function).block_mut(caps[index]).instrs, + Instr::Block(Block { seq: child }), + ); + emit_handler_adapter( + module, + function, + caps[index], + catch_handlers[index + 1], + outer, + &handlers[&catch_handlers[index + 1]], + &handler_shims[&catch_handlers[index + 1]], + ); + } + + push( + &mut local_mut(module, function).block_mut(outer).instrs, + Instr::Block(Block { seq: caps[0] }), + ); + emit_handler_adapter( + module, + function, + outer, + catch_handlers[0], + outer, + &handlers[&catch_handlers[0]], + &handler_shims[&catch_handlers[0]], + ); + + local_mut(module, function).block_mut(parent).instrs[index] = + (Instr::Block(Block { seq: outer }), loc); + Ok(()) +} + +fn emit_handler_adapter( + module: &mut Module, + function: FunctionId, + container: InstrSeqId, + handler: InstrSeqId, + normal_target: InstrSeqId, + meta: &HandlerMeta, + shims: &[ExitShim], +) { + push( + &mut local_mut(module, function).block_mut(container).instrs, + Instr::LocalSet(LocalSet { + local: meta.exception, + }), + ); + + let execution = shims.last().map(|shim| shim.seq).unwrap_or(container); + emit_handler_execution( + module, + function, + execution, + handler, + meta.exception, + normal_target, + ); + + if !shims.is_empty() { + for index in (0..shims.len() - 1).rev() { + let child = shims[index + 1].seq; + let parent = shims[index].seq; + let instrs = &mut local_mut(module, function).block_mut(parent).instrs; + instrs.insert( + 0, + (Instr::Block(Block { seq: child }), InstrLocId::default()), + ); + emit_clear_and_branch(instrs, &shims[index + 1]); + } + let instrs = &mut local_mut(module, function).block_mut(container).instrs; + push(instrs, Instr::Block(Block { seq: shims[0].seq })); + emit_clear_and_branch(instrs, &shims[0]); + } +} + +fn emit_handler_execution( + module: &mut Module, + function: FunctionId, + container: InstrSeqId, + handler: InstrSeqId, + exception: LocalId, + normal_target: InstrSeqId, +) { + let handler_ty = local(module, function).block(handler).ty; + let (handler_params, _) = params_results(module, handler_ty); + let cleanup_cap_ty = InstrSeqType::new( + &mut module.types, + &handler_params, + &[ValType::Ref(NON_NULL_EXNREF)], + ); + let cleanup_cap = local_mut(module, function) + .builder_mut() + .dangling_instr_seq(cleanup_cap_ty) + .id(); + { + let instrs = &mut local_mut(module, function).block_mut(cleanup_cap).instrs; + push( + instrs, + Instr::TryTable(TryTable { + seq: handler, + catches: vec![TryTableCatch::CatchAllRef { label: cleanup_cap }], + }), + ); + emit_clear(instrs, exception); + push( + instrs, + Instr::Br(Br { + block: normal_target, + }), + ); + } + + let instrs = &mut local_mut(module, function).block_mut(container).instrs; + push(instrs, Instr::Block(Block { seq: cleanup_cap })); + emit_clear(instrs, exception); + push(instrs, Instr::ThrowRef(ThrowRef {})); +} + +fn emit_clear_and_branch(instrs: &mut Vec<(Instr, InstrLocId)>, shim: &ExitShim) { + for local in &shim.clear { + emit_clear(instrs, *local); + } + push(instrs, Instr::Br(Br { block: shim.target })); +} + +fn emit_clear(instrs: &mut Vec<(Instr, InstrLocId)>, local: LocalId) { + push( + instrs, + Instr::RefNull(RefNull { + ty: NULLABLE_EXNREF, + }), + ); + push(instrs, Instr::LocalSet(LocalSet { local })); +} + +fn branch_value_types( + module: &Module, + function: FunctionId, + target: InstrSeqId, + label_kinds: &HashMap, +) -> Result> { + let ty = local(module, function).block(target).ty; + let (params, results) = params_results(module, ty); + let kind = label_kinds + .get(&target) + .copied() + .ok_or_else(|| anyhow::anyhow!("fork-instrument: branch target has no label kind"))?; + Ok(if kind == LabelKind::Loop { + params + } else { + results + }) +} + +fn params_results(module: &Module, ty: InstrSeqType) -> (Vec, Vec) { + match ty { + InstrSeqType::Simple(None) => (Vec::new(), Vec::new()), + InstrSeqType::Simple(Some(result)) => (Vec::new(), vec![result]), + InstrSeqType::MultiValue(ty) => ( + module.types.get(ty).params().to_vec(), + module.types.get(ty).results().to_vec(), + ), + } +} + +fn find_try( + local: &LocalFunction, + seq: InstrSeqId, + body: InstrSeqId, +) -> Option<(InstrSeqId, usize, InstrLocId, Try)> { + for (index, (instr, loc)) in local.block(seq).instrs.iter().enumerate() { + if let Instr::Try(try_) = instr + && try_.seq == body + { + return Some((seq, index, *loc, try_.clone())); + } + for child in children(instr) { + if let Some(site) = find_try(local, child, body) { + return Some(site); + } + } + } + None +} + +fn children(instr: &Instr) -> Vec { + match instr { + Instr::Block(block) => vec![block.seq], + Instr::Loop(loop_) => vec![loop_.seq], + Instr::IfElse(if_) => vec![if_.consequent, if_.alternative], + Instr::TryTable(table) => vec![table.seq], + Instr::Try(try_) => { + let mut children = vec![try_.seq]; + for catch in &try_.catches { + match catch { + LegacyCatch::Catch { handler, .. } | LegacyCatch::CatchAll { handler } => { + children.push(*handler) + } + LegacyCatch::Delegate { .. } => {} + } + } + children + } + _ => Vec::new(), + } +} + +fn branch_targets(instr: &Instr) -> Vec { + match instr { + Instr::Br(branch) => vec![branch.block], + Instr::BrIf(branch) => vec![branch.block], + Instr::BrTable(table) => table + .blocks + .iter() + .copied() + .chain(std::iter::once(table.default)) + .collect(), + Instr::BrOnNull(branch) => vec![branch.block], + Instr::BrOnNonNull(branch) => vec![branch.block], + Instr::BrOnCast(branch) => vec![branch.block], + Instr::BrOnCastFail(branch) => vec![branch.block], + _ => Vec::new(), + } +} + +fn replace_branch_targets(instr: &mut Instr, replacements: &HashMap) { + let replace = |target: &mut InstrSeqId| { + if let Some(replacement) = replacements.get(target) { + *target = *replacement; + } + }; + match instr { + Instr::Br(branch) => replace(&mut branch.block), + Instr::BrIf(branch) => replace(&mut branch.block), + Instr::BrTable(table) => { + for target in &mut table.blocks { + replace(target); + } + replace(&mut table.default); + } + Instr::BrOnNull(branch) => replace(&mut branch.block), + Instr::BrOnNonNull(branch) => replace(&mut branch.block), + Instr::BrOnCast(branch) => replace(&mut branch.block), + Instr::BrOnCastFail(branch) => replace(&mut branch.block), + _ => {} + } +} + +fn push(instrs: &mut Vec<(Instr, InstrLocId)>, instr: Instr) { + instrs.push((instr, InstrLocId::default())); +} + +fn local(module: &Module, function: FunctionId) -> &LocalFunction { + match &module.funcs.get(function).kind { + FunctionKind::Local(local) => local, + _ => unreachable!("fork-path legacy EH normalization requires a local function"), + } +} + +fn local_mut(module: &mut Module, function: FunctionId) -> &mut LocalFunction { + match &mut module.funcs.get_mut(function).kind { + FunctionKind::Local(local) => local, + _ => unreachable!("fork-path legacy EH normalization requires a local function"), + } +} diff --git a/crates/fork-instrument/src/lib.rs b/crates/fork-instrument/src/lib.rs index a8014f80f9..6cf8333377 100644 --- a/crates/fork-instrument/src/lib.rs +++ b/crates/fork-instrument/src/lib.rs @@ -4,31 +4,138 @@ //! See `docs/plans/2026-04-20-fork-instrumentation-design.md` for the //! full design. //! -//! Phase 1 (current): skeleton only. Parses a wasm binary, validates -//! it, and emits it unchanged. Subsequent phases add: -//! -//! - Phase 2: direct-call graph discovery -//! - Phase 3: indirect-call graph discovery -//! - Phase 4: core instrumentation (state machine, frame save/restore) -//! - Phase 5: reference-typed local spilling -//! - Phase 6: catch-handler region support -//! - Phase 7: production rollout +//! The ABI 43 transform discovers the direct/indirect fork closure, assigns +//! every replay value to activation-owned bytes or a versioned reconstruction +//! recipe, and emits the linked-frame state machine. Scalars are serialized in +//! the activation frame; references, complete exceptions, mutable reference +//! globals, and table entries are reconstructed from a process-owned typed +//! recipe graph in each fresh module instance. use anyhow::{Context, Result, bail, ensure}; -use walrus::RawCustomSection; +use walrus::{ + ElementItems, ElementKind, FunctionId, RawCustomSection, RefType, TableId, ir::Value, +}; +use wasm_posix_shared::abi::{ + WPK_FORK_CAP_ACTIVATION_STATE_SAFE, WPK_FORK_CAP_DYLINK_MAIN, WPK_FORK_CAP_SIDE_ENTRY, + WPK_FORK_CAPABILITIES_SECTION, WPK_FORK_CAPABILITIES_VERSION, + WPK_FORK_GLOBAL_CATALOG_EXPORT_PREFIX, WPK_FORK_IMPORTED_TABLES_SECTION, + WPK_FORK_MODULE_STATE_ARENA_VERSION, WPK_FORK_MODULE_STATE_DESCRIPTOR_SIZE, + WPK_FORK_MODULE_STATE_FORMAT_MAGIC, WPK_FORK_MODULE_STATE_FORMAT_SECTION, + WPK_FORK_MODULE_STATE_FORMAT_VERSION, WPK_FORK_MODULE_STATE_RECORD_ALIGNMENT, + WPK_FORK_MODULE_STATE_RECORD_VERSION, WPK_FORK_MODULE_STATE_REQUIRED_FLAGS, + WPK_FORK_MODULE_STATE_ROOT_POINTER_WORD_OFFSET, WPK_FORK_REQUIRED_EXPORTS, + WPK_FORK_REQUIRED_IMPORTS, WPK_FORK_TABLE_CATALOG_EXPORT_PREFIX, + WPK_FORK_UNWIND_TRANSPORT_PAYLOAD_ARITY, WPK_FORK_UNWIND_TRANSPORT_SECTION, + WPK_FORK_UNWIND_TRANSPORT_VERSION, +}; use wasmparser::{Parser, Payload}; pub mod call_graph; +pub mod contract_inventory; pub mod instrument; +pub mod legacy_eh; +pub mod legacy_dlopen; pub mod linked_frames; +pub mod module_exception_codec; +pub mod module_gc_codec; +pub mod module_state; +pub mod reference_analysis; pub mod runtime; +pub mod static_reference_catalog; + +/// Fresh instances rebuild this fixed catalog from the module's static element +/// segment, so a funcref recipe needs only a module activation and ordinal. +pub const FUNCTION_CATALOG_EXPORT: &str = "__wpk_fork_function_catalog"; + +/// Declares that unwind completion is transported by the private +/// `env.__wpk_fork_unwind` zero-payload tag rather than synthesized function +/// results. +pub const UNWIND_TRANSPORT_SECTION: &str = WPK_FORK_UNWIND_TRANSPORT_SECTION; +pub const UNWIND_TRANSPORT_VERSION: u8 = WPK_FORK_UNWIND_TRANSPORT_VERSION; + +fn reject_preinstrumented_artifact(module: &walrus::Module) -> Result<()> { + let has_control_export = module.exports.iter().any(|export| { + WPK_FORK_REQUIRED_EXPORTS + .iter() + .any(|requirement| requirement.name == export.name) + }); + let has_frame_import = module.imports.iter().any(|import| { + WPK_FORK_REQUIRED_IMPORTS.iter().any(|requirement| { + requirement.module == import.module && requirement.name == import.name + }) + }); + let has_fork_metadata = module.customs.iter().any(|(_, section)| { + matches!( + section.name(), + WPK_FORK_CAPABILITIES_SECTION + | linked_frames::LINKED_FRAME_FORMAT_SECTION + | module_exception_codec::FORMAT_SECTION + | WPK_FORK_MODULE_STATE_FORMAT_SECTION + | WPK_FORK_IMPORTED_TABLES_SECTION + | instrument::RESUME_CATALOG_SECTION + | static_reference_catalog::FORMAT_SECTION + | UNWIND_TRANSPORT_SECTION + ) + }) || module.exports.iter().any(|export| { + matches!( + export.name.as_str(), + FUNCTION_CATALOG_EXPORT + | instrument::RESUME_CATALOG_EXPORT + | instrument::RESUME_START_EXPORT + | instrument::RESUME_THREAD_EXPORT + | static_reference_catalog::EXPORT + | static_reference_catalog::HARVEST_EXPORT + ) + }) || module.exports.iter().any(|export| { + export + .name + .starts_with(WPK_FORK_GLOBAL_CATALOG_EXPORT_PREFIX) + || export + .name + .starts_with(WPK_FORK_TABLE_CATALOG_EXPORT_PREFIX) + }); + if has_control_export || has_frame_import || has_fork_metadata { + // WHY: restamping an ABI 42 transform would certify code whose frames + // may still name parent-instance reference-table slots. Always rebuild + // from the raw linker output so the ABI 43 validator sees the original + // activation and table state. + bail!( + "fork-instrument: input already contains wasm-fork-instrument \ + imports, exports, or metadata; rebuild and instrument the raw \ + linker output instead of restamping an older artifact" + ); + } + Ok(()) +} -/// Versioned artifact claim emitted by `wasm-fork-instrument` and consumed by -/// the host before it enables cross-module fork coordination. -pub const FORK_CAPABILITIES_SECTION: &str = "kandelo.wpk_fork.capabilities"; -pub const FORK_CAPABILITIES_VERSION: u8 = 1; -pub const FORK_CAP_SIDE_ENTRY: u8 = 1 << 0; -pub const FORK_CAP_DYLINK_MAIN: u8 = 1 << 1; +fn reject_reserved_unwind_import(module: &walrus::Module) -> Result<()> { + let collides = module.imports.iter().any(|import| { + if import.module != runtime::names::IMPORT_UNWIND_TAG_MODULE { + return false; + } + module_exception_codec::is_reserved_host_import(&import.name) + || matches!( + import.name.as_str(), + runtime::names::IMPORT_UNWIND_TAG + | runtime::names::IMPORT_REF_ENCODE_FUNCREF + | runtime::names::IMPORT_REF_DECODE_FUNCREF + | runtime::names::IMPORT_REF_ENCODE_EXTERNREF + | runtime::names::IMPORT_REF_DECODE_EXTERNREF + | runtime::names::IMPORT_REF_ENCODE_EXNREF + | runtime::names::IMPORT_REF_DECODE_EXNREF + | runtime::names::IMPORT_REF_ENCODE_ANYREF + | runtime::names::IMPORT_REF_DECODE_ANYREF + ) + }); + ensure!( + !collides, + "fork-instrument: input already imports a reserved private fork runtime \ + hook from `{}`; the instrumenter must own unwind transport and reference \ + reconstruction imports", + runtime::names::IMPORT_UNWIND_TAG_MODULE, + ); + Ok(()) +} /// Options controlling instrumentation. Fields will grow as phases /// land; a `Default` implementation keeps call sites stable. @@ -36,8 +143,10 @@ pub const FORK_CAP_DYLINK_MAIN: u8 = 1 << 1; pub struct Options { /// The fully-qualified name of the import whose callers should be /// instrumented. Format: `module.field` (e.g. - /// `kernel.kernel_fork`). Future phases read this to seed the - /// call-graph discovery; Phase 1 ignores it. + /// `kernel.kernel_fork`). This import seeds call-graph discovery in a + /// main module. `env.fork` selects complete side-module boundary coverage: + /// every function import and unresolved reference dispatch becomes a + /// possible cross-instance fork boundary. pub entry_import: String, } @@ -63,48 +172,123 @@ pub struct Analysis { /// Phase 2 scope: direct-call closure only. Phase 3 extends to /// indirect calls. pub fn analyze(input: &[u8], opts: &Options) -> Result { - let module = walrus::Module::from_buffer(input).context("failed to parse input wasm module")?; + let mut module = + walrus::Module::from_buffer(input).context("failed to parse input wasm module")?; + legacy_dlopen::lower(&mut module)?; + let side_boundaries = uses_side_module_boundaries(&module, opts); + let entry_imports = call_graph::find_import_funcs(&module, &opts.entry_import); - let Some(entry) = call_graph::find_import_func(&module, &opts.entry_import) else { + if entry_imports.is_empty() + && !side_boundaries + && !call_graph::has_dynamic_linker_imports(&module) + { bail!( "entry import `{}` not found (or not a function) in the module. \ If this module does not use fork, there is nothing to instrument.", opts.entry_import ); - }; + } - let reaching = call_graph::reaching_closure(&module, entry); - let fork_path = call_graph::summarize(&module, &reaching); + let seeds = fork_boundary_seeds(&module, &entry_imports, side_boundaries); + let reaching = prepare_fork_path( + &module, + &seeds, + side_boundaries || call_graph::has_dynamic_linker_imports(&module), + ); + let fork_path = call_graph::summarize(&module, &reaching.activations); Ok(Analysis { fork_path }) } +fn uses_side_module_boundaries(module: &walrus::Module, opts: &Options) -> bool { + // `--entry env.fork` is the historical side-module invocation. ABI 43 + // broadens that role from one named import to every cross-module call + // boundary. Auto-detecting dylink.0 also protects side modules that do not + // import fork themselves but can remain live above a downstream fork. + opts.entry_import == "env.fork" + || module + .customs + .iter() + .any(|(_, section)| section.name() == "dylink.0") +} + +fn fork_boundary_seeds( + module: &walrus::Module, + entry_imports: &[walrus::FunctionId], + side_boundaries: bool, +) -> Vec { + if side_boundaries { + call_graph::imported_functions(module) + } else { + let mut seeds = entry_imports.to_vec(); + // A raw legacy import is a direct boundary when this helper is used + // before lowering. In the normal ABI 43 pipeline, lowering replaces it + // with an ordinary driver call_indirect; external dynamic-dispatch + // discovery then owns that boundary and every surviving caller. + seeds.extend(call_graph::dynamic_linker_imported_functions(module)); + seeds.sort(); + seeds.dedup(); + seeds + } +} + +/// Compute both the surviving activation set and the full semantic control +/// closure. Tail callers remain transparent and retain their bounded-stack +/// `return_call*` semantics; replay bypasses them when the continuation owns a +/// deeper activation frame. +fn prepare_fork_path( + module: &walrus::Module, + seeds: &[walrus::FunctionId], + external_dynamic_dispatch: bool, +) -> call_graph::ReachingAnalysis { + call_graph::analyze_reaching_closure_from_seeds( + module, + seeds.iter().copied(), + external_dynamic_dispatch, + ) +} + /// Instruments `input` (a complete wasm binary) according to `opts` /// and returns the transformed binary. /// -/// Current scope: Phase 4a (runtime scaffolding) + Phase 4b -/// (per-function structural wrap). Future phases 4c–6 extend the -/// per-function transform with call-site state-machine wrapping, -/// frame save/restore, mutable-global save/restore, ref-typed local -/// spilling, and catch-handler resume. +/// The complete transform includes runtime scaffolding, per-function +/// switch-dispatch, linked-frame save/restore, mutable scalar-global +/// save/restore, and activation-owned tagged-catch replay. /// /// Modules that do not import the configured entry (default -/// `kernel.kernel_fork`) are returned unchanged — there is nothing -/// to instrument. We do **not** treat this as an error because the -/// tool is invoked by build scripts across programs that may or may -/// not use `fork()`. +/// `kernel.kernel_fork`) do not have user functions rewritten, but still +/// receive the uniform activation-state helpers and imports. They may be +/// loaded beside a fork-capable main module, so their mutable globals, tables, +/// and segment lifetimes remain part of the child process image even when no +/// activation from that module is on the fork stack. pub fn instrument(input: &[u8], opts: &Options) -> Result> { let mut module = walrus::Module::from_buffer(input).context("failed to parse input wasm module")?; + reject_preinstrumented_artifact(&module)?; + legacy_dlopen::lower(&mut module)?; // Discover the fork-path closure *before* we mutate the module so // the runtime's own injected functions are not mistaken for // fork-path callers. (They can't reach the seed anyway, but the // earlier-is-simpler ordering keeps the invariant trivially.) - let entry = call_graph::find_import_func(&module, &opts.entry_import); - let fork_path = match entry { - Some(seed) => call_graph::reaching_closure(&module, seed), - None => Default::default(), - }; + let side_boundaries = uses_side_module_boundaries(&module, opts); + let entry_imports = call_graph::find_import_funcs(&module, &opts.entry_import); + let seeds = fork_boundary_seeds(&module, &entry_imports, side_boundaries); + let external_dynamic_dispatch = + side_boundaries || call_graph::has_dynamic_linker_imports(&module); + reject_reserved_unwind_import(&module)?; + let initial_fork_path = + prepare_fork_path(&module, &seeds, external_dynamic_dispatch).activations; + legacy_eh::normalize_fork_path(&mut module, &initial_fork_path)?; + // Legacy EH normalization can replace instruction sequences. Recompute + // the semantic closure afterwards so the exact fork-reaching tail-site + // coordinates used by private-tag transport name the normalized IR. + let reaching = prepare_fork_path(&module, &seeds, external_dynamic_dispatch); + let (fork_path, fork_path_targets, tail_call_sites) = ( + reaching.activations, + reaching.control_reachable, + reaching.tail_call_landings, + ); + instrument::validate_activation_state_with_targets(&module, &fork_path, &fork_path_targets)?; // The five wpk_fork_* exports prove only that some instrumentation runtime // was injected. They do not prove which import seeded the transformed call @@ -112,58 +296,109 @@ pub fn instrument(input: &[u8], opts: &Options) -> Result> { // call_indirect boundary. Emit a separate, versioned claim for exactly the // transformations performed in this invocation so the host can reject // stale or generically instrumented artifacts instead of mis-resuming. - let mut fork_capabilities = 0; - if entry.is_some() && opts.entry_import == "env.fork" { - fork_capabilities |= FORK_CAP_SIDE_ENTRY; + let mut fork_capabilities = WPK_FORK_CAP_ACTIVATION_STATE_SAFE; + if side_boundaries { + // ABI 43 interprets SIDE_ENTRY as complete side-boundary coverage, not + // merely proof that one env.fork import was discovered. + fork_capabilities |= WPK_FORK_CAP_SIDE_ENTRY; } - if entry.is_some() + if !side_boundaries + && !entry_imports.is_empty() && opts.entry_import == "kernel.kernel_fork" && call_graph::has_dynamic_linker_imports(&module) { - fork_capabilities |= FORK_CAP_DYLINK_MAIN; + fork_capabilities |= WPK_FORK_CAP_DYLINK_MAIN; } // Phase 4a: runtime scaffolding. Always injected so the module's // exported ABI is stable regardless of whether any caller was // actually rewritten. // - // Discover supported plain-catch regions before injecting the runtime. + // Discover supported tagged-catch regions before injecting the runtime. // The plan contains only static tag/label/type metadata; activation state // is allocated later as ordinary frame-backed function locals. Sort the // targets to keep local allocation and emitted bytes deterministic. - let mut fork_path_targets: Vec = fork_path + let mut activation_targets: Vec = fork_path .iter() .copied() .filter(|id| matches!(module.funcs.get(*id).kind, walrus::FunctionKind::Local(_))) .collect(); - fork_path_targets.sort(); - let plain_catch_plan = instrument::plan_plain_catches(&module, &fork_path_targets); - // Only modules with the configured fork seed need linked-frame imports. - // Runtime exports and metadata remain stable for no-seed modules, but - // adding unused host imports would make an otherwise inert side module - // impossible to instantiate through the dynamic linker. - let runtime = if entry.is_some() { - runtime::inject_linked_runtime(&mut module) - } else { - runtime::inject_runtime(&mut module) - }; + activation_targets.sort(); + let plain_catch_plan = instrument::plan_plain_catches(&module, &activation_targets); + let static_reference_plan = static_reference_catalog::plan(&mut module); + let module_state_plan = module_state::plan(&mut module); + // Capture only original module functions. Runtime and transform helpers + // injected below are implementation details and cannot have appeared in a + // source-level ref.func. A fixed-size table plus an active element segment + // is deterministic across fresh instantiation and does not depend on + // mutable guest table state. + let function_catalog = inject_function_catalog(&mut module); + static_reference_catalog::inject(&mut module, static_reference_plan); + + // Every loaded module is part of the process image even when none of its + // functions can be on the active fork stack. Give no-memory modules the + // shared process-memory staging contract before deriving the pointer ABI, + // and give every artifact the same linked imports and state helpers. + // + // WHY: an inactive side module can still own mutated globals, tables, or + // dropped segments referenced by the main process. Limiting these helpers + // to the module that imports fork would silently reset that state in a + // fresh child. + let staging_memory = module_state::ensure_staging_memory(&mut module); + let gc_codec = module_gc_codec::declare(&mut module, staging_memory)?; + let exception_codec = module_exception_codec::inject_with_reference_overrides( + &mut module, + staging_memory, + Some((gc_codec.encode_externref, gc_codec.decode_externref)), + Some((gc_codec.encode_anyref, gc_codec.decode_anyref)), + )?; + let runtime = runtime::inject_linked_runtime_with_reference_overrides( + &mut module, + runtime::ReferenceCodecOverrides { + funcref: Some(( + exception_codec.references.encode_funcref, + exception_codec.references.decode_funcref, + )), + externref: Some(( + exception_codec.references.encode_externref, + exception_codec.references.decode_externref, + )), + exnref: Some((exception_codec.encode, exception_codec.decode)), + anyref: Some((gc_codec.encode_anyref, gc_codec.decode_anyref)), + cleanup: Some(exception_codec.clear), + }, + ); + let _gc_codec = + module_gc_codec::finish_declaration(&mut module, gc_codec, exception_codec, &runtime)?; // Phase 4b: structural wrap of each fork-path function's body. // No-op when `fork_path` is empty (module doesn't use fork). - instrument::instrument_functions(&mut module, &runtime, &fork_path, &plain_catch_plan); + instrument::instrument_functions_with_targets_and_tail_sites( + &mut module, + &runtime, + &fork_path, + &fork_path_targets, + &tail_call_sites, + &plain_catch_plan, + ); + // Dirty-page instrumentation uses short-lived scalar/reference + // temporaries. Add them after continuation frame planning so they neither + // enlarge saved frames nor survive as stale activation roots. + let module_bootstrap = module_state::inject(&mut module, &runtime, module_state_plan)?; + append_function_catalog_entry(&mut module, function_catalog, module_bootstrap); loop { let existing = module .customs .iter() - .find(|(_, section)| section.name() == FORK_CAPABILITIES_SECTION) + .find(|(_, section)| section.name() == WPK_FORK_CAPABILITIES_SECTION) .map(|(id, _)| id); let Some(existing) = existing else { break }; module.customs.delete(existing); } module.customs.add(RawCustomSection { - name: FORK_CAPABILITIES_SECTION.into(), - data: vec![FORK_CAPABILITIES_VERSION, fork_capabilities], + name: WPK_FORK_CAPABILITIES_SECTION.into(), + data: vec![WPK_FORK_CAPABILITIES_VERSION, fork_capabilities], }); loop { @@ -190,6 +425,26 @@ pub fn instrument(input: &[u8], opts: &Options) -> Result> { .to_vec(), }); + replace_custom_section( + &mut module, + WPK_FORK_MODULE_STATE_FORMAT_SECTION, + module_state_descriptor(pointer_width), + ); + + // Every ABI 43 activation imports the private tag as part of its uniform + // state helpers, including a side module with no local fork entry. Keep + // the versioned descriptor equally uniform so a state-only activation can + // be admitted without weakening the host's exact-tag validation. + module.customs.add(RawCustomSection { + name: UNWIND_TRANSPORT_SECTION.into(), + // Byte 1 is the tag payload arity. Version 1 deliberately fixes it + // at zero so host validation can reject a lookalike tag import. + data: vec![ + UNWIND_TRANSPORT_VERSION, + WPK_FORK_UNWIND_TRANSPORT_PAYLOAD_ARITY, + ], + }); + // Historical phase list (Phase 4b/4c/4d/4e/4f/5/6) was an artefact // of guard-dispatch's body-rewriting approach. Post-commit-4 those // phases are folded into `instrument::instrument_functions` itself; @@ -200,6 +455,92 @@ pub fn instrument(input: &[u8], opts: &Options) -> Result> { restore_leading_dylink_section(input, output) } +fn inject_function_catalog(module: &mut walrus::Module) -> TableId { + let mut functions: Vec = + module.funcs.iter().map(|func| func.id()).collect(); + functions.sort(); + let size = functions.len() as u64; + let table = module + .tables + .add_local(false, size, Some(size), RefType::FUNCREF); + module.tables.get_mut(table).name = Some(FUNCTION_CATALOG_EXPORT.into()); + if !functions.is_empty() { + module.elements.add( + ElementKind::Active { + table, + offset: walrus::ConstExpr::Value(Value::I32(0)), + }, + ElementItems::Functions(functions), + ); + } + // WHY: fixed min/max prevents guest growth. The host treats this export as + // immutable catalog input and never uses it as mutable replay storage. + module.exports.add(FUNCTION_CATALOG_EXPORT, table); + table +} + +fn append_function_catalog_entry( + module: &mut walrus::Module, + table: TableId, + function: FunctionId, +) { + let ordinal = module.tables.get(table).initial; + let size = ordinal + .checked_add(1) + .expect("fork function catalog length fits u64"); + let catalog = module.tables.get_mut(table); + catalog.initial = size; + catalog.maximum = Some(size); + module.elements.add( + ElementKind::Active { + table, + offset: walrus::ConstExpr::Value(Value::I32( + i32::try_from(ordinal).expect("fork function catalog ordinal fits i32"), + )), + }, + ElementItems::Functions(vec![function]), + ); +} + +fn module_state_descriptor(pointer_width: linked_frames::PointerWidth) -> Vec { + let pointer_width = match pointer_width { + linked_frames::PointerWidth::Wasm32 => u8::try_from(u32::BITS / u8::BITS).unwrap(), + linked_frames::PointerWidth::Wasm64 => u8::try_from(u64::BITS / u8::BITS).unwrap(), + }; + let mut data = Vec::with_capacity(usize::from(WPK_FORK_MODULE_STATE_DESCRIPTOR_SIZE)); + data.extend_from_slice(&WPK_FORK_MODULE_STATE_FORMAT_MAGIC); + data.extend_from_slice(&WPK_FORK_MODULE_STATE_FORMAT_VERSION.to_le_bytes()); + data.extend_from_slice(&WPK_FORK_MODULE_STATE_DESCRIPTOR_SIZE.to_le_bytes()); + data.push(pointer_width); + data.push(WPK_FORK_MODULE_STATE_RECORD_ALIGNMENT); + data.extend_from_slice(&WPK_FORK_MODULE_STATE_REQUIRED_FLAGS.to_le_bytes()); + data.extend_from_slice(&WPK_FORK_MODULE_STATE_ARENA_VERSION.to_le_bytes()); + data.extend_from_slice(&WPK_FORK_MODULE_STATE_RECORD_VERSION.to_le_bytes()); + data.extend_from_slice(&WPK_FORK_MODULE_STATE_ROOT_POINTER_WORD_OFFSET.to_le_bytes()); + data.extend_from_slice(&u32::default().to_le_bytes()); + debug_assert_eq!( + data.len(), + usize::from(WPK_FORK_MODULE_STATE_DESCRIPTOR_SIZE) + ); + data +} + +fn replace_custom_section(module: &mut walrus::Module, name: &str, data: Vec) { + loop { + let existing = module + .customs + .iter() + .find(|(_, section)| section.name() == name) + .map(|(id, _)| id); + let Some(existing) = existing else { break }; + module.customs.delete(existing); + } + module.customs.add(RawCustomSection { + name: name.into(), + data, + }); +} + /// Walrus emits raw custom sections after the standard sections. That is /// normally valid, but the WebAssembly dynamic-linking convention requires a /// shared module's `dylink.0` custom section to be first. Preserve that input diff --git a/crates/fork-instrument/src/linked_frames.rs b/crates/fork-instrument/src/linked_frames.rs index 534de03d6d..d63c419005 100644 --- a/crates/fork-instrument/src/linked_frames.rs +++ b/crates/fork-instrument/src/linked_frames.rs @@ -14,7 +14,7 @@ pub const WASM_PAGE_SIZE: u64 = 64 * 1024; /// Alignment used for chunk and node records. pub const RECORD_ALIGNMENT: u64 = abi::WPK_FORK_LINKED_FRAME_RECORD_ALIGNMENT as u64; -/// Linked-frame artifact metadata version used by ABI 42. +/// Linked-frame artifact metadata version used by ABI 42 and later. pub const LINKED_FRAME_FORMAT_VERSION: u16 = abi::WPK_FORK_LINKED_FRAME_FORMAT_VERSION; pub const LINKED_FRAME_FORMAT_SECTION: &str = abi::WPK_FORK_LINKED_FRAME_FORMAT_SECTION; diff --git a/crates/fork-instrument/src/main.rs b/crates/fork-instrument/src/main.rs index 8f63f54983..9238c4cd54 100644 --- a/crates/fork-instrument/src/main.rs +++ b/crates/fork-instrument/src/main.rs @@ -17,7 +17,14 @@ use std::fs; use std::os::unix::fs::PermissionsExt; use std::path::{Path, PathBuf}; -use fork_instrument::{Options, analyze, instrument}; +use fork_instrument::{ + Options, analyze, + contract_inventory::{ + artifact_identity, fork_capability_section_hex, fork_contract_inventory, + linked_frame_descriptor_section_hex, + }, + instrument, +}; #[derive(Debug, Parser)] #[command( @@ -29,13 +36,15 @@ struct Cli { /// Input wasm file to instrument. input: PathBuf, - /// Output path for the instrumented wasm file. Required unless - /// `--discover-only` is set (analysis-only mode). + /// Output path for the instrumented wasm file. Required unless an + /// analysis or contract-inspection mode is set. #[arg(short, long)] output: Option, /// The fully-qualified name of the import that triggers unwind. /// Format: `module.field`. Defaults to `kernel.kernel_fork`. + /// `env.fork` selects complete dynamically linked side-module boundary + /// coverage, including downstream fork in another side module. #[arg(long, default_value = "kernel.kernel_fork")] entry: String, @@ -45,27 +54,102 @@ struct Cli { /// hand-maintained onlylists. #[arg(long)] discover_only: bool, + + /// Print the fork-artifact structural inventory as one TSV row. + /// This mode performs no instrumentation and emits no output file. + #[arg( + long, + conflicts_with_all = ["discover_only", "artifact_identity", "output"] + )] + contract_inventory: bool, + + /// Print relocatable, memory, and strict ABI-export identity as one TSV row. + /// This mode performs no instrumentation and emits no output file. + #[arg( + long, + conflicts_with_all = [ + "discover_only", + "contract_inventory", + "fork_capability_hex", + "linked_frame_descriptor_hex", + "output" + ] + )] + artifact_identity: bool, + + /// Print the unique fork-capability custom section as lowercase hex. + #[arg( + long, + conflicts_with_all = [ + "discover_only", + "contract_inventory", + "artifact_identity", + "linked_frame_descriptor_hex", + "output" + ] + )] + fork_capability_hex: bool, + + /// Print the unique linked-frame descriptor custom section as lowercase hex. + #[arg( + long, + conflicts_with_all = [ + "discover_only", + "contract_inventory", + "artifact_identity", + "fork_capability_hex", + "output" + ] + )] + linked_frame_descriptor_hex: bool, } fn main() -> Result<()> { let cli = Cli::parse(); - let input = fs::read(&cli.input) - .with_context(|| format!("reading input: {}", cli.input.display()))?; + let input = + fs::read(&cli.input).with_context(|| format!("reading input: {}", cli.input.display()))?; + + if cli.contract_inventory { + let inventory = fork_contract_inventory(&input) + .with_context(|| format!("inventorying {}", cli.input.display()))?; + println!("{inventory}"); + return Ok(()); + } + if cli.artifact_identity { + let identity = artifact_identity(&input) + .with_context(|| format!("inspecting artifact identity: {}", cli.input.display()))?; + println!("{identity}"); + return Ok(()); + } + if cli.fork_capability_hex { + let hex = fork_capability_section_hex(&input) + .with_context(|| format!("reading fork capability: {}", cli.input.display()))?; + println!("{hex}"); + return Ok(()); + } + if cli.linked_frame_descriptor_hex { + let hex = linked_frame_descriptor_section_hex(&input) + .with_context(|| format!("reading linked-frame descriptor: {}", cli.input.display()))?; + println!("{hex}"); + return Ok(()); + } let opts = Options { entry_import: cli.entry, }; if cli.discover_only { - let analysis = analyze(&input, &opts) - .with_context(|| format!("analyzing {}", cli.input.display()))?; + let analysis = + analyze(&input, &opts).with_context(|| format!("analyzing {}", cli.input.display()))?; print_analysis_json(&analysis); return Ok(()); } let output_path = cli.output.as_ref().ok_or_else(|| { - anyhow::anyhow!("--output is required unless --discover-only is set") + anyhow::anyhow!( + "--output is required unless an analysis or contract-inspection mode is set" + ) })?; // Capture this before writing: `--output` is allowed to name the input // file, and output creation/truncation must not become the source of truth @@ -115,7 +199,11 @@ fn print_analysis_json(analysis: &fork_instrument::Analysis) { println!("{{"); println!(" \"fork_path\": ["); for (i, entry) in analysis.fork_path.iter().enumerate() { - let comma = if i + 1 == analysis.fork_path.len() { "" } else { "," }; + let comma = if i + 1 == analysis.fork_path.len() { + "" + } else { + "," + }; println!( " {{ \"name\": {}, \"is_import\": {} }}{}", json_string(&entry.name), diff --git a/crates/fork-instrument/src/module_exception_codec.rs b/crates/fork-instrument/src/module_exception_codec.rs new file mode 100644 index 0000000000..b14fcefe64 --- /dev/null +++ b/crates/fork-instrument/src/module_exception_codec.rs @@ -0,0 +1,1594 @@ +//! Exact-tag exception codecs injected into the owning Wasm module. +//! +//! A separate provider module cannot import a module's local tags until that +//! module exists, while the module cannot import an `exnref` codec from that +//! provider before it is instantiated. Injecting the codec here removes that +//! bootstrap cycle and preserves canonical concrete tag and payload types. +//! +//! Cross-activation routing still has scalar-only JavaScript signatures: +//! +//! * encode delegates an unknown exception by asking the host broker to catch +//! `throw_slot(slot)` and call the selected owner's `encode_ingress(token)`; +//! * decode asks the broker to call the selected owner's `throw_recipe(id)`, +//! then catches that thrown value with `CatchAllRef` in the requesting +//! module. +//! +//! Recipe identity is claimed before recursive payload encoding. Decode caches +//! every materialized exception by shared recipe ID, preserving aliases. + +use anyhow::{Result, ensure}; +use walrus::{ + AbstractHeapType, FunctionBuilder, FunctionId, FunctionKind, GlobalId, HeapType, ImportKind, + LocalFunction, LocalId, MemoryId, Module, RawCustomSection, RefType, TableId, TagId, ValType, + ir::{ + BinaryOp, Binop, Block, Call, Const, Drop, GlobalGet, IfElse, Instr, InstrLocId, + InstrSeqId, InstrSeqType, Load, LoadKind, LocalGet, LocalSet, LocalTee, MemArg, + RefAsNonNull, RefCast, RefIsNull, RefNull, Return, Store, StoreKind, TableFill, TableGet, + TableGrow, TableSet, TableSize, Throw, ThrowRef, TryTable, TryTableCatch, UnaryOp, Unop, + Unreachable, Value, + }, +}; +use wasm_posix_shared::abi::{ + WPK_FORK_EXCEPTION_CODEC_HEADER_SIZE, WPK_FORK_EXCEPTION_CODEC_IMPORT_MODULE, + WPK_FORK_EXCEPTION_CODEC_SECTION, WPK_FORK_EXCEPTION_CODEC_TAG_RECORD_SIZE, + WPK_FORK_EXCEPTION_CODEC_VERSION, WPK_FORK_EXCEPTION_EXPORT_ABORT, + WPK_FORK_EXCEPTION_EXPORT_CLEAR, WPK_FORK_EXCEPTION_EXPORT_DECODE, + WPK_FORK_EXCEPTION_EXPORT_ENCODE, WPK_FORK_EXCEPTION_EXPORT_ENCODE_INGRESS, + WPK_FORK_EXCEPTION_EXPORT_MATERIALIZE, WPK_FORK_EXCEPTION_EXPORT_THROW_RECIPE, + WPK_FORK_EXCEPTION_EXPORT_THROW_SLOT, WPK_FORK_EXCEPTION_IMPORT_ACTIVATION, + WPK_FORK_EXCEPTION_IMPORT_BROKER_ENCODE, WPK_FORK_EXCEPTION_IMPORT_BROKER_THROW_RECIPE, + WPK_FORK_EXCEPTION_IMPORT_CACHE_INDEX, WPK_FORK_EXCEPTION_IMPORT_CLAIM, + WPK_FORK_EXCEPTION_IMPORT_DEFINE, WPK_FORK_EXCEPTION_IMPORT_INGRESS_THROW, + WPK_FORK_EXCEPTION_IMPORT_LOAD, WPK_FORK_EXCEPTION_IMPORT_LOOKUP, + WPK_FORK_EXCEPTION_IMPORT_ROUTE, WPK_FORK_REFERENCE_IMPORT_SCRATCH_RELEASE, + WPK_FORK_REFERENCE_IMPORT_SCRATCH_RESERVE, +}; + +use crate::runtime::{ReferenceCodecClass, names as runtime_names}; + +pub const FORMAT_SECTION: &str = WPK_FORK_EXCEPTION_CODEC_SECTION; +pub const FORMAT_VERSION: u8 = WPK_FORK_EXCEPTION_CODEC_VERSION; +pub const FORMAT_HEADER_SIZE: usize = WPK_FORK_EXCEPTION_CODEC_HEADER_SIZE as usize; +pub const FORMAT_TAG_RECORD_SIZE: usize = WPK_FORK_EXCEPTION_CODEC_TAG_RECORD_SIZE as usize; + +pub const HOST_IMPORT_MODULE: &str = WPK_FORK_EXCEPTION_CODEC_IMPORT_MODULE; +pub const IMPORT_ACTIVATION: &str = WPK_FORK_EXCEPTION_IMPORT_ACTIVATION; +pub const IMPORT_LOOKUP: &str = WPK_FORK_EXCEPTION_IMPORT_LOOKUP; +pub const IMPORT_CLAIM: &str = WPK_FORK_EXCEPTION_IMPORT_CLAIM; +pub const IMPORT_DEFINE: &str = WPK_FORK_EXCEPTION_IMPORT_DEFINE; +pub const IMPORT_LOAD: &str = WPK_FORK_EXCEPTION_IMPORT_LOAD; +pub const IMPORT_ROUTE: &str = WPK_FORK_EXCEPTION_IMPORT_ROUTE; +pub const IMPORT_CACHE_INDEX: &str = WPK_FORK_EXCEPTION_IMPORT_CACHE_INDEX; +pub const IMPORT_BROKER_ENCODE: &str = WPK_FORK_EXCEPTION_IMPORT_BROKER_ENCODE; +pub const IMPORT_BROKER_THROW_RECIPE: &str = WPK_FORK_EXCEPTION_IMPORT_BROKER_THROW_RECIPE; +pub const IMPORT_INGRESS_THROW: &str = WPK_FORK_EXCEPTION_IMPORT_INGRESS_THROW; +pub const IMPORT_SCRATCH_RESERVE: &str = WPK_FORK_REFERENCE_IMPORT_SCRATCH_RESERVE; +pub const IMPORT_SCRATCH_RELEASE: &str = WPK_FORK_REFERENCE_IMPORT_SCRATCH_RELEASE; + +pub const EXPORT_ENCODE: &str = WPK_FORK_EXCEPTION_EXPORT_ENCODE; +pub const EXPORT_DECODE: &str = WPK_FORK_EXCEPTION_EXPORT_DECODE; +pub const EXPORT_THROW_SLOT: &str = WPK_FORK_EXCEPTION_EXPORT_THROW_SLOT; +pub const EXPORT_THROW_RECIPE: &str = WPK_FORK_EXCEPTION_EXPORT_THROW_RECIPE; +pub const EXPORT_ENCODE_INGRESS: &str = WPK_FORK_EXCEPTION_EXPORT_ENCODE_INGRESS; +pub const EXPORT_MATERIALIZE: &str = WPK_FORK_EXCEPTION_EXPORT_MATERIALIZE; +pub const EXPORT_CLEAR: &str = WPK_FORK_EXCEPTION_EXPORT_CLEAR; +pub const EXPORT_ABORT: &str = WPK_FORK_EXCEPTION_EXPORT_ABORT; + +const MAX_RECIPE_ID: i32 = 0x7fff_fffe; + +pub fn is_reserved_host_import(name: &str) -> bool { + matches!( + name, + IMPORT_ACTIVATION + | IMPORT_LOOKUP + | IMPORT_CLAIM + | IMPORT_DEFINE + | IMPORT_LOAD + | IMPORT_ROUTE + | IMPORT_CACHE_INDEX + | IMPORT_BROKER_ENCODE + | IMPORT_BROKER_THROW_RECIPE + | IMPORT_INGRESS_THROW + | IMPORT_SCRATCH_RESERVE + | IMPORT_SCRATCH_RELEASE + ) +} + +#[derive(Debug, Clone, Copy)] +pub struct ReferenceDependencies { + pub encode_funcref: FunctionId, + pub decode_funcref: FunctionId, + pub encode_externref: FunctionId, + pub decode_externref: FunctionId, + pub encode_anyref: FunctionId, + pub decode_anyref: FunctionId, +} + +#[derive(Debug, Clone, Copy)] +pub struct InjectedExceptionCodec { + pub encode: FunctionId, + pub decode: FunctionId, + pub throw_slot: FunctionId, + pub throw_recipe: FunctionId, + pub encode_ingress: FunctionId, + pub materialize: FunctionId, + pub clear: FunctionId, + pub abort: FunctionId, + pub memory: MemoryId, + pub references: ReferenceDependencies, +} + +#[derive(Debug, Clone, Copy)] +struct HostImports { + activation: GlobalId, + lookup: FunctionId, + claim: FunctionId, + define: FunctionId, + load: FunctionId, + route: FunctionId, + cache_index: FunctionId, + broker_encode: FunctionId, + broker_throw_recipe: FunctionId, + ingress_throw: FunctionId, + scratch_reserve: FunctionId, + scratch_release: FunctionId, +} + +#[derive(Debug, Clone)] +struct PayloadLayout { + ty: ValType, + scalar_offset: Option, + reference_offset: Option, +} + +#[derive(Debug, Clone)] +struct TagLayout { + tag: TagId, + ordinal: u32, + layout_id: u32, + scalar_len: u32, + references_ptr: u32, + reference_count: u32, + payloads: Vec, +} + +impl TagLayout { + fn staging_len(&self) -> u32 { + self.references_ptr + .checked_add(self.reference_count.saturating_mul(4)) + .expect("validated exception staging layout") + .max(1) + } +} + +#[derive(Debug, Clone, Copy)] +struct PayloadLocals { + value: LocalId, + recipe: Option, +} + +#[derive(Debug, Clone)] +struct HandlerLocals { + payloads: Vec, +} + +const NULLABLE_EXNREF: RefType = RefType { + nullable: true, + heap_type: HeapType::Abstract(AbstractHeapType::Exn), +}; +const NON_NULL_EXNREF: RefType = RefType { + nullable: false, + heap_type: HeapType::Abstract(AbstractHeapType::Exn), +}; + +/// Inject the exact-tag codec before the generic continuation runtime. +/// +/// The module-state plan and function catalog must already be frozen: codec +/// tables are temporary reconstruction caches, not guest mutable table state, +/// and codec helper functions are not source-level `ref.func` targets. +pub fn inject(module: &mut Module, memory: MemoryId) -> Result { + inject_with_reference_overrides(module, memory, None, None) +} + +pub fn inject_with_anyref( + module: &mut Module, + memory: MemoryId, + anyref: Option<(FunctionId, FunctionId)>, +) -> Result { + inject_with_reference_overrides(module, memory, None, anyref) +} + +pub fn inject_with_reference_overrides( + module: &mut Module, + memory: MemoryId, + externref: Option<(FunctionId, FunctionId)>, + anyref: Option<(FunctionId, FunctionId)>, +) -> Result { + let layouts = plan_tags(module)?; + let ptr_ty = if module.memories.get(memory).memory64 { + ValType::I64 + } else { + ValType::I32 + }; + let scratch = module.tables.add_local(false, 1, Some(1), NULLABLE_EXNREF); + module.tables.get_mut(scratch).name = Some("__wpk_fork_ref_exn_scratch".into()); + let replay = module.tables.add_local(false, 1, None, NULLABLE_EXNREF); + module.tables.get_mut(replay).name = Some("__wpk_fork_ref_exn_replay".into()); + + let imports = inject_host_imports(module, ptr_ty); + let references = inject_reference_dependencies(module, externref, anyref); + + let (encode, encode_args) = add_stub( + module, + &[ValType::Ref(NULLABLE_EXNREF)], + &[ValType::I32], + EXPORT_ENCODE, + ); + let (decode, decode_args) = add_stub( + module, + &[ValType::I32], + &[ValType::Ref(NULLABLE_EXNREF)], + EXPORT_DECODE, + ); + let (throw_slot, throw_slot_args) = add_stub(module, &[ValType::I32], &[], EXPORT_THROW_SLOT); + let (throw_recipe, throw_recipe_args) = + add_stub(module, &[ValType::I32], &[], EXPORT_THROW_RECIPE); + let (encode_ingress, encode_ingress_args) = add_stub( + module, + &[ValType::I32], + &[ValType::I32], + EXPORT_ENCODE_INGRESS, + ); + let (materialize, materialize_args) = + add_stub(module, &[ValType::I32], &[], EXPORT_MATERIALIZE); + let (clear, _) = add_stub(module, &[], &[], EXPORT_CLEAR); + let (abort, _) = add_stub(module, &[], &[], EXPORT_ABORT); + + emit_encode( + module, + encode, + encode_args[0], + scratch, + memory, + ptr_ty, + imports, + references, + &layouts, + ); + emit_decode( + module, + decode, + decode_args[0], + replay, + memory, + ptr_ty, + imports, + references, + &layouts, + ); + emit_throw_slot(module, throw_slot, throw_slot_args[0], scratch); + emit_throw_recipe(module, throw_recipe, throw_recipe_args[0], decode); + emit_encode_ingress( + module, + encode_ingress, + encode_ingress_args[0], + imports.ingress_throw, + encode, + ); + emit_materialize(module, materialize, materialize_args[0], decode); + emit_clear(module, clear, scratch, replay); + emit_clear(module, abort, scratch, replay); + + for (name, function) in [ + (EXPORT_ENCODE, encode), + (EXPORT_DECODE, decode), + (EXPORT_THROW_SLOT, throw_slot), + (EXPORT_THROW_RECIPE, throw_recipe), + (EXPORT_ENCODE_INGRESS, encode_ingress), + (EXPORT_MATERIALIZE, materialize), + (EXPORT_CLEAR, clear), + (EXPORT_ABORT, abort), + ] { + module.exports.add(name, function); + } + replace_descriptor(module, &layouts); + + Ok(InjectedExceptionCodec { + encode, + decode, + throw_slot, + throw_recipe, + encode_ingress, + materialize, + clear, + abort, + memory, + references, + }) +} + +fn plan_tags(module: &Module) -> Result> { + let mut layouts = Vec::new(); + for (ordinal, tag) in module.tags.iter().enumerate() { + let ty = module.types.get(tag.ty()); + ensure!( + ty.results().is_empty(), + "fork-instrument: exception tag {ordinal} unexpectedly has results" + ); + let mut scalar_len = 0u32; + let mut reference_count = 0u32; + let mut payloads = Vec::new(); + for payload in ty.params().iter().copied() { + match payload { + ValType::I32 | ValType::F32 => { + let offset = scalar_len; + scalar_len = scalar_len + .checked_add(4) + .ok_or_else(|| anyhow::anyhow!("exception scalar layout overflow"))?; + payloads.push(PayloadLayout { + ty: payload, + scalar_offset: Some(offset), + reference_offset: None, + }); + } + ValType::I64 | ValType::F64 => { + let offset = scalar_len; + scalar_len = scalar_len + .checked_add(8) + .ok_or_else(|| anyhow::anyhow!("exception scalar layout overflow"))?; + payloads.push(PayloadLayout { + ty: payload, + scalar_offset: Some(offset), + reference_offset: None, + }); + } + ValType::V128 => { + let offset = scalar_len; + scalar_len = scalar_len + .checked_add(16) + .ok_or_else(|| anyhow::anyhow!("exception scalar layout overflow"))?; + payloads.push(PayloadLayout { + ty: payload, + scalar_offset: Some(offset), + reference_offset: None, + }); + } + ValType::Ref(_) => { + payloads.push(PayloadLayout { + ty: payload, + scalar_offset: None, + reference_offset: Some(reference_count), + }); + reference_count = reference_count + .checked_add(1) + .ok_or_else(|| anyhow::anyhow!("exception reference layout overflow"))?; + } + } + } + let references_ptr = align_up(scalar_len, 4)?; + layouts.push(TagLayout { + tag: tag.id(), + ordinal: ordinal as u32, + layout_id: ordinal as u32, + scalar_len, + references_ptr, + reference_count, + payloads, + }); + } + Ok(layouts) +} + +fn align_up(value: u32, alignment: u32) -> Result { + let mask = alignment - 1; + value + .checked_add(mask) + .map(|value| value & !mask) + .ok_or_else(|| anyhow::anyhow!("exception staging layout overflow")) +} + +fn inject_host_imports(module: &mut Module, ptr_ty: ValType) -> HostImports { + let existing_activation = module.imports.iter().find_map(|import| { + (import.module == HOST_IMPORT_MODULE && import.name == IMPORT_ACTIVATION) + .then_some(&import.kind) + .and_then(|kind| match kind { + ImportKind::Global(global) => Some(*global), + _ => None, + }) + }); + let activation = match existing_activation { + Some(activation) => activation, + None => { + module + .add_import_global( + HOST_IMPORT_MODULE, + IMPORT_ACTIVATION, + ValType::I32, + false, + false, + ) + .0 + } + }; + let lookup = import_function(module, IMPORT_LOOKUP, &[ValType::I32], &[ValType::I32]); + let claim = import_function(module, IMPORT_CLAIM, &[ValType::I32], &[ValType::I32]); + let define = import_function( + module, + IMPORT_DEFINE, + &[ + ValType::I32, + ValType::I32, + ValType::I32, + ValType::I32, + ptr_ty, + ValType::I32, + ptr_ty, + ValType::I32, + ], + &[], + ); + let load = import_function( + module, + IMPORT_LOAD, + &[ + ValType::I32, + ValType::I32, + ValType::I32, + ValType::I32, + ptr_ty, + ValType::I32, + ptr_ty, + ValType::I32, + ], + &[ValType::I32], + ); + let route = import_function( + module, + IMPORT_ROUTE, + &[ValType::I32, ValType::I32], + &[ValType::I32], + ); + let cache_index = import_function(module, IMPORT_CACHE_INDEX, &[ValType::I32], &[ValType::I32]); + let broker_encode = import_function( + module, + IMPORT_BROKER_ENCODE, + &[ValType::I32], + &[ValType::I32], + ); + let broker_throw_recipe = + import_function(module, IMPORT_BROKER_THROW_RECIPE, &[ValType::I32], &[]); + let ingress_throw = import_function(module, IMPORT_INGRESS_THROW, &[ValType::I32], &[]); + let scratch_reserve = import_function(module, IMPORT_SCRATCH_RESERVE, &[ptr_ty], &[ptr_ty]); + let scratch_release = import_function(module, IMPORT_SCRATCH_RELEASE, &[ptr_ty, ptr_ty], &[]); + HostImports { + activation, + lookup, + claim, + define, + load, + route, + cache_index, + broker_encode, + broker_throw_recipe, + ingress_throw, + scratch_reserve, + scratch_release, + } +} + +fn inject_reference_dependencies( + module: &mut Module, + externref: Option<(FunctionId, FunctionId)>, + anyref: Option<(FunctionId, FunctionId)>, +) -> ReferenceDependencies { + fn pair( + module: &mut Module, + reference: RefType, + encode_name: &str, + decode_name: &str, + ) -> (FunctionId, FunctionId) { + let value = ValType::Ref(reference); + ( + import_function(module, encode_name, &[value], &[ValType::I32]), + import_function(module, decode_name, &[ValType::I32], &[value]), + ) + } + let (encode_funcref, decode_funcref) = pair( + module, + RefType::FUNCREF, + runtime_names::IMPORT_REF_ENCODE_FUNCREF, + runtime_names::IMPORT_REF_DECODE_FUNCREF, + ); + let (encode_externref, decode_externref) = externref.unwrap_or_else(|| { + pair( + module, + RefType::EXTERNREF, + runtime_names::IMPORT_REF_ENCODE_EXTERNREF, + runtime_names::IMPORT_REF_DECODE_EXTERNREF, + ) + }); + let (encode_anyref, decode_anyref) = anyref.unwrap_or_else(|| { + pair( + module, + RefType::ANYREF, + runtime_names::IMPORT_REF_ENCODE_ANYREF, + runtime_names::IMPORT_REF_DECODE_ANYREF, + ) + }); + ReferenceDependencies { + encode_funcref, + decode_funcref, + encode_externref, + decode_externref, + encode_anyref, + decode_anyref, + } +} + +fn import_function( + module: &mut Module, + name: &str, + params: &[ValType], + results: &[ValType], +) -> FunctionId { + let ty = module.types.add(params, results); + module.add_import_func(HOST_IMPORT_MODULE, name, ty).0 +} + +fn add_stub( + module: &mut Module, + params: &[ValType], + results: &[ValType], + name: &str, +) -> (FunctionId, Vec) { + let args: Vec<_> = params + .iter() + .copied() + .map(|ty| module.locals.add(ty)) + .collect(); + let mut builder = FunctionBuilder::new(&mut module.types, params, results); + builder.name(name.into()); + let function = builder.finish(args.clone(), &mut module.funcs); + (function, args) +} + +#[allow(clippy::too_many_arguments)] +fn emit_encode( + module: &mut Module, + function: FunctionId, + exception: LocalId, + scratch: TableId, + memory: MemoryId, + ptr_ty: ValType, + imports: HostImports, + references: ReferenceDependencies, + layouts: &[TagLayout], +) { + let recipe = module.locals.add(ValType::I32); + let staging = module.locals.add(ptr_ty); + let handler_locals: Vec<_> = layouts + .iter() + .map(|layout| HandlerLocals { + payloads: layout + .payloads + .iter() + .map(|payload| PayloadLocals { + value: module.locals.add(storage_type(payload.ty)), + recipe: matches!(payload.ty, ValType::Ref(_)) + .then(|| module.locals.add(ValType::I32)), + }) + .collect(), + }) + .collect(); + + let null_then = dangling(module, function, InstrSeqType::Simple(None)); + { + let instrs = instrs_mut(module, function, null_then); + constant_i32(instrs, 0); + push(instrs, Instr::Return(Return {})); + } + let empty_else = dangling(module, function, InstrSeqType::Simple(None)); + + let existing_then = dangling(module, function, InstrSeqType::Simple(None)); + { + let instrs = instrs_mut(module, function, existing_then); + emit_clear_scratch(instrs, scratch); + local_get(instrs, recipe); + push(instrs, Instr::Return(Return {})); + } + let existing_else = dangling(module, function, InstrSeqType::Simple(None)); + + let outer = dangling(module, function, InstrSeqType::Simple(Some(ValType::I32))); + let mut caps = Vec::new(); + for layout in layouts { + let mut results: Vec<_> = layout.payloads.iter().map(|payload| payload.ty).collect(); + results.push(ValType::Ref(NON_NULL_EXNREF)); + let ty = InstrSeqType::new(&mut module.types, &[], &results); + caps.push(dangling(module, function, ty)); + } + let fallback_ty = InstrSeqType::new(&mut module.types, &[], &[ValType::Ref(NON_NULL_EXNREF)]); + let fallback = dangling(module, function, fallback_ty); + caps.push(fallback); + + let throw_body = dangling(module, function, InstrSeqType::Simple(None)); + { + let instrs = instrs_mut(module, function, throw_body); + local_get(instrs, exception); + push(instrs, Instr::RefAsNonNull(RefAsNonNull {})); + push(instrs, Instr::ThrowRef(ThrowRef {})); + } + let catches: Vec<_> = layouts + .iter() + .zip(caps.iter()) + .map(|(layout, cap)| TryTableCatch::CatchRef { + tag: layout.tag, + label: *cap, + }) + .chain(std::iter::once(TryTableCatch::CatchAllRef { + label: fallback, + })) + .collect(); + { + let innermost = *caps.last().expect("fallback cap always exists"); + let instrs = instrs_mut(module, function, innermost); + push( + instrs, + Instr::TryTable(TryTable { + seq: throw_body, + catches, + }), + ); + push(instrs, Instr::Unreachable(Unreachable {})); + } + + for index in (0..caps.len() - 1).rev() { + let child = caps[index + 1]; + push( + instrs_mut(module, function, caps[index]), + Instr::Block(Block { seq: child }), + ); + if index + 1 == layouts.len() { + emit_unknown_encode_handler( + module, + function, + caps[index], + scratch, + imports.broker_encode, + recipe, + ); + } else { + emit_known_encode_handler( + module, + function, + caps[index], + &layouts[index + 1], + &handler_locals[index + 1], + scratch, + memory, + ptr_ty, + staging, + imports, + references, + function, + recipe, + ); + } + } + push( + instrs_mut(module, function, outer), + Instr::Block(Block { seq: caps[0] }), + ); + if layouts.is_empty() { + emit_unknown_encode_handler( + module, + function, + outer, + scratch, + imports.broker_encode, + recipe, + ); + } else { + emit_known_encode_handler( + module, + function, + outer, + &layouts[0], + &handler_locals[0], + scratch, + memory, + ptr_ty, + staging, + imports, + references, + function, + recipe, + ); + } + + let entry = entry(function, module); + let instrs = instrs_mut(module, function, entry); + local_get(instrs, exception); + push(instrs, Instr::RefIsNull(RefIsNull {})); + push( + instrs, + Instr::IfElse(IfElse { + consequent: null_then, + alternative: empty_else, + }), + ); + constant_i32(instrs, 0); + local_get(instrs, exception); + push(instrs, Instr::TableSet(TableSet { table: scratch })); + constant_i32(instrs, 0); + call(instrs, imports.lookup); + push(instrs, Instr::LocalTee(LocalTee { local: recipe })); + push( + instrs, + Instr::IfElse(IfElse { + consequent: existing_then, + alternative: existing_else, + }), + ); + push(instrs, Instr::Block(Block { seq: outer })); +} + +#[allow(clippy::too_many_arguments)] +fn emit_known_encode_handler( + module: &mut Module, + function: FunctionId, + seq: InstrSeqId, + layout: &TagLayout, + locals: &HandlerLocals, + scratch: TableId, + memory: MemoryId, + ptr_ty: ValType, + staging: LocalId, + imports: HostImports, + references: ReferenceDependencies, + encode_exnref: FunctionId, + recipe: LocalId, +) { + let encoders: Vec<_> = layout + .payloads + .iter() + .map(|payload| match payload.ty { + ValType::Ref(reference) => Some(reference_encoder( + module, + references, + encode_exnref, + reference, + )), + _ => None, + }) + .collect(); + let instrs = instrs_mut(module, function, seq); + push(instrs, Instr::Drop(Drop {})); + for payload in locals.payloads.iter().rev() { + local_set(instrs, payload.value); + } + constant_i32(instrs, 0); + call(instrs, imports.claim); + local_set(instrs, recipe); + emit_clear_scratch(instrs, scratch); + + // WHY: transaction scratch is disjoint for recursive payload codecs and + // lives in the one process memory copied into the child. It is transient + // exchange storage, never continuation evidence. + constant_ptr(instrs, ptr_ty, u64::from(layout.staging_len())); + call(instrs, imports.scratch_reserve); + local_set(instrs, staging); + + for ((payload, local), encoder) in layout.payloads.iter().zip(&locals.payloads).zip(encoders) { + let ValType::Ref(reference) = payload.ty else { + continue; + }; + local_get(instrs, local.value); + let _ = reference; + call(instrs, encoder.expect("reference payload encoder")); + local_set( + instrs, + local.recipe.expect("reference payload has recipe local"), + ); + } + for (payload, local) in layout.payloads.iter().zip(&locals.payloads) { + if let Some(offset) = payload.scalar_offset { + emit_staging_address(instrs, staging, ptr_ty, 0); + local_get(instrs, local.value); + push( + instrs, + Instr::Store(Store { + memory, + kind: scalar_store(payload.ty), + arg: MemArg { + align: 1, + offset: u64::from(offset), + }, + }), + ); + } else { + let index = payload.reference_offset.expect("reference payload index"); + emit_staging_address(instrs, staging, ptr_ty, 0); + local_get( + instrs, + local.recipe.expect("reference payload recipe local"), + ); + push( + instrs, + Instr::Store(Store { + memory, + kind: StoreKind::I32 { atomic: false }, + arg: MemArg { + align: 4, + offset: u64::from(layout.references_ptr + index * 4), + }, + }), + ); + } + } + local_get(instrs, recipe); + push( + instrs, + Instr::GlobalGet(GlobalGet { + global: imports.activation, + }), + ); + constant_i32(instrs, layout.ordinal as i32); + constant_i32(instrs, layout.layout_id as i32); + emit_staging_address(instrs, staging, ptr_ty, 0); + constant_i32(instrs, layout.scalar_len as i32); + emit_staging_address(instrs, staging, ptr_ty, layout.references_ptr); + constant_i32(instrs, layout.reference_count as i32); + call(instrs, imports.define); + local_get(instrs, staging); + constant_ptr(instrs, ptr_ty, u64::from(layout.staging_len())); + call(instrs, imports.scratch_release); + local_get(instrs, recipe); + push(instrs, Instr::Return(Return {})); +} + +fn emit_unknown_encode_handler( + module: &mut Module, + function: FunctionId, + seq: InstrSeqId, + scratch: TableId, + broker_encode: FunctionId, + recipe: LocalId, +) { + let instrs = instrs_mut(module, function, seq); + push(instrs, Instr::Drop(Drop {})); + constant_i32(instrs, 0); + call(instrs, broker_encode); + local_set(instrs, recipe); + emit_clear_scratch(instrs, scratch); + local_get(instrs, recipe); + push(instrs, Instr::Return(Return {})); +} + +#[allow(clippy::too_many_arguments)] +fn emit_decode( + module: &mut Module, + function: FunctionId, + recipe: LocalId, + replay: TableId, + memory: MemoryId, + ptr_ty: ValType, + imports: HostImports, + references: ReferenceDependencies, + layouts: &[TagLayout], +) { + let route = module.locals.add(ValType::I32); + let cache_index = module.locals.add(ValType::I32); + let cached = module.locals.add(ValType::Ref(NULLABLE_EXNREF)); + let staging = module.locals.add(ptr_ty); + let null_then = dangling(module, function, InstrSeqType::Simple(None)); + { + let instrs = instrs_mut(module, function, null_then); + push( + instrs, + Instr::RefNull(RefNull { + ty: NULLABLE_EXNREF, + }), + ); + push(instrs, Instr::Return(Return {})); + } + let empty_else = dangling(module, function, InstrSeqType::Simple(None)); + let invalid_then = dangling(module, function, InstrSeqType::Simple(None)); + push( + instrs_mut(module, function, invalid_then), + Instr::Unreachable(Unreachable {}), + ); + let invalid_else = dangling(module, function, InstrSeqType::Simple(None)); + let grow_then = dangling(module, function, InstrSeqType::Simple(None)); + { + let failed_then = dangling(module, function, InstrSeqType::Simple(None)); + push( + instrs_mut(module, function, failed_then), + Instr::Unreachable(Unreachable {}), + ); + let failed_else = dangling(module, function, InstrSeqType::Simple(None)); + let instrs = instrs_mut(module, function, grow_then); + push( + instrs, + Instr::RefNull(RefNull { + ty: NULLABLE_EXNREF, + }), + ); + local_get(instrs, cache_index); + constant_i32(instrs, 1); + binop(instrs, BinaryOp::I32Add); + push(instrs, Instr::TableSize(TableSize { table: replay })); + binop(instrs, BinaryOp::I32Sub); + push(instrs, Instr::TableGrow(TableGrow { table: replay })); + constant_i32(instrs, -1); + binop(instrs, BinaryOp::I32Eq); + push( + instrs, + Instr::IfElse(IfElse { + consequent: failed_then, + alternative: failed_else, + }), + ); + } + let grow_else = dangling(module, function, InstrSeqType::Simple(None)); + let cached_then = dangling(module, function, InstrSeqType::Simple(None)); + { + let instrs = instrs_mut(module, function, cached_then); + local_get(instrs, cached); + push(instrs, Instr::Return(Return {})); + } + let cached_else = dangling(module, function, InstrSeqType::Simple(None)); + + let broker_then = dangling(module, function, InstrSeqType::Simple(None)); + emit_broker_decode( + module, + function, + broker_then, + recipe, + cache_index, + cached, + replay, + imports.broker_throw_recipe, + ); + let broker_else = dangling(module, function, InstrSeqType::Simple(None)); + + let known_locals: Vec<_> = layouts + .iter() + .map(|layout| HandlerLocals { + payloads: layout + .payloads + .iter() + .map(|payload| PayloadLocals { + value: module.locals.add(storage_type(payload.ty)), + recipe: matches!(payload.ty, ValType::Ref(_)) + .then(|| module.locals.add(ValType::I32)), + }) + .collect(), + }) + .collect(); + let known_blocks: Vec<_> = layouts + .iter() + .zip(&known_locals) + .map(|(layout, locals)| { + let seq = dangling(module, function, InstrSeqType::Simple(None)); + emit_known_decode( + module, + function, + seq, + recipe, + cache_index, + cached, + replay, + memory, + ptr_ty, + staging, + imports, + references, + function, + layout, + locals, + ); + seq + }) + .collect(); + + let entry = entry(function, module); + let instrs = instrs_mut(module, function, entry); + local_get(instrs, recipe); + unop(instrs, UnaryOp::I32Eqz); + push( + instrs, + Instr::IfElse(IfElse { + consequent: null_then, + alternative: empty_else, + }), + ); + local_get(instrs, recipe); + constant_i32(instrs, 0); + binop(instrs, BinaryOp::I32LtS); + local_get(instrs, recipe); + constant_i32(instrs, MAX_RECIPE_ID); + binop(instrs, BinaryOp::I32GtU); + binop(instrs, BinaryOp::I32Or); + push( + instrs, + Instr::IfElse(IfElse { + consequent: invalid_then, + alternative: invalid_else, + }), + ); + local_get(instrs, recipe); + call(instrs, imports.cache_index); + push(instrs, Instr::LocalTee(LocalTee { local: cache_index })); + unop(instrs, UnaryOp::I32Eqz); + push( + instrs, + Instr::IfElse(IfElse { + consequent: invalid_then, + alternative: invalid_else, + }), + ); + local_get(instrs, cache_index); + push(instrs, Instr::TableSize(TableSize { table: replay })); + binop(instrs, BinaryOp::I32GeU); + push( + instrs, + Instr::IfElse(IfElse { + consequent: grow_then, + alternative: grow_else, + }), + ); + local_get(instrs, cache_index); + push(instrs, Instr::TableGet(TableGet { table: replay })); + push(instrs, Instr::LocalTee(LocalTee { local: cached })); + push(instrs, Instr::RefIsNull(RefIsNull {})); + unop(instrs, UnaryOp::I32Eqz); + push( + instrs, + Instr::IfElse(IfElse { + consequent: cached_then, + alternative: cached_else, + }), + ); + local_get(instrs, recipe); + push( + instrs, + Instr::GlobalGet(GlobalGet { + global: imports.activation, + }), + ); + call(instrs, imports.route); + push(instrs, Instr::LocalTee(LocalTee { local: route })); + constant_i32(instrs, -1); + binop(instrs, BinaryOp::I32Eq); + push( + instrs, + Instr::IfElse(IfElse { + consequent: broker_then, + alternative: broker_else, + }), + ); + for (layout, block) in layouts.iter().zip(known_blocks) { + let next = dangling(module, function, InstrSeqType::Simple(None)); + local_get(instrs_mut(module, function, entry), route); + constant_i32(instrs_mut(module, function, entry), layout.layout_id as i32); + binop(instrs_mut(module, function, entry), BinaryOp::I32Eq); + push( + instrs_mut(module, function, entry), + Instr::IfElse(IfElse { + consequent: block, + alternative: next, + }), + ); + } + push( + instrs_mut(module, function, entry), + Instr::Unreachable(Unreachable {}), + ); +} + +fn emit_broker_decode( + module: &mut Module, + function: FunctionId, + seq: InstrSeqId, + recipe: LocalId, + cache_index: LocalId, + cached: LocalId, + replay: TableId, + broker_throw: FunctionId, +) { + let cap_ty = InstrSeqType::new(&mut module.types, &[], &[ValType::Ref(NON_NULL_EXNREF)]); + let cap = dangling(module, function, cap_ty); + let body = dangling(module, function, InstrSeqType::Simple(None)); + { + let instrs = instrs_mut(module, function, body); + local_get(instrs, recipe); + call(instrs, broker_throw); + push(instrs, Instr::Unreachable(Unreachable {})); + } + { + let instrs = instrs_mut(module, function, cap); + push( + instrs, + Instr::TryTable(TryTable { + seq: body, + catches: vec![TryTableCatch::CatchAllRef { label: cap }], + }), + ); + push(instrs, Instr::Unreachable(Unreachable {})); + } + let instrs = instrs_mut(module, function, seq); + push(instrs, Instr::Block(Block { seq: cap })); + local_set(instrs, cached); + emit_cache_and_return(instrs, cache_index, cached, replay); +} + +#[allow(clippy::too_many_arguments)] +fn emit_known_decode( + module: &mut Module, + function: FunctionId, + seq: InstrSeqId, + recipe: LocalId, + cache_index: LocalId, + cached: LocalId, + replay: TableId, + memory: MemoryId, + ptr_ty: ValType, + staging: LocalId, + imports: HostImports, + references: ReferenceDependencies, + decode_exnref: FunctionId, + layout: &TagLayout, + locals: &HandlerLocals, +) { + let invalid = dangling(module, function, InstrSeqType::Simple(None)); + push( + instrs_mut(module, function, invalid), + Instr::Unreachable(Unreachable {}), + ); + let valid = dangling(module, function, InstrSeqType::Simple(None)); + let classes: Vec<_> = layout + .payloads + .iter() + .map(|payload| match payload.ty { + ValType::Ref(reference) => Some(ReferenceCodecClass::of(module, reference)), + _ => None, + }) + .collect(); + { + let instrs = instrs_mut(module, function, seq); + constant_ptr(instrs, ptr_ty, u64::from(layout.staging_len())); + call(instrs, imports.scratch_reserve); + local_set(instrs, staging); + local_get(instrs, recipe); + push( + instrs, + Instr::GlobalGet(GlobalGet { + global: imports.activation, + }), + ); + constant_i32(instrs, layout.ordinal as i32); + constant_i32(instrs, layout.layout_id as i32); + emit_staging_address(instrs, staging, ptr_ty, 0); + constant_i32(instrs, layout.scalar_len as i32); + emit_staging_address(instrs, staging, ptr_ty, layout.references_ptr); + constant_i32(instrs, layout.reference_count as i32); + call(instrs, imports.load); + unop(instrs, UnaryOp::I32Eqz); + push( + instrs, + Instr::IfElse(IfElse { + consequent: invalid, + alternative: valid, + }), + ); + + for (payload, local) in layout.payloads.iter().zip(&locals.payloads) { + emit_staging_address(instrs, staging, ptr_ty, 0); + if let Some(offset) = payload.scalar_offset { + push( + instrs, + Instr::Load(Load { + memory, + kind: scalar_load(payload.ty), + arg: MemArg { + align: 1, + offset: u64::from(offset), + }, + }), + ); + local_set(instrs, local.value); + } else { + let index = payload.reference_offset.expect("reference payload index"); + push( + instrs, + Instr::Load(Load { + memory, + kind: LoadKind::I32 { atomic: false }, + arg: MemArg { + align: 4, + offset: u64::from(layout.references_ptr + index * 4), + }, + }), + ); + local_set( + instrs, + local.recipe.expect("reference payload recipe local"), + ); + } + } + // Scalar bits and child recipe IDs are now in typed locals. Releasing + // here lets recursive decoders reserve disjoint ranges and guarantees + // the transaction zeroes exchange bytes before reuse. + local_get(instrs, staging); + constant_ptr(instrs, ptr_ty, u64::from(layout.staging_len())); + call(instrs, imports.scratch_release); + for ((payload, local), class) in layout.payloads.iter().zip(&locals.payloads).zip(&classes) + { + let ValType::Ref(reference) = payload.ty else { + continue; + }; + local_get( + instrs, + local.recipe.expect("reference payload recipe local"), + ); + let class = class.expect("reference payload class"); + call(instrs, reference_decoder(references, decode_exnref, class)); + emit_narrow(instrs, class, reference); + local_set(instrs, local.value); + } + } + + let cap_ty = InstrSeqType::new(&mut module.types, &[], &[ValType::Ref(NON_NULL_EXNREF)]); + let cap = dangling(module, function, cap_ty); + let throw_body = dangling(module, function, InstrSeqType::Simple(None)); + { + let throw = instrs_mut(module, function, throw_body); + for (payload, local) in layout.payloads.iter().zip(&locals.payloads) { + local_get(throw, local.value); + if let ValType::Ref(reference) = payload.ty + && !reference.nullable + { + push(throw, Instr::RefAsNonNull(RefAsNonNull {})); + } + } + push(throw, Instr::Throw(Throw { tag: layout.tag })); + } + { + let capture = instrs_mut(module, function, cap); + push( + capture, + Instr::TryTable(TryTable { + seq: throw_body, + catches: vec![TryTableCatch::CatchAllRef { label: cap }], + }), + ); + push(capture, Instr::Unreachable(Unreachable {})); + } + push( + instrs_mut(module, function, seq), + Instr::Block(Block { seq: cap }), + ); + local_set(instrs_mut(module, function, seq), cached); + emit_cache_and_return( + instrs_mut(module, function, seq), + cache_index, + cached, + replay, + ); +} + +fn emit_cache_and_return( + instrs: &mut Vec<(Instr, InstrLocId)>, + cache_index: LocalId, + exception: LocalId, + replay: TableId, +) { + local_get(instrs, cache_index); + local_get(instrs, exception); + push(instrs, Instr::TableSet(TableSet { table: replay })); + local_get(instrs, exception); + push(instrs, Instr::Return(Return {})); +} + +fn emit_throw_slot(module: &mut Module, function: FunctionId, slot: LocalId, scratch: TableId) { + let entry = entry(function, module); + let instrs = instrs_mut(module, function, entry); + local_get(instrs, slot); + push(instrs, Instr::TableGet(TableGet { table: scratch })); + push(instrs, Instr::RefAsNonNull(RefAsNonNull {})); + push(instrs, Instr::ThrowRef(ThrowRef {})); +} + +fn emit_throw_recipe( + module: &mut Module, + function: FunctionId, + recipe: LocalId, + decode: FunctionId, +) { + let entry = entry(function, module); + let instrs = instrs_mut(module, function, entry); + local_get(instrs, recipe); + call(instrs, decode); + push(instrs, Instr::RefAsNonNull(RefAsNonNull {})); + push(instrs, Instr::ThrowRef(ThrowRef {})); +} + +fn emit_materialize( + module: &mut Module, + function: FunctionId, + recipe: LocalId, + decode: FunctionId, +) { + let entry = entry(function, module); + let instrs = instrs_mut(module, function, entry); + local_get(instrs, recipe); + call(instrs, decode); + // WHY: the JavaScript embedding rejects calls whose result contains + // `exnref`. Decode and cache entirely inside the owning instance, then + // cross the host boundary with a void result. + push(instrs, Instr::Drop(Drop {})); +} + +fn emit_encode_ingress( + module: &mut Module, + function: FunctionId, + token: LocalId, + ingress_throw: FunctionId, + encode: FunctionId, +) { + let cap_ty = InstrSeqType::new(&mut module.types, &[], &[ValType::Ref(NON_NULL_EXNREF)]); + let cap = dangling(module, function, cap_ty); + let body = dangling(module, function, InstrSeqType::Simple(None)); + { + let instrs = instrs_mut(module, function, body); + local_get(instrs, token); + call(instrs, ingress_throw); + push(instrs, Instr::Unreachable(Unreachable {})); + } + { + let instrs = instrs_mut(module, function, cap); + push( + instrs, + Instr::TryTable(TryTable { + seq: body, + catches: vec![TryTableCatch::CatchAllRef { label: cap }], + }), + ); + push(instrs, Instr::Unreachable(Unreachable {})); + } + let entry = entry(function, module); + let instrs = instrs_mut(module, function, entry); + push(instrs, Instr::Block(Block { seq: cap })); + call(instrs, encode); +} + +fn emit_clear(module: &mut Module, function: FunctionId, scratch: TableId, replay: TableId) { + let entry = entry(function, module); + let instrs = instrs_mut(module, function, entry); + emit_clear_scratch(instrs, scratch); + constant_i32(instrs, 0); + push( + instrs, + Instr::RefNull(RefNull { + ty: NULLABLE_EXNREF, + }), + ); + push(instrs, Instr::TableSize(TableSize { table: replay })); + push(instrs, Instr::TableFill(TableFill { table: replay })); +} + +fn emit_clear_scratch(instrs: &mut Vec<(Instr, InstrLocId)>, scratch: TableId) { + constant_i32(instrs, 0); + push( + instrs, + Instr::RefNull(RefNull { + ty: NULLABLE_EXNREF, + }), + ); + push(instrs, Instr::TableSet(TableSet { table: scratch })); +} + +fn reference_encoder( + module: &Module, + references: ReferenceDependencies, + encode_exnref: FunctionId, + reference: RefType, +) -> FunctionId { + match ReferenceCodecClass::of(module, reference) { + ReferenceCodecClass::Func => references.encode_funcref, + ReferenceCodecClass::Extern => references.encode_externref, + ReferenceCodecClass::Exn => encode_exnref, + ReferenceCodecClass::Any => references.encode_anyref, + } +} + +fn reference_decoder( + references: ReferenceDependencies, + decode_exnref: FunctionId, + class: ReferenceCodecClass, +) -> FunctionId { + match class { + ReferenceCodecClass::Func => references.decode_funcref, + ReferenceCodecClass::Extern => references.decode_externref, + ReferenceCodecClass::Exn => decode_exnref, + ReferenceCodecClass::Any => references.decode_anyref, + } +} + +fn emit_narrow( + instrs: &mut Vec<(Instr, InstrLocId)>, + class: ReferenceCodecClass, + expected: RefType, +) { + let broad = class.nullable_type(); + if expected.heap_type != broad.heap_type { + push( + instrs, + Instr::RefCast(RefCast { + nullable: expected.nullable, + heap_type: expected.heap_type, + }), + ); + } else if !expected.nullable { + push(instrs, Instr::RefAsNonNull(RefAsNonNull {})); + } +} + +fn storage_type(ty: ValType) -> ValType { + match ty { + ValType::Ref(mut reference) => { + reference.nullable = true; + ValType::Ref(reference) + } + scalar => scalar, + } +} + +fn scalar_store(ty: ValType) -> StoreKind { + match ty { + ValType::I32 => StoreKind::I32 { atomic: false }, + ValType::I64 => StoreKind::I64 { atomic: false }, + ValType::F32 => StoreKind::F32, + ValType::F64 => StoreKind::F64, + ValType::V128 => StoreKind::V128, + ValType::Ref(_) => unreachable!("reference payload uses a recipe ID"), + } +} + +fn scalar_load(ty: ValType) -> LoadKind { + match ty { + ValType::I32 => LoadKind::I32 { atomic: false }, + ValType::I64 => LoadKind::I64 { atomic: false }, + ValType::F32 => LoadKind::F32, + ValType::F64 => LoadKind::F64, + ValType::V128 => LoadKind::V128, + ValType::Ref(_) => unreachable!("reference payload uses a recipe ID"), + } +} + +fn replace_descriptor(module: &mut Module, layouts: &[TagLayout]) { + loop { + let existing = module + .customs + .iter() + .find(|(_, section)| section.name() == FORMAT_SECTION) + .map(|(id, _)| id); + let Some(existing) = existing else { break }; + module.customs.delete(existing); + } + let mut data = Vec::with_capacity(FORMAT_HEADER_SIZE + layouts.len() * FORMAT_TAG_RECORD_SIZE); + data.push(FORMAT_VERSION); + data.push(0); + data.extend_from_slice(&0u16.to_le_bytes()); + data.extend_from_slice(&(layouts.len() as u32).to_le_bytes()); + for layout in layouts { + data.extend_from_slice(&layout.ordinal.to_le_bytes()); + data.extend_from_slice(&layout.layout_id.to_le_bytes()); + data.extend_from_slice(&layout.scalar_len.to_le_bytes()); + data.extend_from_slice(&layout.reference_count.to_le_bytes()); + } + module.customs.add(RawCustomSection { + name: FORMAT_SECTION.into(), + data, + }); +} + +fn dangling(module: &mut Module, function: FunctionId, ty: InstrSeqType) -> InstrSeqId { + local_mut(module, function) + .builder_mut() + .dangling_instr_seq(ty) + .id() +} + +fn entry(function: FunctionId, module: &Module) -> InstrSeqId { + local(module, function).entry_block() +} + +fn instrs_mut( + module: &mut Module, + function: FunctionId, + seq: InstrSeqId, +) -> &mut Vec<(Instr, InstrLocId)> { + &mut local_mut(module, function).block_mut(seq).instrs +} + +fn local(module: &Module, function: FunctionId) -> &LocalFunction { + match &module.funcs.get(function).kind { + FunctionKind::Local(local) => local, + _ => unreachable!("injected exception codec function is local"), + } +} + +fn local_mut(module: &mut Module, function: FunctionId) -> &mut LocalFunction { + match &mut module.funcs.get_mut(function).kind { + FunctionKind::Local(local) => local, + _ => unreachable!("injected exception codec function is local"), + } +} + +fn push(instrs: &mut Vec<(Instr, InstrLocId)>, instr: Instr) { + instrs.push((instr, InstrLocId::default())); +} + +fn constant_i32(instrs: &mut Vec<(Instr, InstrLocId)>, value: i32) { + push( + instrs, + Instr::Const(Const { + value: Value::I32(value), + }), + ); +} + +fn constant_ptr(instrs: &mut Vec<(Instr, InstrLocId)>, ptr_ty: ValType, value: u64) { + match ptr_ty { + ValType::I32 => constant_i32(instrs, value as u32 as i32), + ValType::I64 => push( + instrs, + Instr::Const(Const { + value: Value::I64(value as i64), + }), + ), + other => unreachable!("unsupported exception staging pointer type {other:?}"), + } +} + +fn emit_staging_address( + instrs: &mut Vec<(Instr, InstrLocId)>, + staging: LocalId, + ptr_ty: ValType, + offset: u32, +) { + local_get(instrs, staging); + if offset == 0 { + return; + } + constant_ptr(instrs, ptr_ty, u64::from(offset)); + binop( + instrs, + match ptr_ty { + ValType::I32 => BinaryOp::I32Add, + ValType::I64 => BinaryOp::I64Add, + other => unreachable!("unsupported exception staging pointer type {other:?}"), + }, + ); +} + +fn local_get(instrs: &mut Vec<(Instr, InstrLocId)>, local: LocalId) { + push(instrs, Instr::LocalGet(LocalGet { local })); +} + +fn local_set(instrs: &mut Vec<(Instr, InstrLocId)>, local: LocalId) { + push(instrs, Instr::LocalSet(LocalSet { local })); +} + +fn call(instrs: &mut Vec<(Instr, InstrLocId)>, function: FunctionId) { + push(instrs, Instr::Call(Call { func: function })); +} + +fn binop(instrs: &mut Vec<(Instr, InstrLocId)>, op: BinaryOp) { + push(instrs, Instr::Binop(Binop { op })); +} + +fn unop(instrs: &mut Vec<(Instr, InstrLocId)>, op: UnaryOp) { + push(instrs, Instr::Unop(Unop { op })); +} diff --git a/crates/fork-instrument/src/module_gc_codec.rs b/crates/fork-instrument/src/module_gc_codec.rs new file mode 100644 index 0000000000..39e3040f16 --- /dev/null +++ b/crates/fork-instrument/src/module_gc_codec.rs @@ -0,0 +1,3836 @@ +//! Activation-owned codecs for WebAssembly GC references. +//! +//! The durable representation is a scalar recipe graph in copied linear +//! memory. A process-owned `anyref` table is only a transaction-local routing +//! bus: slot zero carries one synchronous probe value and slot `recipe + 1` +//! carries the parent or freshly reconstructed child identity. The host clears +//! every slot on successful replay, abort, and exec. +//! +//! Immutable arrays need constructor provenance. Unlike structs, an arbitrary +//! immutable array cannot be populated after allocation, and +//! `array.new_fixed` has a statically encoded arity. Planning therefore gives +//! every non-generic constructor site a deterministic layout id so replay can +//! execute the same typed constructor in the fresh instance. + +use std::collections::{HashMap, HashSet}; + +use anyhow::{Result, ensure}; +use walrus::{ + AbstractHeapType, CompositeType, DataId, ElementId, FieldType, FunctionBuilder, FunctionId, + FunctionKind, GlobalId, HeapType, ImportKind, LocalFunction, LocalId, MemoryId, Module, + RawCustomSection, RefType, StorageType, TableId, TypeId, ValType, + ir::{ + AnyConvertExtern, ArrayGet, ArrayLen, ArrayNew, ArrayNewData, ArrayNewDefault, + ArrayNewElem, ArrayNewFixed, ArraySet, BinaryOp, Binop, Br, BrIf, Call, Const, + ExternConvertAny, GlobalGet, IfElse, Instr, InstrLocId, Load, LoadKind, LocalGet, LocalSet, + LocalTee, Loop, MemArg, RefAsNonNull, RefCast, RefFunc, RefI31, RefIsNull, RefNull, + RefTest, Return, Store, StoreKind, StructGet, StructGetU, StructNew, StructNewDefault, + StructSet, TableGet, TableSet, Throw, TryTable, TryTableCatch, UnaryOp, Unop, Unreachable, + Value, Visitor, VisitorMut, dfs_in_order, dfs_pre_order_mut, + }, +}; + +use crate::{module_exception_codec, runtime}; +use wasm_posix_shared::abi::{ + WPK_FORK_EXCEPTION_IMPORT_ACTIVATION, WPK_FORK_GC_CODEC_FIELD_RECORD_SIZE, + WPK_FORK_GC_CODEC_HEADER_SIZE, WPK_FORK_GC_CODEC_LAYOUT_RECORD_SIZE, WPK_FORK_GC_CODEC_MAGIC, + WPK_FORK_GC_CODEC_SECTION, WPK_FORK_GC_CODEC_VERSION, WPK_FORK_REFERENCE_CODEC_IMPORT_MODULE, + WPK_FORK_REFERENCE_EXPORT_GC_ALLOCATE, WPK_FORK_REFERENCE_EXPORT_GC_ENCODE_SLOT, + WPK_FORK_REFERENCE_EXPORT_GC_FILL, WPK_FORK_REFERENCE_EXPORT_GC_PROBE, + WPK_FORK_REFERENCE_EXPORT_GC_PUBLISH_EXTERNREF, WPK_FORK_REFERENCE_IMPORT_GC_BROKER_ENCODE, + WPK_FORK_REFERENCE_IMPORT_GC_CAPTURE_LAYOUT, WPK_FORK_REFERENCE_IMPORT_GC_CLAIM, + WPK_FORK_REFERENCE_IMPORT_GC_DEFINE, WPK_FORK_REFERENCE_IMPORT_GC_I31, + WPK_FORK_REFERENCE_IMPORT_GC_LOAD, WPK_FORK_REFERENCE_IMPORT_GC_LOOKUP, + WPK_FORK_REFERENCE_IMPORT_GC_PAYLOAD_LEN, WPK_FORK_REFERENCE_IMPORT_GC_PROVENANCE_BEGIN, + WPK_FORK_REFERENCE_IMPORT_GC_PROVENANCE_END, WPK_FORK_REFERENCE_IMPORT_GC_PROVENANCE_REF, + WPK_FORK_REFERENCE_IMPORT_GC_ROUTE, WPK_FORK_REFERENCE_IMPORT_GC_TRANSIT, +}; + +pub const FORMAT_SECTION: &str = WPK_FORK_GC_CODEC_SECTION; +pub const FORMAT_MAGIC: [u8; 4] = WPK_FORK_GC_CODEC_MAGIC; +pub const FORMAT_VERSION: u16 = WPK_FORK_GC_CODEC_VERSION; +pub const FORMAT_HEADER_SIZE: u16 = WPK_FORK_GC_CODEC_HEADER_SIZE; +pub const FORMAT_LAYOUT_RECORD_SIZE: u16 = WPK_FORK_GC_CODEC_LAYOUT_RECORD_SIZE; +pub const FORMAT_FIELD_RECORD_SIZE: u16 = WPK_FORK_GC_CODEC_FIELD_RECORD_SIZE; + +pub const KIND_STRUCT: u8 = 1; +pub const KIND_ARRAY: u8 = 2; + +pub const CONSTRUCTOR_STRUCT: u8 = 0; +pub const CONSTRUCTOR_ARRAY_GENERIC: u8 = 1; +pub const CONSTRUCTOR_ARRAY_NEW: u8 = 2; +pub const CONSTRUCTOR_ARRAY_DEFAULT: u8 = 3; +pub const CONSTRUCTOR_ARRAY_FIXED: u8 = 4; +pub const CONSTRUCTOR_ARRAY_DATA: u8 = 5; +pub const CONSTRUCTOR_ARRAY_ELEMENT: u8 = 6; + +pub const LAYOUT_FLAG_REQUIRES_PROVENANCE: u16 = 1 << 0; +pub const LAYOUT_FLAG_DEFAULTABLE_SHELL: u16 = 1 << 1; +pub const LAYOUT_KNOWN_FLAGS: u16 = LAYOUT_FLAG_REQUIRES_PROVENANCE | LAYOUT_FLAG_DEFAULTABLE_SHELL; + +pub const FIELD_FLAG_MUTABLE: u8 = 1 << 0; +pub const FIELD_FLAG_NULLABLE: u8 = 1 << 1; +pub const FIELD_FLAG_REFERENCE: u8 = 1 << 2; +pub const FIELD_FLAG_ALLOCATION_DEPENDENCY: u8 = 1 << 3; +pub const FIELD_KNOWN_FLAGS: u8 = FIELD_FLAG_MUTABLE + | FIELD_FLAG_NULLABLE + | FIELD_FLAG_REFERENCE + | FIELD_FLAG_ALLOCATION_DEPENDENCY; + +pub const STORAGE_I8: u8 = 1; +pub const STORAGE_I16: u8 = 2; +pub const STORAGE_I32: u8 = 3; +pub const STORAGE_I64: u8 = 4; +pub const STORAGE_F32: u8 = 5; +pub const STORAGE_F64: u8 = 6; +pub const STORAGE_V128: u8 = 7; +pub const STORAGE_REFERENCE: u8 = 8; + +const NO_ORDINAL: u32 = u32::MAX; + +pub const HOST_IMPORT_MODULE: &str = WPK_FORK_REFERENCE_CODEC_IMPORT_MODULE; +pub const IMPORT_TRANSIT_TABLE: &str = WPK_FORK_REFERENCE_IMPORT_GC_TRANSIT; +pub const IMPORT_LOOKUP: &str = WPK_FORK_REFERENCE_IMPORT_GC_LOOKUP; +pub const IMPORT_CLAIM: &str = WPK_FORK_REFERENCE_IMPORT_GC_CLAIM; +pub const IMPORT_I31: &str = WPK_FORK_REFERENCE_IMPORT_GC_I31; +pub const IMPORT_DEFINE: &str = WPK_FORK_REFERENCE_IMPORT_GC_DEFINE; +pub const IMPORT_ROUTE: &str = WPK_FORK_REFERENCE_IMPORT_GC_ROUTE; +pub const IMPORT_PAYLOAD_LEN: &str = WPK_FORK_REFERENCE_IMPORT_GC_PAYLOAD_LEN; +pub const IMPORT_LOAD: &str = WPK_FORK_REFERENCE_IMPORT_GC_LOAD; +pub const IMPORT_BROKER_ENCODE: &str = WPK_FORK_REFERENCE_IMPORT_GC_BROKER_ENCODE; +pub const IMPORT_CAPTURE_LAYOUT: &str = WPK_FORK_REFERENCE_IMPORT_GC_CAPTURE_LAYOUT; +pub const IMPORT_PROVENANCE_BEGIN: &str = WPK_FORK_REFERENCE_IMPORT_GC_PROVENANCE_BEGIN; +pub const IMPORT_PROVENANCE_REF: &str = WPK_FORK_REFERENCE_IMPORT_GC_PROVENANCE_REF; +pub const IMPORT_PROVENANCE_END: &str = WPK_FORK_REFERENCE_IMPORT_GC_PROVENANCE_END; + +pub const EXPORT_PROBE: &str = WPK_FORK_REFERENCE_EXPORT_GC_PROBE; +pub const EXPORT_ENCODE_SLOT: &str = WPK_FORK_REFERENCE_EXPORT_GC_ENCODE_SLOT; +pub const EXPORT_ALLOCATE: &str = WPK_FORK_REFERENCE_EXPORT_GC_ALLOCATE; +pub const EXPORT_FILL: &str = WPK_FORK_REFERENCE_EXPORT_GC_FILL; +pub const EXPORT_PUBLISH_EXTERNREF: &str = WPK_FORK_REFERENCE_EXPORT_GC_PUBLISH_EXTERNREF; +pub const LOCAL_ENCODE_ANYREF: &str = "__wpk_fork_ref_encode_anyref"; +pub const LOCAL_DECODE_ANYREF: &str = "__wpk_fork_ref_decode_anyref"; +pub const LOCAL_ENCODE_EXTERNREF: &str = "__wpk_fork_ref_encode_externref"; +pub const LOCAL_DECODE_EXTERNREF: &str = "__wpk_fork_ref_decode_externref"; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum GcLayoutKind { + Struct, + Array, +} + +impl GcLayoutKind { + fn wire(self) -> u8 { + match self { + Self::Struct => KIND_STRUCT, + Self::Array => KIND_ARRAY, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum GcConstructorKind { + Struct, + ArrayGeneric, + ArrayNew, + ArrayDefault, + ArrayFixed { len: u32 }, + ArrayData { segment_ordinal: u32 }, + ArrayElement { segment_ordinal: u32 }, +} + +impl GcConstructorKind { + fn wire(self) -> u8 { + match self { + Self::Struct => CONSTRUCTOR_STRUCT, + Self::ArrayGeneric => CONSTRUCTOR_ARRAY_GENERIC, + Self::ArrayNew => CONSTRUCTOR_ARRAY_NEW, + Self::ArrayDefault => CONSTRUCTOR_ARRAY_DEFAULT, + Self::ArrayFixed { .. } => CONSTRUCTOR_ARRAY_FIXED, + Self::ArrayData { .. } => CONSTRUCTOR_ARRAY_DATA, + Self::ArrayElement { .. } => CONSTRUCTOR_ARRAY_ELEMENT, + } + } + + fn auxiliary(self) -> u32 { + match self { + Self::Struct | Self::ArrayGeneric | Self::ArrayNew | Self::ArrayDefault => 0, + Self::ArrayFixed { len } => len, + Self::ArrayData { segment_ordinal } | Self::ArrayElement { segment_ordinal } => { + segment_ordinal + } + } + } + + fn requires_provenance(self) -> bool { + !matches!(self, Self::Struct | Self::ArrayGeneric) + } +} + +#[derive(Debug, Clone, Copy)] +pub struct GcFieldLayout { + pub field: FieldType, + pub scalar_offset: Option, + pub reference_ordinal: Option, + pub allocation_dependency: bool, +} + +impl GcFieldLayout { + pub fn is_allocation_dependency(self) -> bool { + self.allocation_dependency + } +} + +#[derive(Debug, Clone)] +pub struct GcLayout { + pub id: u32, + pub type_id: TypeId, + pub type_ordinal: u32, + /// Base type layout accepted by `capture_layout` for this constructor. + pub base_layout_id: u32, + pub kind: GcLayoutKind, + pub constructor: GcConstructorKind, + pub scalar_len_or_stride: u32, + pub fields: Vec, + pub super_type_ordinal: Option, + pub subtype_depth: u32, + pub defaultable_shell: bool, + /// The current fields are not sufficient to allocate a safe shell. + /// + /// For immutable fields the final snapshot values are constructor inputs. + /// Mutable non-null fields can have diverged from their constructor inputs, + /// so their original seed references are retained by a weak-keyed + /// provenance record and serialized only if the aggregate reaches fork. + pub requires_provenance: bool, + /// Constructor-only scalar bytes prepended to the snapshot scalar payload. + pub provenance_scalar_len: u32, + /// Constructor-only recipe ids prepended to the snapshot edge vector. + pub provenance_reference_count: u32, +} + +#[derive(Debug, Clone)] +pub struct GcCodecPlan { + layouts: Vec, + dispatch_layouts: Vec, +} + +impl GcCodecPlan { + pub fn layouts(&self) -> &[GcLayout] { + &self.layouts + } + + /// Base layouts ordered most-specific-first for exact dynamic dispatch. + pub fn dispatch_layouts(&self) -> &[u32] { + &self.dispatch_layouts + } + + pub fn descriptor(&self) -> Vec { + encode_descriptor(self) + } +} + +#[derive(Debug, Clone, Copy)] +struct HostImports { + activation: GlobalId, + lookup: FunctionId, + claim: FunctionId, + i31: FunctionId, + define: FunctionId, + route: FunctionId, + payload_len: FunctionId, + load: FunctionId, + broker_encode: FunctionId, + capture_layout: FunctionId, + provenance_begin: FunctionId, + provenance_ref: FunctionId, + provenance_end: FunctionId, +} + +/// Stubs are declared before the exception codec so an exception payload that +/// contains an internal GC reference calls back into this module-local codec, +/// never through a typed JavaScript `anyref` import. +#[derive(Debug)] +pub struct DeclaredGcCodec { + pub encode_anyref: FunctionId, + pub decode_anyref: FunctionId, + pub encode_externref: FunctionId, + pub decode_externref: FunctionId, + pub probe: FunctionId, + pub encode_slot: FunctionId, + pub allocate: FunctionId, + pub fill: FunctionId, + pub publish_externref: FunctionId, + pub transit: TableId, + pub memory: MemoryId, + pub ptr_ty: ValType, + plan: GcCodecPlan, + imports: HostImports, + probe_args: Vec, + encode_anyref_args: Vec, + decode_anyref_args: Vec, + encode_externref_args: Vec, + decode_externref_args: Vec, + encode_slot_args: Vec, + allocate_args: Vec, + fill_args: Vec, + publish_externref_args: Vec, +} + +#[derive(Debug, Clone, Copy)] +pub struct InjectedGcCodec { + pub encode_anyref: FunctionId, + pub decode_anyref: FunctionId, + pub encode_externref: FunctionId, + pub decode_externref: FunctionId, + pub probe: FunctionId, + pub encode_slot: FunctionId, + pub allocate: FunctionId, + pub fill: FunctionId, + pub publish_externref: FunctionId, + pub transit: TableId, +} + +/// Freeze source GC types and install the versioned host surface. +/// +/// Emission is split from declaration because exception payloads and GC +/// fields can recursively refer to each other. The two codecs first exchange +/// typed local function ids and only then emit their bodies. +pub fn declare(module: &mut Module, memory: MemoryId) -> Result { + let plan = plan(module)?; + let mut source_functions: Vec<_> = module + .funcs + .iter() + .filter_map(|function| { + matches!(function.kind, FunctionKind::Local(_)).then_some(function.id()) + }) + .collect(); + source_functions.sort(); + let ptr_ty = if module.memories.get(memory).memory64 { + ValType::I64 + } else { + ValType::I32 + }; + let (transit, _) = module.add_import_table( + HOST_IMPORT_MODULE, + IMPORT_TRANSIT_TABLE, + false, + 1, + None, + RefType::ANYREF, + ); + let imports = inject_host_imports(module, ptr_ty); + inject_provenance_wrappers(module, &plan, transit, imports, &source_functions)?; + let (encode_anyref, encode_anyref_args) = add_stub( + module, + &[ValType::Ref(RefType::ANYREF)], + &[ValType::I32], + LOCAL_ENCODE_ANYREF, + ); + let (decode_anyref, decode_anyref_args) = add_stub( + module, + &[ValType::I32], + &[ValType::Ref(RefType::ANYREF)], + LOCAL_DECODE_ANYREF, + ); + let (encode_externref, encode_externref_args) = add_stub( + module, + &[ValType::Ref(RefType::EXTERNREF)], + &[ValType::I32], + LOCAL_ENCODE_EXTERNREF, + ); + let (decode_externref, decode_externref_args) = add_stub( + module, + &[ValType::I32], + &[ValType::Ref(RefType::EXTERNREF)], + LOCAL_DECODE_EXTERNREF, + ); + let (probe, probe_args) = add_stub(module, &[ValType::I32], &[ValType::I64], EXPORT_PROBE); + let (encode_slot, encode_slot_args) = + add_stub(module, &[ValType::I32], &[ValType::I32], EXPORT_ENCODE_SLOT); + let (allocate, allocate_args) = add_stub(module, &[ValType::I32], &[], EXPORT_ALLOCATE); + let (fill, fill_args) = add_stub(module, &[ValType::I32], &[], EXPORT_FILL); + let (publish_externref, publish_externref_args) = add_stub( + module, + &[ValType::I32, ValType::Ref(RefType::EXTERNREF)], + &[], + EXPORT_PUBLISH_EXTERNREF, + ); + + for (name, function) in [ + (EXPORT_PROBE, probe), + (EXPORT_ENCODE_SLOT, encode_slot), + (EXPORT_ALLOCATE, allocate), + (EXPORT_FILL, fill), + (EXPORT_PUBLISH_EXTERNREF, publish_externref), + ] { + module.exports.add(name, function); + } + replace_descriptor(module, &plan); + + Ok(DeclaredGcCodec { + encode_anyref, + decode_anyref, + encode_externref, + decode_externref, + probe, + encode_slot, + allocate, + fill, + publish_externref, + transit, + memory, + ptr_ty, + plan, + imports, + probe_args, + encode_anyref_args, + decode_anyref_args, + encode_externref_args, + decode_externref_args, + encode_slot_args, + allocate_args, + fill_args, + publish_externref_args, + }) +} + +fn inject_provenance_wrappers( + module: &mut Module, + plan: &GcCodecPlan, + transit: TableId, + imports: HostImports, + source_functions: &[FunctionId], +) -> Result<()> { + let mut struct_wrappers = HashMap::new(); + let mut array_wrappers = HashMap::new(); + for layout in plan.layouts() { + let needs_wrapper = match layout.constructor { + GcConstructorKind::Struct => layout.provenance_reference_count != 0, + GcConstructorKind::ArrayGeneric => false, + _ => true, + }; + if !needs_wrapper { + continue; + } + let wrapper = add_provenance_wrapper(module, layout, transit, imports)?; + match layout.constructor { + GcConstructorKind::Struct => { + struct_wrappers.insert(layout.type_id, wrapper); + } + constructor => { + array_wrappers.insert( + (layout.type_id, constructor.wire(), constructor.auxiliary()), + wrapper, + ); + } + } + } + + let data_ordinals: HashMap = module + .data + .iter() + .enumerate() + .map(|(ordinal, data)| (data.id(), ordinal as u32)) + .collect(); + let element_ordinals: HashMap = module + .elements + .iter() + .enumerate() + .map(|(ordinal, element)| (element.id(), ordinal as u32)) + .collect(); + struct Rewrite { + structs: HashMap, + arrays: HashMap<(TypeId, u8, u32), FunctionId>, + data_ordinals: HashMap, + element_ordinals: HashMap, + } + impl VisitorMut for Rewrite { + fn visit_instr_mut(&mut self, instr: &mut Instr, _loc: &mut InstrLocId) { + let wrapper = match instr { + Instr::StructNew(StructNew { ty }) => self.structs.get(ty).copied(), + Instr::ArrayNew(ArrayNew { ty }) => { + self.arrays.get(&(*ty, CONSTRUCTOR_ARRAY_NEW, 0)).copied() + } + Instr::ArrayNewDefault(ArrayNewDefault { ty }) => self + .arrays + .get(&(*ty, CONSTRUCTOR_ARRAY_DEFAULT, 0)) + .copied(), + Instr::ArrayNewFixed(ArrayNewFixed { ty, len }) => self + .arrays + .get(&(*ty, CONSTRUCTOR_ARRAY_FIXED, *len)) + .copied(), + Instr::ArrayNewData(ArrayNewData { ty, data }) => self + .data_ordinals + .get(data) + .and_then(|ordinal| self.arrays.get(&(*ty, CONSTRUCTOR_ARRAY_DATA, *ordinal))) + .copied(), + Instr::ArrayNewElem(ArrayNewElem { ty, elem }) => self + .element_ordinals + .get(elem) + .and_then(|ordinal| { + self.arrays.get(&(*ty, CONSTRUCTOR_ARRAY_ELEMENT, *ordinal)) + }) + .copied(), + _ => None, + }; + if let Some(wrapper) = wrapper { + *instr = Instr::Call(Call { func: wrapper }); + } + } + } + let mut rewrite = Rewrite { + structs: struct_wrappers, + arrays: array_wrappers, + data_ordinals, + element_ordinals, + }; + for &function in source_functions { + let local = local_mut(module, function); + let entry = local.entry_block(); + dfs_pre_order_mut(&mut rewrite, local, entry); + } + Ok(()) +} + +fn add_provenance_wrapper( + module: &mut Module, + layout: &GcLayout, + transit: TableId, + imports: HostImports, +) -> Result { + let params: Vec = match layout.constructor { + GcConstructorKind::Struct => layout + .fields + .iter() + .map(|field| field.field.element_type.unpack()) + .collect(), + GcConstructorKind::ArrayNew => { + vec![layout.fields[0].field.element_type.unpack(), ValType::I32] + } + GcConstructorKind::ArrayDefault + | GcConstructorKind::ArrayData { .. } + | GcConstructorKind::ArrayElement { .. } => vec![ValType::I32, ValType::I32], + GcConstructorKind::ArrayFixed { len } => { + vec![layout.fields[0].field.element_type.unpack(); len as usize] + } + GcConstructorKind::ArrayGeneric => unreachable!( + "generic GC array layouts never receive provenance wrappers" + ), + }; + // `array.new_default` has only its dynamic length operand. + let params = if matches!(layout.constructor, GcConstructorKind::ArrayDefault) { + vec![ValType::I32] + } else { + params + }; + let result_ty = ValType::Ref(RefType { + nullable: false, + heap_type: HeapType::Concrete(layout.type_id), + }); + let name = format!("__wpk_fork_ref_gc_construct_{}", layout.id); + let (wrapper, args) = add_stub(module, ¶ms, &[result_ty], &name); + let result = module.locals.add(ValType::Ref(RefType { + nullable: true, + heap_type: HeapType::Concrete(layout.type_id), + })); + let token = module.locals.add(ValType::I32); + let constructor = match layout.constructor { + GcConstructorKind::Struct => Instr::StructNew(StructNew { ty: layout.type_id }), + GcConstructorKind::ArrayNew => Instr::ArrayNew(ArrayNew { ty: layout.type_id }), + GcConstructorKind::ArrayDefault => { + Instr::ArrayNewDefault(ArrayNewDefault { ty: layout.type_id }) + } + GcConstructorKind::ArrayFixed { len } => Instr::ArrayNewFixed(ArrayNewFixed { + ty: layout.type_id, + len, + }), + GcConstructorKind::ArrayData { segment_ordinal } => { + let data = module + .data + .iter() + .nth(segment_ordinal as usize) + .ok_or_else(|| anyhow::anyhow!("GC provenance data segment disappeared"))? + .id(); + Instr::ArrayNewData(ArrayNewData { + ty: layout.type_id, + data, + }) + } + GcConstructorKind::ArrayElement { segment_ordinal } => { + let elem = module + .elements + .iter() + .nth(segment_ordinal as usize) + .ok_or_else(|| anyhow::anyhow!("GC provenance element segment disappeared"))? + .id(); + Instr::ArrayNewElem(ArrayNewElem { + ty: layout.type_id, + elem, + }) + } + GcConstructorKind::ArrayGeneric => unreachable!(), + }; + let entry = entry(wrapper, module); + { + let instrs = instrs_mut(module, wrapper, entry); + for &arg in &args { + local_get(instrs, arg); + } + push(instrs, constructor); + local_set(instrs, result); + + constant_i32(instrs, 0); + local_get(instrs, result); + push(instrs, Instr::TableSet(TableSet { table: transit })); + constant_i32(instrs, 0); + push( + instrs, + Instr::GlobalGet(GlobalGet { + global: imports.activation, + }), + ); + constant_i32(instrs, layout.base_layout_id as i32); + constant_i32(instrs, layout.id as i32); + emit_provenance_scalars(instrs, layout, &args); + constant_i32(instrs, layout.provenance_reference_count as i32); + call(instrs, imports.provenance_begin); + local_set(instrs, token); + clear_transit_slot(instrs, transit, 0); + } + + let reference_args = provenance_reference_args(module, layout); + for (ordinal, arg) in reference_args.into_iter().enumerate() { + let instrs = instrs_mut(module, wrapper, entry); + constant_i32(instrs, 0); + local_get(instrs, args[arg]); + push(instrs, Instr::TableSet(TableSet { table: transit })); + local_get(instrs, token); + constant_i32(instrs, ordinal as i32); + constant_i32(instrs, 0); + call(instrs, imports.provenance_ref); + clear_transit_slot(instrs, transit, 0); + } + let instrs = instrs_mut(module, wrapper, entry); + local_get(instrs, token); + call(instrs, imports.provenance_end); + local_get(instrs, result); + push(instrs, Instr::RefAsNonNull(RefAsNonNull {})); + Ok(wrapper) +} + +fn provenance_reference_args(module: &Module, layout: &GcLayout) -> Vec { + match layout.constructor { + GcConstructorKind::Struct => layout + .fields + .iter() + .enumerate() + .filter_map(|(index, field)| match field.field.element_type { + StorageType::Val(ValType::Ref(reference)) + if field.field.mutable + && !reference.nullable + && is_internal_gc_reference(module, reference) => + { + Some(index) + } + _ => None, + }) + .collect(), + GcConstructorKind::ArrayNew => match layout.fields[0].field.element_type { + StorageType::Val(ValType::Ref(reference)) + if is_internal_gc_reference(module, reference) => + { + vec![0] + } + _ => Vec::new(), + }, + GcConstructorKind::ArrayFixed { len } => match layout.fields[0].field.element_type { + StorageType::Val(ValType::Ref(reference)) + if layout.fields[0].field.mutable + && !reference.nullable + && is_internal_gc_reference(module, reference) => + { + (0..len as usize).collect() + } + _ => Vec::new(), + }, + _ => Vec::new(), + } +} + +fn emit_provenance_scalars( + instrs: &mut Vec<(Instr, InstrLocId)>, + layout: &GcLayout, + args: &[LocalId], +) { + match layout.constructor { + GcConstructorKind::ArrayNew => match layout.fields[0].field.element_type { + StorageType::Val(ValType::Ref(_)) => { + constant_i64(instrs, 0); + constant_i64(instrs, 0); + } + storage => { + emit_scalar_as_i64_pair(instrs, storage, args[0]); + } + }, + GcConstructorKind::ArrayData { .. } | GcConstructorKind::ArrayElement { .. } => { + // WHY: the wire record owns exactly eight provenance bytes for the + // two i32 constructor operands. Pack both into scalarLo; the host + // intentionally truncates scalarLo/scalarHi to that declared + // length and would otherwise discard the second operand. + local_get(instrs, args[0]); + push( + instrs, + Instr::Unop(Unop { + op: UnaryOp::I64ExtendUI32, + }), + ); + local_get(instrs, args[1]); + push( + instrs, + Instr::Unop(Unop { + op: UnaryOp::I64ExtendUI32, + }), + ); + constant_i64(instrs, 32); + binop(instrs, BinaryOp::I64Shl); + binop(instrs, BinaryOp::I64Or); + constant_i64(instrs, 0); + } + _ => { + constant_i64(instrs, 0); + constant_i64(instrs, 0); + } + } +} + +fn emit_scalar_as_i64_pair( + instrs: &mut Vec<(Instr, InstrLocId)>, + storage: StorageType, + value: LocalId, +) { + match storage { + StorageType::I8 | StorageType::I16 | StorageType::Val(ValType::I32) => { + local_get(instrs, value); + push( + instrs, + Instr::Unop(Unop { + op: UnaryOp::I64ExtendUI32, + }), + ); + constant_i64(instrs, 0); + } + StorageType::Val(ValType::I64) => { + local_get(instrs, value); + constant_i64(instrs, 0); + } + StorageType::Val(ValType::F32) => { + local_get(instrs, value); + push( + instrs, + Instr::Unop(Unop { + op: UnaryOp::I32ReinterpretF32, + }), + ); + push( + instrs, + Instr::Unop(Unop { + op: UnaryOp::I64ExtendUI32, + }), + ); + constant_i64(instrs, 0); + } + StorageType::Val(ValType::F64) => { + local_get(instrs, value); + push( + instrs, + Instr::Unop(Unop { + op: UnaryOp::I64ReinterpretF64, + }), + ); + constant_i64(instrs, 0); + } + StorageType::Val(ValType::V128) => { + local_get(instrs, value); + push( + instrs, + Instr::Unop(Unop { + op: UnaryOp::I64x2ExtractLane { idx: 0 }, + }), + ); + local_get(instrs, value); + push( + instrs, + Instr::Unop(Unop { + op: UnaryOp::I64x2ExtractLane { idx: 1 }, + }), + ); + } + StorageType::Val(ValType::Ref(_)) => unreachable!(), + } +} + +/// Emit the type-test probe. The remaining functions are emitted once their +/// recursive reference codec dependencies have been declared. +pub fn emit_probe(module: &mut Module, codec: &DeclaredGcCodec) { + let value = module.locals.add(ValType::Ref(RefType::ANYREF)); + let entry = entry(codec.probe, module); + { + let instrs = instrs_mut(module, codec.probe, entry); + local_get(instrs, codec.probe_args[0]); + push( + instrs, + Instr::TableGet(TableGet { + table: codec.transit, + }), + ); + local_set(instrs, value); + } + + for &layout_id in codec.plan.dispatch_layouts() { + let layout = &codec.plan.layouts()[(layout_id - 1) as usize]; + let yes = dangling(module, codec.probe, walrus::ir::InstrSeqType::Simple(None)); + { + let instrs = instrs_mut(module, codec.probe, yes); + let packed = (u64::from(layout.type_ordinal) << 32) | u64::from(layout.id); + push( + instrs, + Instr::Const(Const { + value: Value::I64(packed as i64), + }), + ); + push(instrs, Instr::Return(Return {})); + } + let no = dangling(module, codec.probe, walrus::ir::InstrSeqType::Simple(None)); + let instrs = instrs_mut(module, codec.probe, entry); + local_get(instrs, value); + push( + instrs, + Instr::RefTest(RefTest { + nullable: false, + heap_type: HeapType::Concrete(layout.type_id), + }), + ); + push( + instrs, + Instr::IfElse(IfElse { + consequent: yes, + alternative: no, + }), + ); + } + let instrs = instrs_mut(module, codec.probe, entry); + push( + instrs, + Instr::Const(Const { + value: Value::I64(0), + }), + ); +} + +pub fn finish_declaration( + module: &mut Module, + codec: DeclaredGcCodec, + exception: module_exception_codec::InjectedExceptionCodec, + runtime: &runtime::Runtime, +) -> Result { + let deps = emit_dependencies(module, exception, runtime)?; + let seeds = ReferenceSeeds::inject(module, &codec.plan); + emit_probe(module, &codec); + emit_encode_anyref(module, &codec, deps); + emit_decode_anyref(module, &codec); + emit_externref_bridge(module, &codec); + emit_encode_slot(module, &codec); + emit_publish_externref(module, &codec); + // Allocation/fill are deliberately separate. Mutable/defaultable shells + // are allocated for the entire graph before any edge is filled; immutable + // and non-defaultable layouts are constructed in dependency order. + emit_allocate(module, &codec, deps, &seeds)?; + emit_fill(module, &codec, deps)?; + Ok(InjectedGcCodec { + encode_anyref: codec.encode_anyref, + decode_anyref: codec.decode_anyref, + encode_externref: codec.encode_externref, + decode_externref: codec.decode_externref, + probe: codec.probe, + encode_slot: codec.encode_slot, + allocate: codec.allocate, + fill: codec.fill, + publish_externref: codec.publish_externref, + transit: codec.transit, + }) +} + +#[derive(Debug, Clone, Copy)] +struct EmitDependencies { + activation: GlobalId, + codecs: runtime::ReferenceCodecs, + vector_begin: FunctionId, + vector_append: FunctionId, + vector_finish: FunctionId, + vector_get: FunctionId, + scratch_reserve: FunctionId, + scratch_release: FunctionId, +} + +#[derive(Debug)] +struct ReferenceSeeds { + abstract_func: FunctionId, + concrete_funcs: HashMap, + exn: FunctionId, +} + +impl ReferenceSeeds { + fn inject(module: &mut Module, plan: &GcCodecPlan) -> Self { + let abstract_func = add_trapping_function(module, &[], &[], "__wpk_fork_ref_seed_func"); + let mut concrete_funcs = HashMap::new(); + for layout in plan.layouts() { + for field in &layout.fields { + let StorageType::Val(ValType::Ref(reference)) = field.field.element_type else { + continue; + }; + let ty = match reference.heap_type { + HeapType::Concrete(ty) | HeapType::Exact(ty) + if module.types.get(ty).is_function() => + { + ty + } + _ => continue, + }; + concrete_funcs.entry(ty).or_insert_with(|| { + let signature = module.types.get(ty); + let params = signature.params().to_vec(); + let results = signature.results().to_vec(); + add_trapping_function( + module, + ¶ms, + &results, + &format!("__wpk_fork_ref_seed_func_{}", ty.index()), + ) + }); + } + } + let exn = add_seed_exception_function(module); + Self { + abstract_func, + concrete_funcs, + exn, + } + } +} + +fn add_trapping_function( + module: &mut Module, + params: &[ValType], + results: &[ValType], + name: &str, +) -> FunctionId { + let (function, _) = add_stub(module, params, results, name); + push( + instrs_mut(module, function, entry(function, module)), + Instr::Unreachable(Unreachable {}), + ); + function +} + +fn add_seed_exception_function(module: &mut Module) -> FunctionId { + let tag_ty = module.types.add(&[], &[]); + let tag = module.tags.add(tag_ty); + module.tags.get_mut(tag).name = Some("__wpk_fork_ref_seed_tag".into()); + let result_ty = ValType::Ref(RefType { + nullable: false, + heap_type: HeapType::Abstract(AbstractHeapType::Exn), + }); + let (function, _) = add_stub(module, &[], &[result_ty], "__wpk_fork_ref_seed_exn"); + let capture = dangling( + module, + function, + walrus::ir::InstrSeqType::Simple(Some(result_ty)), + ); + let body = dangling(module, function, walrus::ir::InstrSeqType::Simple(None)); + push( + instrs_mut(module, function, body), + Instr::Throw(Throw { tag }), + ); + { + let instrs = instrs_mut(module, function, capture); + push( + instrs, + Instr::TryTable(TryTable { + seq: body, + catches: vec![TryTableCatch::CatchAllRef { label: capture }], + }), + ); + push(instrs, Instr::Unreachable(Unreachable {})); + } + push( + instrs_mut(module, function, entry(function, module)), + Instr::Block(walrus::ir::Block { seq: capture }), + ); + function +} + +fn emit_dependencies( + module: &Module, + exception: module_exception_codec::InjectedExceptionCodec, + runtime: &runtime::Runtime, +) -> Result { + let activation = find_import_global( + module, + module_exception_codec::HOST_IMPORT_MODULE, + module_exception_codec::IMPORT_ACTIVATION, + )?; + let mut codecs = runtime + .reference_codecs + .ok_or_else(|| anyhow::anyhow!("GC codec requires linked reference codecs"))?; + // The exception codec is the exact local owner even if a caller assembled + // runtime overrides differently. + codecs.encode_exnref = exception.encode; + codecs.decode_exnref = exception.decode; + let vector_begin = runtime + .reference_vector_begin + .ok_or_else(|| anyhow::anyhow!("GC codec requires reference-vector begin"))?; + let vector_append = runtime + .reference_vector_append + .ok_or_else(|| anyhow::anyhow!("GC codec requires reference-vector append"))?; + let vector_finish = runtime + .reference_vector_finish + .ok_or_else(|| anyhow::anyhow!("GC codec requires reference-vector finish"))?; + let vector_get = runtime + .reference_vector_get + .ok_or_else(|| anyhow::anyhow!("GC codec requires reference-vector get"))?; + Ok(EmitDependencies { + activation, + codecs, + vector_begin, + vector_append, + vector_finish, + vector_get, + scratch_reserve: find_import_function( + module, + module_exception_codec::HOST_IMPORT_MODULE, + module_exception_codec::IMPORT_SCRATCH_RESERVE, + )?, + scratch_release: find_import_function( + module, + module_exception_codec::HOST_IMPORT_MODULE, + module_exception_codec::IMPORT_SCRATCH_RELEASE, + )?, + }) +} + +fn find_import_global(module: &Module, import_module: &str, name: &str) -> Result { + module + .imports + .iter() + .find_map(|import| { + (import.module == import_module && import.name == name) + .then_some(&import.kind) + .and_then(|kind| match kind { + ImportKind::Global(global) => Some(*global), + _ => None, + }) + }) + .ok_or_else(|| anyhow::anyhow!("missing generated import `{import_module}.{name}`")) +} + +fn find_import_function(module: &Module, import_module: &str, name: &str) -> Result { + module + .imports + .iter() + .find_map(|import| { + (import.module == import_module && import.name == name) + .then_some(&import.kind) + .and_then(|kind| match kind { + ImportKind::Function(function) => Some(*function), + _ => None, + }) + }) + .ok_or_else(|| anyhow::anyhow!("missing generated import `{import_module}.{name}`")) +} + +fn emit_decode_anyref(module: &mut Module, codec: &DeclaredGcCodec) { + let recipe = codec.decode_anyref_args[0]; + let null = dangling( + module, + codec.decode_anyref, + walrus::ir::InstrSeqType::Simple(None), + ); + { + let instrs = instrs_mut(module, codec.decode_anyref, null); + push( + instrs, + Instr::RefNull(RefNull { + ty: RefType::ANYREF, + }), + ); + push(instrs, Instr::Return(Return {})); + } + let nonnull = dangling( + module, + codec.decode_anyref, + walrus::ir::InstrSeqType::Simple(None), + ); + let entry = entry(codec.decode_anyref, module); + let instrs = instrs_mut(module, codec.decode_anyref, entry); + local_get(instrs, recipe); + push( + instrs, + Instr::Unop(Unop { + op: UnaryOp::I32Eqz, + }), + ); + push( + instrs, + Instr::IfElse(IfElse { + consequent: null, + alternative: nonnull, + }), + ); + local_get(instrs, recipe); + constant_i32(instrs, 1); + binop(instrs, BinaryOp::I32Add); + push( + instrs, + Instr::TableGet(TableGet { + table: codec.transit, + }), + ); +} + +fn emit_externref_bridge(module: &mut Module, codec: &DeclaredGcCodec) { + { + let entry = entry(codec.encode_externref, module); + let instrs = instrs_mut(module, codec.encode_externref, entry); + local_get(instrs, codec.encode_externref_args[0]); + // WHY: extern.convert_any may have exposed a module-local GC identity + // as externref. Convert it back inside Wasm before classification so + // the ordinary typed graph owns it instead of an opaque host handle. + push(instrs, Instr::AnyConvertExtern(AnyConvertExtern {})); + call(instrs, codec.encode_anyref); + } + { + let entry = entry(codec.decode_externref, module); + let instrs = instrs_mut(module, codec.decode_externref, entry); + local_get(instrs, codec.decode_externref_args[0]); + call(instrs, codec.decode_anyref); + push(instrs, Instr::ExternConvertAny(ExternConvertAny {})); + } +} + +fn emit_encode_slot(module: &mut Module, codec: &DeclaredGcCodec) { + let entry = entry(codec.encode_slot, module); + let instrs = instrs_mut(module, codec.encode_slot, entry); + local_get(instrs, codec.encode_slot_args[0]); + push( + instrs, + Instr::TableGet(TableGet { + table: codec.transit, + }), + ); + call(instrs, codec.encode_anyref); +} + +fn emit_publish_externref(module: &mut Module, codec: &DeclaredGcCodec) { + let entry = entry(codec.publish_externref, module); + let instrs = instrs_mut(module, codec.publish_externref, entry); + local_get(instrs, codec.publish_externref_args[0]); + constant_i32(instrs, 1); + binop(instrs, BinaryOp::I32Add); + local_get(instrs, codec.publish_externref_args[1]); + // WHY: JavaScript cannot directly manufacture an `anyref` transit value. + // The process owner supplies only its canonical externref token; this + // module-local conversion creates the child-side host reference consumed + // by the same anyref decoder used for GC graph edges. + push(instrs, Instr::AnyConvertExtern(AnyConvertExtern {})); + push( + instrs, + Instr::TableSet(TableSet { + table: codec.transit, + }), + ); +} + +fn emit_encode_anyref(module: &mut Module, codec: &DeclaredGcCodec, deps: EmitDependencies) { + let value = codec.encode_anyref_args[0]; + let recipe = module.locals.add(ValType::I32); + let selected_layout = module.locals.add(ValType::I32); + + let null = dangling( + module, + codec.encode_anyref, + walrus::ir::InstrSeqType::Simple(None), + ); + { + let instrs = instrs_mut(module, codec.encode_anyref, null); + constant_i32(instrs, 0); + push(instrs, Instr::Return(Return {})); + } + let nonnull = dangling( + module, + codec.encode_anyref, + walrus::ir::InstrSeqType::Simple(None), + ); + + let i31 = dangling( + module, + codec.encode_anyref, + walrus::ir::InstrSeqType::Simple(None), + ); + { + let instrs = instrs_mut(module, codec.encode_anyref, i31); + local_get(instrs, value); + push( + instrs, + Instr::RefCast(RefCast { + nullable: false, + heap_type: HeapType::Abstract(AbstractHeapType::I31), + }), + ); + push(instrs, Instr::I31GetS(walrus::ir::I31GetS {})); + call(instrs, codec.imports.i31); + local_set(instrs, recipe); + // Parent replay reads the same process-owned transit table as child + // replay. Publish i31 identity here because JavaScript receives only + // its scalar payload and cannot manufacture an `i31ref`. + local_get(instrs, recipe); + constant_i32(instrs, 1); + binop(instrs, BinaryOp::I32Add); + local_get(instrs, value); + push( + instrs, + Instr::TableSet(TableSet { + table: codec.transit, + }), + ); + local_get(instrs, recipe); + push(instrs, Instr::Return(Return {})); + } + let not_i31 = dangling( + module, + codec.encode_anyref, + walrus::ir::InstrSeqType::Simple(None), + ); + + let existing = dangling( + module, + codec.encode_anyref, + walrus::ir::InstrSeqType::Simple(None), + ); + { + let instrs = instrs_mut(module, codec.encode_anyref, existing); + clear_transit_slot(instrs, codec.transit, 0); + local_get(instrs, recipe); + push(instrs, Instr::Return(Return {})); + } + let fresh = dangling( + module, + codec.encode_anyref, + walrus::ir::InstrSeqType::Simple(None), + ); + + let entry = entry(codec.encode_anyref, module); + { + let instrs = instrs_mut(module, codec.encode_anyref, entry); + local_get(instrs, value); + push(instrs, Instr::RefIsNull(RefIsNull {})); + push( + instrs, + Instr::IfElse(IfElse { + consequent: null, + alternative: nonnull, + }), + ); + local_get(instrs, value); + push( + instrs, + Instr::RefTest(RefTest { + nullable: false, + heap_type: HeapType::Abstract(AbstractHeapType::I31), + }), + ); + push( + instrs, + Instr::IfElse(IfElse { + consequent: i31, + alternative: not_i31, + }), + ); + constant_i32(instrs, 0); + local_get(instrs, value); + push( + instrs, + Instr::TableSet(TableSet { + table: codec.transit, + }), + ); + constant_i32(instrs, 0); + call(instrs, codec.imports.lookup); + push(instrs, Instr::LocalTee(LocalTee { local: recipe })); + push( + instrs, + Instr::IfElse(IfElse { + consequent: existing, + alternative: fresh, + }), + ); + } + + for &layout_id in codec.plan.dispatch_layouts() { + let layout = codec.plan.layouts()[(layout_id - 1) as usize].clone(); + let yes = dangling( + module, + codec.encode_anyref, + walrus::ir::InstrSeqType::Simple(None), + ); + emit_encode_layout( + module, + codec, + deps, + yes, + &layout, + value, + recipe, + selected_layout, + ); + let no = dangling( + module, + codec.encode_anyref, + walrus::ir::InstrSeqType::Simple(None), + ); + let instrs = instrs_mut(module, codec.encode_anyref, entry); + local_get(instrs, value); + push( + instrs, + Instr::RefTest(RefTest { + nullable: false, + heap_type: HeapType::Concrete(layout.type_id), + }), + ); + push( + instrs, + Instr::IfElse(IfElse { + consequent: yes, + alternative: no, + }), + ); + } + + let instrs = instrs_mut(module, codec.encode_anyref, entry); + // A structurally canonical GC value may have entered through another + // module activation. The broker probes registered module-local codecs and + // routes the shared transit slot without exposing `anyref` to JavaScript. + constant_i32(instrs, 0); + call(instrs, codec.imports.broker_encode); + local_set(instrs, recipe); + // `broker_encode` grows the shared table through recipe+1. Publish the + // original internal identity before clearing slot zero so parent replay + // and a later anyref/externref alias both use the canonical recipe. + local_get(instrs, recipe); + constant_i32(instrs, 1); + binop(instrs, BinaryOp::I32Add); + local_get(instrs, value); + push( + instrs, + Instr::TableSet(TableSet { + table: codec.transit, + }), + ); + clear_transit_slot(instrs, codec.transit, 0); + local_get(instrs, recipe); +} + +#[allow(clippy::too_many_arguments)] +fn emit_encode_layout( + module: &mut Module, + codec: &DeclaredGcCodec, + deps: EmitDependencies, + seq: walrus::ir::InstrSeqId, + layout: &GcLayout, + value: LocalId, + recipe: LocalId, + selected_layout: LocalId, +) { + let concrete = RefType { + nullable: true, + heap_type: HeapType::Concrete(layout.type_id), + }; + let typed = module.locals.add(ValType::Ref(concrete)); + let staging = module.locals.add(codec.ptr_ty); + let scalar_len = module.locals.add(ValType::I32); + let vector = module.locals.add(ValType::I32); + let array_len = (layout.kind == GcLayoutKind::Array).then(|| module.locals.add(ValType::I32)); + + { + let instrs = instrs_mut(module, codec.encode_anyref, seq); + constant_i32(instrs, 0); + call(instrs, codec.imports.claim); + local_set(instrs, recipe); + + // Claim grows the process-owned transit table through recipe+1 before + // returning. Publishing the source identity before recursive fields + // is what makes aliases and cycles terminate deterministically. + local_get(instrs, recipe); + constant_i32(instrs, 1); + binop(instrs, BinaryOp::I32Add); + local_get(instrs, value); + push( + instrs, + Instr::TableSet(TableSet { + table: codec.transit, + }), + ); + + constant_i32(instrs, 0); + push( + instrs, + Instr::GlobalGet(GlobalGet { + global: deps.activation, + }), + ); + constant_i32(instrs, layout.id as i32); + call(instrs, codec.imports.capture_layout); + local_set(instrs, selected_layout); + + local_get(instrs, value); + push( + instrs, + Instr::RefCast(RefCast { + nullable: true, + heap_type: HeapType::Concrete(layout.type_id), + }), + ); + local_set(instrs, typed); + clear_transit_slot(instrs, codec.transit, 0); + } + + match layout.kind { + GcLayoutKind::Struct => emit_encode_struct_payload( + module, + codec, + deps, + seq, + layout, + typed, + recipe, + selected_layout, + staging, + scalar_len, + vector, + ), + GcLayoutKind::Array => emit_encode_array_payload( + module, + codec, + deps, + seq, + layout, + typed, + recipe, + selected_layout, + staging, + scalar_len, + vector, + array_len.expect("array length local"), + ), + } + let instrs = instrs_mut(module, codec.encode_anyref, seq); + local_get(instrs, recipe); + push(instrs, Instr::Return(Return {})); +} + +#[allow(clippy::too_many_arguments)] +fn emit_encode_struct_payload( + module: &mut Module, + codec: &DeclaredGcCodec, + deps: EmitDependencies, + seq: walrus::ir::InstrSeqId, + layout: &GcLayout, + typed: LocalId, + recipe: LocalId, + selected_layout: LocalId, + staging: LocalId, + scalar_len: LocalId, + vector: LocalId, +) { + let reference_count = layout + .fields + .iter() + .filter(|field| field.reference_ordinal.is_some()) + .count() as u32; + let reservation_len = layout.scalar_len_or_stride.max(1); + let encoders: Vec<_> = layout + .fields + .iter() + .map(|field| match field.field.element_type { + StorageType::Val(ValType::Ref(reference)) => { + Some(reference_encoder(module, deps.codecs, reference)) + } + _ => None, + }) + .collect(); + + let instrs = instrs_mut(module, codec.encode_anyref, seq); + constant_i32(instrs, layout.scalar_len_or_stride as i32); + local_set(instrs, scalar_len); + constant_ptr(instrs, codec.ptr_ty, u64::from(reservation_len)); + call(instrs, deps.scratch_reserve); + local_set(instrs, staging); + if reference_count == 0 { + constant_i32(instrs, 0); + } else { + constant_i32(instrs, reference_count as i32); + call(instrs, deps.vector_begin); + } + local_set(instrs, vector); + + for (index, (field, encoder)) in layout.fields.iter().zip(encoders).enumerate() { + if let Some(offset) = field.scalar_offset { + local_get(instrs, staging); + local_get(instrs, typed); + emit_struct_get( + instrs, + layout.type_id, + index as u32, + field.field.element_type, + ); + push( + instrs, + Instr::Store(Store { + memory: codec.memory, + kind: scalar_store(field.field.element_type), + arg: MemArg { + align: 1, + offset: u64::from(offset), + }, + }), + ); + } else { + local_get(instrs, vector); + local_get(instrs, typed); + emit_struct_get( + instrs, + layout.type_id, + index as u32, + field.field.element_type, + ); + call(instrs, encoder.expect("reference struct field encoder")); + call(instrs, deps.vector_append); + } + } + if reference_count != 0 { + // WHY: vector_begin returns a transaction-local builder handle. Only + // finish publishes a canonical wire ordinal suitable for durable GC + // recipes and deduplicates identical recursive activation vectors. + local_get(instrs, vector); + call(instrs, deps.vector_finish); + local_set(instrs, vector); + } + emit_define( + instrs, + codec, + deps, + recipe, + selected_layout, + layout, + KIND_STRUCT, + staging, + scalar_len, + vector, + ); + local_get(instrs, staging); + constant_ptr(instrs, codec.ptr_ty, u64::from(reservation_len)); + call(instrs, deps.scratch_release); +} + +#[allow(clippy::too_many_arguments)] +fn emit_encode_array_payload( + module: &mut Module, + codec: &DeclaredGcCodec, + deps: EmitDependencies, + seq: walrus::ir::InstrSeqId, + layout: &GcLayout, + typed: LocalId, + recipe: LocalId, + selected_layout: LocalId, + staging: LocalId, + scalar_len: LocalId, + vector: LocalId, + array_len: LocalId, +) { + let index = module.locals.add(ValType::I32); + let field = layout.fields[0]; + let reference = match field.field.element_type { + StorageType::Val(ValType::Ref(reference)) => Some(reference), + _ => None, + }; + + { + let instrs = instrs_mut(module, codec.encode_anyref, seq); + local_get(instrs, typed); + push(instrs, Instr::ArrayLen(ArrayLen {})); + local_set(instrs, array_len); + emit_array_scalar_len( + module, + codec, + seq, + field.field.element_type, + layout.scalar_len_or_stride, + array_len, + scalar_len, + ); + let instrs = instrs_mut(module, codec.encode_anyref, seq); + local_get(instrs, scalar_len); + emit_i32_to_ptr(instrs, codec.ptr_ty); + call(instrs, deps.scratch_reserve); + local_set(instrs, staging); + local_get(instrs, staging); + local_get(instrs, array_len); + push( + instrs, + Instr::Store(Store { + memory: codec.memory, + kind: StoreKind::I32 { atomic: false }, + arg: MemArg { + align: 1, + offset: 0, + }, + }), + ); + } + + if reference.is_some() { + let yes = dangling( + module, + codec.encode_anyref, + walrus::ir::InstrSeqType::Simple(Some(ValType::I32)), + ); + { + let instrs = instrs_mut(module, codec.encode_anyref, yes); + local_get(instrs, array_len); + call(instrs, deps.vector_begin); + } + let no = dangling( + module, + codec.encode_anyref, + walrus::ir::InstrSeqType::Simple(Some(ValType::I32)), + ); + constant_i32(instrs_mut(module, codec.encode_anyref, no), 0); + let instrs = instrs_mut(module, codec.encode_anyref, seq); + local_get(instrs, array_len); + push( + instrs, + Instr::IfElse(IfElse { + consequent: yes, + alternative: no, + }), + ); + } else { + constant_i32(instrs_mut(module, codec.encode_anyref, seq), 0); + } + local_set(instrs_mut(module, codec.encode_anyref, seq), vector); + constant_i32(instrs_mut(module, codec.encode_anyref, seq), 0); + local_set(instrs_mut(module, codec.encode_anyref, seq), index); + + let outer = dangling( + module, + codec.encode_anyref, + walrus::ir::InstrSeqType::Simple(None), + ); + let body = dangling( + module, + codec.encode_anyref, + walrus::ir::InstrSeqType::Simple(None), + ); + let reference_encoder = + reference.map(|reference| reference_encoder(module, deps.codecs, reference)); + { + let instrs = instrs_mut(module, codec.encode_anyref, body); + local_get(instrs, index); + local_get(instrs, array_len); + binop(instrs, BinaryOp::I32GeU); + push(instrs, Instr::BrIf(BrIf { block: outer })); + if reference.is_some() { + local_get(instrs, vector); + local_get(instrs, typed); + local_get(instrs, index); + push(instrs, Instr::ArrayGet(ArrayGet { ty: layout.type_id })); + call( + instrs, + reference_encoder.expect("reference array element encoder"), + ); + call(instrs, deps.vector_append); + } else { + emit_array_scalar_address( + instrs, + staging, + codec.ptr_ty, + index, + layout.scalar_len_or_stride, + ); + local_get(instrs, typed); + local_get(instrs, index); + emit_array_get(instrs, layout.type_id, field.field.element_type); + push( + instrs, + Instr::Store(Store { + memory: codec.memory, + kind: scalar_store(field.field.element_type), + arg: MemArg { + align: 1, + offset: 0, + }, + }), + ); + } + local_get(instrs, index); + constant_i32(instrs, 1); + binop(instrs, BinaryOp::I32Add); + local_set(instrs, index); + push(instrs, Instr::Br(Br { block: body })); + } + push( + instrs_mut(module, codec.encode_anyref, outer), + Instr::Loop(Loop { seq: body }), + ); + push( + instrs_mut(module, codec.encode_anyref, seq), + Instr::Block(walrus::ir::Block { seq: outer }), + ); + + if reference.is_some() { + let finish = dangling( + module, + codec.encode_anyref, + walrus::ir::InstrSeqType::Simple(None), + ); + { + let instrs = instrs_mut(module, codec.encode_anyref, finish); + local_get(instrs, vector); + call(instrs, deps.vector_finish); + local_set(instrs, vector); + } + let empty = dangling( + module, + codec.encode_anyref, + walrus::ir::InstrSeqType::Simple(None), + ); + let instrs = instrs_mut(module, codec.encode_anyref, seq); + local_get(instrs, array_len); + push( + instrs, + Instr::IfElse(IfElse { + consequent: finish, + alternative: empty, + }), + ); + } + + let instrs = instrs_mut(module, codec.encode_anyref, seq); + emit_define( + instrs, + codec, + deps, + recipe, + selected_layout, + layout, + KIND_ARRAY, + staging, + scalar_len, + vector, + ); + local_get(instrs, staging); + local_get(instrs, scalar_len); + emit_i32_to_ptr(instrs, codec.ptr_ty); + call(instrs, deps.scratch_release); +} + +#[allow(clippy::too_many_arguments)] +fn emit_define( + instrs: &mut Vec<(Instr, InstrLocId)>, + codec: &DeclaredGcCodec, + deps: EmitDependencies, + recipe: LocalId, + selected_layout: LocalId, + layout: &GcLayout, + kind: u8, + staging: LocalId, + scalar_len: LocalId, + vector: LocalId, +) { + local_get(instrs, recipe); + push( + instrs, + Instr::GlobalGet(GlobalGet { + global: deps.activation, + }), + ); + constant_i32(instrs, layout.type_ordinal as i32); + local_get(instrs, selected_layout); + constant_i32(instrs, i32::from(kind)); + local_get(instrs, staging); + local_get(instrs, scalar_len); + local_get(instrs, vector); + call(instrs, codec.imports.define); +} + +fn reference_encoder( + module: &Module, + codecs: runtime::ReferenceCodecs, + reference: RefType, +) -> FunctionId { + runtime::ReferenceCodecClass::of(module, reference).encoder(codecs) +} + +fn emit_struct_get( + instrs: &mut Vec<(Instr, InstrLocId)>, + ty: TypeId, + field: u32, + storage: StorageType, +) { + match storage { + StorageType::I8 | StorageType::I16 => { + push(instrs, Instr::StructGetU(StructGetU { ty, field })) + } + StorageType::Val(_) => push(instrs, Instr::StructGet(StructGet { ty, field })), + } +} + +fn emit_array_get(instrs: &mut Vec<(Instr, InstrLocId)>, ty: TypeId, storage: StorageType) { + match storage { + StorageType::I8 | StorageType::I16 => { + push(instrs, Instr::ArrayGetU(walrus::ir::ArrayGetU { ty })) + } + StorageType::Val(_) => push(instrs, Instr::ArrayGet(ArrayGet { ty })), + } +} + +fn scalar_store(storage: StorageType) -> StoreKind { + match storage { + StorageType::I8 => StoreKind::I32_8 { atomic: false }, + StorageType::I16 => StoreKind::I32_16 { atomic: false }, + StorageType::Val(ValType::I32) => StoreKind::I32 { atomic: false }, + StorageType::Val(ValType::I64) => StoreKind::I64 { atomic: false }, + StorageType::Val(ValType::F32) => StoreKind::F32, + StorageType::Val(ValType::F64) => StoreKind::F64, + StorageType::Val(ValType::V128) => StoreKind::V128, + StorageType::Val(ValType::Ref(_)) => unreachable!("reference field uses recipe vector"), + } +} + +fn emit_array_scalar_len( + module: &mut Module, + codec: &DeclaredGcCodec, + seq: walrus::ir::InstrSeqId, + storage: StorageType, + stride: u32, + length: LocalId, + destination: LocalId, +) { + if matches!(storage, StorageType::Val(ValType::Ref(_))) { + let instrs = instrs_mut(module, codec.encode_anyref, seq); + constant_i32(instrs, 4); + local_set(instrs, destination); + return; + } + let maximum_length = (u32::MAX - 4) / stride; + let too_large = dangling( + module, + codec.encode_anyref, + walrus::ir::InstrSeqType::Simple(None), + ); + push( + instrs_mut(module, codec.encode_anyref, too_large), + Instr::Unreachable(Unreachable {}), + ); + let okay = dangling( + module, + codec.encode_anyref, + walrus::ir::InstrSeqType::Simple(None), + ); + let instrs = instrs_mut(module, codec.encode_anyref, seq); + local_get(instrs, length); + constant_i32(instrs, maximum_length as i32); + binop(instrs, BinaryOp::I32GtU); + push( + instrs, + Instr::IfElse(IfElse { + consequent: too_large, + alternative: okay, + }), + ); + local_get(instrs, length); + constant_i32(instrs, stride as i32); + binop(instrs, BinaryOp::I32Mul); + constant_i32(instrs, 4); + binop(instrs, BinaryOp::I32Add); + local_set(instrs, destination); +} + +fn emit_array_scalar_address( + instrs: &mut Vec<(Instr, InstrLocId)>, + staging: LocalId, + ptr_ty: ValType, + index: LocalId, + stride: u32, +) { + local_get(instrs, staging); + constant_ptr(instrs, ptr_ty, 4); + binop(instrs, pointer_add(ptr_ty)); + local_get(instrs, index); + constant_i32(instrs, stride as i32); + binop(instrs, BinaryOp::I32Mul); + emit_i32_to_ptr(instrs, ptr_ty); + binop(instrs, pointer_add(ptr_ty)); +} + +#[derive(Debug, Clone, Copy)] +struct BaseType { + type_id: TypeId, + type_ordinal: u32, + layout_id: u32, + needs_constructor_provenance: bool, +} + +/// Freeze original GC types and constructor sites before module-state/runtime +/// helpers add synthetic functions and types. +pub fn plan(module: &Module) -> Result { + let mut type_ordinals = HashMap::new(); + for ty in module.types.iter() { + if matches!( + ty.kind(), + CompositeType::Struct(_) | CompositeType::Array(_) + ) { + let ordinal = u32::try_from(type_ordinals.len()) + .map_err(|_| anyhow::anyhow!("GC type catalog exceeds u32"))?; + type_ordinals.insert(ty.id(), ordinal); + } + } + + let mut layouts = Vec::with_capacity(type_ordinals.len()); + let mut base_types = HashMap::new(); + for ty in module.types.iter() { + let Some(&type_ordinal) = type_ordinals.get(&ty.id()) else { + continue; + }; + let id = u32::try_from(layouts.len() + 1) + .map_err(|_| anyhow::anyhow!("GC layout catalog exceeds u31"))?; + ensure!(id <= 0x7fff_ffff, "GC layout catalog exceeds u31"); + let super_type_ordinal = ty + .supertype + .and_then(|supertype| type_ordinals.get(&supertype).copied()); + let subtype_depth = subtype_depth(module, ty.id())?; + match ty.kind() { + CompositeType::Struct(structure) => { + let (fields, scalar_len) = layout_fields(module, &structure.fields)?; + let defaultable_shell = structure + .fields + .iter() + .all(|field| field.mutable && defaultable_field(field)); + let requires_provenance = structure.fields.iter().any(|field| { + field.mutable + && matches!( + field.element_type, + StorageType::Val(ValType::Ref(reference)) + if !reference.nullable + && is_internal_gc_reference(module, reference) + ) + }); + let provenance_reference_count = structure + .fields + .iter() + .filter(|field| { + field.mutable + && matches!( + field.element_type, + StorageType::Val(ValType::Ref(reference)) + if !reference.nullable + && is_internal_gc_reference(module, reference) + ) + }) + .count() as u32; + layouts.push(GcLayout { + id, + type_id: ty.id(), + type_ordinal, + base_layout_id: id, + kind: GcLayoutKind::Struct, + constructor: GcConstructorKind::Struct, + scalar_len_or_stride: scalar_len, + fields, + super_type_ordinal, + subtype_depth, + defaultable_shell, + requires_provenance, + provenance_scalar_len: 0, + provenance_reference_count, + }); + base_types.insert( + ty.id(), + BaseType { + type_id: ty.id(), + type_ordinal, + layout_id: id, + needs_constructor_provenance: requires_provenance, + }, + ); + } + CompositeType::Array(array) => { + let (fields, stride) = layout_fields(module, std::slice::from_ref(&array.field))?; + let defaultable_shell = defaultable_field(&array.field) && array.field.mutable; + let needs_constructor_provenance = + !array.field.mutable || !defaultable_field(&array.field); + layouts.push(GcLayout { + id, + type_id: ty.id(), + type_ordinal, + base_layout_id: id, + kind: GcLayoutKind::Array, + constructor: GcConstructorKind::ArrayGeneric, + scalar_len_or_stride: stride, + fields, + super_type_ordinal, + subtype_depth, + defaultable_shell, + requires_provenance: needs_constructor_provenance, + provenance_scalar_len: 0, + provenance_reference_count: 0, + }); + base_types.insert( + ty.id(), + BaseType { + type_id: ty.id(), + type_ordinal, + layout_id: id, + needs_constructor_provenance, + }, + ); + } + CompositeType::Function(_) => unreachable!(), + } + } + + let data_ordinals: HashMap<_, _> = module + .data + .iter() + .enumerate() + .map(|(ordinal, data)| (data.id(), ordinal as u32)) + .collect(); + let element_ordinals: HashMap<_, _> = module + .elements + .iter() + .enumerate() + .map(|(ordinal, element)| (element.id(), ordinal as u32)) + .collect(); + + #[derive(Default)] + struct Constructors { + sites: Vec<(TypeId, GcConstructorKind)>, + } + impl<'instr> Visitor<'instr> for Constructors { + fn visit_instr(&mut self, instr: &'instr Instr, _loc: &'instr InstrLocId) { + match instr { + Instr::ArrayNew(ArrayNew { ty }) => { + self.sites.push((*ty, GcConstructorKind::ArrayNew)) + } + Instr::ArrayNewDefault(ArrayNewDefault { ty }) => { + self.sites.push((*ty, GcConstructorKind::ArrayDefault)) + } + Instr::ArrayNewFixed(ArrayNewFixed { ty, len }) => self + .sites + .push((*ty, GcConstructorKind::ArrayFixed { len: *len })), + Instr::ArrayNewData(ArrayNewData { ty, data }) => self.sites.push(( + *ty, + GcConstructorKind::ArrayData { + segment_ordinal: data.index() as u32, + }, + )), + Instr::ArrayNewElem(ArrayNewElem { ty, elem }) => self.sites.push(( + *ty, + GcConstructorKind::ArrayElement { + segment_ordinal: elem.index() as u32, + }, + )), + _ => {} + } + } + } + + let mut constructors = Constructors::default(); + let mut functions: Vec<_> = module + .funcs + .iter() + .filter_map(|function| match &function.kind { + FunctionKind::Local(local) => Some((function.id(), local)), + FunctionKind::Import(_) | FunctionKind::Uninitialized(_) => None, + }) + .collect(); + functions.sort_by_key(|(id, _)| *id); + for (_, function) in functions { + dfs_in_order(&mut constructors, function, function.entry_block()); + } + + let mut emitted = HashSet::new(); + for (type_id, mut constructor) in constructors.sites { + let Some(base) = base_types.get(&type_id).copied() else { + continue; + }; + if !base.needs_constructor_provenance { + continue; + } + constructor = match constructor { + GcConstructorKind::ArrayData { segment_ordinal } => { + let data = module + .data + .iter() + .find(|data| data.id().index() as u32 == segment_ordinal) + .and_then(|data| data_ordinals.get(&data.id()).copied()) + .ok_or_else(|| anyhow::anyhow!("GC array data segment is not catalogued"))?; + GcConstructorKind::ArrayData { + segment_ordinal: data, + } + } + GcConstructorKind::ArrayElement { segment_ordinal } => { + let element = module + .elements + .iter() + .find(|element| element.id().index() as u32 == segment_ordinal) + .and_then(|element| element_ordinals.get(&element.id()).copied()) + .ok_or_else(|| anyhow::anyhow!("GC array element segment is not catalogued"))?; + GcConstructorKind::ArrayElement { + segment_ordinal: element, + } + } + other => other, + }; + let key = (type_id, constructor.wire(), constructor.auxiliary()); + if !emitted.insert(key) { + continue; + } + let id = u32::try_from(layouts.len() + 1) + .map_err(|_| anyhow::anyhow!("GC constructor catalog exceeds u31"))?; + ensure!(id <= 0x7fff_ffff, "GC constructor catalog exceeds u31"); + let base_layout = &layouts[(base.layout_id - 1) as usize]; + let (provenance_scalar_len, provenance_reference_count) = + constructor_provenance(module, constructor, base_layout.fields[0].field); + layouts.push(GcLayout { + id, + type_id: base.type_id, + type_ordinal: base.type_ordinal, + base_layout_id: base.layout_id, + kind: GcLayoutKind::Array, + constructor, + scalar_len_or_stride: base_layout.scalar_len_or_stride, + fields: base_layout.fields.clone(), + super_type_ordinal: base_layout.super_type_ordinal, + subtype_depth: base_layout.subtype_depth, + defaultable_shell: base_layout.defaultable_shell, + requires_provenance: true, + provenance_scalar_len, + provenance_reference_count, + }); + } + + let mut dispatch: Vec<_> = base_types.values().copied().collect(); + dispatch.sort_by(|left, right| { + let left_layout = &layouts[(left.layout_id - 1) as usize]; + let right_layout = &layouts[(right.layout_id - 1) as usize]; + right_layout + .subtype_depth + .cmp(&left_layout.subtype_depth) + .then_with(|| left.type_ordinal.cmp(&right.type_ordinal)) + }); + Ok(GcCodecPlan { + layouts, + dispatch_layouts: dispatch.into_iter().map(|entry| entry.layout_id).collect(), + }) +} + +fn subtype_depth(module: &Module, mut ty: TypeId) -> Result { + let mut depth = 0u32; + let limit = module.types.iter().count(); + for _ in 0..=limit { + let Some(supertype) = module.types.get(ty).supertype else { + return Ok(depth); + }; + depth = depth + .checked_add(1) + .ok_or_else(|| anyhow::anyhow!("GC subtype depth overflow"))?; + ty = supertype; + } + anyhow::bail!("fork-instrument: cyclic GC supertype chain") +} + +fn defaultable_field(field: &FieldType) -> bool { + match field.element_type { + StorageType::I8 | StorageType::I16 => true, + StorageType::Val(ValType::Ref(reference)) => reference.nullable, + StorageType::Val(_) => true, + } +} + +fn constructor_provenance( + module: &Module, + constructor: GcConstructorKind, + field: FieldType, +) -> (u32, u32) { + match constructor { + GcConstructorKind::ArrayNew => match field.element_type { + StorageType::Val(ValType::Ref(reference)) + if is_internal_gc_reference(module, reference) => + { + (0, 1) + } + StorageType::Val(ValType::Ref(_)) => (0, 0), + scalar => (storage_size(scalar), 0), + }, + GcConstructorKind::ArrayFixed { len } + if field.mutable + && matches!( + field.element_type, + StorageType::Val(ValType::Ref(reference)) + if !reference.nullable + && is_internal_gc_reference(module, reference) + ) => + { + (0, len) + } + GcConstructorKind::ArrayData { .. } | GcConstructorKind::ArrayElement { .. } => (8, 0), + GcConstructorKind::Struct + | GcConstructorKind::ArrayGeneric + | GcConstructorKind::ArrayDefault + | GcConstructorKind::ArrayFixed { .. } => (0, 0), + } +} + +fn layout_fields(_module: &Module, fields: &[FieldType]) -> Result<(Vec, u32)> { + let mut scalar_offset = 0u32; + let mut reference_ordinal = 0u32; + let mut layouts = Vec::with_capacity(fields.len()); + for field in fields { + match field.element_type { + StorageType::Val(ValType::Ref(reference)) => { + layouts.push(GcFieldLayout { + field: *field, + scalar_offset: None, + reference_ordinal: Some(reference_ordinal), + // Immutable edges are constructor values. Mutable + // internal non-null edges use the separately recorded + // constructor seed; other hierarchies have generated + // temporary seeds and are filled in phase two. + allocation_dependency: !field.mutable + && !matches!( + reference.heap_type, + HeapType::Abstract(AbstractHeapType::None) + ), + }); + reference_ordinal = reference_ordinal + .checked_add(1) + .ok_or_else(|| anyhow::anyhow!("GC reference layout overflow"))?; + } + storage => { + let size = storage_size(storage); + let aligned = align_up(scalar_offset, size.min(16))?; + layouts.push(GcFieldLayout { + field: *field, + scalar_offset: Some(aligned), + reference_ordinal: None, + allocation_dependency: false, + }); + scalar_offset = aligned + .checked_add(size) + .ok_or_else(|| anyhow::anyhow!("GC scalar layout overflow"))?; + } + } + } + Ok((layouts, scalar_offset)) +} + +fn is_internal_gc_reference(module: &Module, reference: RefType) -> bool { + runtime::ReferenceCodecClass::of(module, reference) == runtime::ReferenceCodecClass::Any +} + +fn storage_size(storage: StorageType) -> u32 { + match storage { + StorageType::I8 => 1, + StorageType::I16 => 2, + StorageType::Val(ValType::I32 | ValType::F32) => 4, + StorageType::Val(ValType::I64 | ValType::F64) => 8, + StorageType::Val(ValType::V128) => 16, + StorageType::Val(ValType::Ref(_)) => 4, + } +} + +fn align_up(value: u32, alignment: u32) -> Result { + let mask = alignment - 1; + value + .checked_add(mask) + .map(|value| value & !mask) + .ok_or_else(|| anyhow::anyhow!("GC scalar layout overflow")) +} + +fn storage_code(storage: StorageType) -> u8 { + match storage { + StorageType::I8 => STORAGE_I8, + StorageType::I16 => STORAGE_I16, + StorageType::Val(ValType::I32) => STORAGE_I32, + StorageType::Val(ValType::I64) => STORAGE_I64, + StorageType::Val(ValType::F32) => STORAGE_F32, + StorageType::Val(ValType::F64) => STORAGE_F64, + StorageType::Val(ValType::V128) => STORAGE_V128, + StorageType::Val(ValType::Ref(_)) => STORAGE_REFERENCE, + } +} + +pub fn encode_descriptor(plan: &GcCodecPlan) -> Vec { + let field_count: usize = plan.layouts.iter().map(|layout| layout.fields.len()).sum(); + let mut data = Vec::with_capacity( + usize::from(FORMAT_HEADER_SIZE) + + plan.layouts.len() * usize::from(FORMAT_LAYOUT_RECORD_SIZE) + + field_count * usize::from(FORMAT_FIELD_RECORD_SIZE), + ); + data.extend_from_slice(&FORMAT_MAGIC); + data.extend_from_slice(&FORMAT_VERSION.to_le_bytes()); + data.extend_from_slice(&FORMAT_HEADER_SIZE.to_le_bytes()); + data.extend_from_slice(&(plan.layouts.len() as u32).to_le_bytes()); + data.extend_from_slice(&(field_count as u32).to_le_bytes()); + + let mut field_start = 0u32; + for layout in &plan.layouts { + let flags = (if layout.requires_provenance || layout.constructor.requires_provenance() { + LAYOUT_FLAG_REQUIRES_PROVENANCE + } else { + 0 + }) | (if layout.defaultable_shell { + LAYOUT_FLAG_DEFAULTABLE_SHELL + } else { + 0 + }); + data.extend_from_slice(&layout.id.to_le_bytes()); + data.extend_from_slice(&layout.type_ordinal.to_le_bytes()); + data.push(layout.kind.wire()); + data.push(layout.constructor.wire()); + data.extend_from_slice(&flags.to_le_bytes()); + data.extend_from_slice(&layout.scalar_len_or_stride.to_le_bytes()); + data.extend_from_slice(&field_start.to_le_bytes()); + data.extend_from_slice(&(layout.fields.len() as u32).to_le_bytes()); + data.extend_from_slice( + &layout + .super_type_ordinal + .unwrap_or(NO_ORDINAL) + .to_le_bytes(), + ); + data.extend_from_slice(&layout.base_layout_id.to_le_bytes()); + data.extend_from_slice(&layout.constructor.auxiliary().to_le_bytes()); + data.extend_from_slice(&layout.provenance_scalar_len.to_le_bytes()); + data.extend_from_slice(&layout.provenance_reference_count.to_le_bytes()); + field_start += layout.fields.len() as u32; + } + for layout in &plan.layouts { + for field in &layout.fields { + let reference = matches!(field.field.element_type, StorageType::Val(ValType::Ref(_))); + let nullable = matches!( + field.field.element_type, + StorageType::Val(ValType::Ref(reference)) if reference.nullable + ); + let flags = (if field.field.mutable { + FIELD_FLAG_MUTABLE + } else { + 0 + }) | (if nullable { FIELD_FLAG_NULLABLE } else { 0 }) + | (if reference { FIELD_FLAG_REFERENCE } else { 0 }) + | (if field.is_allocation_dependency() { + FIELD_FLAG_ALLOCATION_DEPENDENCY + } else { + 0 + }); + data.push(storage_code(field.field.element_type)); + data.push(flags); + data.extend_from_slice(&0u16.to_le_bytes()); + data.extend_from_slice(&field.scalar_offset.unwrap_or(NO_ORDINAL).to_le_bytes()); + data.extend_from_slice(&field.reference_ordinal.unwrap_or(NO_ORDINAL).to_le_bytes()); + } + } + data +} + +fn inject_host_imports(module: &mut Module, ptr_ty: ValType) -> HostImports { + HostImports { + activation: module + .add_import_global( + HOST_IMPORT_MODULE, + WPK_FORK_EXCEPTION_IMPORT_ACTIVATION, + ValType::I32, + false, + false, + ) + .0, + lookup: import_function(module, IMPORT_LOOKUP, &[ValType::I32], &[ValType::I32]), + claim: import_function(module, IMPORT_CLAIM, &[ValType::I32], &[ValType::I32]), + i31: import_function(module, IMPORT_I31, &[ValType::I32], &[ValType::I32]), + define: import_function( + module, + IMPORT_DEFINE, + &[ + ValType::I32, + ValType::I32, + ValType::I32, + ValType::I32, + ValType::I32, + ptr_ty, + ValType::I32, + ValType::I32, + ], + &[], + ), + route: import_function( + module, + IMPORT_ROUTE, + &[ValType::I32, ValType::I32], + &[ValType::I32], + ), + payload_len: import_function( + module, + IMPORT_PAYLOAD_LEN, + &[ValType::I32, ValType::I32, ValType::I32], + &[ValType::I32], + ), + load: import_function( + module, + IMPORT_LOAD, + &[ + ValType::I32, + ValType::I32, + ValType::I32, + ValType::I32, + ValType::I32, + ptr_ty, + ValType::I32, + ], + &[ValType::I32], + ), + broker_encode: import_function( + module, + IMPORT_BROKER_ENCODE, + &[ValType::I32], + &[ValType::I32], + ), + capture_layout: import_function( + module, + IMPORT_CAPTURE_LAYOUT, + &[ValType::I32, ValType::I32, ValType::I32], + &[ValType::I32], + ), + provenance_begin: import_function( + module, + IMPORT_PROVENANCE_BEGIN, + &[ + ValType::I32, + ValType::I32, + ValType::I32, + ValType::I32, + ValType::I64, + ValType::I64, + ValType::I32, + ], + &[ValType::I32], + ), + provenance_ref: import_function( + module, + IMPORT_PROVENANCE_REF, + &[ValType::I32, ValType::I32, ValType::I32], + &[], + ), + provenance_end: import_function(module, IMPORT_PROVENANCE_END, &[ValType::I32], &[]), + } +} + +fn import_function( + module: &mut Module, + name: &str, + params: &[ValType], + results: &[ValType], +) -> FunctionId { + let ty = module.types.add(params, results); + module.add_import_func(HOST_IMPORT_MODULE, name, ty).0 +} + +fn add_stub( + module: &mut Module, + params: &[ValType], + results: &[ValType], + name: &str, +) -> (FunctionId, Vec) { + let args: Vec<_> = params + .iter() + .copied() + .map(|ty| module.locals.add(ty)) + .collect(); + let mut builder = FunctionBuilder::new(&mut module.types, params, results); + builder.name(name.into()); + let function = builder.finish(args.clone(), &mut module.funcs); + (function, args) +} + +fn replace_descriptor(module: &mut Module, plan: &GcCodecPlan) { + loop { + let existing = module + .customs + .iter() + .find(|(_, section)| section.name() == FORMAT_SECTION) + .map(|(id, _)| id); + let Some(existing) = existing else { break }; + module.customs.delete(existing); + } + module.customs.add(RawCustomSection { + name: FORMAT_SECTION.into(), + data: encode_descriptor(plan), + }); +} + +fn dangling( + module: &mut Module, + function: FunctionId, + ty: walrus::ir::InstrSeqType, +) -> walrus::ir::InstrSeqId { + local_mut(module, function) + .builder_mut() + .dangling_instr_seq(ty) + .id() +} + +fn entry(function: FunctionId, module: &Module) -> walrus::ir::InstrSeqId { + local(module, function).entry_block() +} + +fn instrs_mut( + module: &mut Module, + function: FunctionId, + seq: walrus::ir::InstrSeqId, +) -> &mut Vec<(Instr, InstrLocId)> { + &mut local_mut(module, function).block_mut(seq).instrs +} + +fn local(module: &Module, function: FunctionId) -> &LocalFunction { + match &module.funcs.get(function).kind { + FunctionKind::Local(local) => local, + _ => unreachable!("injected GC codec function is local"), + } +} + +fn local_mut(module: &mut Module, function: FunctionId) -> &mut LocalFunction { + match &mut module.funcs.get_mut(function).kind { + FunctionKind::Local(local) => local, + _ => unreachable!("injected GC codec function is local"), + } +} + +fn push(instrs: &mut Vec<(Instr, InstrLocId)>, instr: Instr) { + instrs.push((instr, InstrLocId::default())); +} + +fn local_get(instrs: &mut Vec<(Instr, InstrLocId)>, local: LocalId) { + push(instrs, Instr::LocalGet(LocalGet { local })); +} + +fn local_set(instrs: &mut Vec<(Instr, InstrLocId)>, local: LocalId) { + push(instrs, Instr::LocalSet(LocalSet { local })); +} + +fn constant_i32(instrs: &mut Vec<(Instr, InstrLocId)>, value: i32) { + push( + instrs, + Instr::Const(Const { + value: Value::I32(value), + }), + ); +} + +fn constant_i64(instrs: &mut Vec<(Instr, InstrLocId)>, value: i64) { + push( + instrs, + Instr::Const(Const { + value: Value::I64(value), + }), + ); +} + +fn constant_ptr(instrs: &mut Vec<(Instr, InstrLocId)>, ptr_ty: ValType, value: u64) { + match ptr_ty { + ValType::I32 => constant_i32(instrs, value as u32 as i32), + ValType::I64 => push( + instrs, + Instr::Const(Const { + value: Value::I64(value as i64), + }), + ), + other => unreachable!("unsupported GC staging pointer type {other:?}"), + } +} + +fn call(instrs: &mut Vec<(Instr, InstrLocId)>, function: FunctionId) { + push(instrs, Instr::Call(Call { func: function })); +} + +fn binop(instrs: &mut Vec<(Instr, InstrLocId)>, op: BinaryOp) { + push(instrs, Instr::Binop(Binop { op })); +} + +fn pointer_add(ptr_ty: ValType) -> BinaryOp { + match ptr_ty { + ValType::I32 => BinaryOp::I32Add, + ValType::I64 => BinaryOp::I64Add, + other => unreachable!("unsupported GC staging pointer type {other:?}"), + } +} + +fn emit_i32_to_ptr(instrs: &mut Vec<(Instr, InstrLocId)>, ptr_ty: ValType) { + match ptr_ty { + ValType::I32 => {} + ValType::I64 => push( + instrs, + Instr::Unop(Unop { + op: UnaryOp::I64ExtendUI32, + }), + ), + other => unreachable!("unsupported GC staging pointer type {other:?}"), + } +} + +fn clear_transit_slot(instrs: &mut Vec<(Instr, InstrLocId)>, transit: TableId, slot: i32) { + constant_i32(instrs, slot); + push( + instrs, + Instr::RefNull(RefNull { + ty: RefType::ANYREF, + }), + ); + push(instrs, Instr::TableSet(TableSet { table: transit })); +} + +fn emit_allocate( + module: &mut Module, + codec: &DeclaredGcCodec, + deps: EmitDependencies, + seeds: &ReferenceSeeds, +) -> Result<()> { + let recipe = codec.allocate_args[0]; + let routed_layout = module.locals.add(ValType::I32); + let scalar_len = module.locals.add(ValType::I32); + let reservation_len = module.locals.add(ValType::I32); + let staging = module.locals.add(codec.ptr_ty); + let vector = module.locals.add(ValType::I32); + // Replay-created aggregates must become valid parents of a later fork. + // This local exists only in the generated replay helper, never in a saved + // user activation frame. + let provenance_token = module.locals.add(ValType::I32); + let entry = entry(codec.allocate, module); + { + let instrs = instrs_mut(module, codec.allocate, entry); + local_get(instrs, recipe); + push( + instrs, + Instr::GlobalGet(GlobalGet { + global: deps.activation, + }), + ); + call(instrs, codec.imports.route); + local_set(instrs, routed_layout); + } + + let i31 = dangling( + module, + codec.allocate, + walrus::ir::InstrSeqType::Simple(None), + ); + emit_allocate_i31( + module, + codec, + deps, + i31, + recipe, + scalar_len, + reservation_len, + staging, + vector, + ); + let not_i31 = dangling( + module, + codec.allocate, + walrus::ir::InstrSeqType::Simple(None), + ); + { + let instrs = instrs_mut(module, codec.allocate, entry); + local_get(instrs, routed_layout); + push( + instrs, + Instr::Unop(Unop { + op: UnaryOp::I32Eqz, + }), + ); + push( + instrs, + Instr::IfElse(IfElse { + consequent: i31, + alternative: not_i31, + }), + ); + } + + for layout in codec.plan.layouts().iter().cloned() { + let yes = dangling( + module, + codec.allocate, + walrus::ir::InstrSeqType::Simple(None), + ); + emit_allocate_layout( + module, + codec, + deps, + yes, + &layout, + recipe, + scalar_len, + reservation_len, + staging, + vector, + provenance_token, + seeds, + )?; + let no = dangling( + module, + codec.allocate, + walrus::ir::InstrSeqType::Simple(None), + ); + let instrs = instrs_mut(module, codec.allocate, entry); + local_get(instrs, routed_layout); + constant_i32(instrs, layout.id as i32); + binop(instrs, BinaryOp::I32Eq); + push( + instrs, + Instr::IfElse(IfElse { + consequent: yes, + alternative: no, + }), + ); + } + push( + instrs_mut(module, codec.allocate, entry), + Instr::Unreachable(Unreachable {}), + ); + Ok(()) +} + +fn emit_fill(module: &mut Module, codec: &DeclaredGcCodec, deps: EmitDependencies) -> Result<()> { + let recipe = codec.fill_args[0]; + let routed_layout = module.locals.add(ValType::I32); + let scalar_len = module.locals.add(ValType::I32); + let reservation_len = module.locals.add(ValType::I32); + let staging = module.locals.add(codec.ptr_ty); + let vector = module.locals.add(ValType::I32); + let entry = entry(codec.fill, module); + { + let instrs = instrs_mut(module, codec.fill, entry); + local_get(instrs, recipe); + push( + instrs, + Instr::GlobalGet(GlobalGet { + global: deps.activation, + }), + ); + call(instrs, codec.imports.route); + local_set(instrs, routed_layout); + } + for layout in codec.plan.layouts().iter().cloned() { + let yes = dangling(module, codec.fill, walrus::ir::InstrSeqType::Simple(None)); + emit_fill_layout( + module, + codec, + deps, + yes, + &layout, + recipe, + scalar_len, + reservation_len, + staging, + vector, + ); + let no = dangling(module, codec.fill, walrus::ir::InstrSeqType::Simple(None)); + let instrs = instrs_mut(module, codec.fill, entry); + local_get(instrs, routed_layout); + constant_i32(instrs, layout.id as i32); + binop(instrs, BinaryOp::I32Eq); + push( + instrs, + Instr::IfElse(IfElse { + consequent: yes, + alternative: no, + }), + ); + } + push( + instrs_mut(module, codec.fill, entry), + Instr::Unreachable(Unreachable {}), + ); + Ok(()) +} + +#[allow(clippy::too_many_arguments)] +fn emit_load_payload( + module: &mut Module, + function: FunctionId, + seq: walrus::ir::InstrSeqId, + codec: &DeclaredGcCodec, + deps: EmitDependencies, + recipe: LocalId, + layout_id: u32, + type_ordinal: u32, + kind: u8, + scalar_len: LocalId, + reservation_len: LocalId, + staging: LocalId, + vector: LocalId, +) { + { + let instrs = instrs_mut(module, function, seq); + local_get(instrs, recipe); + push( + instrs, + Instr::GlobalGet(GlobalGet { + global: deps.activation, + }), + ); + constant_i32(instrs, layout_id as i32); + call(instrs, codec.imports.payload_len); + local_set(instrs, scalar_len); + } + let nonzero = dangling( + module, + function, + walrus::ir::InstrSeqType::Simple(Some(ValType::I32)), + ); + local_get(instrs_mut(module, function, nonzero), scalar_len); + let zero = dangling( + module, + function, + walrus::ir::InstrSeqType::Simple(Some(ValType::I32)), + ); + constant_i32(instrs_mut(module, function, zero), 1); + { + let instrs = instrs_mut(module, function, seq); + local_get(instrs, scalar_len); + push( + instrs, + Instr::IfElse(IfElse { + consequent: nonzero, + alternative: zero, + }), + ); + local_set(instrs, reservation_len); + local_get(instrs, reservation_len); + emit_i32_to_ptr(instrs, codec.ptr_ty); + call(instrs, deps.scratch_reserve); + local_set(instrs, staging); + + local_get(instrs, recipe); + push( + instrs, + Instr::GlobalGet(GlobalGet { + global: deps.activation, + }), + ); + constant_i32(instrs, type_ordinal as i32); + constant_i32(instrs, layout_id as i32); + constant_i32(instrs, i32::from(kind)); + local_get(instrs, staging); + local_get(instrs, scalar_len); + call(instrs, codec.imports.load); + local_set(instrs, vector); + } +} + +fn emit_release_payload( + instrs: &mut Vec<(Instr, InstrLocId)>, + codec: &DeclaredGcCodec, + deps: EmitDependencies, + staging: LocalId, + reservation_len: LocalId, +) { + local_get(instrs, staging); + local_get(instrs, reservation_len); + emit_i32_to_ptr(instrs, codec.ptr_ty); + call(instrs, deps.scratch_release); +} + +#[allow(clippy::too_many_arguments)] +fn emit_allocate_i31( + module: &mut Module, + codec: &DeclaredGcCodec, + deps: EmitDependencies, + seq: walrus::ir::InstrSeqId, + recipe: LocalId, + scalar_len: LocalId, + reservation_len: LocalId, + staging: LocalId, + vector: LocalId, +) { + emit_load_payload( + module, + codec.allocate, + seq, + codec, + deps, + recipe, + 0, + NO_ORDINAL, + 0, + scalar_len, + reservation_len, + staging, + vector, + ); + let instrs = instrs_mut(module, codec.allocate, seq); + local_get(instrs, recipe); + constant_i32(instrs, 1); + binop(instrs, BinaryOp::I32Add); + local_get(instrs, staging); + push( + instrs, + Instr::Load(Load { + memory: codec.memory, + kind: LoadKind::I32 { atomic: false }, + arg: MemArg { + align: 1, + offset: 0, + }, + }), + ); + push(instrs, Instr::RefI31(RefI31 {})); + push( + instrs, + Instr::TableSet(TableSet { + table: codec.transit, + }), + ); + emit_release_payload(instrs, codec, deps, staging, reservation_len); + push(instrs, Instr::Return(Return {})); +} + +#[allow(clippy::too_many_arguments)] +fn emit_allocate_layout( + module: &mut Module, + codec: &DeclaredGcCodec, + deps: EmitDependencies, + seq: walrus::ir::InstrSeqId, + layout: &GcLayout, + recipe: LocalId, + scalar_len: LocalId, + reservation_len: LocalId, + staging: LocalId, + vector: LocalId, + provenance_token: LocalId, + seeds: &ReferenceSeeds, +) -> Result<()> { + emit_load_payload( + module, + codec.allocate, + seq, + codec, + deps, + recipe, + layout.id, + layout.type_ordinal, + layout.kind.wire(), + scalar_len, + reservation_len, + staging, + vector, + ); + { + let instrs = instrs_mut(module, codec.allocate, seq); + local_get(instrs, recipe); + constant_i32(instrs, 1); + binop(instrs, BinaryOp::I32Add); + } + match layout.kind { + GcLayoutKind::Struct => { + emit_allocate_struct(module, codec, deps, seq, layout, staging, vector, seeds) + } + GcLayoutKind::Array => { + emit_allocate_array(module, codec, deps, seq, layout, staging, vector, seeds)? + } + } + { + let instrs = instrs_mut(module, codec.allocate, seq); + push( + instrs, + Instr::TableSet(TableSet { + table: codec.transit, + }), + ); + } + if layout.requires_provenance { + emit_replay_provenance_registration( + module, + codec, + deps, + seq, + layout, + recipe, + staging, + vector, + provenance_token, + ); + } + let instrs = instrs_mut(module, codec.allocate, seq); + emit_release_payload(instrs, codec, deps, staging, reservation_len); + push(instrs, Instr::Return(Return {})); + Ok(()) +} + +#[allow(clippy::too_many_arguments)] +fn emit_replay_provenance_registration( + module: &mut Module, + codec: &DeclaredGcCodec, + deps: EmitDependencies, + seq: walrus::ir::InstrSeqId, + layout: &GcLayout, + recipe: LocalId, + staging: LocalId, + vector: LocalId, + token: LocalId, +) { + { + let instrs = instrs_mut(module, codec.allocate, seq); + local_get(instrs, recipe); + constant_i32(instrs, 1); + binop(instrs, BinaryOp::I32Add); + push( + instrs, + Instr::GlobalGet(GlobalGet { + global: deps.activation, + }), + ); + constant_i32(instrs, layout.base_layout_id as i32); + constant_i32(instrs, layout.id as i32); + emit_replayed_provenance_scalars(instrs, codec, layout, staging); + constant_i32(instrs, layout.provenance_reference_count as i32); + call(instrs, codec.imports.provenance_begin); + local_set(instrs, token); + } + for index in 0..layout.provenance_reference_count { + let instrs = instrs_mut(module, codec.allocate, seq); + local_get(instrs, token); + constant_i32(instrs, index as i32); + local_get(instrs, vector); + constant_i32(instrs, index as i32); + call(instrs, deps.vector_get); + constant_i32(instrs, 1); + binop(instrs, BinaryOp::I32Add); + call(instrs, codec.imports.provenance_ref); + } + let instrs = instrs_mut(module, codec.allocate, seq); + local_get(instrs, token); + call(instrs, codec.imports.provenance_end); +} + +fn emit_replayed_provenance_scalars( + instrs: &mut Vec<(Instr, InstrLocId)>, + codec: &DeclaredGcCodec, + layout: &GcLayout, + staging: LocalId, +) { + let load = |instrs: &mut Vec<(Instr, InstrLocId)>, + kind: LoadKind, + offset: u64| { + local_get(instrs, staging); + push( + instrs, + Instr::Load(Load { + memory: codec.memory, + kind, + arg: MemArg { + align: 1, + offset, + }, + }), + ); + }; + match layout.provenance_scalar_len { + 0 => { + constant_i64(instrs, 0); + constant_i64(instrs, 0); + } + 1 => { + load( + instrs, + LoadKind::I32_8 { + kind: walrus::ir::ExtendedLoad::ZeroExtend, + }, + 0, + ); + push( + instrs, + Instr::Unop(Unop { + op: UnaryOp::I64ExtendUI32, + }), + ); + constant_i64(instrs, 0); + } + 2 => { + load( + instrs, + LoadKind::I32_16 { + kind: walrus::ir::ExtendedLoad::ZeroExtend, + }, + 0, + ); + push( + instrs, + Instr::Unop(Unop { + op: UnaryOp::I64ExtendUI32, + }), + ); + constant_i64(instrs, 0); + } + 4 => { + load(instrs, LoadKind::I32 { atomic: false }, 0); + push( + instrs, + Instr::Unop(Unop { + op: UnaryOp::I64ExtendUI32, + }), + ); + constant_i64(instrs, 0); + } + 8 => { + load(instrs, LoadKind::I64 { atomic: false }, 0); + constant_i64(instrs, 0); + } + 16 => { + load(instrs, LoadKind::I64 { atomic: false }, 0); + load(instrs, LoadKind::I64 { atomic: false }, 8); + } + length => unreachable!("invalid GC provenance scalar length {length}"), + } +} + +fn emit_allocate_struct( + module: &mut Module, + codec: &DeclaredGcCodec, + deps: EmitDependencies, + seq: walrus::ir::InstrSeqId, + layout: &GcLayout, + staging: LocalId, + vector: LocalId, + seeds: &ReferenceSeeds, +) { + if layout.defaultable_shell { + push( + instrs_mut(module, codec.allocate, seq), + Instr::StructNewDefault(StructNewDefault { ty: layout.type_id }), + ); + return; + } + + let reference_decoders: Vec<_> = layout + .fields + .iter() + .map(|field| match field.field.element_type { + StorageType::Val(ValType::Ref(reference)) => { + let class = runtime::ReferenceCodecClass::of(module, reference); + Some(( + class, + class.decoder(deps.codecs), + class.nullable_type(), + reference_seed(module, seeds, reference), + )) + } + _ => None, + }) + .collect(); + let mut provenance_reference = 0u32; + for (index, field) in layout.fields.iter().enumerate() { + let decoder = reference_decoders[index]; + let instrs = instrs_mut(module, codec.allocate, seq); + match field.field.element_type { + StorageType::Val(ValType::Ref(reference)) if !field.field.mutable => { + local_get(instrs, vector); + constant_i32( + instrs, + (layout.provenance_reference_count + + field.reference_ordinal.expect("reference ordinal")) + as i32, + ); + call(instrs, deps.vector_get); + let (_, decoder, broad, _) = decoder.expect("reference decoder"); + call(instrs, decoder); + emit_narrow(instrs, broad, reference); + } + StorageType::Val(ValType::Ref(reference)) + if field.field.mutable && !reference.nullable => + { + let (class, decoder, broad, seed) = decoder.expect("reference decoder"); + if class == runtime::ReferenceCodecClass::Any { + local_get(instrs, vector); + constant_i32(instrs, provenance_reference as i32); + call(instrs, deps.vector_get); + provenance_reference += 1; + call(instrs, decoder); + emit_narrow(instrs, broad, reference); + } else { + emit_seed_reference(instrs, seed); + } + } + StorageType::Val(ValType::Ref(reference)) => { + push(instrs, Instr::RefNull(RefNull { ty: reference })); + } + storage if !field.field.mutable => { + local_get(instrs, staging); + push( + instrs, + Instr::Load(Load { + memory: codec.memory, + kind: scalar_load(storage), + arg: MemArg { + align: 1, + offset: u64::from( + layout.provenance_scalar_len + + field.scalar_offset.expect("scalar offset"), + ), + }, + }), + ); + } + storage => emit_default_scalar(instrs, storage), + } + } + push( + instrs_mut(module, codec.allocate, seq), + Instr::StructNew(StructNew { ty: layout.type_id }), + ); +} + +#[allow(clippy::too_many_arguments)] +fn emit_array_new_reference_seed( + module: &mut Module, + codec: &DeclaredGcCodec, + deps: EmitDependencies, + seq: walrus::ir::InstrSeqId, + layout: &GcLayout, + staging: LocalId, + vector: LocalId, + reference: RefType, + class: runtime::ReferenceCodecClass, + decoder: FunctionId, + broad: RefType, + seed: ReferenceSeed, +) { + if class == runtime::ReferenceCodecClass::Any { + let instrs = instrs_mut(module, codec.allocate, seq); + local_get(instrs, vector); + constant_i32(instrs, 0); + call(instrs, deps.vector_get); + call(instrs, decoder); + emit_narrow(instrs, broad, reference); + return; + } + if layout.fields[0].field.mutable { + emit_seed_reference(instrs_mut(module, codec.allocate, seq), seed); + return; + } + + // Immutable array.new values equal the constructor seed whenever length + // is nonzero. A zero-length array has no observable element, but the Wasm + // instruction still requires a typed seed; synthesize one locally. + let result_ty = ValType::Ref(reference); + let nonempty = dangling( + module, + codec.allocate, + walrus::ir::InstrSeqType::Simple(Some(result_ty)), + ); + { + let instrs = instrs_mut(module, codec.allocate, nonempty); + local_get(instrs, vector); + constant_i32(instrs, layout.provenance_reference_count as i32); + call(instrs, deps.vector_get); + call(instrs, decoder); + emit_narrow(instrs, broad, reference); + } + let empty = dangling( + module, + codec.allocate, + walrus::ir::InstrSeqType::Simple(Some(result_ty)), + ); + if reference.nullable { + push( + instrs_mut(module, codec.allocate, empty), + Instr::RefNull(RefNull { ty: reference }), + ); + } else { + emit_seed_reference(instrs_mut(module, codec.allocate, empty), seed); + } + let instrs = instrs_mut(module, codec.allocate, seq); + emit_load_i32(instrs, codec.memory, staging, layout.provenance_scalar_len); + push( + instrs, + Instr::IfElse(IfElse { + consequent: nonempty, + alternative: empty, + }), + ); +} + +fn emit_allocate_array( + module: &mut Module, + codec: &DeclaredGcCodec, + deps: EmitDependencies, + seq: walrus::ir::InstrSeqId, + layout: &GcLayout, + staging: LocalId, + vector: LocalId, + seeds: &ReferenceSeeds, +) -> Result<()> { + let field = layout.fields[0].field; + match layout.constructor { + GcConstructorKind::ArrayGeneric if layout.defaultable_shell => { + emit_load_i32( + instrs_mut(module, codec.allocate, seq), + codec.memory, + staging, + layout.provenance_scalar_len, + ); + push( + instrs_mut(module, codec.allocate, seq), + Instr::ArrayNewDefault(ArrayNewDefault { ty: layout.type_id }), + ); + } + GcConstructorKind::ArrayNew => { + let reference_decoder = match field.element_type { + StorageType::Val(ValType::Ref(reference)) => { + let class = runtime::ReferenceCodecClass::of(module, reference); + Some(( + reference, + class, + class.decoder(deps.codecs), + class.nullable_type(), + reference_seed(module, seeds, reference), + )) + } + _ => None, + }; + if let Some((reference, class, decoder, broad, seed)) = reference_decoder { + emit_array_new_reference_seed( + module, codec, deps, seq, layout, staging, vector, reference, class, decoder, + broad, seed, + ); + } + { + let instrs = instrs_mut(module, codec.allocate, seq); + match field.element_type { + StorageType::Val(ValType::Ref(_)) => {} + storage => { + local_get(instrs, staging); + push( + instrs, + Instr::Load(Load { + memory: codec.memory, + kind: scalar_load(storage), + arg: MemArg { + align: 1, + offset: 0, + }, + }), + ); + } + } + emit_load_i32(instrs, codec.memory, staging, layout.provenance_scalar_len); + push(instrs, Instr::ArrayNew(ArrayNew { ty: layout.type_id })); + } + } + GcConstructorKind::ArrayDefault => { + emit_load_i32( + instrs_mut(module, codec.allocate, seq), + codec.memory, + staging, + layout.provenance_scalar_len, + ); + push( + instrs_mut(module, codec.allocate, seq), + Instr::ArrayNewDefault(ArrayNewDefault { ty: layout.type_id }), + ); + } + GcConstructorKind::ArrayFixed { len } => { + for index in 0..len { + emit_load_array_snapshot_element( + module, codec, deps, seq, layout, staging, vector, index, seeds, + ); + } + push( + instrs_mut(module, codec.allocate, seq), + Instr::ArrayNewFixed(ArrayNewFixed { + ty: layout.type_id, + len, + }), + ); + } + GcConstructorKind::ArrayData { segment_ordinal } => { + let data = module + .data + .iter() + .nth(segment_ordinal as usize) + .ok_or_else(|| anyhow::anyhow!("GC data constructor segment disappeared"))? + .id(); + let instrs = instrs_mut(module, codec.allocate, seq); + emit_load_i32(instrs, codec.memory, staging, 0); + emit_load_i32(instrs, codec.memory, staging, 4); + push( + instrs, + Instr::ArrayNewData(ArrayNewData { + ty: layout.type_id, + data, + }), + ); + } + GcConstructorKind::ArrayElement { segment_ordinal } => { + let elem = module + .elements + .iter() + .nth(segment_ordinal as usize) + .ok_or_else(|| anyhow::anyhow!("GC element constructor segment disappeared"))? + .id(); + let instrs = instrs_mut(module, codec.allocate, seq); + emit_load_i32(instrs, codec.memory, staging, 0); + emit_load_i32(instrs, codec.memory, staging, 4); + push( + instrs, + Instr::ArrayNewElem(ArrayNewElem { + ty: layout.type_id, + elem, + }), + ); + } + GcConstructorKind::Struct | GcConstructorKind::ArrayGeneric => { + push( + instrs_mut(module, codec.allocate, seq), + Instr::Unreachable(Unreachable {}), + ); + } + } + Ok(()) +} + +#[allow(clippy::too_many_arguments)] +fn emit_fill_layout( + module: &mut Module, + codec: &DeclaredGcCodec, + deps: EmitDependencies, + seq: walrus::ir::InstrSeqId, + layout: &GcLayout, + recipe: LocalId, + scalar_len: LocalId, + reservation_len: LocalId, + staging: LocalId, + vector: LocalId, +) { + let has_mutable = layout.fields.iter().any(|field| field.field.mutable); + if !has_mutable { + push( + instrs_mut(module, codec.fill, seq), + Instr::Return(Return {}), + ); + return; + } + emit_load_payload( + module, + codec.fill, + seq, + codec, + deps, + recipe, + layout.id, + layout.type_ordinal, + layout.kind.wire(), + scalar_len, + reservation_len, + staging, + vector, + ); + match layout.kind { + GcLayoutKind::Struct => { + emit_fill_struct(module, codec, deps, seq, layout, recipe, staging, vector) + } + GcLayoutKind::Array => { + emit_fill_array(module, codec, deps, seq, layout, recipe, staging, vector) + } + } + let instrs = instrs_mut(module, codec.fill, seq); + emit_release_payload(instrs, codec, deps, staging, reservation_len); + push(instrs, Instr::Return(Return {})); +} + +#[allow(clippy::too_many_arguments)] +fn emit_fill_struct( + module: &mut Module, + codec: &DeclaredGcCodec, + deps: EmitDependencies, + seq: walrus::ir::InstrSeqId, + layout: &GcLayout, + recipe: LocalId, + staging: LocalId, + vector: LocalId, +) { + let concrete = RefType { + nullable: true, + heap_type: HeapType::Concrete(layout.type_id), + }; + let object = module.locals.add(ValType::Ref(concrete)); + { + let instrs = instrs_mut(module, codec.fill, seq); + local_get(instrs, recipe); + constant_i32(instrs, 1); + binop(instrs, BinaryOp::I32Add); + push( + instrs, + Instr::TableGet(TableGet { + table: codec.transit, + }), + ); + push( + instrs, + Instr::RefCast(RefCast { + nullable: true, + heap_type: HeapType::Concrete(layout.type_id), + }), + ); + local_set(instrs, object); + } + let reference_decoders: Vec<_> = layout + .fields + .iter() + .map(|field| match field.field.element_type { + StorageType::Val(ValType::Ref(reference)) => { + let class = runtime::ReferenceCodecClass::of(module, reference); + Some((class.decoder(deps.codecs), class.nullable_type())) + } + _ => None, + }) + .collect(); + for (index, field) in layout.fields.iter().enumerate() { + if !field.field.mutable { + continue; + } + let instrs = instrs_mut(module, codec.fill, seq); + local_get(instrs, object); + match field.field.element_type { + StorageType::Val(ValType::Ref(reference)) => { + local_get(instrs, vector); + constant_i32( + instrs, + (layout.provenance_reference_count + + field.reference_ordinal.expect("reference ordinal")) + as i32, + ); + call(instrs, deps.vector_get); + let (decoder, broad) = reference_decoders[index].expect("reference decoder"); + call(instrs, decoder); + emit_narrow(instrs, broad, reference); + } + storage => { + local_get(instrs, staging); + push( + instrs, + Instr::Load(Load { + memory: codec.memory, + kind: scalar_load(storage), + arg: MemArg { + align: 1, + offset: u64::from( + layout.provenance_scalar_len + + field.scalar_offset.expect("scalar offset"), + ), + }, + }), + ); + } + } + push( + instrs, + Instr::StructSet(StructSet { + ty: layout.type_id, + field: index as u32, + }), + ); + } +} + +#[allow(clippy::too_many_arguments)] +fn emit_fill_array( + module: &mut Module, + codec: &DeclaredGcCodec, + deps: EmitDependencies, + seq: walrus::ir::InstrSeqId, + layout: &GcLayout, + recipe: LocalId, + staging: LocalId, + vector: LocalId, +) { + let concrete = RefType { + nullable: true, + heap_type: HeapType::Concrete(layout.type_id), + }; + let object = module.locals.add(ValType::Ref(concrete)); + let length = module.locals.add(ValType::I32); + let index = module.locals.add(ValType::I32); + { + let instrs = instrs_mut(module, codec.fill, seq); + local_get(instrs, recipe); + constant_i32(instrs, 1); + binop(instrs, BinaryOp::I32Add); + push( + instrs, + Instr::TableGet(TableGet { + table: codec.transit, + }), + ); + push( + instrs, + Instr::RefCast(RefCast { + nullable: true, + heap_type: HeapType::Concrete(layout.type_id), + }), + ); + local_set(instrs, object); + emit_load_i32(instrs, codec.memory, staging, layout.provenance_scalar_len); + local_set(instrs, length); + constant_i32(instrs, 0); + local_set(instrs, index); + } + let outer = dangling(module, codec.fill, walrus::ir::InstrSeqType::Simple(None)); + let body = dangling(module, codec.fill, walrus::ir::InstrSeqType::Simple(None)); + let reference_decoder = match layout.fields[0].field.element_type { + StorageType::Val(ValType::Ref(reference)) => { + let class = runtime::ReferenceCodecClass::of(module, reference); + Some((reference, class.decoder(deps.codecs), class.nullable_type())) + } + _ => None, + }; + { + let instrs = instrs_mut(module, codec.fill, body); + local_get(instrs, index); + local_get(instrs, length); + binop(instrs, BinaryOp::I32GeU); + push(instrs, Instr::BrIf(BrIf { block: outer })); + local_get(instrs, object); + local_get(instrs, index); + if let Some((reference, decoder, broad)) = reference_decoder { + local_get(instrs, vector); + local_get(instrs, index); + constant_i32(instrs, layout.provenance_reference_count as i32); + binop(instrs, BinaryOp::I32Add); + call(instrs, deps.vector_get); + call(instrs, decoder); + emit_narrow(instrs, broad, reference); + } else { + emit_array_scalar_address( + instrs, + staging, + codec.ptr_ty, + index, + layout.scalar_len_or_stride, + ); + push( + instrs, + Instr::Load(Load { + memory: codec.memory, + kind: scalar_load(layout.fields[0].field.element_type), + arg: MemArg { + align: 1, + offset: u64::from(layout.provenance_scalar_len), + }, + }), + ); + } + push(instrs, Instr::ArraySet(ArraySet { ty: layout.type_id })); + local_get(instrs, index); + constant_i32(instrs, 1); + binop(instrs, BinaryOp::I32Add); + local_set(instrs, index); + push(instrs, Instr::Br(Br { block: body })); + } + push( + instrs_mut(module, codec.fill, outer), + Instr::Loop(Loop { seq: body }), + ); + push( + instrs_mut(module, codec.fill, seq), + Instr::Block(walrus::ir::Block { seq: outer }), + ); +} + +fn emit_load_array_snapshot_element( + module: &mut Module, + codec: &DeclaredGcCodec, + deps: EmitDependencies, + seq: walrus::ir::InstrSeqId, + layout: &GcLayout, + staging: LocalId, + vector: LocalId, + index: u32, + seeds: &ReferenceSeeds, +) { + let reference_decoder = match layout.fields[0].field.element_type { + StorageType::Val(ValType::Ref(reference)) => { + let class = runtime::ReferenceCodecClass::of(module, reference); + Some(( + reference, + class, + class.decoder(deps.codecs), + class.nullable_type(), + reference_seed(module, seeds, reference), + )) + } + _ => None, + }; + let instrs = instrs_mut(module, codec.allocate, seq); + match layout.fields[0].field.element_type { + StorageType::Val(ValType::Ref(reference)) => { + let (_, class, decoder, broad, seed) = reference_decoder.expect("reference decoder"); + if layout.fields[0].field.mutable { + if class == runtime::ReferenceCodecClass::Any { + local_get(instrs, vector); + constant_i32(instrs, index as i32); + call(instrs, deps.vector_get); + call(instrs, decoder); + emit_narrow(instrs, broad, reference); + } else { + emit_seed_reference(instrs, seed); + } + } else { + local_get(instrs, vector); + constant_i32(instrs, (layout.provenance_reference_count + index) as i32); + call(instrs, deps.vector_get); + call(instrs, decoder); + emit_narrow(instrs, broad, reference); + } + } + storage => { + local_get(instrs, staging); + push( + instrs, + Instr::Load(Load { + memory: codec.memory, + kind: scalar_load(storage), + arg: MemArg { + align: 1, + offset: u64::from( + layout.provenance_scalar_len + 4 + index * layout.scalar_len_or_stride, + ), + }, + }), + ); + } + } +} + +fn emit_load_i32( + instrs: &mut Vec<(Instr, InstrLocId)>, + memory: MemoryId, + staging: LocalId, + offset: u32, +) { + local_get(instrs, staging); + push( + instrs, + Instr::Load(Load { + memory, + kind: LoadKind::I32 { atomic: false }, + arg: MemArg { + align: 1, + offset: u64::from(offset), + }, + }), + ); +} + +fn scalar_load(storage: StorageType) -> LoadKind { + match storage { + StorageType::I8 => LoadKind::I32_8 { + kind: walrus::ir::ExtendedLoad::ZeroExtend, + }, + StorageType::I16 => LoadKind::I32_16 { + kind: walrus::ir::ExtendedLoad::ZeroExtend, + }, + StorageType::Val(ValType::I32) => LoadKind::I32 { atomic: false }, + StorageType::Val(ValType::I64) => LoadKind::I64 { atomic: false }, + StorageType::Val(ValType::F32) => LoadKind::F32, + StorageType::Val(ValType::F64) => LoadKind::F64, + StorageType::Val(ValType::V128) => LoadKind::V128, + StorageType::Val(ValType::Ref(_)) => unreachable!("reference field uses recipe vector"), + } +} + +fn emit_default_scalar(instrs: &mut Vec<(Instr, InstrLocId)>, storage: StorageType) { + let value = match storage { + StorageType::I8 | StorageType::I16 | StorageType::Val(ValType::I32) => Value::I32(0), + StorageType::Val(ValType::I64) => Value::I64(0), + StorageType::Val(ValType::F32) => Value::F32(0.0), + StorageType::Val(ValType::F64) => Value::F64(0.0), + StorageType::Val(ValType::V128) => Value::V128(0), + StorageType::Val(ValType::Ref(_)) => unreachable!(), + }; + push(instrs, Instr::Const(Const { value })); +} + +#[derive(Debug, Clone, Copy)] +enum ReferenceSeed { + Func(FunctionId), + Extern, + Exn(FunctionId), + Uninhabited, +} + +fn reference_seed(module: &Module, seeds: &ReferenceSeeds, reference: RefType) -> ReferenceSeed { + match reference.heap_type { + HeapType::Abstract(AbstractHeapType::Func) => ReferenceSeed::Func(seeds.abstract_func), + HeapType::Concrete(ty) | HeapType::Exact(ty) if module.types.get(ty).is_function() => { + let func = seeds + .concrete_funcs + .get(&ty) + .copied() + .expect("planned concrete function seed"); + ReferenceSeed::Func(func) + } + HeapType::Abstract(AbstractHeapType::Extern) => ReferenceSeed::Extern, + HeapType::Abstract(AbstractHeapType::Exn) => ReferenceSeed::Exn(seeds.exn), + _ => ReferenceSeed::Uninhabited, + } +} + +fn emit_seed_reference(instrs: &mut Vec<(Instr, InstrLocId)>, seed: ReferenceSeed) { + match seed { + ReferenceSeed::Func(func) => push(instrs, Instr::RefFunc(RefFunc { func })), + ReferenceSeed::Extern => { + constant_i32(instrs, 0); + push(instrs, Instr::RefI31(RefI31 {})); + push(instrs, Instr::ExternConvertAny(ExternConvertAny {})); + } + ReferenceSeed::Exn(function) => call(instrs, function), + // Bottom reference types have no runtime inhabitant. If an aggregate + // with such a field is somehow routed here, trap before installing a + // partially reconstructed identity. + ReferenceSeed::Uninhabited => push(instrs, Instr::Unreachable(Unreachable {})), + } +} + +fn emit_narrow(instrs: &mut Vec<(Instr, InstrLocId)>, broad: RefType, expected: RefType) { + if expected.heap_type != broad.heap_type { + push( + instrs, + Instr::RefCast(RefCast { + nullable: expected.nullable, + heap_type: expected.heap_type, + }), + ); + } else if !expected.nullable { + push(instrs, Instr::RefAsNonNull(RefAsNonNull {})); + } +} diff --git a/crates/fork-instrument/src/module_state.rs b/crates/fork-instrument/src/module_state.rs new file mode 100644 index 0000000000..3a9ccc573f --- /dev/null +++ b/crates/fork-instrument/src/module_state.rs @@ -0,0 +1,3395 @@ +//! Guest-owned snapshot and reconstruction of module-instance state. +//! +//! A fork child is a fresh WebAssembly instance. Mutable globals, tables, and +//! passive-segment lifetime therefore do not survive merely because linear +//! memory was copied. This module emits typed guest helpers: +//! +//! * `wpk_fork_module_state_save(activation_id)` encodes every reference root +//! and table entry into the process-wide recipe transaction and writes only +//! recipe IDs/metadata into the KFMS arena. +//! * `wpk_fork_module_state_restore(activation_id)` decodes those IDs inside +//! the fresh instance, restores globals, and replays static table baselines +//! while passive segments are still available to later activations. +//! * `wpk_fork_module_state_finish_restore(activation_id)` restores the one +//! canonical sparse overlay for each physical table, then reapplies the +//! parent instance's `elem.drop`/`data.drop` state after the complete +//! activation graph has consumed every segment-backed initializer. +//! +//! The helpers never expose references through the JavaScript Table/Global +//! APIs. `exnref` and GC values cross only typed Wasm-to-Wasm codec imports. + +use std::collections::{HashMap, HashSet}; + +use anyhow::Result; +use sha2::{Digest, Sha256}; +use walrus::{ + ConstExpr, DataId, DataKind, ElementId, ElementItems, ElementKind, ExportItem, FunctionBuilder, + FunctionId, FunctionKind, GlobalId, InstrSeqBuilder, LocalFunction, LocalId, MemoryId, Module, + RawCustomSection, RefType, TableId, TypeId, ValType, + ir::{ + BinaryOp, Block, Br, BrIf, Call, CallIndirect, DataDrop, ElemDrop, ExtendedLoad, IfElse, + Instr, InstrLocId, InstrSeqId, LegacyCatch, LoadKind, Loop, MemArg, MemoryInit, + RefAsNonNull, RefCast, ReturnCall, ReturnCallIndirect, StoreKind, TableCopy, TableFill, + TableGet, TableGrow, TableInit, TableSet, TableSize, TryTable, UnaryOp, Unreachable, Value, + }, +}; +use wasm_posix_shared::abi::{ + WPK_FORK_EXPORT_MODULE_BOOTSTRAP, WPK_FORK_EXPORT_MODULE_STATE_FINISH_RESTORE, + WPK_FORK_EXPORT_MODULE_STATE_RESTORE, WPK_FORK_EXPORT_MODULE_STATE_SAVE, + WPK_FORK_EXPORT_MODULE_TABLE_STATE_RESTORE, WPK_FORK_EXPORT_MODULE_TABLE_STATE_SAVE, + WPK_FORK_EXPORT_MODULE_THREAD_BOOTSTRAP, WPK_FORK_GLOBAL_CATALOG_EXPORT_PREFIX, + WPK_FORK_IMPORTED_GLOBAL_FLAG_MUTABLE, WPK_FORK_IMPORTED_GLOBAL_FLAG_SHARED, + WPK_FORK_IMPORTED_GLOBALS_HEADER_SIZE, WPK_FORK_IMPORTED_GLOBALS_MAGIC, + WPK_FORK_IMPORTED_GLOBALS_RECORD_HEADER_SIZE, WPK_FORK_IMPORTED_GLOBALS_SECTION, + WPK_FORK_IMPORTED_GLOBALS_VERSION, WPK_FORK_IMPORTED_TABLE_FLAG_TABLE64, + WPK_FORK_IMPORTED_TABLES_HEADER_SIZE, WPK_FORK_IMPORTED_TABLES_MAGIC, + WPK_FORK_IMPORTED_TABLES_RECORD_HEADER_SIZE, WPK_FORK_IMPORTED_TABLES_SECTION, + WPK_FORK_IMPORTED_TABLES_VERSION, WPK_FORK_MODULE_STATE_DATA_SEGMENT_HEADER_SIZE, + WPK_FORK_MODULE_STATE_ELEMENT_SEGMENT_HEADER_SIZE, WPK_FORK_MODULE_STATE_GLOBAL_HEADER_SIZE, + WPK_FORK_MODULE_STATE_GLOBAL_TYPE_ANYREF, WPK_FORK_MODULE_STATE_GLOBAL_TYPE_EXNREF, + WPK_FORK_MODULE_STATE_GLOBAL_TYPE_EXTERNREF, WPK_FORK_MODULE_STATE_GLOBAL_TYPE_F32, + WPK_FORK_MODULE_STATE_GLOBAL_TYPE_F64, WPK_FORK_MODULE_STATE_GLOBAL_TYPE_FUNCREF, + WPK_FORK_MODULE_STATE_GLOBAL_TYPE_I32, WPK_FORK_MODULE_STATE_GLOBAL_TYPE_I64, + WPK_FORK_MODULE_STATE_GLOBAL_TYPE_V128, WPK_FORK_MODULE_STATE_IMPORT_MODULE, + WPK_FORK_MODULE_STATE_IMPORT_RECORD_COMMIT, WPK_FORK_MODULE_STATE_IMPORT_RECORD_FIND, + WPK_FORK_MODULE_STATE_IMPORT_RECORD_RESERVE, WPK_FORK_MODULE_STATE_IMPORT_TABLE_DIRTY_COUNT, + WPK_FORK_MODULE_STATE_IMPORT_TABLE_DIRTY_MARK, WPK_FORK_MODULE_STATE_IMPORT_TABLE_DIRTY_PAGE, + WPK_FORK_MODULE_STATE_IMPORT_TABLE_GENERATION_ADDR, + WPK_FORK_MODULE_STATE_IMPORT_TABLE_MUTATION_ABORT, + WPK_FORK_MODULE_STATE_IMPORT_TABLE_MUTATION_BEGIN, + WPK_FORK_MODULE_STATE_IMPORT_TABLE_MUTATION_COMMIT, + WPK_FORK_MODULE_STATE_IMPORT_TABLE_RECONCILE, WPK_FORK_MODULE_STATE_IMPORT_TABLE_STATE_OWNED, + WPK_FORK_MODULE_STATE_RECORD_KIND_DATA_SEGMENTS, + WPK_FORK_MODULE_STATE_RECORD_KIND_ELEMENT_SEGMENTS, + WPK_FORK_MODULE_STATE_RECORD_KIND_MUTABLE_GLOBAL, WPK_FORK_MODULE_STATE_RECORD_KIND_TABLE, + WPK_FORK_MODULE_STATE_RECORD_KIND_TABLE_PAGE, + WPK_FORK_MODULE_STATE_TABLE_DESCRIPTOR_PAYLOAD_SIZE, + WPK_FORK_MODULE_STATE_TABLE_FLAG_SPARSE_OVERRIDES, + WPK_FORK_MODULE_STATE_TABLE_PAGE_HEADER_SIZE, WPK_FORK_MODULE_STATE_TABLE_PAGE_SHIFT, + WPK_FORK_MODULE_STATE_TABLE_RUN_HEADER_SIZE, WPK_FORK_RESUME_IMPORT_TABLE, + WPK_FORK_TABLE_CATALOG_EXPORT_PREFIX, +}; + +use crate::runtime::{ReferenceCodecClass, ReferenceCodecs, Runtime}; + +const TABLE_PAGE_SHIFT: u32 = WPK_FORK_MODULE_STATE_TABLE_PAGE_SHIFT as u32; +const TABLE_PAGE_SIZE: u64 = 1 << TABLE_PAGE_SHIFT; +const GLOBAL_RECIPE_PAYLOAD_SIZE: u32 = 4; +const WASM32_MAX_PAGES: u64 = 1 << 16; + +#[derive(Debug, Clone, Copy)] +struct GlobalState { + id: GlobalId, + owner: u32, + ty: ValType, + restore: bool, +} + +#[derive(Debug, Clone)] +struct ImportedGlobalState { + module: String, + name: String, + import_ordinal: u32, + owner: u32, + ty: ValType, + mutable: bool, + shared: bool, +} + +#[derive(Debug, Clone)] +struct ImportedTableState { + module: String, + name: String, + import_ordinal: u32, + owner: u32, + table64: bool, + ty: RefType, +} + +#[derive(Debug, Clone, Copy)] +struct TableState { + id: TableId, + owner: u32, + table64: bool, + ty: RefType, + baseline_len: u64, + baseline_fingerprint: [u8; 32], + synchronized: bool, +} + +#[derive(Debug, Clone)] +struct ActiveElement { + id: ElementId, + table: TableId, + offset: ConstExpr, + offset_global: GlobalId, + len: u64, +} + +#[derive(Debug, Clone)] +struct ActiveData { + id: DataId, + memory: MemoryId, + offset_global: GlobalId, + len: u64, +} + +/// Original module-instance state captured before the instrumenter adds its +/// private function catalog, imports, globals, and helper functions. +#[derive(Debug, Default)] +pub struct ModuleStatePlan { + /// Every source global, including immutable locals and child-only host + /// bindings, exported under a deterministic private name. + /// + /// WHY: a WebAssembly.Global is the only JavaScript API value that can + /// carry an exnref binding into a fresh instance. The loader records which + /// consumer import aliases which provider cell, then resolves the same + /// cell from this catalog before instantiating the consumer. + global_catalog: Vec<(GlobalId, u32)>, + table_catalog: Vec<(TableId, u32)>, + globals: Vec, + imported_globals: Vec, + imported_tables: Vec, + tables: Vec, + elements: Vec<(ElementId, bool)>, + data: Vec<(DataId, bool)>, + active_elements: Vec, + active_data: Vec, + original_start: Option, + original_functions: Vec, +} + +/// Imported record operations shared by every generated helper. +#[derive(Debug, Clone, Copy)] +pub struct ModuleStateImports { + pub reserve: FunctionId, + pub commit: FunctionId, + pub find: FunctionId, + pub table_dirty_mark: FunctionId, + pub table_dirty_count: FunctionId, + pub table_dirty_page: FunctionId, + pub table_state_owned: FunctionId, + pub table_mutation_begin: FunctionId, + pub table_mutation_commit: FunctionId, + pub table_mutation_abort: FunctionId, + pub table_reconcile: FunctionId, + pub table_generation_addr: GlobalId, +} + +/// Plan all state whose owner is a WebAssembly module activation. +pub fn plan(module: &mut Module) -> ModuleStatePlan { + let import_ordinals: HashMap<_, _> = module + .imports + .iter() + .enumerate() + .map(|(ordinal, import)| { + ( + import.id(), + u32::try_from(ordinal).expect("import ordinal fits u32"), + ) + }) + .collect(); + let mut original_functions: Vec<_> = module + .funcs + .iter() + .filter_map(|func| matches!(func.kind, FunctionKind::Local(_)).then_some(func.id())) + .collect(); + original_functions.sort(); + + let mut globals: Vec<_> = module.globals.iter().map(|global| global.id()).collect(); + globals.sort(); + let global_catalog = globals + .iter() + .copied() + .enumerate() + .map(|(ordinal, id)| { + ( + id, + u32::try_from(ordinal + 1).expect("global owner ordinal fits u32"), + ) + }) + .collect(); + let mut state_globals = Vec::new(); + let mut imported_globals = Vec::new(); + for (ordinal, id) in globals.into_iter().enumerate() { + let global = module.globals.get(id); + let imported = match global.kind { + walrus::GlobalKind::Import(import_id) => Some(module.imports.get(import_id)), + walrus::GlobalKind::Local(_) => None, + }; + if imported_global_is_child_binding(module, global) + || (!global.mutable && imported.is_none()) + { + continue; + } + let owner = u32::try_from(ordinal + 1).expect("global owner ordinal fits u32"); + state_globals.push(GlobalState { + id, + owner, + ty: global.ty, + restore: global.mutable, + }); + if let Some(import) = imported { + // Preserve every declaration, including repeated JS property + // identities. Immutable raw imports are coerced independently by + // their declared Wasm types, so collapsing aliases here can lose + // information even though one import-object property supplies all + // declarations. + imported_globals.push(ImportedGlobalState { + module: import.module.to_owned(), + name: import.name.to_owned(), + import_ordinal: import_ordinals[&import.id()], + owner, + ty: global.ty, + mutable: global.mutable, + shared: global.shared, + }); + } + } + + let mut element_ids: Vec<_> = module.elements.iter().map(|elem| elem.id()).collect(); + element_ids.sort(); + let mut elements = Vec::with_capacity(element_ids.len()); + let mut active_elements = Vec::new(); + for id in element_ids { + let initially_dropped = !matches!(module.elements.get(id).kind, ElementKind::Passive); + elements.push((id, initially_dropped)); + let active = match &module.elements.get(id).kind { + ElementKind::Active { table, offset } => Some(( + *table, + offset.clone(), + match &module.elements.get(id).items { + ElementItems::Functions(items) => items.len() as u64, + ElementItems::Expressions(_, items) => items.len() as u64, + }, + )), + _ => None, + }; + if let Some((table, offset, len)) = active { + let offset_global = module.globals.add_local( + if module.tables.get(table).table64 { + ValType::I64 + } else { + ValType::I32 + }, + false, + false, + offset.clone(), + ); + module.elements.get_mut(id).kind = ElementKind::Passive; + active_elements.push(ActiveElement { + id, + table, + offset, + offset_global, + len, + }); + } + } + + let mut data_ids: Vec<_> = module.data.iter().map(|data| data.id()).collect(); + data_ids.sort(); + let mut data = Vec::with_capacity(data_ids.len()); + let mut active_data = Vec::new(); + for id in data_ids { + let initially_dropped = !matches!(module.data.get(id).kind, DataKind::Passive); + data.push((id, initially_dropped)); + let active = match &module.data.get(id).kind { + DataKind::Active { memory, offset } => Some(( + *memory, + offset.clone(), + module.data.get(id).value.len() as u64, + )), + DataKind::Passive => None, + }; + if let Some((memory, offset, len)) = active { + let offset_global = module.globals.add_local( + if module.memories.get(memory).memory64 { + ValType::I64 + } else { + ValType::I32 + }, + false, + false, + offset.clone(), + ); + module.data.get_mut(id).kind = DataKind::Passive; + active_data.push(ActiveData { + id, + memory, + offset_global, + len, + }); + } + } + + let mut table_ids: Vec<_> = module.tables.iter().map(|table| table.id()).collect(); + table_ids.sort(); + let runtime_mutated_tables = collect_source_mutated_tables(module, &original_functions); + let process_indirect_tables: HashSet<_> = + if crate::call_graph::has_dynamic_linker_imports(module) { + module + .exports + .iter() + .filter_map(|export| { + (export.name == "__indirect_function_table") + .then_some(export.item) + .and_then(|item| match item { + ExportItem::Table(table) => Some(table), + _ => None, + }) + }) + .collect() + } else { + HashSet::new() + }; + let table_catalog = table_ids + .iter() + .copied() + .enumerate() + .map(|(ordinal, id)| { + ( + id, + u32::try_from(ordinal + 1).expect("table owner ordinal fits u32"), + ) + }) + .collect(); + let mut imported_tables = Vec::new(); + let tables = table_ids + .into_iter() + .enumerate() + .filter_map(|(ordinal, id)| { + let table = module.tables.get(id); + if imported_table_is_resume_binding(module, table) { + return None; + } + let owner = u32::try_from(ordinal + 1).expect("table owner ordinal fits u32"); + if let Some(import_id) = table.import { + let import = module.imports.get(import_id); + imported_tables.push(ImportedTableState { + module: import.module.to_owned(), + name: import.name.to_owned(), + import_ordinal: import_ordinals[&import.id()], + owner, + table64: table.table64, + ty: table.element_ty, + }); + } + Some(TableState { + id, + owner, + table64: table.table64, + ty: table.element_ty, + baseline_len: table.initial, + baseline_fingerprint: table_baseline_fingerprint( + module, + id, + owner, + &active_elements, + ), + // WHY: a local table with no runtime writer and no host-owned + // process-table role is fully reconstructed by its declared + // minimum plus static element initializers. Synchronizing it + // would add a generation fence to ordinary table reads while + // carrying no state that can differ between Workers. + // + // Imported tables stay synchronized because another + // activation can mutate the aliased physical table. Kandelo's + // dynamic linker mutates only the exact process indirect + // table, and only an artifact importing its dlopen surface can + // activate that writer. wasm-ld exports this table broadly, + // so its name alone is not evidence of mutable state. + synchronized: table.import.is_some() + || process_indirect_tables.contains(&id) + || runtime_mutated_tables.contains(&id), + }) + }) + .collect(); + + ModuleStatePlan { + global_catalog, + table_catalog, + globals: state_globals, + imported_globals, + imported_tables, + tables, + elements, + data, + active_elements, + active_data, + original_start: module.start.take(), + original_functions, + } +} + +fn table_baseline_fingerprint( + module: &Module, + table_id: TableId, + owner: u32, + active_elements: &[ActiveElement], +) -> [u8; 32] { + let table = module.tables.get(table_id); + let mut hasher = Sha256::new(); + hasher.update(b"kandelo-kfms-table-baseline-v1\0"); + hasher.update(owner.to_le_bytes()); + hasher.update([u8::from(table.table64)]); + hasher.update(table.initial.to_le_bytes()); + hasher.update(table.maximum.unwrap_or(u64::MAX).to_le_bytes()); + hasher.update(format!("{:?}", table.element_ty).as_bytes()); + hasher.update(format!("{:?}", table.init).as_bytes()); + for active in active_elements + .iter() + .filter(|active| active.table == table_id) + { + hasher.update(format!("{:?}", active.offset).as_bytes()); + hasher.update(active.len.to_le_bytes()); + hasher.update(format!("{:?}", module.elements.get(active.id).items).as_bytes()); + } + hasher.finalize().into() +} + +/// Return the process-memory view used to exchange KFMS record payloads with +/// the host, adding Kandelo's standard side-module memory import when the +/// original module has no memory. +/// +/// WHY: a module without guest loads/stores can still own mutable globals and +/// tables. The record callbacks return addresses in the process memory copied +/// to a fork child, so a private local memory would write the same numeric +/// address in the wrong allocation. `env.memory` gives every module activation +/// the same staging address space without imposing an artifact-shape failure. +pub fn ensure_staging_memory(module: &mut Module) -> MemoryId { + if let Some(memory) = module.memories.iter().next() { + return memory.id(); + } + let (memory, _) = module.add_import_memory( + "env", + "memory", + true, + false, + 0, + Some(WASM32_MAX_PAGES), + None, + ); + memory +} + +fn imported_global_is_child_binding(module: &Module, global: &walrus::Global) -> bool { + let walrus::GlobalKind::Import(import_id) = global.kind else { + return false; + }; + let import = module.imports.get(import_id); + import.module == "env" && import.name == "__channel_base" +} + +fn imported_table_is_resume_binding(module: &Module, table: &walrus::Table) -> bool { + let Some(import_id) = table.import else { + return false; + }; + let import = module.imports.get(import_id); + import.module == WPK_FORK_MODULE_STATE_IMPORT_MODULE + && import.name == WPK_FORK_RESUME_IMPORT_TABLE +} + +fn scalar_size(ty: ValType) -> u32 { + match ty { + ValType::I32 | ValType::F32 => 4, + ValType::I64 | ValType::F64 => 8, + ValType::V128 => 16, + ValType::Ref(_) => unreachable!("reference globals store a recipe id"), + } +} + +fn scalar_align(ty: ValType) -> u32 { + scalar_size(ty) +} + +fn scalar_store_kind(ty: ValType) -> StoreKind { + match ty { + ValType::I32 => StoreKind::I32 { atomic: false }, + ValType::I64 => StoreKind::I64 { atomic: false }, + ValType::F32 => StoreKind::F32, + ValType::F64 => StoreKind::F64, + ValType::V128 => StoreKind::V128, + ValType::Ref(_) => unreachable!("reference globals store a recipe id"), + } +} + +fn scalar_load_kind(ty: ValType) -> LoadKind { + match ty { + ValType::I32 => LoadKind::I32 { atomic: false }, + ValType::I64 => LoadKind::I64 { atomic: false }, + ValType::F32 => LoadKind::F32, + ValType::F64 => LoadKind::F64, + ValType::V128 => LoadKind::V128, + ValType::Ref(_) => unreachable!("reference globals load a recipe id"), + } +} + +fn global_type_code(ty: ValType, class: Option) -> u8 { + match (ty, class) { + (ValType::I32, None) => WPK_FORK_MODULE_STATE_GLOBAL_TYPE_I32, + (ValType::I64, None) => WPK_FORK_MODULE_STATE_GLOBAL_TYPE_I64, + (ValType::F32, None) => WPK_FORK_MODULE_STATE_GLOBAL_TYPE_F32, + (ValType::F64, None) => WPK_FORK_MODULE_STATE_GLOBAL_TYPE_F64, + (ValType::V128, None) => WPK_FORK_MODULE_STATE_GLOBAL_TYPE_V128, + (ValType::Ref(_), Some(ReferenceCodecClass::Func)) => { + WPK_FORK_MODULE_STATE_GLOBAL_TYPE_FUNCREF + } + (ValType::Ref(_), Some(ReferenceCodecClass::Extern)) => { + WPK_FORK_MODULE_STATE_GLOBAL_TYPE_EXTERNREF + } + (ValType::Ref(_), Some(ReferenceCodecClass::Exn)) => { + WPK_FORK_MODULE_STATE_GLOBAL_TYPE_EXNREF + } + (ValType::Ref(_), Some(ReferenceCodecClass::Any)) => { + WPK_FORK_MODULE_STATE_GLOBAL_TYPE_ANYREF + } + _ => unreachable!("global type and codec class disagree"), + } +} + +#[derive(Debug)] +struct SegmentTracker { + segments: Vec, + globals: Vec, +} + +#[derive(Debug)] +struct Trackers { + elements: SegmentTracker, + data: SegmentTracker, +} + +/// Inject the KFMS record imports and the two typed guest helpers. +pub fn inject( + module: &mut Module, + runtime: &Runtime, + plan: ModuleStatePlan, +) -> Result { + let memory = module + .memories + .iter() + .next() + .expect("module-state staging memory is injected before the runtime") + .id(); + let codecs = runtime + .reference_codecs + .expect("linked module-state helpers require reference codecs"); + let imports = inject_record_imports(module, runtime.buf_type); + let table_markers = inject_table_dirty_markers(module, imports, &plan); + let (table_reconcile_guard, table_mutation_begin) = + inject_table_reconcile_guard(module, memory, runtime.buf_type, imports); + rewrite_table_mutations( + module, + &plan, + &table_markers, + table_reconcile_guard, + table_mutation_begin, + runtime.resume_table, + ); + let trackers = inject_segment_trackers(module, &plan); + rewrite_segment_drops(module, &trackers); + let bootstrap_done = + module + .globals + .add_local(ValType::I32, true, false, ConstExpr::Value(Value::I32(0))); + let save = emit_save_helper( + module, + memory, + runtime.buf_type, + codecs, + imports, + &plan, + &trackers, + ); + let restore = emit_restore_helper( + module, + memory, + runtime.buf_type, + codecs, + imports, + &plan, + bootstrap_done, + )?; + let finish_restore = emit_finish_restore_helper( + module, + memory, + runtime.buf_type, + codecs, + imports, + &plan, + &trackers, + bootstrap_done, + ); + let table_save = + emit_table_save_helper(module, memory, runtime.buf_type, codecs, imports, &plan); + let table_restore = + emit_table_restore_helper(module, memory, runtime.buf_type, codecs, imports, &plan); + let bootstrap = emit_bootstrap_helper(module, &plan, bootstrap_done)?; + let thread_bootstrap = emit_thread_bootstrap_helper(module, &plan, bootstrap_done)?; + export_helpers( + module, + bootstrap, + thread_bootstrap, + save, + restore, + finish_restore, + table_save, + table_restore, + ); + export_global_catalog(module, &plan.global_catalog); + export_table_catalog(module, &plan.table_catalog); + replace_imported_globals_section(module, &plan.imported_globals); + replace_imported_tables_section(module, &plan.imported_tables); + Ok(bootstrap) +} + +fn export_global_catalog(module: &mut Module, globals: &[(GlobalId, u32)]) { + for (global, owner) in globals { + let name = format!("{WPK_FORK_GLOBAL_CATALOG_EXPORT_PREFIX}{owner}"); + module.exports.add(&name, *global); + } +} + +fn export_table_catalog(module: &mut Module, tables: &[(TableId, u32)]) { + for (table, owner) in tables { + let name = format!("{WPK_FORK_TABLE_CATALOG_EXPORT_PREFIX}{owner}"); + module.exports.add(&name, *table); + } +} + +fn replace_imported_globals_section(module: &mut Module, globals: &[ImportedGlobalState]) { + loop { + let existing = module + .customs + .iter() + .find(|(_, section)| section.name() == WPK_FORK_IMPORTED_GLOBALS_SECTION) + .map(|(id, _)| id); + let Some(existing) = existing else { break }; + module.customs.delete(existing); + } + + let records: Vec<_> = globals + .iter() + .map(|global| { + let class = match global.ty { + ValType::Ref(ty) => Some(ReferenceCodecClass::of(module, ty)), + _ => None, + }; + let module_len = + u32::try_from(global.module.len()).expect("Wasm import module name fits u32"); + let name_len = u32::try_from(global.name.len()).expect("Wasm import name fits u32"); + let record_size = + u32::from(WPK_FORK_IMPORTED_GLOBALS_RECORD_HEADER_SIZE) + module_len + name_len; + ( + global, + global_type_code(global.ty, class), + module_len, + name_len, + record_size, + ) + }) + .collect(); + let capacity = usize::from(WPK_FORK_IMPORTED_GLOBALS_HEADER_SIZE) + + records + .iter() + .map(|record| usize::try_from(record.4).unwrap()) + .sum::(); + let mut data = Vec::with_capacity(capacity); + data.extend_from_slice(&WPK_FORK_IMPORTED_GLOBALS_MAGIC); + data.extend_from_slice(&WPK_FORK_IMPORTED_GLOBALS_VERSION.to_le_bytes()); + data.extend_from_slice(&WPK_FORK_IMPORTED_GLOBALS_HEADER_SIZE.to_le_bytes()); + data.extend_from_slice( + &u32::try_from(records.len()) + .expect("imported global count fits u32") + .to_le_bytes(), + ); + data.extend_from_slice(&0u32.to_le_bytes()); + for (global, type_code, module_len, name_len, record_size) in records { + data.extend_from_slice(&record_size.to_le_bytes()); + data.extend_from_slice(&global.owner.to_le_bytes()); + data.push(type_code); + data.push( + (if global.mutable { + WPK_FORK_IMPORTED_GLOBAL_FLAG_MUTABLE + } else { + 0 + }) | (if global.shared { + WPK_FORK_IMPORTED_GLOBAL_FLAG_SHARED + } else { + 0 + }), + ); + data.extend_from_slice(&0u16.to_le_bytes()); + data.extend_from_slice(&module_len.to_le_bytes()); + data.extend_from_slice(&name_len.to_le_bytes()); + data.extend_from_slice(&global.import_ordinal.to_le_bytes()); + data.extend_from_slice(global.module.as_bytes()); + data.extend_from_slice(global.name.as_bytes()); + } + debug_assert_eq!(data.len(), capacity); + module.customs.add(RawCustomSection { + name: WPK_FORK_IMPORTED_GLOBALS_SECTION.into(), + data, + }); +} + +fn replace_imported_tables_section(module: &mut Module, tables: &[ImportedTableState]) { + loop { + let existing = module + .customs + .iter() + .find(|(_, section)| section.name() == WPK_FORK_IMPORTED_TABLES_SECTION) + .map(|(id, _)| id); + let Some(existing) = existing else { break }; + module.customs.delete(existing); + } + + let records: Vec<_> = tables + .iter() + .map(|table| { + let type_code = global_type_code( + ValType::Ref(table.ty), + Some(ReferenceCodecClass::of(module, table.ty)), + ); + let module_len = + u32::try_from(table.module.len()).expect("Wasm import module name fits u32"); + let name_len = u32::try_from(table.name.len()).expect("Wasm import name fits u32"); + let record_size = + u32::from(WPK_FORK_IMPORTED_TABLES_RECORD_HEADER_SIZE) + module_len + name_len; + (table, type_code, module_len, name_len, record_size) + }) + .collect(); + let capacity = usize::from(WPK_FORK_IMPORTED_TABLES_HEADER_SIZE) + + records + .iter() + .map(|record| usize::try_from(record.4).unwrap()) + .sum::(); + let mut data = Vec::with_capacity(capacity); + data.extend_from_slice(&WPK_FORK_IMPORTED_TABLES_MAGIC); + data.extend_from_slice(&WPK_FORK_IMPORTED_TABLES_VERSION.to_le_bytes()); + data.extend_from_slice(&WPK_FORK_IMPORTED_TABLES_HEADER_SIZE.to_le_bytes()); + data.extend_from_slice( + &u32::try_from(records.len()) + .expect("imported table count fits u32") + .to_le_bytes(), + ); + data.extend_from_slice(&0u32.to_le_bytes()); + for (table, type_code, module_len, name_len, record_size) in records { + data.extend_from_slice(&record_size.to_le_bytes()); + data.extend_from_slice(&table.owner.to_le_bytes()); + data.push(type_code); + data.push(if table.table64 { + WPK_FORK_IMPORTED_TABLE_FLAG_TABLE64 + } else { + 0 + }); + data.extend_from_slice(&0u16.to_le_bytes()); + data.extend_from_slice(&module_len.to_le_bytes()); + data.extend_from_slice(&name_len.to_le_bytes()); + data.extend_from_slice(&table.import_ordinal.to_le_bytes()); + data.extend_from_slice(table.module.as_bytes()); + data.extend_from_slice(table.name.as_bytes()); + } + debug_assert_eq!(data.len(), capacity); + module.customs.add(RawCustomSection { + name: WPK_FORK_IMPORTED_TABLES_SECTION.into(), + data, + }); +} + +fn export_helpers( + module: &mut Module, + bootstrap: FunctionId, + thread_bootstrap: FunctionId, + save: FunctionId, + restore: FunctionId, + finish_restore: FunctionId, + table_save: FunctionId, + table_restore: FunctionId, +) { + module + .exports + .add(WPK_FORK_EXPORT_MODULE_BOOTSTRAP, bootstrap); + module + .exports + .add(WPK_FORK_EXPORT_MODULE_THREAD_BOOTSTRAP, thread_bootstrap); + module.exports.add(WPK_FORK_EXPORT_MODULE_STATE_SAVE, save); + module + .exports + .add(WPK_FORK_EXPORT_MODULE_STATE_RESTORE, restore); + module + .exports + .add(WPK_FORK_EXPORT_MODULE_STATE_FINISH_RESTORE, finish_restore); + module + .exports + .add(WPK_FORK_EXPORT_MODULE_TABLE_STATE_SAVE, table_save); + module + .exports + .add(WPK_FORK_EXPORT_MODULE_TABLE_STATE_RESTORE, table_restore); + module.funcs.get_mut(bootstrap).name = Some(WPK_FORK_EXPORT_MODULE_BOOTSTRAP.into()); + module.funcs.get_mut(thread_bootstrap).name = + Some(WPK_FORK_EXPORT_MODULE_THREAD_BOOTSTRAP.into()); + module.funcs.get_mut(save).name = Some(WPK_FORK_EXPORT_MODULE_STATE_SAVE.into()); + module.funcs.get_mut(restore).name = Some(WPK_FORK_EXPORT_MODULE_STATE_RESTORE.into()); + module.funcs.get_mut(finish_restore).name = + Some(WPK_FORK_EXPORT_MODULE_STATE_FINISH_RESTORE.into()); + module.funcs.get_mut(table_save).name = Some(WPK_FORK_EXPORT_MODULE_TABLE_STATE_SAVE.into()); + module.funcs.get_mut(table_restore).name = + Some(WPK_FORK_EXPORT_MODULE_TABLE_STATE_RESTORE.into()); +} + +fn inject_record_imports(module: &mut Module, ptr_ty: ValType) -> ModuleStateImports { + let reserve_ty = module.types.add( + &[ValType::I32, ValType::I32, ValType::I32, ptr_ty], + &[ptr_ty], + ); + let commit_ty = module.types.add(&[ptr_ty], &[]); + let find_ty = module.types.add( + &[ValType::I32, ValType::I32, ValType::I32, ValType::I32], + &[ptr_ty], + ); + let table_dirty_mark_ty = module + .types + .add(&[ValType::I32, ValType::I64, ValType::I64], &[]); + let table_dirty_count_ty = module.types.add(&[ValType::I32], &[ValType::I32]); + let table_dirty_page_ty = module + .types + .add(&[ValType::I32, ValType::I32], &[ValType::I64]); + let table_state_owned_ty = module.types.add(&[ValType::I32], &[ValType::I32]); + let table_mutation_begin_ty = module.types.add(&[], &[ValType::I64]); + let table_mutation_commit_ty = module + .types + .add(&[ValType::I32, ValType::I64, ValType::I64], &[]); + let table_mutation_abort_ty = module.types.add(&[], &[]); + let table_reconcile_ty = module.types.add(&[], &[ValType::I64]); + let (reserve, _) = module.add_import_func( + WPK_FORK_MODULE_STATE_IMPORT_MODULE, + WPK_FORK_MODULE_STATE_IMPORT_RECORD_RESERVE, + reserve_ty, + ); + let (commit, _) = module.add_import_func( + WPK_FORK_MODULE_STATE_IMPORT_MODULE, + WPK_FORK_MODULE_STATE_IMPORT_RECORD_COMMIT, + commit_ty, + ); + let (find, _) = module.add_import_func( + WPK_FORK_MODULE_STATE_IMPORT_MODULE, + WPK_FORK_MODULE_STATE_IMPORT_RECORD_FIND, + find_ty, + ); + let (table_dirty_count, _) = module.add_import_func( + WPK_FORK_MODULE_STATE_IMPORT_MODULE, + WPK_FORK_MODULE_STATE_IMPORT_TABLE_DIRTY_COUNT, + table_dirty_count_ty, + ); + let (table_dirty_mark, _) = module.add_import_func( + WPK_FORK_MODULE_STATE_IMPORT_MODULE, + WPK_FORK_MODULE_STATE_IMPORT_TABLE_DIRTY_MARK, + table_dirty_mark_ty, + ); + let (table_dirty_page, _) = module.add_import_func( + WPK_FORK_MODULE_STATE_IMPORT_MODULE, + WPK_FORK_MODULE_STATE_IMPORT_TABLE_DIRTY_PAGE, + table_dirty_page_ty, + ); + let (table_state_owned, _) = module.add_import_func( + WPK_FORK_MODULE_STATE_IMPORT_MODULE, + WPK_FORK_MODULE_STATE_IMPORT_TABLE_STATE_OWNED, + table_state_owned_ty, + ); + let (table_mutation_begin, _) = module.add_import_func( + WPK_FORK_MODULE_STATE_IMPORT_MODULE, + WPK_FORK_MODULE_STATE_IMPORT_TABLE_MUTATION_BEGIN, + table_mutation_begin_ty, + ); + let (table_mutation_commit, _) = module.add_import_func( + WPK_FORK_MODULE_STATE_IMPORT_MODULE, + WPK_FORK_MODULE_STATE_IMPORT_TABLE_MUTATION_COMMIT, + table_mutation_commit_ty, + ); + let (table_mutation_abort, _) = module.add_import_func( + WPK_FORK_MODULE_STATE_IMPORT_MODULE, + WPK_FORK_MODULE_STATE_IMPORT_TABLE_MUTATION_ABORT, + table_mutation_abort_ty, + ); + let (table_reconcile, _) = module.add_import_func( + WPK_FORK_MODULE_STATE_IMPORT_MODULE, + WPK_FORK_MODULE_STATE_IMPORT_TABLE_RECONCILE, + table_reconcile_ty, + ); + let (table_generation_addr, _) = module.add_import_global( + WPK_FORK_MODULE_STATE_IMPORT_MODULE, + WPK_FORK_MODULE_STATE_IMPORT_TABLE_GENERATION_ADDR, + ValType::I64, + false, + false, + ); + ModuleStateImports { + reserve, + commit, + find, + table_dirty_mark, + table_dirty_count, + table_dirty_page, + table_state_owned, + table_mutation_begin, + table_mutation_commit, + table_mutation_abort, + table_reconcile, + table_generation_addr, + } +} + +#[derive(Debug, Clone, Copy)] +struct TableDirtyMarker { + mark: FunctionId, + grow: FunctionId, +} + +fn inject_table_dirty_markers( + module: &mut Module, + imports: ModuleStateImports, + plan: &ModuleStatePlan, +) -> HashMap { + let mut markers = HashMap::new(); + for table in plan.tables.iter().filter(|table| table.synchronized) { + let last_page = + module + .globals + .add_local(ValType::I64, true, false, ConstExpr::Value(Value::I64(-1))); + let start = module.locals.add(ValType::I64); + let count = module.locals.add(ValType::I64); + let first_page = module.locals.add(ValType::I64); + let last_page_local = module.locals.add(ValType::I64); + let mut mark_builder = + FunctionBuilder::new(&mut module.types, &[ValType::I64, ValType::I64], &[]); + { + let mut body = mark_builder.func_body(); + body.local_get(count) + .i64_const(0) + .binop(BinaryOp::I64Ne) + .if_else( + None, + |nonempty| { + nonempty + .local_get(start) + .i64_const(i64::from(TABLE_PAGE_SHIFT)) + .binop(BinaryOp::I64ShrU) + .local_set(first_page) + .local_get(start) + .local_get(count) + .binop(BinaryOp::I64Add) + .i64_const(1) + .binop(BinaryOp::I64Sub) + .i64_const(i64::from(TABLE_PAGE_SHIFT)) + .binop(BinaryOp::I64ShrU) + .local_set(last_page_local) + .local_get(first_page) + .local_get(last_page_local) + .binop(BinaryOp::I64Eq) + .global_get(last_page) + .local_get(last_page_local) + .binop(BinaryOp::I64Eq) + .binop(BinaryOp::I32And) + .unop(UnaryOp::I32Eqz) + .if_else( + None, + |uncached| { + uncached + .i32_const(table.owner as i32) + .local_get(first_page) + .local_get(last_page_local) + .local_get(first_page) + .binop(BinaryOp::I64Sub) + .i64_const(1) + .binop(BinaryOp::I64Add) + .call(imports.table_dirty_mark) + .local_get(last_page_local) + .global_set(last_page); + }, + |_| {}, + ) + // WHY: dirty-page caching is sufficient for one + // later fork capture, but another pthread owns a + // different Table object and may consume this + // mutation immediately. Commit while the process + // writer lock is still held. + .i32_const(table.owner as i32) + .local_get(start) + .local_get(count) + .call(imports.table_mutation_commit); + }, + |empty| { + // A zero-length fill/copy/init/grow has no state to + // publish, but its pre-op reconciliation still owns + // one writer-lock depth that must be balanced. + empty.call(imports.table_mutation_abort); + }, + ); + } + let mark = mark_builder.finish(vec![start, count], &mut module.funcs); + + let old = module.locals.add(ValType::I64); + let delta = module.locals.add(ValType::I64); + let failed = if table.table64 { + u64::MAX + } else { + u64::from(u32::MAX) + }; + let mut grow_builder = + FunctionBuilder::new(&mut module.types, &[ValType::I64, ValType::I64], &[]); + { + let mut body = grow_builder.func_body(); + body.local_get(old) + .i64_const(failed as i64) + .binop(BinaryOp::I64Ne) + .if_else( + None, + |succeeded| { + succeeded.local_get(old).local_get(delta).call(mark); + }, + |failed| { + // table.grow reports failure instead of trapping. End + // the mutation transaction without publishing state. + failed.call(imports.table_mutation_abort); + }, + ); + } + let grow = grow_builder.finish(vec![old, delta], &mut module.funcs); + markers.insert(table.id, TableDirtyMarker { mark, grow }); + } + markers +} + +fn inject_table_reconcile_guard( + module: &mut Module, + memory: MemoryId, + ptr_ty: ValType, + imports: ModuleStateImports, +) -> (FunctionId, FunctionId) { + let last_generation = + module + .globals + .add_local(ValType::I64, true, false, ConstExpr::Value(Value::I64(0))); + let memory_is_shared = module.memories.get(memory).shared; + let mut builder = FunctionBuilder::new(&mut module.types, &[], &[]); + { + let mut body = builder.func_body(); + body.global_get(imports.table_generation_addr); + if ptr_ty == ValType::I32 { + body.unop(UnaryOp::I32WrapI64); + } + body.load( + memory, + LoadKind::I64 { + // Atomic loads are required for cross-Worker publication. + // Keep standalone/unshared test modules valid: there is no + // peer Agent in that shape, so an ordinary aligned load is + // already race-free. + atomic: memory_is_shared, + }, + MemArg { + align: 8, + offset: 0, + }, + ) + .global_get(last_generation) + .binop(BinaryOp::I64Ne) + .if_else( + None, + |changed| { + // The host returns the exact generation it applied. Do + // not reread the shared fence here: a writer may publish a + // newer generation after reconcile returns, and caching + // that unapplied value would skip the next guard. + changed + .call(imports.table_reconcile) + .global_set(last_generation); + }, + |_| {}, + ); + } + let guard = builder.finish(Vec::new(), &mut module.funcs); + module.funcs.get_mut(guard).name = Some("__wpk_fork_table_generation_guard".into()); + let mut mutation_builder = FunctionBuilder::new(&mut module.types, &[], &[]); + mutation_builder + .func_body() + .call(imports.table_mutation_begin) + .global_set(last_generation); + let mutation_begin = mutation_builder.finish(Vec::new(), &mut module.funcs); + module.funcs.get_mut(mutation_begin).name = Some("__wpk_fork_table_mutation_begin".into()); + (guard, mutation_begin) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +enum TableOperation { + Set(TableId), + Fill(TableId), + Copy { src: TableId, dst: TableId }, + Init { table: TableId, elem: ElementId }, + Grow(TableId), +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +enum TableConsumer { + Get(TableId), + Size(TableId), + CallIndirect { table: TableId, ty: TypeId }, + ReturnCallIndirect { table: TableId, ty: TypeId }, +} + +fn rewrite_table_mutations( + module: &mut Module, + plan: &ModuleStatePlan, + markers: &HashMap, + reconcile_guard: FunctionId, + mutation_begin: FunctionId, + resume_table: Option, +) { + let table_states: HashMap<_, _> = plan.tables.iter().map(|table| (table.id, *table)).collect(); + let synchronized_tables: HashSet<_> = plan + .tables + .iter() + .filter(|table| table.synchronized) + .map(|table| table.id) + .collect(); + let mut operations = HashSet::new(); + let mut consumers = HashSet::new(); + for function in &plan.original_functions { + let FunctionKind::Local(local) = &module.funcs.get(*function).kind else { + continue; + }; + collect_table_operations( + local, + local.entry_block(), + resume_table, + &synchronized_tables, + &mut operations, + &mut consumers, + ); + } + let helpers: HashMap<_, _> = operations + .into_iter() + .map(|operation| { + let helper = emit_table_operation_helper( + module, + &table_states, + markers, + mutation_begin, + operation, + ); + (operation, helper) + }) + .collect(); + let consumer_helpers: HashMap<_, _> = consumers + .into_iter() + .map(|consumer| { + let helper = + emit_table_consumer_helper(module, &table_states, reconcile_guard, consumer); + (consumer, helper) + }) + .collect(); + for function in &plan.original_functions { + let FunctionKind::Local(local) = &mut module.funcs.get_mut(*function).kind else { + continue; + }; + rewrite_table_mutation_seq( + local, + local.entry_block(), + &helpers, + &consumer_helpers, + resume_table, + false, + ); + } + + // Fork call-transport helpers are emitted after module_state::plan, so + // they are not in original_functions. Guard them at entry, before their + // parameters are reloaded for the one indirect call, and leave that call + // in place to avoid reintroducing an operand-stack scratch local. + let transport_helpers: Vec<_> = module + .funcs + .iter() + .filter(|function| { + function + .name + .as_deref() + .is_some_and(|name| name.starts_with("__wpk_fork_unwind_transport_indirect_")) + && matches!(function.kind, FunctionKind::Local(_)) + }) + .map(|function| function.id()) + .collect(); + for function in transport_helpers { + let FunctionKind::Local(local) = &module.funcs.get(function).kind else { + unreachable!() + }; + let mut transport_operations = HashSet::new(); + let mut transport_consumers = HashSet::new(); + collect_table_operations( + local, + local.entry_block(), + resume_table, + &synchronized_tables, + &mut transport_operations, + &mut transport_consumers, + ); + let guard_at_entry = !transport_consumers.is_empty(); + let FunctionKind::Local(local) = &mut module.funcs.get_mut(function).kind else { + unreachable!() + }; + let entry = local.entry_block(); + if guard_at_entry { + let loc = local + .block(entry) + .instrs + .first() + .map(|(_, loc)| *loc) + .unwrap_or_default(); + local.block_mut(entry).instrs.insert( + 0, + ( + Call { + func: reconcile_guard, + } + .into(), + loc, + ), + ); + } + rewrite_table_mutation_seq( + local, + entry, + &helpers, + &consumer_helpers, + resume_table, + guard_at_entry, + ); + } +} + +fn collect_table_operations( + local: &LocalFunction, + seq: InstrSeqId, + resume_table: Option, + synchronized_tables: &HashSet, + operations: &mut HashSet, + consumers: &mut HashSet, +) { + for (instr, _) in &local.block(seq).instrs { + for child in nested_seqs(instr) { + collect_table_operations( + local, + child, + resume_table, + synchronized_tables, + operations, + consumers, + ); + } + let operation = match instr { + Instr::TableSet(set) => Some(TableOperation::Set(set.table)), + Instr::TableFill(fill) => Some(TableOperation::Fill(fill.table)), + Instr::TableCopy(copy) => Some(TableOperation::Copy { + src: copy.src, + dst: copy.dst, + }), + Instr::TableInit(init) => Some(TableOperation::Init { + table: init.table, + elem: init.elem, + }), + Instr::TableGrow(grow) => Some(TableOperation::Grow(grow.table)), + _ => None, + }; + if let Some(operation) = operation { + operations.insert(operation); + } + let consumer = match instr { + Instr::TableGet(get) + if Some(get.table) != resume_table && synchronized_tables.contains(&get.table) => + { + Some(TableConsumer::Get(get.table)) + } + Instr::TableSize(size) + if Some(size.table) != resume_table + && synchronized_tables.contains(&size.table) => + { + Some(TableConsumer::Size(size.table)) + } + Instr::CallIndirect(call) + if Some(call.table) != resume_table + && synchronized_tables.contains(&call.table) => + { + Some(TableConsumer::CallIndirect { + table: call.table, + ty: call.ty, + }) + } + Instr::ReturnCallIndirect(call) + if Some(call.table) != resume_table + && synchronized_tables.contains(&call.table) => + { + Some(TableConsumer::ReturnCallIndirect { + table: call.table, + ty: call.ty, + }) + } + _ => None, + }; + if let Some(consumer) = consumer { + consumers.insert(consumer); + } + } +} + +fn collect_source_mutated_tables(module: &Module, functions: &[FunctionId]) -> HashSet { + fn visit(local: &LocalFunction, seq: InstrSeqId, mutated: &mut HashSet) { + for (instr, _) in &local.block(seq).instrs { + for child in nested_seqs(instr) { + visit(local, child, mutated); + } + match instr { + Instr::TableSet(set) => { + mutated.insert(set.table); + } + Instr::TableFill(fill) => { + mutated.insert(fill.table); + } + Instr::TableCopy(copy) => { + mutated.insert(copy.dst); + } + Instr::TableInit(init) => { + mutated.insert(init.table); + } + Instr::TableGrow(grow) => { + mutated.insert(grow.table); + } + _ => {} + } + } + } + + let mut mutated = HashSet::new(); + for function in functions { + let FunctionKind::Local(local) = &module.funcs.get(*function).kind else { + continue; + }; + visit(local, local.entry_block(), &mut mutated); + } + mutated +} + +fn emit_table_operation_helper( + module: &mut Module, + tables: &HashMap, + markers: &HashMap, + mutation_begin: FunctionId, + operation: TableOperation, +) -> FunctionId { + let (function, name) = match operation { + TableOperation::Set(table_id) => { + let table = tables[&table_id]; + let marker = markers[&table_id]; + let index_ty = table_index_type(table); + let index = module.locals.add(index_ty); + let reference = module.locals.add(ValType::Ref(table.ty)); + let mut builder = + FunctionBuilder::new(&mut module.types, &[index_ty, ValType::Ref(table.ty)], &[]); + { + let mut body = builder.func_body(); + body.call(mutation_begin) + .local_get(index) + .local_get(reference) + .instr(TableSet { table: table_id }); + builder_index_as_i64(&mut body, index_ty, index); + body.i64_const(1).call(marker.mark); + } + ( + builder.finish(vec![index, reference], &mut module.funcs), + format!("__wpk_fork_table_set_{}", table.owner), + ) + } + TableOperation::Fill(table_id) => { + let table = tables[&table_id]; + let marker = markers[&table_id]; + let index_ty = table_index_type(table); + let dst = module.locals.add(index_ty); + let reference = module.locals.add(ValType::Ref(table.ty)); + let count = module.locals.add(index_ty); + let mut builder = FunctionBuilder::new( + &mut module.types, + &[index_ty, ValType::Ref(table.ty), index_ty], + &[], + ); + { + let mut body = builder.func_body(); + body.call(mutation_begin) + .local_get(dst) + .local_get(reference) + .local_get(count) + .instr(TableFill { table: table_id }); + builder_index_as_i64(&mut body, index_ty, dst); + builder_index_as_i64(&mut body, index_ty, count); + body.call(marker.mark); + } + ( + builder.finish(vec![dst, reference, count], &mut module.funcs), + format!("__wpk_fork_table_fill_{}", table.owner), + ) + } + TableOperation::Copy { src, dst } => { + let src_table = tables[&src]; + let dst_table = tables[&dst]; + let marker = markers[&dst]; + let src_ty = table_index_type(src_table); + let dst_ty = table_index_type(dst_table); + let count_ty = if src_ty == ValType::I32 || dst_ty == ValType::I32 { + ValType::I32 + } else { + ValType::I64 + }; + let dst_index = module.locals.add(dst_ty); + let src_index = module.locals.add(src_ty); + let count = module.locals.add(count_ty); + let mut builder = + FunctionBuilder::new(&mut module.types, &[dst_ty, src_ty, count_ty], &[]); + { + let mut body = builder.func_body(); + body.call(mutation_begin) + .local_get(dst_index) + .local_get(src_index) + .local_get(count) + .instr(TableCopy { src, dst }); + builder_index_as_i64(&mut body, dst_ty, dst_index); + builder_index_as_i64(&mut body, count_ty, count); + body.call(marker.mark); + } + ( + builder.finish(vec![dst_index, src_index, count], &mut module.funcs), + format!( + "__wpk_fork_table_copy_{}_from_{}", + dst_table.owner, src_table.owner + ), + ) + } + TableOperation::Init { + table: table_id, + elem, + } => { + let table = tables[&table_id]; + let marker = markers[&table_id]; + let index_ty = table_index_type(table); + let dst = module.locals.add(index_ty); + let src = module.locals.add(ValType::I32); + let count = module.locals.add(ValType::I32); + let mut builder = FunctionBuilder::new( + &mut module.types, + &[index_ty, ValType::I32, ValType::I32], + &[], + ); + { + let mut body = builder.func_body(); + body.call(mutation_begin) + .local_get(dst) + .local_get(src) + .local_get(count) + .instr(TableInit { + table: table_id, + elem, + }); + builder_index_as_i64(&mut body, index_ty, dst); + builder_index_as_i64(&mut body, ValType::I32, count); + body.call(marker.mark); + } + ( + builder.finish(vec![dst, src, count], &mut module.funcs), + format!("__wpk_fork_table_init_{}_{}", table.owner, elem.index()), + ) + } + TableOperation::Grow(table_id) => { + let table = tables[&table_id]; + let marker = markers[&table_id]; + let index_ty = table_index_type(table); + let reference = module.locals.add(ValType::Ref(table.ty)); + let delta = module.locals.add(index_ty); + let result = module.locals.add(index_ty); + let mut builder = FunctionBuilder::new( + &mut module.types, + &[ValType::Ref(table.ty), index_ty], + &[index_ty], + ); + { + let mut body = builder.func_body(); + body.call(mutation_begin) + .local_get(reference) + .local_get(delta) + .instr(TableGrow { table: table_id }) + .local_set(result); + builder_index_as_i64(&mut body, index_ty, result); + builder_index_as_i64(&mut body, index_ty, delta); + body.call(marker.grow).local_get(result); + } + ( + builder.finish(vec![reference, delta], &mut module.funcs), + format!("__wpk_fork_table_grow_{}", table.owner), + ) + } + }; + module.funcs.get_mut(function).name = Some(name); + function +} + +fn emit_table_consumer_helper( + module: &mut Module, + tables: &HashMap, + reconcile_guard: FunctionId, + consumer: TableConsumer, +) -> FunctionId { + let (function, name) = match consumer { + TableConsumer::Get(table_id) => { + let table = tables[&table_id]; + let index_ty = table_index_type(table); + let index = module.locals.add(index_ty); + let mut builder = + FunctionBuilder::new(&mut module.types, &[index_ty], &[ValType::Ref(table.ty)]); + builder + .func_body() + .call(reconcile_guard) + .local_get(index) + .instr(TableGet { table: table_id }); + ( + builder.finish(vec![index], &mut module.funcs), + format!("__wpk_fork_table_get_{}", table.owner), + ) + } + TableConsumer::Size(table_id) => { + let table = tables[&table_id]; + let index_ty = table_index_type(table); + let mut builder = FunctionBuilder::new(&mut module.types, &[], &[index_ty]); + builder + .func_body() + .call(reconcile_guard) + .instr(TableSize { table: table_id }); + ( + builder.finish(Vec::new(), &mut module.funcs), + format!("__wpk_fork_table_size_{}", table.owner), + ) + } + TableConsumer::CallIndirect { table, ty } => { + let table_state = tables[&table]; + let signature = module.types.get(ty); + let params = signature.params().to_vec(); + let results = signature.results().to_vec(); + let mut args: Vec<_> = params + .iter() + .map(|param| module.locals.add(*param)) + .collect(); + let index_ty = table_index_type(table_state); + let index = module.locals.add(index_ty); + let mut helper_params = params; + helper_params.push(index_ty); + let mut builder = FunctionBuilder::new(&mut module.types, &helper_params, &results); + { + let mut body = builder.func_body(); + body.call(reconcile_guard); + for arg in &args { + body.local_get(*arg); + } + body.local_get(index).instr(CallIndirect { table, ty }); + } + args.push(index); + ( + builder.finish(args, &mut module.funcs), + format!( + "__wpk_fork_table_call_indirect_{}_{}", + table_state.owner, + ty.index() + ), + ) + } + TableConsumer::ReturnCallIndirect { table, ty } => { + let table_state = tables[&table]; + let signature = module.types.get(ty); + let params = signature.params().to_vec(); + let results = signature.results().to_vec(); + let mut args: Vec<_> = params + .iter() + .map(|param| module.locals.add(*param)) + .collect(); + let index_ty = table_index_type(table_state); + let index = module.locals.add(index_ty); + let mut helper_params = params; + helper_params.push(index_ty); + let mut builder = FunctionBuilder::new(&mut module.types, &helper_params, &results); + { + let mut body = builder.func_body(); + body.call(reconcile_guard); + for arg in &args { + body.local_get(*arg); + } + body.local_get(index) + .instr(ReturnCallIndirect { table, ty }); + } + args.push(index); + ( + builder.finish(args, &mut module.funcs), + format!( + "__wpk_fork_table_return_call_indirect_{}_{}", + table_state.owner, + ty.index() + ), + ) + } + }; + module.funcs.get_mut(function).name = Some(name); + function +} + +fn builder_index_as_i64(body: &mut InstrSeqBuilder, ty: ValType, local: LocalId) { + body.local_get(local); + if ty == ValType::I32 { + body.unop(UnaryOp::I64ExtendUI32); + } +} + +fn rewrite_table_mutation_seq( + local: &mut LocalFunction, + seq: InstrSeqId, + helpers: &HashMap, + consumer_helpers: &HashMap, + resume_table: Option, + consumers_guarded_at_entry: bool, +) { + let old = std::mem::take(&mut local.block_mut(seq).instrs); + let mut rewritten = Vec::with_capacity(old.len()); + for (instr, loc) in old { + for child in nested_seqs(&instr) { + rewrite_table_mutation_seq( + local, + child, + helpers, + consumer_helpers, + resume_table, + consumers_guarded_at_entry, + ); + } + match instr { + Instr::TableSet(set) => { + push( + &mut rewritten, + Call { + func: helpers[&TableOperation::Set(set.table)], + }, + loc, + ); + } + Instr::TableFill(fill) => { + push( + &mut rewritten, + Call { + func: helpers[&TableOperation::Fill(fill.table)], + }, + loc, + ); + } + Instr::TableCopy(copy) => { + push( + &mut rewritten, + Call { + func: helpers[&TableOperation::Copy { + src: copy.src, + dst: copy.dst, + }], + }, + loc, + ); + } + Instr::TableInit(init) => { + push( + &mut rewritten, + Call { + func: helpers[&TableOperation::Init { + table: init.table, + elem: init.elem, + }], + }, + loc, + ); + } + Instr::TableGrow(grow) => { + push( + &mut rewritten, + Call { + func: helpers[&TableOperation::Grow(grow.table)], + }, + loc, + ); + } + other @ Instr::CallIndirect(CallIndirect { table, .. }) + if Some(table) == resume_table || consumers_guarded_at_entry => + { + // WHY: this private dispatch table is host-built, immutable, + // and already excluded from module-state ownership. Keeping + // the guard out of resume_peek -> call_indirect also preserves + // the operand-stack shape that Binaryen can lower without a + // per-call-site scratch local. + rewritten.push((other, loc)); + } + other @ Instr::ReturnCallIndirect(ReturnCallIndirect { table, .. }) + if Some(table) == resume_table || consumers_guarded_at_entry => + { + rewritten.push((other, loc)); + } + other @ Instr::TableGet(TableGet { table }) + if Some(table) == resume_table || consumers_guarded_at_entry => + { + rewritten.push((other, loc)); + } + other @ Instr::TableSize(TableSize { table }) + if Some(table) == resume_table || consumers_guarded_at_entry => + { + rewritten.push((other, loc)); + } + Instr::CallIndirect(call) => { + let consumer = TableConsumer::CallIndirect { + table: call.table, + ty: call.ty, + }; + if let Some(&func) = consumer_helpers.get(&consumer) { + push(&mut rewritten, Call { func }, loc); + } else { + rewritten.push((Instr::CallIndirect(call), loc)); + } + } + Instr::ReturnCallIndirect(call) => { + let consumer = TableConsumer::ReturnCallIndirect { + table: call.table, + ty: call.ty, + }; + if let Some(&func) = consumer_helpers.get(&consumer) { + push(&mut rewritten, ReturnCall { func }, loc); + } else { + rewritten.push((Instr::ReturnCallIndirect(call), loc)); + } + } + Instr::TableGet(get) => { + if let Some(&func) = consumer_helpers.get(&TableConsumer::Get(get.table)) { + push(&mut rewritten, Call { func }, loc); + } else { + rewritten.push((Instr::TableGet(get), loc)); + } + } + Instr::TableSize(size) => { + if let Some(&func) = consumer_helpers.get(&TableConsumer::Size(size.table)) { + push(&mut rewritten, Call { func }, loc); + } else { + rewritten.push((Instr::TableSize(size), loc)); + } + } + other => rewritten.push((other, loc)), + } + } + local.block_mut(seq).instrs = rewritten; +} + +fn push>(out: &mut Vec<(Instr, InstrLocId)>, instr: T, loc: InstrLocId) { + out.push((instr.into(), loc)); +} + +fn inject_segment_trackers(module: &mut Module, plan: &ModuleStatePlan) -> Trackers { + fn tracker(module: &mut Module, segments: &[(T, bool)]) -> SegmentTracker { + let mut globals = Vec::with_capacity(segments.len().div_ceil(32)); + for chunk in segments.chunks(32) { + let mut initial = 0u32; + for (bit, (_, dropped)) in chunk.iter().enumerate() { + if *dropped { + initial |= 1 << bit; + } + } + globals.push(module.globals.add_local( + ValType::I32, + true, + false, + ConstExpr::Value(Value::I32(initial as i32)), + )); + } + SegmentTracker { + segments: segments.iter().map(|(id, _)| *id).collect(), + globals, + } + } + + Trackers { + elements: tracker(module, &plan.elements), + data: tracker(module, &plan.data), + } +} + +fn rewrite_segment_drops(module: &mut Module, trackers: &Trackers) { + let element_bits: HashMap<_, _> = trackers + .elements + .segments + .iter() + .enumerate() + .map(|(index, id)| (*id, index)) + .collect(); + let data_bits: HashMap<_, _> = trackers + .data + .segments + .iter() + .enumerate() + .map(|(index, id)| (*id, index)) + .collect(); + let funcs: Vec<_> = module + .funcs + .iter() + .filter_map(|func| matches!(func.kind, FunctionKind::Local(_)).then_some(func.id())) + .collect(); + for func in funcs { + let FunctionKind::Local(local) = &mut module.funcs.get_mut(func).kind else { + unreachable!() + }; + rewrite_drop_seq( + local, + local.entry_block(), + &element_bits, + &trackers.elements.globals, + &data_bits, + &trackers.data.globals, + ); + } +} + +fn rewrite_drop_seq( + local: &mut LocalFunction, + seq: InstrSeqId, + element_bits: &HashMap, + element_globals: &[GlobalId], + data_bits: &HashMap, + data_globals: &[GlobalId], +) { + let old = std::mem::take(&mut local.block_mut(seq).instrs); + let mut rewritten = Vec::with_capacity(old.len()); + for (instr, loc) in old { + for child in nested_seqs(&instr) { + rewrite_drop_seq( + local, + child, + element_bits, + element_globals, + data_bits, + data_globals, + ); + } + let tracked = match &instr { + Instr::ElemDrop(ElemDrop { elem }) => element_bits + .get(elem) + .map(|index| (*index, element_globals)), + Instr::DataDrop(DataDrop { data }) => { + data_bits.get(data).map(|index| (*index, data_globals)) + } + _ => None, + }; + rewritten.push((instr, loc)); + if let Some((index, globals)) = tracked { + let word = globals[index / 32]; + rewritten.push(( + Instr::GlobalGet(walrus::ir::GlobalGet { global: word }), + loc, + )); + rewritten.push(( + Instr::Const(walrus::ir::Const { + value: Value::I32((1u32 << (index % 32)) as i32), + }), + loc, + )); + rewritten.push(( + Instr::Binop(walrus::ir::Binop { + op: BinaryOp::I32Or, + }), + loc, + )); + rewritten.push(( + Instr::GlobalSet(walrus::ir::GlobalSet { global: word }), + loc, + )); + } + } + local.block_mut(seq).instrs = rewritten; +} + +fn nested_seqs(instr: &Instr) -> Vec { + match instr { + Instr::Block(Block { seq }) => vec![*seq], + Instr::Loop(Loop { seq }) => vec![*seq], + Instr::IfElse(IfElse { + consequent, + alternative, + }) => vec![*consequent, *alternative], + Instr::TryTable(TryTable { seq, .. }) => vec![*seq], + Instr::Try(try_) => { + let mut ids = vec![try_.seq]; + for catch in &try_.catches { + match catch { + LegacyCatch::Catch { handler, .. } | LegacyCatch::CatchAll { handler } => { + ids.push(*handler) + } + LegacyCatch::Delegate { .. } => {} + } + } + ids + } + _ => Vec::new(), + } +} + +#[derive(Debug, Clone, Copy)] +struct TableLocals { + payload: LocalId, + len: LocalId, + current_len: LocalId, + page_count: LocalId, + record_page_count: LocalId, + page_ordinal: LocalId, + page_index: LocalId, + page_start_wide: LocalId, + page_start: LocalId, + count: LocalId, + index: LocalId, +} + +fn allocate_table_locals(module: &mut Module, ptr_ty: ValType, table64: bool) -> TableLocals { + let index_ty = if table64 { ValType::I64 } else { ValType::I32 }; + TableLocals { + payload: module.locals.add(ptr_ty), + len: module.locals.add(index_ty), + current_len: module.locals.add(index_ty), + page_count: module.locals.add(ValType::I32), + record_page_count: module.locals.add(ValType::I32), + page_ordinal: module.locals.add(ValType::I32), + page_index: module.locals.add(ValType::I64), + page_start_wide: module.locals.add(ValType::I64), + page_start: module.locals.add(index_ty), + count: module.locals.add(index_ty), + index: module.locals.add(index_ty), + } +} + +fn emit_save_helper( + module: &mut Module, + memory: MemoryId, + ptr_ty: ValType, + codecs: ReferenceCodecs, + imports: ModuleStateImports, + plan: &ModuleStatePlan, + trackers: &Trackers, +) -> FunctionId { + let global_classes: Vec<_> = plan + .globals + .iter() + .map(|global| match global.ty { + ValType::Ref(ty) => Some(ReferenceCodecClass::of(module, ty)), + _ => None, + }) + .collect(); + let synchronized_tables: Vec<_> = plan + .tables + .iter() + .filter(|table| table.synchronized) + .copied() + .collect(); + let table_classes: Vec<_> = synchronized_tables + .iter() + .map(|table| ReferenceCodecClass::of(module, table.ty)) + .collect(); + let table_locals: Vec<_> = synchronized_tables + .iter() + .map(|table| allocate_table_locals(module, ptr_ty, table.table64)) + .collect(); + let payload = module.locals.add(ptr_ty); + let activation = module.locals.add(ValType::I32); + let mut builder = FunctionBuilder::new(&mut module.types, &[ValType::I32], &[]); + { + let mut body = builder.func_body(); + for (global, class) in plan.globals.iter().zip(global_classes) { + let value_size = class + .map(|_| GLOBAL_RECIPE_PAYLOAD_SIZE) + .unwrap_or_else(|| scalar_size(global.ty)); + let payload_size = u32::from(WPK_FORK_MODULE_STATE_GLOBAL_HEADER_SIZE) + value_size; + reserve_static_record( + &mut body, + imports, + ptr_ty, + activation, + WPK_FORK_MODULE_STATE_RECORD_KIND_MUTABLE_GLOBAL, + global.owner, + payload_size, + payload, + ); + body.local_get(payload) + .i32_const(i32::from(global_type_code(global.ty, class))) + .store( + memory, + StoreKind::I32_8 { atomic: false }, + MemArg { + align: 1, + offset: 0, + }, + ) + .local_get(payload) + .i32_const(value_size as i32) + .store( + memory, + StoreKind::I32_8 { atomic: false }, + MemArg { + align: 1, + offset: 1, + }, + ) + .local_get(payload) + .i32_const(0) + .store( + memory, + StoreKind::I32_16 { atomic: false }, + MemArg { + align: 2, + offset: 2, + }, + ) + .local_get(payload) + .i32_const(0) + .store( + memory, + StoreKind::I32 { atomic: false }, + MemArg { + align: 4, + offset: 4, + }, + ); + body.local_get(payload).global_get(global.id); + match class { + Some(class) => { + body.call(class.encoder(codecs)).store( + memory, + StoreKind::I32 { atomic: false }, + MemArg { + align: 4, + offset: u64::from(WPK_FORK_MODULE_STATE_GLOBAL_HEADER_SIZE), + }, + ); + } + None => { + body.store( + memory, + scalar_store_kind(global.ty), + MemArg { + align: scalar_align(global.ty), + offset: u64::from(WPK_FORK_MODULE_STATE_GLOBAL_HEADER_SIZE), + }, + ); + } + } + body.local_get(payload).call(imports.commit); + } + + emit_save_segments( + &mut body, + memory, + ptr_ty, + imports, + activation, + payload, + WPK_FORK_MODULE_STATE_RECORD_KIND_ELEMENT_SEGMENTS, + u32::from(WPK_FORK_MODULE_STATE_ELEMENT_SEGMENT_HEADER_SIZE), + &trackers.elements, + ); + emit_save_segments( + &mut body, + memory, + ptr_ty, + imports, + activation, + payload, + WPK_FORK_MODULE_STATE_RECORD_KIND_DATA_SEGMENTS, + u32::from(WPK_FORK_MODULE_STATE_DATA_SEGMENT_HEADER_SIZE), + &trackers.data, + ); + + for ((table, class), locals) in synchronized_tables + .iter() + .zip(table_classes) + .zip(table_locals) + { + body.i32_const(table.owner as i32) + .call(imports.table_state_owned) + .if_else( + None, + |owned| { + // WHY: imported aliases name one physical Table. Only + // its canonical activation writes sparse state, while + // every alias still contributes mutation marks to the + // shared journal. + emit_save_table( + owned, memory, ptr_ty, codecs, imports, activation, *table, class, + locals, + ); + }, + |_| {}, + ); + } + } + builder.finish(vec![activation], &mut module.funcs) +} + +fn emit_restore_helper( + module: &mut Module, + memory: MemoryId, + ptr_ty: ValType, + codecs: ReferenceCodecs, + imports: ModuleStateImports, + plan: &ModuleStatePlan, + bootstrap_done: GlobalId, +) -> Result { + let global_classes: Vec<_> = plan + .globals + .iter() + .map(|global| match global.ty { + ValType::Ref(ty) => Some(ReferenceCodecClass::of(module, ty)), + _ => None, + }) + .collect(); + let payload = module.locals.add(ptr_ty); + let activation = module.locals.add(ValType::I32); + let mut builder = FunctionBuilder::new(&mut module.types, &[ValType::I32], &[]); + { + let mut body = builder.func_body(); + for (global, class) in plan.globals.iter().zip(global_classes) { + if !global.restore { + continue; + } + find_record( + &mut body, + imports, + activation, + WPK_FORK_MODULE_STATE_RECORD_KIND_MUTABLE_GLOBAL, + global.owner, + 0, + payload, + ); + body.local_get(payload); + match class { + Some(class) => { + body.load( + memory, + LoadKind::I32 { atomic: false }, + MemArg { + align: 4, + offset: u64::from(WPK_FORK_MODULE_STATE_GLOBAL_HEADER_SIZE), + }, + ) + .call(class.decoder(codecs)); + let ValType::Ref(reference) = global.ty else { + unreachable!("reference codec assigned to scalar global") + }; + emit_narrow_reference(&mut body, class, reference); + } + None => { + body.load( + memory, + scalar_load_kind(global.ty), + MemArg { + align: scalar_align(global.ty), + offset: u64::from(WPK_FORK_MODULE_STATE_GLOBAL_HEADER_SIZE), + }, + ); + } + } + body.global_set(global.id); + } + + // WHY: active element offsets can depend on globals. Restore globals + // first and recreate only this activation's deterministic table + // baseline. Sparse overlays are intentionally deferred until every + // activation has replayed its baseline, matching parent + // instantiation order without serializing shared tables per alias. + // Data initialization and the original start are deliberately absent + // because child linear memory was copied. + body.global_get(bootstrap_done) + .unop(UnaryOp::I32Eqz) + .if_else( + None, + |baseline| { + emit_active_element_initializers(baseline, &plan.active_elements, false) + .expect("active element segment length fits u32"); + }, + |_| {}, + ); + + // WHY: reference recipes can contain array.new_data/array.new_elem + // nodes owned by another activation. Keep every passive segment + // physically live until all activations have restored values/tables; + // the finish helper reapplies the parent-visible drop state globally. + body.i32_const(1).global_set(bootstrap_done); + } + Ok(builder.finish(vec![activation], &mut module.funcs)) +} + +fn emit_finish_restore_helper( + module: &mut Module, + memory: MemoryId, + ptr_ty: ValType, + codecs: ReferenceCodecs, + imports: ModuleStateImports, + plan: &ModuleStatePlan, + trackers: &Trackers, + bootstrap_done: GlobalId, +) -> FunctionId { + let synchronized_tables: Vec<_> = plan + .tables + .iter() + .filter(|table| table.synchronized) + .copied() + .collect(); + let table_classes: Vec<_> = synchronized_tables + .iter() + .map(|table| ReferenceCodecClass::of(module, table.ty)) + .collect(); + let table_locals: Vec<_> = synchronized_tables + .iter() + .map(|table| allocate_table_locals(module, ptr_ty, table.table64)) + .collect(); + let payload = module.locals.add(ptr_ty); + let activation = module.locals.add(ValType::I32); + let mut builder = FunctionBuilder::new(&mut module.types, &[ValType::I32], &[]); + { + let mut body = builder.func_body(); + body.global_get(bootstrap_done) + .unop(UnaryOp::I32Eqz) + .if_else( + None, + |invalid| { + // A pre-restore finish would destroy constructor inputs + // before globals/tables had a chance to decode them. + invalid.unreachable(); + }, + |_| {}, + ); + for ((table, class), locals) in synchronized_tables + .iter() + .zip(table_classes) + .zip(table_locals) + { + body.i32_const(table.owner as i32) + .call(imports.table_state_owned) + .if_else( + None, + |owned| { + // WHY: all activation baselines now exist. Reapply the + // final sparse overlay exactly once for the physical + // Table, so later aliases cannot overwrite it. + emit_restore_table( + owned, memory, ptr_ty, codecs, imports, activation, *table, class, + locals, + ); + }, + |_| {}, + ); + } + emit_restore_segments( + &mut body, + memory, + imports, + activation, + payload, + WPK_FORK_MODULE_STATE_RECORD_KIND_ELEMENT_SEGMENTS, + &trackers.elements, + |body, elem| { + body.instr(ElemDrop { elem }); + }, + ); + emit_restore_segments( + &mut body, + memory, + imports, + activation, + payload, + WPK_FORK_MODULE_STATE_RECORD_KIND_DATA_SEGMENTS, + &trackers.data, + |body, data| { + body.instr(DataDrop { data }); + }, + ); + } + builder.finish(vec![activation], &mut module.funcs) +} + +fn emit_table_save_helper( + module: &mut Module, + memory: MemoryId, + ptr_ty: ValType, + codecs: ReferenceCodecs, + imports: ModuleStateImports, + plan: &ModuleStatePlan, +) -> FunctionId { + let synchronized_tables: Vec<_> = plan + .tables + .iter() + .filter(|table| table.synchronized) + .copied() + .collect(); + let table_classes: Vec<_> = synchronized_tables + .iter() + .map(|table| ReferenceCodecClass::of(module, table.ty)) + .collect(); + let table_locals: Vec<_> = synchronized_tables + .iter() + .map(|table| allocate_table_locals(module, ptr_ty, table.table64)) + .collect(); + let activation = module.locals.add(ValType::I32); + let mut builder = FunctionBuilder::new(&mut module.types, &[ValType::I32], &[]); + { + let mut body = builder.func_body(); + for ((table, class), locals) in synchronized_tables + .iter() + .zip(table_classes) + .zip(table_locals) + { + body.i32_const(table.owner as i32) + .call(imports.table_state_owned) + .if_else( + None, + |owned| { + emit_save_table( + owned, memory, ptr_ty, codecs, imports, activation, *table, class, + locals, + ); + }, + |_| {}, + ); + } + } + builder.finish(vec![activation], &mut module.funcs) +} + +fn emit_table_restore_helper( + module: &mut Module, + memory: MemoryId, + ptr_ty: ValType, + codecs: ReferenceCodecs, + imports: ModuleStateImports, + plan: &ModuleStatePlan, +) -> FunctionId { + let synchronized_tables: Vec<_> = plan + .tables + .iter() + .filter(|table| table.synchronized) + .copied() + .collect(); + let table_classes: Vec<_> = synchronized_tables + .iter() + .map(|table| ReferenceCodecClass::of(module, table.ty)) + .collect(); + let table_locals: Vec<_> = synchronized_tables + .iter() + .map(|table| allocate_table_locals(module, ptr_ty, table.table64)) + .collect(); + let activation = module.locals.add(ValType::I32); + let mut builder = FunctionBuilder::new(&mut module.types, &[ValType::I32], &[]); + { + let mut body = builder.func_body(); + for ((table, class), locals) in synchronized_tables + .iter() + .zip(table_classes) + .zip(table_locals) + { + body.i32_const(table.owner as i32) + .call(imports.table_state_owned) + .if_else( + None, + |owned| { + emit_restore_table( + owned, memory, ptr_ty, codecs, imports, activation, *table, class, + locals, + ); + }, + |_| {}, + ); + } + } + builder.finish(vec![activation], &mut module.funcs) +} + +fn emit_bootstrap_helper( + module: &mut Module, + plan: &ModuleStatePlan, + bootstrap_done: GlobalId, +) -> Result { + let mut builder = FunctionBuilder::new(&mut module.types, &[], &[]); + { + let mut body = builder.func_body(); + body.global_get(bootstrap_done) + .unop(UnaryOp::I32Eqz) + .if_else( + None, + |initialize| { + // Native instantiation applies element segments, then data + // segments, then invokes the start function. Keep that + // ordering observable while allowing pthread instances + // and fork children to skip the parent-only phase. + emit_active_element_initializers(initialize, &plan.active_elements, true) + .expect("active element segment length fits u32"); + emit_active_data_initializers(initialize, &plan.active_data) + .expect("active data segment length fits u32"); + initialize.i32_const(1).global_set(bootstrap_done); + if let Some(start) = plan.original_start { + initialize.call(start); + } + }, + |_| {}, + ); + } + Ok(builder.finish(Vec::new(), &mut module.funcs)) +} + +fn emit_thread_bootstrap_helper( + module: &mut Module, + plan: &ModuleStatePlan, + bootstrap_done: GlobalId, +) -> Result { + let mut builder = FunctionBuilder::new(&mut module.types, &[], &[]); + { + let mut body = builder.func_body(); + body.global_get(bootstrap_done) + .unop(UnaryOp::I32Eqz) + .if_else( + None, + |initialize| { + // WHY: pthread instances have instance-local tables but + // share the parent's already-initialized linear memory. + // Recreate and consume only the element baseline; consume + // converted active data without copying or rerunning start. + emit_active_element_initializers(initialize, &plan.active_elements, true) + .expect("active element segment length fits u32"); + for active in &plan.active_data { + initialize.instr(DataDrop { data: active.id }); + } + initialize.i32_const(1).global_set(bootstrap_done); + }, + |_| {}, + ); + } + Ok(builder.finish(Vec::new(), &mut module.funcs)) +} + +fn emit_active_element_initializers( + body: &mut InstrSeqBuilder<'_>, + active_elements: &[ActiveElement], + drop_after: bool, +) -> Result<()> { + for active in active_elements { + // WHY: retaining the original const expression in an immutable + // global makes Walrus/WebAssembly own extended-const semantics. The + // runtime helper reads the already-evaluated index and therefore does + // not impose an instruction whitelist or evaluate the expression more + // than once when active segments are converted to passive segments. + body.global_get(active.offset_global); + body.i32_const(0) + .i32_const( + u32::try_from(active.len) + .map_err(|_| anyhow::anyhow!("element segment length exceeds u32"))? + as i32, + ) + .instr(TableInit { + table: active.table, + elem: active.id, + }); + if drop_after { + body.instr(ElemDrop { elem: active.id }); + } + } + Ok(()) +} + +fn emit_active_data_initializers( + body: &mut InstrSeqBuilder<'_>, + active_data: &[ActiveData], +) -> Result<()> { + for active in active_data { + body.global_get(active.offset_global); + body.i32_const(0) + .i32_const( + u32::try_from(active.len) + .map_err(|_| anyhow::anyhow!("data segment length exceeds u32"))? + as i32, + ) + .instr(MemoryInit { + memory: active.memory, + data: active.id, + }) + .instr(DataDrop { data: active.id }); + } + Ok(()) +} + +fn reserve_static_record( + body: &mut InstrSeqBuilder<'_>, + imports: ModuleStateImports, + ptr_ty: ValType, + activation: LocalId, + kind: u16, + owner: u32, + size: u32, + payload: LocalId, +) { + body.i32_const(i32::from(kind)) + .local_get(activation) + .i32_const(owner as i32); + emit_ptr_const(body, ptr_ty, u64::from(size)); + body.call(imports.reserve).local_set(payload); +} + +fn find_record( + body: &mut InstrSeqBuilder<'_>, + imports: ModuleStateImports, + activation: LocalId, + kind: u16, + owner: u32, + ordinal: u32, + payload: LocalId, +) { + body.i32_const(i32::from(kind)) + .local_get(activation) + .i32_const(owner as i32) + .i32_const(ordinal as i32) + .call(imports.find) + .local_set(payload); +} + +fn emit_save_segments( + body: &mut InstrSeqBuilder<'_>, + memory: MemoryId, + ptr_ty: ValType, + imports: ModuleStateImports, + activation: LocalId, + payload: LocalId, + kind: u16, + header_size: u32, + tracker: &SegmentTracker, +) { + if tracker.segments.is_empty() { + return; + } + let bitmap_bytes = tracker.segments.len().div_ceil(8); + reserve_static_record( + body, + imports, + ptr_ty, + activation, + kind, + 1, + header_size + bitmap_bytes as u32, + payload, + ); + body.local_get(payload) + .i32_const(tracker.segments.len() as i32) + .store( + memory, + StoreKind::I32 { atomic: false }, + MemArg { + align: 4, + offset: 0, + }, + ) + .local_get(payload) + .i32_const(bitmap_bytes as i32) + .store( + memory, + StoreKind::I32 { atomic: false }, + MemArg { + align: 4, + offset: 4, + }, + ); + for byte in 0..bitmap_bytes { + let word = tracker.globals[byte / 4]; + body.local_get(payload).global_get(word); + if byte % 4 != 0 { + body.i32_const((byte % 4 * 8) as i32) + .binop(BinaryOp::I32ShrU); + } + body.store( + memory, + StoreKind::I32_8 { atomic: false }, + MemArg { + align: 1, + offset: u64::from(header_size) + byte as u64, + }, + ); + } + body.local_get(payload).call(imports.commit); +} + +fn emit_restore_segments( + body: &mut InstrSeqBuilder<'_>, + memory: MemoryId, + imports: ModuleStateImports, + activation: LocalId, + payload: LocalId, + kind: u16, + tracker: &SegmentTracker, + mut emit_drop: impl FnMut(&mut InstrSeqBuilder<'_>, T), +) { + if tracker.segments.is_empty() { + return; + } + find_record(body, imports, activation, kind, 1, 0, payload); + let bitmap_bytes = tracker.segments.len().div_ceil(8); + for (word_index, global) in tracker.globals.iter().copied().enumerate() { + // Assemble the bitmap word from exact byte loads. The KFMS validator + // guarantees the declared payload length, but a guest helper must not + // make correctness depend on allocator padding beyond that payload. + body.i32_const(0); + let first_byte = word_index * 4; + let word_bytes = bitmap_bytes.saturating_sub(first_byte).min(4); + for byte in 0..word_bytes { + body.local_get(payload).load( + memory, + LoadKind::I32_8 { + kind: ExtendedLoad::ZeroExtend, + }, + MemArg { + align: 1, + offset: 8 + (first_byte + byte) as u64, + }, + ); + if byte != 0 { + body.i32_const((byte * 8) as i32).binop(BinaryOp::I32Shl); + } + body.binop(BinaryOp::I32Or); + } + let remaining = tracker.segments.len().saturating_sub(word_index * 32); + if remaining < 32 { + let mask = if remaining == 0 { + 0 + } else { + (1u32 << remaining) - 1 + }; + body.i32_const(mask as i32).binop(BinaryOp::I32And); + } + body.global_set(global); + } + for (index, segment) in tracker.segments.iter().copied().enumerate() { + body.global_get(tracker.globals[index / 32]) + .i32_const((1u32 << (index % 32)) as i32) + .binop(BinaryOp::I32And) + .if_else(None, |then| emit_drop(then, segment), |_| {}); + } +} + +fn emit_save_table( + body: &mut InstrSeqBuilder<'_>, + memory: MemoryId, + ptr_ty: ValType, + codecs: ReferenceCodecs, + imports: ModuleStateImports, + activation: LocalId, + table: TableState, + class: ReferenceCodecClass, + locals: TableLocals, +) { + body.instr(TableSize { table: table.id }) + .local_set(locals.len) + .i32_const(table.owner as i32) + .call(imports.table_dirty_count) + .local_set(locals.page_count); + + reserve_static_record( + body, + imports, + ptr_ty, + activation, + WPK_FORK_MODULE_STATE_RECORD_KIND_TABLE, + table.owner, + u32::from(WPK_FORK_MODULE_STATE_TABLE_DESCRIPTOR_PAYLOAD_SIZE), + locals.payload, + ); + body.local_get(locals.payload) + .i32_const(if table.table64 { 8 } else { 4 }) + .store( + memory, + StoreKind::I32_8 { atomic: false }, + MemArg { + align: 1, + offset: 0, + }, + ) + .local_get(locals.payload) + .i32_const(TABLE_PAGE_SHIFT as i32) + .store( + memory, + StoreKind::I32_8 { atomic: false }, + MemArg { + align: 1, + offset: 1, + }, + ) + .local_get(locals.payload) + .i32_const(WPK_FORK_MODULE_STATE_TABLE_FLAG_SPARSE_OVERRIDES as i32) + .store( + memory, + StoreKind::I32_16 { atomic: false }, + MemArg { + align: 2, + offset: 2, + }, + ) + .local_get(locals.payload) + .local_get(locals.page_count) + .store( + memory, + StoreKind::I32 { atomic: false }, + MemArg { + align: 4, + offset: 4, + }, + ) + .local_get(locals.payload) + .local_get(locals.len); + emit_index_to_i64(body, table); + body.store( + memory, + StoreKind::I64 { atomic: false }, + MemArg { + align: 8, + offset: 8, + }, + ) + .local_get(locals.payload) + .i64_const(table.baseline_len as i64) + .store( + memory, + StoreKind::I64 { atomic: false }, + MemArg { + align: 8, + offset: 16, + }, + ); + for (chunk, bytes) in table.baseline_fingerprint.chunks_exact(8).enumerate() { + body.local_get(locals.payload) + .i64_const(i64::from_le_bytes(bytes.try_into().unwrap())) + .store( + memory, + StoreKind::I64 { atomic: false }, + MemArg { + align: 8, + offset: 24 + (chunk as u64 * 8), + }, + ); + } + body.local_get(locals.payload) + .call(imports.commit) + .i32_const(0) + .local_set(locals.page_ordinal); + + body.block(None, |done| { + let done_id = done.id(); + done.loop_(None, |page_loop| { + let loop_id = page_loop.id(); + page_loop + .local_get(locals.page_ordinal) + .local_get(locals.page_count) + .binop(BinaryOp::I32GeU) + .instr(BrIf { block: done_id }); + + page_loop + .i32_const(table.owner as i32) + .local_get(locals.page_ordinal) + .call(imports.table_dirty_page) + .local_set(locals.page_index) + .local_get(locals.page_index) + .i64_const(i64::from(TABLE_PAGE_SHIFT)) + .binop(BinaryOp::I64Shl); + if !table.table64 { + page_loop + .local_set(locals.page_start_wide) + .local_get(locals.page_start_wide) + .i64_const(i64::from(u32::MAX)) + .binop(BinaryOp::I64GtU); + emit_trap_if(page_loop); + page_loop + .local_get(locals.page_start_wide) + .unop(UnaryOp::I32WrapI64); + } + page_loop.local_set(locals.page_start); + page_loop.local_get(locals.page_start).local_get(locals.len); + emit_index_binop(page_loop, table, BinaryOp::I32GeU, BinaryOp::I64GeU); + emit_trap_if(page_loop); + emit_page_entry_count(page_loop, table, locals); + page_loop + .i32_const(i32::from(WPK_FORK_MODULE_STATE_RECORD_KIND_TABLE_PAGE)) + .local_get(activation) + .i32_const(table.owner as i32) + .local_get(locals.count); + emit_index_to_ptr(page_loop, table, ptr_ty); + emit_ptr_const(page_loop, ptr_ty, 4); + emit_ptr_binop(page_loop, ptr_ty, BinaryOp::I32Mul, BinaryOp::I64Mul); + emit_ptr_const( + page_loop, + ptr_ty, + u64::from( + WPK_FORK_MODULE_STATE_TABLE_PAGE_HEADER_SIZE + + WPK_FORK_MODULE_STATE_TABLE_RUN_HEADER_SIZE, + ), + ); + emit_ptr_binop(page_loop, ptr_ty, BinaryOp::I32Add, BinaryOp::I64Add); + page_loop.call(imports.reserve).local_set(locals.payload); + + page_loop + .local_get(locals.payload) + .local_get(locals.page_index) + .store( + memory, + StoreKind::I64 { atomic: false }, + MemArg { + align: 8, + offset: 0, + }, + ) + .local_get(locals.payload) + .i32_const(1) + .store( + memory, + StoreKind::I32 { atomic: false }, + MemArg { + align: 4, + offset: 8, + }, + ) + .local_get(locals.payload) + .local_get(locals.count); + emit_index_to_i32(page_loop, table); + page_loop + .store( + memory, + StoreKind::I32 { atomic: false }, + MemArg { + align: 4, + offset: 12, + }, + ) + .local_get(locals.payload) + .i32_const(0) + .store( + memory, + StoreKind::I32 { atomic: false }, + MemArg { + align: 4, + offset: 16, + }, + ) + .local_get(locals.payload) + .local_get(locals.count); + emit_index_to_i32(page_loop, table); + page_loop.store( + memory, + StoreKind::I32 { atomic: false }, + MemArg { + align: 4, + offset: 20, + }, + ); + + emit_index_const(page_loop, table, 0); + page_loop.local_set(locals.index); + page_loop.block(None, |entries_done| { + let entries_done_id = entries_done.id(); + entries_done.loop_(None, |entry_loop| { + let entry_loop_id = entry_loop.id(); + entry_loop.local_get(locals.index).local_get(locals.count); + emit_index_binop(entry_loop, table, BinaryOp::I32GeU, BinaryOp::I64GeU); + entry_loop.instr(BrIf { + block: entries_done_id, + }); + emit_table_recipe_addr(entry_loop, ptr_ty, table, locals.payload, locals.index); + entry_loop + .local_get(locals.page_start) + .local_get(locals.index); + emit_index_binop(entry_loop, table, BinaryOp::I32Add, BinaryOp::I64Add); + entry_loop + .instr(TableGet { table: table.id }) + .call(class.encoder(codecs)) + .store( + memory, + StoreKind::I32 { atomic: false }, + MemArg { + align: 4, + offset: 0, + }, + ) + .local_get(locals.index); + emit_index_const(entry_loop, table, 1); + emit_index_binop(entry_loop, table, BinaryOp::I32Add, BinaryOp::I64Add); + entry_loop.local_set(locals.index).instr(Br { + block: entry_loop_id, + }); + }); + }); + page_loop + .local_get(locals.payload) + .call(imports.commit) + .local_get(locals.page_ordinal) + .i32_const(1) + .binop(BinaryOp::I32Add) + .local_set(locals.page_ordinal) + .instr(Br { block: loop_id }); + }); + }); +} + +fn emit_restore_table( + body: &mut InstrSeqBuilder<'_>, + memory: MemoryId, + ptr_ty: ValType, + codecs: ReferenceCodecs, + imports: ModuleStateImports, + activation: LocalId, + table: TableState, + class: ReferenceCodecClass, + locals: TableLocals, +) { + find_record( + body, + imports, + activation, + WPK_FORK_MODULE_STATE_RECORD_KIND_TABLE, + table.owner, + 0, + locals.payload, + ); + body.local_get(locals.payload).load( + memory, + LoadKind::I64 { atomic: false }, + MemArg { + align: 8, + offset: 8, + }, + ); + emit_i64_to_index(body, table); + body.local_set(locals.len) + .local_get(locals.payload) + .load( + memory, + LoadKind::I32 { atomic: false }, + MemArg { + align: 4, + offset: 4, + }, + ) + .local_set(locals.record_page_count); + body.local_get(locals.payload) + .load( + memory, + LoadKind::I64 { atomic: false }, + MemArg { + align: 8, + offset: 16, + }, + ) + .i64_const(table.baseline_len as i64) + .binop(BinaryOp::I64Ne); + emit_trap_if(body); + for (chunk, bytes) in table.baseline_fingerprint.chunks_exact(8).enumerate() { + body.local_get(locals.payload) + .load( + memory, + LoadKind::I64 { atomic: false }, + MemArg { + align: 8, + offset: 24 + (chunk as u64 * 8), + }, + ) + .i64_const(i64::from_le_bytes(bytes.try_into().unwrap())) + .binop(BinaryOp::I64Ne); + emit_trap_if(body); + } + body.instr(TableSize { table: table.id }) + .local_set(locals.current_len) + .local_get(locals.current_len) + .local_get(locals.len); + emit_index_binop(body, table, BinaryOp::I32GtU, BinaryOp::I64GtU); + body.if_else( + None, + |then| { + then.instr(Unreachable {}); + }, + |_| {}, + ); + + body.i32_const(0).local_set(locals.page_ordinal); + body.block(None, |done| { + let done_id = done.id(); + done.loop_(None, |page_loop| { + let loop_id = page_loop.id(); + page_loop + .local_get(locals.page_ordinal) + .local_get(locals.record_page_count) + .binop(BinaryOp::I32GeU) + .instr(BrIf { block: done_id }); + page_loop + .i32_const(i32::from(WPK_FORK_MODULE_STATE_RECORD_KIND_TABLE_PAGE)) + .local_get(activation) + .i32_const(table.owner as i32) + .local_get(locals.page_ordinal) + .call(imports.find) + .local_set(locals.payload); + page_loop + .local_get(locals.payload) + .load( + memory, + LoadKind::I64 { atomic: false }, + MemArg { + align: 8, + offset: 0, + }, + ) + .local_set(locals.page_index) + .local_get(locals.page_index) + .i64_const(i64::from(TABLE_PAGE_SHIFT)) + .binop(BinaryOp::I64Shl); + if table.table64 { + page_loop.local_set(locals.page_start); + } else { + page_loop + .local_set(locals.page_start_wide) + .local_get(locals.page_start_wide) + .i64_const(i64::from(u32::MAX)) + .binop(BinaryOp::I64GtU); + emit_trap_if(page_loop); + page_loop + .local_get(locals.page_start_wide) + .unop(UnaryOp::I32WrapI64) + .local_set(locals.page_start); + } + page_loop.local_get(locals.page_start).local_get(locals.len); + emit_index_binop(page_loop, table, BinaryOp::I32GeU, BinaryOp::I64GeU); + emit_trap_if(page_loop); + emit_page_entry_count(page_loop, table, locals); + emit_validate_sparse_page(page_loop, memory, table, locals); + + // A grown table needs one valid reference before its entries can + // be overlaid. Every successful grow dirties the page containing + // the old end, so the first page spanning current_len owns that + // initializer without a full-table scan. + page_loop + .local_get(locals.current_len) + .local_get(locals.len); + emit_index_binop(page_loop, table, BinaryOp::I32LtU, BinaryOp::I64LtU); + page_loop.if_else( + None, + |needs_growth| { + needs_growth + .local_get(locals.current_len) + .local_get(locals.page_start); + emit_index_binop(needs_growth, table, BinaryOp::I32GeU, BinaryOp::I64GeU); + needs_growth + .local_get(locals.current_len) + .local_get(locals.page_start) + .local_get(locals.count); + emit_index_binop(needs_growth, table, BinaryOp::I32Add, BinaryOp::I64Add); + emit_index_binop(needs_growth, table, BinaryOp::I32LtU, BinaryOp::I64LtU); + needs_growth.binop(BinaryOp::I32And).if_else( + None, + |spans_old_end| { + spans_old_end + .local_get(locals.current_len) + .local_get(locals.page_start); + emit_index_binop( + spans_old_end, + table, + BinaryOp::I32Sub, + BinaryOp::I64Sub, + ); + spans_old_end.local_set(locals.index); + emit_table_recipe_addr( + spans_old_end, + ptr_ty, + table, + locals.payload, + locals.index, + ); + spans_old_end + .load( + memory, + LoadKind::I32 { atomic: false }, + MemArg { + align: 4, + offset: 0, + }, + ) + .call(class.decoder(codecs)); + emit_narrow_reference(spans_old_end, class, table.ty); + spans_old_end + .local_get(locals.len) + .local_get(locals.current_len); + emit_index_binop( + spans_old_end, + table, + BinaryOp::I32Sub, + BinaryOp::I64Sub, + ); + spans_old_end.instr(TableGrow { table: table.id }); + emit_index_const(spans_old_end, table, u64::MAX); + emit_index_binop( + spans_old_end, + table, + BinaryOp::I32Eq, + BinaryOp::I64Eq, + ); + emit_trap_if(spans_old_end); + spans_old_end + .local_get(locals.len) + .local_set(locals.current_len); + }, + |_| {}, + ); + }, + |_| {}, + ); + + emit_index_const(page_loop, table, 0); + page_loop.local_set(locals.index); + page_loop.block(None, |entries_done| { + let entries_done_id = entries_done.id(); + entries_done.loop_(None, |entry_loop| { + let entry_loop_id = entry_loop.id(); + entry_loop.local_get(locals.index).local_get(locals.count); + emit_index_binop(entry_loop, table, BinaryOp::I32GeU, BinaryOp::I64GeU); + entry_loop.instr(BrIf { + block: entries_done_id, + }); + entry_loop + .local_get(locals.page_start) + .local_get(locals.index); + emit_index_binop(entry_loop, table, BinaryOp::I32Add, BinaryOp::I64Add); + emit_table_recipe_addr(entry_loop, ptr_ty, table, locals.payload, locals.index); + entry_loop + .load( + memory, + LoadKind::I32 { atomic: false }, + MemArg { + align: 4, + offset: 0, + }, + ) + .call(class.decoder(codecs)); + emit_narrow_reference(entry_loop, class, table.ty); + entry_loop + .instr(TableSet { table: table.id }) + .local_get(locals.index); + emit_index_const(entry_loop, table, 1); + emit_index_binop(entry_loop, table, BinaryOp::I32Add, BinaryOp::I64Add); + entry_loop.local_set(locals.index).instr(Br { + block: entry_loop_id, + }); + }); + }); + page_loop + .i32_const(table.owner as i32) + .local_get(locals.page_index) + .i64_const(1) + .call(imports.table_dirty_mark) + .local_get(locals.page_ordinal) + .i32_const(1) + .binop(BinaryOp::I32Add) + .local_set(locals.page_ordinal) + .instr(Br { block: loop_id }); + }); + }); + body.local_get(locals.current_len).local_get(locals.len); + emit_index_binop(body, table, BinaryOp::I32Ne, BinaryOp::I64Ne); + emit_trap_if(body); +} + +fn emit_page_entry_count(body: &mut InstrSeqBuilder<'_>, table: TableState, locals: TableLocals) { + // select(page_size, len-page_start, remaining > page_size) + emit_index_const(body, table, TABLE_PAGE_SIZE); + body.local_get(locals.len).local_get(locals.page_start); + emit_index_binop(body, table, BinaryOp::I32Sub, BinaryOp::I64Sub); + body.local_get(locals.len).local_get(locals.page_start); + emit_index_binop(body, table, BinaryOp::I32Sub, BinaryOp::I64Sub); + emit_index_const(body, table, TABLE_PAGE_SIZE); + emit_index_binop(body, table, BinaryOp::I32GtU, BinaryOp::I64GtU); + body.instr(walrus::ir::Select { + ty: Some(table_index_type(table)), + }) + .local_set(locals.count); +} + +fn emit_validate_sparse_page( + body: &mut InstrSeqBuilder<'_>, + memory: MemoryId, + table: TableState, + locals: TableLocals, +) { + body.local_get(locals.payload) + .load( + memory, + LoadKind::I64 { atomic: false }, + MemArg { + align: 8, + offset: 0, + }, + ) + .local_get(locals.page_index) + .binop(BinaryOp::I64Ne); + emit_trap_if(body); + for (offset, expected) in [(8, 1), (16, 0)] { + body.local_get(locals.payload) + .load( + memory, + LoadKind::I32 { atomic: false }, + MemArg { align: 4, offset }, + ) + .i32_const(expected) + .binop(BinaryOp::I32Ne); + emit_trap_if(body); + } + for offset in [12, 20] { + body.local_get(locals.payload).load( + memory, + LoadKind::I32 { atomic: false }, + MemArg { align: 4, offset }, + ); + body.local_get(locals.count); + emit_index_to_i32(body, table); + body.binop(BinaryOp::I32Ne); + emit_trap_if(body); + } +} + +fn emit_trap_if(body: &mut InstrSeqBuilder<'_>) { + body.if_else( + None, + |invalid| { + invalid.instr(Unreachable {}); + }, + |_| {}, + ); +} + +fn emit_table_recipe_addr( + body: &mut InstrSeqBuilder<'_>, + ptr_ty: ValType, + table: TableState, + payload: LocalId, + index: LocalId, +) { + body.local_get(payload).local_get(index); + emit_index_to_ptr(body, table, ptr_ty); + emit_ptr_const(body, ptr_ty, 4); + emit_ptr_binop(body, ptr_ty, BinaryOp::I32Mul, BinaryOp::I64Mul); + emit_ptr_const( + body, + ptr_ty, + u64::from( + WPK_FORK_MODULE_STATE_TABLE_PAGE_HEADER_SIZE + + WPK_FORK_MODULE_STATE_TABLE_RUN_HEADER_SIZE, + ), + ); + emit_ptr_binop(body, ptr_ty, BinaryOp::I32Add, BinaryOp::I64Add); + emit_ptr_binop(body, ptr_ty, BinaryOp::I32Add, BinaryOp::I64Add); +} + +fn emit_narrow_reference( + body: &mut InstrSeqBuilder<'_>, + class: ReferenceCodecClass, + expected: RefType, +) { + let broad = class.nullable_type(); + if expected.heap_type != broad.heap_type { + body.instr(RefCast { + nullable: expected.nullable, + heap_type: expected.heap_type, + }); + } else if !expected.nullable { + body.instr(RefAsNonNull {}); + } +} + +fn table_index_type(table: TableState) -> ValType { + if table.table64 { + ValType::I64 + } else { + ValType::I32 + } +} + +fn emit_ptr_const(body: &mut InstrSeqBuilder<'_>, ptr_ty: ValType, value: u64) { + match ptr_ty { + ValType::I32 => { + body.i32_const(value as u32 as i32); + } + ValType::I64 => { + body.i64_const(value as i64); + } + other => unreachable!("unsupported KFMS pointer type {other:?}"), + } +} + +fn emit_index_const(body: &mut InstrSeqBuilder<'_>, table: TableState, value: u64) { + if table.table64 { + body.i64_const(value as i64); + } else { + body.i32_const(value as u32 as i32); + } +} + +fn emit_ptr_binop(body: &mut InstrSeqBuilder<'_>, ptr_ty: ValType, op32: BinaryOp, op64: BinaryOp) { + body.binop(match ptr_ty { + ValType::I32 => op32, + ValType::I64 => op64, + other => unreachable!("unsupported KFMS pointer type {other:?}"), + }); +} + +fn emit_index_binop( + body: &mut InstrSeqBuilder<'_>, + table: TableState, + op32: BinaryOp, + op64: BinaryOp, +) { + body.binop(if table.table64 { op64 } else { op32 }); +} + +fn emit_index_to_i32(body: &mut InstrSeqBuilder<'_>, table: TableState) { + if table.table64 { + body.unop(UnaryOp::I32WrapI64); + } +} + +fn emit_index_to_i64(body: &mut InstrSeqBuilder<'_>, table: TableState) { + if !table.table64 { + body.unop(UnaryOp::I64ExtendUI32); + } +} + +fn emit_i64_to_index(body: &mut InstrSeqBuilder<'_>, table: TableState) { + if !table.table64 { + // KFMS validation rejects table32 lengths above u32::MAX before this + // helper is called, so the narrowing conversion is exact. + body.unop(UnaryOp::I32WrapI64); + } +} + +fn emit_index_to_ptr(body: &mut InstrSeqBuilder<'_>, table: TableState, ptr_ty: ValType) { + match (table.table64, ptr_ty) { + (false, ValType::I32) | (true, ValType::I64) => {} + (false, ValType::I64) => { + body.unop(UnaryOp::I64ExtendUI32); + } + (true, ValType::I32) => { + body.unop(UnaryOp::I32WrapI64); + } + (_, other) => unreachable!("unsupported KFMS pointer type {other:?}"), + } +} diff --git a/crates/fork-instrument/src/reference_analysis.rs b/crates/fork-instrument/src/reference_analysis.rs new file mode 100644 index 0000000000..9d65f4d143 --- /dev/null +++ b/crates/fork-instrument/src/reference_analysis.rs @@ -0,0 +1,1624 @@ +//! Reference-state analysis for fork continuation planning. +//! +//! This module deliberately analyzes the original Walrus IR. The emission +//! transform splits and nests instruction sequences, so running liveness after +//! rewriting would answer questions about synthetic locals rather than the +//! guest values that are live at a fork landing. +//! +//! The analysis is independent from `instrument.rs` for now. It provides: +//! +//! * stable, depth-first call-site identities; +//! * a structured control-flow graph, including exception edges; +//! * backward reference-local liveness; and +//! * a conservative forward definitely-null analysis. +//! +//! `MaybeNonNull` intentionally includes both non-null values and values whose +//! nullness is unknown. Only `DefinitelyNull` is strong enough to omit a +//! reconstruction recipe. + +use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet, VecDeque}; + +use anyhow::{Result, bail}; +use walrus::{ + AbstractHeapType, FunctionId, FunctionKind, HeapType, LocalFunction, LocalId, Module, RefType, + TableId, TagId, TypeId, ValType, + ir::{ + AtomicWidth, Instr, InstrSeqId, InstrSeqType, LegacyCatch, LoadKind, TryTableCatch, + UnaryOp, Value, + }, +}; + +/// Stable identity assigned before any instruction rewriting. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct OriginalCallSiteId(pub u32); + +/// One instruction in the original Walrus IR. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct OriginalProgramPoint { + pub sequence: InstrSeqId, + pub instruction_index: usize, +} + +/// The invocation form at a fork-relevant call landing. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum OriginalCallKind { + Direct(FunctionId), + Indirect { table: TableId, ty: TypeId }, + Ref { ty: TypeId }, +} + +impl OriginalCallKind { + fn signature(self, module: &Module) -> TypeId { + match self { + Self::Direct(function) => module.funcs.get(function).ty(), + Self::Indirect { ty, .. } | Self::Ref { ty } => ty, + } + } + + fn extra_stack_operands(self) -> usize { + match self { + Self::Indirect { .. } | Self::Ref { .. } => 1, + Self::Direct(_) => 0, + } + } + + fn has_reference_callee(self) -> bool { + matches!(self, Self::Ref { .. }) + } +} + +/// One statically typed reference operand. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct ReferenceOperand { + /// Parameter index for arguments, or stack index for carryovers. + pub index: usize, + pub ty: RefType, +} + +/// Precision of the operand-stack carryover scan. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum CarryoverPrecision { + Exact, + /// At least one carryover slot had a producer this bounded scanner could + /// not type. Consumers must not interpret an empty reference list as a + /// proof that no reference is carried. + ContainsUnknownSlots, + Unavailable, +} + +/// Conservative null provenance for a reference local. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ReferenceNullability { + DefinitelyNull, + MaybeNonNull, +} + +impl ReferenceNullability { + fn join(self, other: Self) -> Self { + if self == Self::DefinitelyNull && other == Self::DefinitelyNull { + Self::DefinitelyNull + } else { + Self::MaybeNonNull + } + } +} + +/// Reference facts at one fork-relevant original call. +#[derive(Clone, Debug)] +pub struct ReferenceCallSite { + pub id: OriginalCallSiteId, + pub point: OriginalProgramPoint, + pub kind: OriginalCallKind, + pub reference_arguments: Vec, + pub reference_results: Vec, + /// `call_ref` and `return_call_ref` consume a function reference in + /// addition to the declared function parameters. + pub has_reference_callee: bool, + pub reference_carryovers: Vec, + pub carryover_precision: CarryoverPrecision, + /// References needed after the call returns normally. This is the set a + /// fork continuation needs after reissuing its active call. + pub live_ref_locals_on_normal_return: BTreeSet, + /// References needed on any normal or exceptional successor. Keeping this + /// separate makes exceptional CFG coverage testable without making a + /// throwing-only cleanup value look live on deterministic fork replay. + pub live_ref_locals_on_any_successor: BTreeSet, + pub local_nullability_before_call: BTreeMap, + pub reachable: bool, +} + +/// Standalone output for one local function. +#[derive(Clone, Debug)] +pub struct FunctionReferenceAnalysis { + pub function: FunctionId, + pub reference_locals: BTreeMap, + pub call_sites: Vec, +} + +/// Analyze reference state for one function. +/// +/// Direct calls are selected only when their target is in +/// `fork_path_targets`. Indirect and reference calls are selected +/// conservatively because their runtime target is not encoded in the +/// instruction. This mirrors the transform's original-IR landing discovery. +pub fn analyze_function_references( + module: &Module, + function: FunctionId, + fork_path_targets: &HashSet, +) -> Result { + let FunctionKind::Local(local) = &module.funcs.get(function).kind else { + bail!("reference analysis requires a local function"); + }; + + let reference_locals = collect_reference_locals(module, local); + let mut cfg = StructuredCfg::build(module, local, fork_path_targets)?; + let (live_in, live_out) = compute_reference_liveness(&cfg, &reference_locals); + let nullability = compute_nullability(module, local, &cfg, &reference_locals); + annotate_stack_carryovers(module, local, &mut cfg.calls); + + let mut call_sites = Vec::with_capacity(cfg.calls.len()); + for call in cfg.calls { + let signature = module.types.get(call.kind.signature(module)); + let reference_arguments = signature + .params() + .iter() + .enumerate() + .filter_map(|(index, ty)| match ty { + ValType::Ref(ty) => Some(ReferenceOperand { index, ty: *ty }), + _ => None, + }) + .collect(); + let reference_results = signature + .results() + .iter() + .enumerate() + .filter_map(|(index, ty)| match ty { + ValType::Ref(ty) => Some(ReferenceOperand { index, ty: *ty }), + _ => None, + }) + .collect(); + + let normal_live = call + .normal_successor + .map(|node| live_in[node].clone()) + .unwrap_or_default(); + let state = nullability[call.node].clone(); + call_sites.push(ReferenceCallSite { + id: call.id, + point: call.point, + kind: call.kind, + reference_arguments, + reference_results, + has_reference_callee: call.kind.has_reference_callee(), + reference_carryovers: call.reference_carryovers, + carryover_precision: call.carryover_precision, + live_ref_locals_on_normal_return: normal_live, + live_ref_locals_on_any_successor: live_out[call.node].clone(), + local_nullability_before_call: state.clone().unwrap_or_default(), + reachable: state.is_some(), + }); + } + + Ok(FunctionReferenceAnalysis { + function, + reference_locals, + call_sites, + }) +} + +type NodeId = usize; + +#[derive(Clone)] +enum ExceptionRegion { + TryTable(Vec), + Legacy(Vec), +} + +#[derive(Clone, Copy)] +enum SequenceOwner { + Function { exit: NodeId }, + Linear { continuation: NodeId }, + Loop { continuation: NodeId }, +} + +struct CfgNode { + point: Option, + successors: Vec, + predecessors: Vec, + active_exceptions: Vec, +} + +struct PendingCall { + id: OriginalCallSiteId, + point: OriginalProgramPoint, + node: NodeId, + kind: OriginalCallKind, + normal_successor: Option, + reference_carryovers: Vec, + carryover_precision: CarryoverPrecision, +} + +struct StructuredCfg<'a> { + local: &'a LocalFunction, + nodes: Vec, + point_nodes: BTreeMap, + sequence_ends: HashMap, + owners: HashMap, + sequence_order: Vec, + function_exit: NodeId, + calls: Vec, +} + +impl<'a> StructuredCfg<'a> { + fn build( + module: &Module, + local: &'a LocalFunction, + fork_path_targets: &HashSet, + ) -> Result { + let mut cfg = Self { + local, + nodes: Vec::new(), + point_nodes: BTreeMap::new(), + sequence_ends: HashMap::new(), + owners: HashMap::new(), + sequence_order: Vec::new(), + function_exit: 0, + calls: Vec::new(), + }; + cfg.function_exit = cfg.add_node(None, Vec::new()); + let entry = local.entry_block(); + cfg.owners.insert( + entry, + SequenceOwner::Function { + exit: cfg.function_exit, + }, + ); + cfg.enumerate_sequence(module, entry, Vec::new(), fork_path_targets, &mut 0)?; + cfg.add_control_flow_edges()?; + cfg.populate_predecessors(); + Ok(cfg) + } + + fn add_node( + &mut self, + point: Option, + active_exceptions: Vec, + ) -> NodeId { + let id = self.nodes.len(); + self.nodes.push(CfgNode { + point, + successors: Vec::new(), + predecessors: Vec::new(), + active_exceptions, + }); + id + } + + fn enumerate_sequence( + &mut self, + module: &Module, + sequence: InstrSeqId, + active_exceptions: Vec, + fork_path_targets: &HashSet, + next_call_id: &mut u32, + ) -> Result<()> { + if self.sequence_ends.contains_key(&sequence) { + return Ok(()); + } + self.sequence_order.push(sequence); + let block = self.local.block(sequence); + for instruction_index in 0..block.instrs.len() { + let point = OriginalProgramPoint { + sequence, + instruction_index, + }; + let node = self.add_node(Some(point), active_exceptions.clone()); + self.point_nodes.insert(point, node); + } + let end = self.add_node(None, active_exceptions.clone()); + self.sequence_ends.insert(sequence, end); + + // Assign calls in the same parent-before-child DFS order used by the + // switch transform's original call discovery. + for (instruction_index, (instruction, _)) in block.instrs.iter().enumerate() { + let point = OriginalProgramPoint { + sequence, + instruction_index, + }; + let node = self.point_nodes[&point]; + if let Some(kind) = selected_call(instruction, fork_path_targets) { + self.calls.push(PendingCall { + id: OriginalCallSiteId(*next_call_id), + point, + node, + kind, + normal_successor: None, + reference_carryovers: Vec::new(), + carryover_precision: CarryoverPrecision::Unavailable, + }); + *next_call_id += 1; + } + + let continuation = self.next_node(sequence, instruction_index); + match instruction { + Instr::Block(block) => { + self.owners + .insert(block.seq, SequenceOwner::Linear { continuation }); + self.enumerate_sequence( + module, + block.seq, + active_exceptions.clone(), + fork_path_targets, + next_call_id, + )?; + } + Instr::Loop(loop_) => { + self.owners + .insert(loop_.seq, SequenceOwner::Loop { continuation }); + self.enumerate_sequence( + module, + loop_.seq, + active_exceptions.clone(), + fork_path_targets, + next_call_id, + )?; + } + Instr::IfElse(if_else) => { + for child in [if_else.consequent, if_else.alternative] { + self.owners + .insert(child, SequenceOwner::Linear { continuation }); + self.enumerate_sequence( + module, + child, + active_exceptions.clone(), + fork_path_targets, + next_call_id, + )?; + } + } + Instr::TryTable(try_table) => { + self.owners + .insert(try_table.seq, SequenceOwner::Linear { continuation }); + let mut nested = active_exceptions.clone(); + nested.push(ExceptionRegion::TryTable(try_table.catches.clone())); + self.enumerate_sequence( + module, + try_table.seq, + nested, + fork_path_targets, + next_call_id, + )?; + } + Instr::Try(try_) => { + self.owners + .insert(try_.seq, SequenceOwner::Linear { continuation }); + let mut nested = active_exceptions.clone(); + nested.push(ExceptionRegion::Legacy(try_.catches.clone())); + self.enumerate_sequence( + module, + try_.seq, + nested, + fork_path_targets, + next_call_id, + )?; + for catch in &try_.catches { + let handler = match catch { + LegacyCatch::Catch { handler, .. } + | LegacyCatch::CatchAll { handler } => *handler, + LegacyCatch::Delegate { .. } => continue, + }; + self.owners + .insert(handler, SequenceOwner::Linear { continuation }); + // Exceptions in a handler propagate to the enclosing + // region, not to a later clause of the same legacy try. + self.enumerate_sequence( + module, + handler, + active_exceptions.clone(), + fork_path_targets, + next_call_id, + )?; + } + } + _ => {} + } + } + Ok(()) + } + + fn next_node(&self, sequence: InstrSeqId, instruction_index: usize) -> NodeId { + let point = OriginalProgramPoint { + sequence, + instruction_index: instruction_index + 1, + }; + self.point_nodes + .get(&point) + .copied() + .unwrap_or(self.sequence_ends[&sequence]) + } + + fn sequence_entry(&self, sequence: InstrSeqId) -> NodeId { + self.point_nodes + .get(&OriginalProgramPoint { + sequence, + instruction_index: 0, + }) + .copied() + .unwrap_or(self.sequence_ends[&sequence]) + } + + fn label_target(&self, sequence: InstrSeqId) -> Result { + let Some(owner) = self.owners.get(&sequence).copied() else { + bail!("branch references an unowned instruction sequence"); + }; + Ok(match owner { + SequenceOwner::Function { exit } => exit, + SequenceOwner::Linear { continuation } => continuation, + SequenceOwner::Loop { .. } => self.sequence_entry(sequence), + }) + } + + fn normal_completion(&self, sequence: InstrSeqId) -> Result { + let Some(owner) = self.owners.get(&sequence).copied() else { + bail!("instruction sequence has no structural owner"); + }; + Ok(match owner { + SequenceOwner::Function { exit } => exit, + SequenceOwner::Linear { continuation } | SequenceOwner::Loop { continuation } => { + continuation + } + }) + } + + fn add_edge(&mut self, from: NodeId, to: NodeId) { + if !self.nodes[from].successors.contains(&to) { + self.nodes[from].successors.push(to); + } + } + + fn add_control_flow_edges(&mut self) -> Result<()> { + let sequences = self.sequence_order.clone(); + for sequence in sequences { + let end = self.sequence_ends[&sequence]; + let completion = self.normal_completion(sequence)?; + self.add_edge(end, completion); + + let instruction_count = self.local.block(sequence).instrs.len(); + for instruction_index in 0..instruction_count { + let point = OriginalProgramPoint { + sequence, + instruction_index, + }; + let node = self.point_nodes[&point]; + let next = self.next_node(sequence, instruction_index); + let instruction = &self.local.block(sequence).instrs[instruction_index].0; + + match instruction { + Instr::Block(block) => self.add_edge(node, self.sequence_entry(block.seq)), + Instr::Loop(loop_) => self.add_edge(node, self.sequence_entry(loop_.seq)), + Instr::IfElse(if_else) => { + self.add_edge(node, self.sequence_entry(if_else.consequent)); + self.add_edge(node, self.sequence_entry(if_else.alternative)); + } + Instr::TryTable(try_table) => { + self.add_edge(node, self.sequence_entry(try_table.seq)); + } + Instr::Try(try_) => self.add_edge(node, self.sequence_entry(try_.seq)), + Instr::Br(branch) => self.add_edge(node, self.label_target(branch.block)?), + Instr::BrIf(branch) => { + self.add_edge(node, self.label_target(branch.block)?); + self.add_edge(node, next); + } + Instr::BrTable(table) => { + for &target in table.blocks.iter() { + self.add_edge(node, self.label_target(target)?); + } + self.add_edge(node, self.label_target(table.default)?); + } + Instr::BrOnNull(branch) => { + self.add_edge(node, self.label_target(branch.block)?); + self.add_edge(node, next); + } + Instr::BrOnNonNull(branch) => { + self.add_edge(node, self.label_target(branch.block)?); + self.add_edge(node, next); + } + Instr::BrOnCast(branch) => { + self.add_edge(node, self.label_target(branch.block)?); + self.add_edge(node, next); + } + Instr::BrOnCastFail(branch) => { + self.add_edge(node, self.label_target(branch.block)?); + self.add_edge(node, next); + } + Instr::Call(_) | Instr::CallIndirect(_) | Instr::CallRef(_) => { + self.add_edge(node, next); + for target in self.exception_successors(node, None)? { + self.add_edge(node, target); + } + } + Instr::ReturnCall(_) + | Instr::ReturnCallIndirect(_) + | Instr::ReturnCallRef(_) + | Instr::Return(_) => self.add_edge(node, self.function_exit), + Instr::Throw(throw_) => { + for target in self.exception_successors(node, Some(throw_.tag))? { + self.add_edge(node, target); + } + } + Instr::ThrowRef(_) | Instr::Rethrow(_) => { + for target in self.exception_successors(node, None)? { + self.add_edge(node, target); + } + } + Instr::Unreachable(_) => {} + _ => self.add_edge(node, next), + } + } + } + + let normal_successors: HashMap = self + .calls + .iter() + .filter_map(|call| { + Some(( + call.node, + self.next_node(call.point.sequence, call.point.instruction_index), + )) + }) + .collect(); + for call in &mut self.calls { + call.normal_successor = normal_successors.get(&call.node).copied(); + } + Ok(()) + } + + fn exception_successors(&self, node: NodeId, tag: Option) -> Result> { + let mut targets = BTreeSet::new(); + for region in self.nodes[node].active_exceptions.iter().rev() { + match region { + ExceptionRegion::TryTable(catches) => { + let mut catches_all = false; + for catch in catches { + let (matches_tag, is_all, label) = match catch { + TryTableCatch::Catch { tag: caught, label } + | TryTableCatch::CatchRef { tag: caught, label } => { + (tag.is_none_or(|tag| tag == *caught), false, *label) + } + TryTableCatch::CatchAll { label } + | TryTableCatch::CatchAllRef { label } => (true, true, *label), + }; + if matches_tag { + targets.insert(self.label_target(label)?); + if tag.is_some() || is_all { + catches_all = true; + break; + } + } + } + if catches_all { + return Ok(targets.into_iter().collect()); + } + } + ExceptionRegion::Legacy(catches) => { + let mut catches_all = false; + for catch in catches { + match catch { + LegacyCatch::Catch { + tag: caught, + handler, + } if tag.is_none_or(|tag| tag == *caught) => { + targets.insert(self.sequence_entry(*handler)); + if tag.is_some() { + catches_all = true; + break; + } + } + LegacyCatch::CatchAll { handler } => { + targets.insert(self.sequence_entry(*handler)); + catches_all = true; + break; + } + LegacyCatch::Catch { .. } | LegacyCatch::Delegate { .. } => {} + } + } + if catches_all { + return Ok(targets.into_iter().collect()); + } + } + } + } + Ok(targets.into_iter().collect()) + } + + fn populate_predecessors(&mut self) { + for node in 0..self.nodes.len() { + let successors = self.nodes[node].successors.clone(); + for successor in successors { + self.nodes[successor].predecessors.push(node); + } + } + } +} + +fn selected_call( + instruction: &Instr, + fork_path_targets: &HashSet, +) -> Option { + match instruction { + Instr::Call(call) if fork_path_targets.contains(&call.func) => { + Some(OriginalCallKind::Direct(call.func)) + } + Instr::CallIndirect(call) => Some(OriginalCallKind::Indirect { + table: call.table, + ty: call.ty, + }), + Instr::CallRef(call) => Some(OriginalCallKind::Ref { ty: call.ty }), + _ => None, + } +} + +fn collect_reference_locals(module: &Module, local: &LocalFunction) -> BTreeMap { + struct Collector { + locals: BTreeSet, + } + + impl<'a> walrus::ir::Visitor<'a> for Collector { + fn visit_local_id(&mut self, local: &LocalId) { + self.locals.insert(*local); + } + } + + let mut collector = Collector { + locals: local.args.iter().copied().collect(), + }; + walrus::ir::dfs_in_order(&mut collector, local, local.entry_block()); + collector + .locals + .into_iter() + .filter_map(|local| match module.locals.get(local).ty() { + ValType::Ref(ty) => Some((local, ty)), + _ => None, + }) + .collect() +} + +fn compute_reference_liveness( + cfg: &StructuredCfg<'_>, + reference_locals: &BTreeMap, +) -> (Vec>, Vec>) { + let mut live_in = vec![BTreeSet::new(); cfg.nodes.len()]; + let mut live_out = vec![BTreeSet::new(); cfg.nodes.len()]; + + loop { + let mut changed = false; + for node in (0..cfg.nodes.len()).rev() { + let mut next_out = BTreeSet::new(); + for &successor in &cfg.nodes[node].successors { + next_out.extend(live_in[successor].iter().copied()); + } + let (used, defined) = cfg.nodes[node] + .point + .map(|point| local_uses_and_defs(cfg.local, point, reference_locals)) + .unwrap_or_default(); + let mut next_in = next_out.clone(); + if let Some(defined) = defined { + next_in.remove(&defined); + } + next_in.extend(used); + if next_in != live_in[node] || next_out != live_out[node] { + live_in[node] = next_in; + live_out[node] = next_out; + changed = true; + } + } + if !changed { + break; + } + } + (live_in, live_out) +} + +fn local_uses_and_defs( + local: &LocalFunction, + point: OriginalProgramPoint, + reference_locals: &BTreeMap, +) -> (BTreeSet, Option) { + let instruction = &local.block(point.sequence).instrs[point.instruction_index].0; + match instruction { + Instr::LocalGet(get) if reference_locals.contains_key(&get.local) => { + (BTreeSet::from([get.local]), None) + } + Instr::LocalSet(set) if reference_locals.contains_key(&set.local) => { + (BTreeSet::new(), Some(set.local)) + } + Instr::LocalTee(tee) if reference_locals.contains_key(&tee.local) => { + (BTreeSet::new(), Some(tee.local)) + } + _ => (BTreeSet::new(), None), + } +} + +type NullState = BTreeMap; + +fn compute_nullability( + module: &Module, + local: &LocalFunction, + cfg: &StructuredCfg<'_>, + reference_locals: &BTreeMap, +) -> Vec> { + let args: BTreeSet = local.args.iter().copied().collect(); + let initial: NullState = reference_locals + .iter() + .map(|(&local, ty)| { + let state = if !args.contains(&local) && ty.nullable { + ReferenceNullability::DefinitelyNull + } else { + ReferenceNullability::MaybeNonNull + }; + (local, state) + }) + .collect(); + let entry = cfg.sequence_entry(local.entry_block()); + let mut states = vec![None; cfg.nodes.len()]; + states[entry] = Some(initial); + let mut queue = VecDeque::from([entry]); + + while let Some(node) = queue.pop_front() { + let Some(input) = states[node].clone() else { + continue; + }; + let output = transfer_nullability(module, local, cfg, node, input, reference_locals); + for &successor in &cfg.nodes[node].successors { + let candidate = refine_nullability_edge(local, cfg, node, successor, output.clone()); + let changed = match &mut states[successor] { + Some(existing) => join_null_states(existing, &candidate), + slot @ None => { + *slot = Some(candidate); + true + } + }; + if changed { + queue.push_back(successor); + } + } + } + states +} + +fn transfer_nullability( + module: &Module, + local: &LocalFunction, + cfg: &StructuredCfg<'_>, + node: NodeId, + mut state: NullState, + reference_locals: &BTreeMap, +) -> NullState { + let Some(point) = cfg.nodes[node].point else { + return state; + }; + let instruction = &local.block(point.sequence).instrs[point.instruction_index].0; + let target = match instruction { + Instr::LocalSet(set) if reference_locals.contains_key(&set.local) => Some(set.local), + Instr::LocalTee(tee) if reference_locals.contains_key(&tee.local) => Some(tee.local), + _ => None, + }; + if let Some(target) = target { + let value = classify_reference_assignment(module, local, cfg, node, &state); + state.insert(target, value); + } + state +} + +fn classify_reference_assignment( + module: &Module, + local: &LocalFunction, + cfg: &StructuredCfg<'_>, + node: NodeId, + state: &NullState, +) -> ReferenceNullability { + let point = cfg.nodes[node].point.expect("instruction node"); + if point.instruction_index == 0 { + return ReferenceNullability::MaybeNonNull; + } + let producer_point = OriginalProgramPoint { + sequence: point.sequence, + instruction_index: point.instruction_index - 1, + }; + let producer_node = cfg.point_nodes[&producer_point]; + // A catch or branch can land on the assignment with values that did not + // come from the lexically previous instruction. In that case syntax is + // not provenance; retain the conservative state. + if cfg.nodes[node].predecessors.as_slice() != [producer_node] { + return ReferenceNullability::MaybeNonNull; + } + match &local.block(point.sequence).instrs[point.instruction_index - 1].0 { + Instr::RefNull(_) => ReferenceNullability::DefinitelyNull, + Instr::LocalGet(get) => state + .get(&get.local) + .copied() + .unwrap_or(ReferenceNullability::MaybeNonNull), + Instr::RefFunc(_) + | Instr::RefI31(_) + | Instr::StructNew(_) + | Instr::StructNewDefault(_) + | Instr::ArrayNew(_) + | Instr::ArrayNewDefault(_) + | Instr::ArrayNewFixed(_) + | Instr::ArrayNewData(_) + | Instr::ArrayNewElem(_) => ReferenceNullability::MaybeNonNull, + Instr::GlobalGet(get) if matches!(module.globals.get(get.global).ty, ValType::Ref(_)) => { + ReferenceNullability::MaybeNonNull + } + _ => ReferenceNullability::MaybeNonNull, + } +} + +fn refine_nullability_edge( + local: &LocalFunction, + cfg: &StructuredCfg<'_>, + node: NodeId, + successor: NodeId, + mut state: NullState, +) -> NullState { + let Some(point) = cfg.nodes[node].point else { + return state; + }; + let instruction = &local.block(point.sequence).instrs[point.instruction_index].0; + let preceding_local = || { + point.instruction_index.checked_sub(1).and_then(|index| { + match &local.block(point.sequence).instrs[index].0 { + Instr::LocalGet(get) => Some(get.local), + _ => None, + } + }) + }; + match instruction { + Instr::BrOnNull(branch) if successor == cfg.label_target(branch.block).ok().unwrap() => { + if let Some(local) = preceding_local() { + state.insert(local, ReferenceNullability::DefinitelyNull); + } + } + Instr::BrOnNonNull(_) + if successor == cfg.next_node(point.sequence, point.instruction_index) => + { + if let Some(local) = preceding_local() { + state.insert(local, ReferenceNullability::DefinitelyNull); + } + } + Instr::BrIf(branch) => { + if point.instruction_index >= 2 + && matches!( + local.block(point.sequence).instrs[point.instruction_index - 1].0, + Instr::RefIsNull(_) + ) + && successor == cfg.label_target(branch.block).ok().unwrap() + { + if let Instr::LocalGet(get) = + &local.block(point.sequence).instrs[point.instruction_index - 2].0 + { + state.insert(get.local, ReferenceNullability::DefinitelyNull); + } + } + } + Instr::IfElse(if_else) => { + if point.instruction_index >= 2 + && matches!( + local.block(point.sequence).instrs[point.instruction_index - 1].0, + Instr::RefIsNull(_) + ) + && successor == cfg.sequence_entry(if_else.consequent) + { + if let Instr::LocalGet(get) = + &local.block(point.sequence).instrs[point.instruction_index - 2].0 + { + state.insert(get.local, ReferenceNullability::DefinitelyNull); + } + } + } + _ => {} + } + state +} + +fn join_null_states(existing: &mut NullState, incoming: &NullState) -> bool { + let mut changed = false; + for (&local, &incoming) in incoming { + let current = existing + .get(&local) + .copied() + .unwrap_or(ReferenceNullability::MaybeNonNull); + let joined = current.join(incoming); + if joined != current { + existing.insert(local, joined); + changed = true; + } + } + changed +} + +fn annotate_stack_carryovers(module: &Module, local: &LocalFunction, calls: &mut [PendingCall]) { + let mut by_point: BTreeMap = calls + .iter() + .enumerate() + .map(|(index, call)| (call.point, index)) + .collect(); + let mut seen = HashSet::new(); + scan_sequence_stack( + module, + local, + local.entry_block(), + calls, + &mut by_point, + &mut seen, + ); +} + +fn scan_sequence_stack( + module: &Module, + local: &LocalFunction, + sequence: InstrSeqId, + calls: &mut [PendingCall], + by_point: &mut BTreeMap, + seen: &mut HashSet, +) { + if !seen.insert(sequence) { + return; + } + let mut stack = Some(sequence_params(module, local, sequence)); + for (instruction_index, (instruction, _)) in local.block(sequence).instrs.iter().enumerate() { + let point = OriginalProgramPoint { + sequence, + instruction_index, + }; + if let Some(&call_index) = by_point.get(&point) { + let call = &mut calls[call_index]; + let signature = module.types.get(call.kind.signature(module)); + let consumed = signature.params().len() + call.kind.extra_stack_operands(); + match &stack { + Some(stack) if stack.len() >= consumed => { + let carryovers = &stack[..stack.len() - consumed]; + call.reference_carryovers = carryovers + .iter() + .enumerate() + .filter_map(|(index, ty)| match ty { + Some(ValType::Ref(ty)) => Some(ReferenceOperand { index, ty: *ty }), + _ => None, + }) + .collect(); + call.carryover_precision = if carryovers.iter().any(Option::is_none) { + CarryoverPrecision::ContainsUnknownSlots + } else { + CarryoverPrecision::Exact + }; + } + _ => call.carryover_precision = CarryoverPrecision::Unavailable, + } + } + + stack = apply_stack_effect(module, local, instruction, stack); + for child in nested_sequences(instruction) { + scan_sequence_stack(module, local, child, calls, by_point, seen); + } + } +} + +fn sequence_params( + module: &Module, + local: &LocalFunction, + sequence: InstrSeqId, +) -> Vec> { + match local.block(sequence).ty { + InstrSeqType::MultiValue(ty) => module + .types + .get(ty) + .params() + .iter() + .copied() + .map(Some) + .collect(), + InstrSeqType::Simple(_) => Vec::new(), + } +} + +enum BoundedStackEffect { + Delta { + pops: usize, + pushes: Vec>, + }, + Terminator, + Unknown, +} + +fn apply_stack_effect( + module: &Module, + local: &LocalFunction, + instruction: &Instr, + stack: Option>>, +) -> Option>> { + let mut stack = stack?; + match bounded_stack_effect(module, local, instruction, &stack) { + BoundedStackEffect::Delta { pops, pushes } if stack.len() >= pops => { + stack.truncate(stack.len() - pops); + stack.extend(pushes); + Some(stack) + } + BoundedStackEffect::Delta { .. } + | BoundedStackEffect::Terminator + | BoundedStackEffect::Unknown => None, + } +} + +fn bounded_stack_effect( + module: &Module, + local: &LocalFunction, + instruction: &Instr, + pre_stack: &[Option], +) -> BoundedStackEffect { + use BoundedStackEffect::{Delta, Terminator, Unknown}; + let unknown = |pops, pushes| Delta { + pops, + pushes: vec![None; pushes], + }; + let exact = |pops, pushes: Vec| Delta { + pops, + pushes: pushes.into_iter().map(Some).collect(), + }; + match instruction { + Instr::Const(constant) => exact( + 0, + vec![match constant.value { + Value::I32(_) => ValType::I32, + Value::I64(_) => ValType::I64, + Value::F32(_) => ValType::F32, + Value::F64(_) => ValType::F64, + Value::V128(_) => ValType::V128, + }], + ), + Instr::LocalGet(get) => exact(0, vec![module.locals.get(get.local).ty()]), + Instr::LocalSet(_) | Instr::GlobalSet(_) | Instr::Drop(_) => exact(1, vec![]), + Instr::LocalTee(tee) => exact(1, vec![module.locals.get(tee.local).ty()]), + Instr::GlobalGet(get) => exact(0, vec![module.globals.get(get.global).ty]), + Instr::RefNull(null) => exact(0, vec![ValType::Ref(null.ty)]), + Instr::RefFunc(reference) => exact( + 0, + vec![ValType::Ref(RefType { + nullable: false, + heap_type: HeapType::Concrete(module.funcs.get(reference.func).ty()), + })], + ), + Instr::RefI31(_) => exact( + 1, + vec![ValType::Ref(RefType { + nullable: false, + heap_type: HeapType::Abstract(AbstractHeapType::I31), + })], + ), + Instr::RefAsNonNull(_) => { + let ty = pre_stack.last().copied().flatten().map(|ty| match ty { + ValType::Ref(mut reference) => { + reference.nullable = false; + ValType::Ref(reference) + } + scalar => scalar, + }); + Delta { + pops: 1, + pushes: vec![ty], + } + } + Instr::RefCast(cast) => exact( + 1, + vec![ValType::Ref(RefType { + nullable: cast.nullable, + heap_type: cast.heap_type, + })], + ), + Instr::AnyConvertExtern(_) => exact(1, vec![ValType::Ref(RefType::ANYREF)]), + Instr::ExternConvertAny(_) => exact(1, vec![ValType::Ref(RefType::EXTERNREF)]), + Instr::RefIsNull(_) | Instr::RefTest(_) => exact(1, vec![ValType::I32]), + Instr::RefEq(_) => exact(2, vec![ValType::I32]), + Instr::I31GetS(_) | Instr::I31GetU(_) => exact(1, vec![ValType::I32]), + Instr::StructNew(new) => exact( + module.types.get(new.ty).kind().unwrap_struct().fields.len(), + vec![concrete_non_null_ref(new.ty)], + ), + Instr::StructNewDefault(new) => exact(0, vec![concrete_non_null_ref(new.ty)]), + Instr::StructGet(get) => exact( + 1, + vec![ + module.types.get(get.ty).kind().unwrap_struct().fields[get.field as usize] + .element_type + .unpack(), + ], + ), + Instr::StructGetS(_) | Instr::StructGetU(_) => exact(1, vec![ValType::I32]), + Instr::StructSet(_) => exact(2, vec![]), + Instr::ArrayNew(new) => exact(2, vec![concrete_non_null_ref(new.ty)]), + Instr::ArrayNewDefault(new) => exact(1, vec![concrete_non_null_ref(new.ty)]), + Instr::ArrayNewFixed(new) => exact(new.len as usize, vec![concrete_non_null_ref(new.ty)]), + Instr::ArrayNewData(new) => exact(2, vec![concrete_non_null_ref(new.ty)]), + Instr::ArrayNewElem(new) => exact(2, vec![concrete_non_null_ref(new.ty)]), + Instr::ArrayGet(get) => exact( + 2, + vec![ + module + .types + .get(get.ty) + .kind() + .unwrap_array() + .field + .element_type + .unpack(), + ], + ), + Instr::ArrayGetS(_) | Instr::ArrayGetU(_) => exact(2, vec![ValType::I32]), + Instr::ArraySet(_) => exact(3, vec![]), + Instr::ArrayLen(_) => exact(1, vec![ValType::I32]), + Instr::ArrayFill(_) => exact(4, vec![]), + Instr::ArrayCopy(_) => exact(5, vec![]), + Instr::ArrayInitData(_) | Instr::ArrayInitElem(_) => exact(4, vec![]), + Instr::TableGet(get) => exact( + 1, + vec![ValType::Ref(module.tables.get(get.table).element_ty)], + ), + Instr::Call(call) => { + let signature = module.types.get(module.funcs.get(call.func).ty()); + exact(signature.params().len(), signature.results().to_vec()) + } + Instr::CallIndirect(call) => { + let signature = module.types.get(call.ty); + exact(signature.params().len() + 1, signature.results().to_vec()) + } + Instr::CallRef(call) => { + let signature = module.types.get(call.ty); + exact(signature.params().len() + 1, signature.results().to_vec()) + } + Instr::Block(block) => structured_effect(module, local, block.seq, 0), + Instr::Loop(loop_) => structured_effect(module, local, loop_.seq, 0), + Instr::IfElse(if_else) => structured_effect(module, local, if_else.consequent, 1), + Instr::TryTable(try_table) => structured_effect(module, local, try_table.seq, 0), + Instr::Try(try_) => structured_effect(module, local, try_.seq, 0), + Instr::BrIf(_) => exact(1, vec![]), + Instr::BrOnNull(_) => { + let ty = pre_stack.last().copied().flatten().map(|ty| match ty { + ValType::Ref(mut reference) => { + reference.nullable = false; + ValType::Ref(reference) + } + scalar => scalar, + }); + Delta { + pops: 1, + pushes: vec![ty], + } + } + Instr::BrOnNonNull(_) => exact(1, vec![]), + Instr::BrOnCast(cast) => exact( + 1, + vec![ValType::Ref(RefType { + nullable: cast.from_nullable, + heap_type: cast.from_heap_type, + })], + ), + Instr::BrOnCastFail(cast) => exact( + 1, + vec![ValType::Ref(RefType { + nullable: cast.to_nullable, + heap_type: cast.to_heap_type, + })], + ), + Instr::Load(load) => exact(1, vec![load_type(load.kind)]), + Instr::LoadSimd(_) => exact(1, vec![ValType::V128]), + Instr::Store(_) | Instr::TableSet(_) => exact(2, vec![]), + Instr::MemorySize(_) | Instr::TableSize(_) => exact(0, vec![ValType::I32]), + Instr::MemoryGrow(_) => exact(1, vec![ValType::I32]), + Instr::TableGrow(_) => exact(2, vec![ValType::I32]), + Instr::Binop(_) => unknown(2, 1), + Instr::Unop(unary) => exact(1, vec![unary_result_type(&unary.op)]), + Instr::Select(select) => { + let ty = select.ty.or_else(|| { + pre_stack + .get(pre_stack.len().saturating_sub(3)) + .copied() + .flatten() + }); + Delta { + pops: 3, + pushes: vec![ty], + } + } + Instr::TernOp(_) | Instr::V128Bitselect { .. } => exact(3, vec![ValType::V128]), + Instr::AtomicRmw(rmw) => exact(2, vec![atomic_type(rmw.width)]), + Instr::Cmpxchg(cmp) => exact(3, vec![atomic_type(cmp.width)]), + Instr::AtomicNotify(_) => exact(2, vec![ValType::I32]), + Instr::AtomicWait(_) => unknown(3, 1), + Instr::MemoryFill(_) + | Instr::MemoryCopy(_) + | Instr::MemoryInit(_) + | Instr::TableFill(_) + | Instr::TableInit(_) + | Instr::TableCopy(_) => exact(3, vec![]), + Instr::DataDrop(_) | Instr::ElemDrop(_) | Instr::AtomicFence(_) => exact(0, vec![]), + Instr::Return(_) + | Instr::Unreachable(_) + | Instr::Br(_) + | Instr::BrTable(_) + | Instr::ReturnCall(_) + | Instr::ReturnCallIndirect(_) + | Instr::ReturnCallRef(_) + | Instr::Throw(_) + | Instr::ThrowRef(_) + | Instr::Rethrow(_) => Terminator, + _ => Unknown, + } +} + +fn concrete_non_null_ref(ty: TypeId) -> ValType { + ValType::Ref(RefType { + nullable: false, + heap_type: HeapType::Concrete(ty), + }) +} + +fn structured_effect( + module: &Module, + local: &LocalFunction, + sequence: InstrSeqId, + extra_pops: usize, +) -> BoundedStackEffect { + let (params, results) = sequence_params_results(module, local, sequence); + BoundedStackEffect::Delta { + pops: params.len() + extra_pops, + pushes: results.into_iter().map(Some).collect(), + } +} + +fn sequence_params_results( + module: &Module, + local: &LocalFunction, + sequence: InstrSeqId, +) -> (Vec, Vec) { + match local.block(sequence).ty { + InstrSeqType::Simple(None) => (Vec::new(), Vec::new()), + InstrSeqType::Simple(Some(result)) => (Vec::new(), vec![result]), + InstrSeqType::MultiValue(ty) => { + let ty = module.types.get(ty); + (ty.params().to_vec(), ty.results().to_vec()) + } + } +} + +fn load_type(kind: LoadKind) -> ValType { + match kind { + LoadKind::I32 { .. } | LoadKind::I32_8 { .. } | LoadKind::I32_16 { .. } => ValType::I32, + LoadKind::I64 { .. } + | LoadKind::I64_8 { .. } + | LoadKind::I64_16 { .. } + | LoadKind::I64_32 { .. } => ValType::I64, + LoadKind::F32 => ValType::F32, + LoadKind::F64 => ValType::F64, + LoadKind::V128 => ValType::V128, + } +} + +fn unary_result_type(op: &UnaryOp) -> ValType { + let name = format!("{op:?}"); + if name.starts_with("I32") || name == "I64Eqz" { + ValType::I32 + } else if name.starts_with("I64") { + ValType::I64 + } else if name.starts_with("F32") { + ValType::F32 + } else if name.starts_with("F64") { + ValType::F64 + } else { + ValType::V128 + } +} + +fn atomic_type(width: AtomicWidth) -> ValType { + match width { + AtomicWidth::I64 | AtomicWidth::I64_8 | AtomicWidth::I64_16 | AtomicWidth::I64_32 => { + ValType::I64 + } + AtomicWidth::I32 | AtomicWidth::I32_8 | AtomicWidth::I32_16 => ValType::I32, + } +} + +fn nested_sequences(instruction: &Instr) -> Vec { + match instruction { + Instr::Block(block) => vec![block.seq], + Instr::Loop(loop_) => vec![loop_.seq], + Instr::IfElse(if_else) => vec![if_else.consequent, if_else.alternative], + Instr::TryTable(try_table) => vec![try_table.seq], + Instr::Try(try_) => { + let mut sequences = vec![try_.seq]; + for catch in &try_.catches { + match catch { + LegacyCatch::Catch { handler, .. } | LegacyCatch::CatchAll { handler } => { + sequences.push(*handler) + } + LegacyCatch::Delegate { .. } => {} + } + } + sequences + } + _ => Vec::new(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn module_and_functions( + wat: &str, + caller: &str, + targets: &[&str], + ) -> (Module, FunctionId, HashSet) { + let bytes = wat::parse_str(wat).expect("wat parses"); + let module = Module::from_buffer(&bytes).expect("walrus parses"); + let find = |name: &str| { + module + .funcs + .iter() + .find(|function| function.name.as_deref() == Some(name)) + .unwrap_or_else(|| panic!("function `{name}` exists")) + .id() + }; + let caller = find(caller); + let targets = targets.iter().map(|name| find(name)).collect(); + (module, caller, targets) + } + + fn only_call(analysis: &FunctionReferenceAnalysis) -> &ReferenceCallSite { + assert_eq!(analysis.call_sites.len(), 1); + &analysis.call_sites[0] + } + + #[test] + fn dash_style_cleanup_exnref_is_dead_and_null_at_fork() { + let (module, caller, targets) = module_and_functions( + r#" + (module + (import "kernel" "kernel_fork" (func $fork (result i32))) + (tag $cleanup) + (func $caller + (local $scratch (ref null exn)) + (block $caught (result exnref) + (try_table (result exnref) (catch_all_ref $caught) + call $fork + drop + ref.null exn)) + local.set $scratch + local.get $scratch + throw_ref)) + "#, + "caller", + &["fork"], + ); + let analysis = analyze_function_references(&module, caller, &targets).unwrap(); + let call = only_call(&analysis); + let (&scratch, _) = analysis.reference_locals.iter().next().unwrap(); + assert!(!call.live_ref_locals_on_normal_return.contains(&scratch)); + assert!(!call.live_ref_locals_on_any_successor.contains(&scratch)); + assert_eq!( + call.local_nullability_before_call[&scratch], + ReferenceNullability::DefinitelyNull + ); + } + + #[test] + fn reference_used_after_fork_is_live() { + let (module, caller, targets) = module_and_functions( + r#" + (module + (import "kernel" "kernel_fork" (func $fork (result i32))) + (func $caller (param $value externref) + call $fork + drop + local.get $value + drop)) + "#, + "caller", + &["fork"], + ); + let analysis = analyze_function_references(&module, caller, &targets).unwrap(); + let call = only_call(&analysis); + let (&value, _) = analysis.reference_locals.iter().next().unwrap(); + assert!(call.live_ref_locals_on_normal_return.contains(&value)); + assert_eq!( + call.local_nullability_before_call[&value], + ReferenceNullability::MaybeNonNull + ); + } + + #[test] + fn reference_arguments_and_carryovers_are_marked_separately() { + let (module, caller, targets) = module_and_functions( + r#" + (module + (func $takes_ref (param externref) (result i32) + i32.const 0) + (func $fork (result i32) + i32.const 0) + (func $caller + ref.null extern + call $takes_ref + drop + ref.null extern + call $fork + drop + drop)) + "#, + "caller", + &["takes_ref", "fork"], + ); + let analysis = analyze_function_references(&module, caller, &targets).unwrap(); + assert_eq!(analysis.call_sites.len(), 2); + assert_eq!(analysis.call_sites[0].id, OriginalCallSiteId(0)); + assert_eq!(analysis.call_sites[0].reference_arguments.len(), 1); + assert!(analysis.call_sites[0].reference_carryovers.is_empty()); + assert!(analysis.call_sites[1].reference_arguments.is_empty()); + assert_eq!(analysis.call_sites[1].reference_carryovers.len(), 1); + assert_eq!( + analysis.call_sites[1].carryover_precision, + CarryoverPrecision::Exact + ); + } + + #[test] + fn inline_struct_new_carryover_has_exact_concrete_type() { + let (module, caller, targets) = module_and_functions( + r#" + (module + (type $pair (struct (field i32) (field i32))) + (func $fork (result i32) + i32.const 0) + (func $caller + i32.const 23 + i32.const 42 + struct.new $pair + call $fork + drop + drop)) + "#, + "caller", + &["fork"], + ); + let analysis = analyze_function_references(&module, caller, &targets).unwrap(); + let call = only_call(&analysis); + assert_eq!(call.carryover_precision, CarryoverPrecision::Exact); + assert_eq!(call.reference_carryovers.len(), 1); + let HeapType::Concrete(pair) = call.reference_carryovers[0].ty.heap_type else { + panic!("struct.new must produce a concrete reference"); + }; + assert!(module.types.get(pair).kind().is_struct()); + assert!(!call.reference_carryovers[0].ty.nullable); + } + + #[test] + fn gc_reference_field_read_remains_an_exact_carryover() { + let (module, caller, targets) = module_and_functions( + r#" + (module + (type $pair (struct (field i32))) + (type $holder (struct (field (ref null $pair)))) + (func $fork (result i32) + i32.const 0) + (func $caller (param $holder (ref $holder)) + local.get $holder + struct.get $holder 0 + call $fork + drop + drop)) + "#, + "caller", + &["fork"], + ); + let analysis = analyze_function_references(&module, caller, &targets).unwrap(); + let call = only_call(&analysis); + assert_eq!(call.carryover_precision, CarryoverPrecision::Exact); + assert_eq!(call.reference_carryovers.len(), 1); + let reference = call.reference_carryovers[0].ty; + assert!(reference.nullable); + let HeapType::Concrete(pair) = reference.heap_type else { + panic!("struct.get must preserve the field's concrete reference type"); + }; + assert!(module.types.get(pair).kind().is_struct()); + } + + #[test] + fn branch_merge_and_null_test_refine_forward_state() { + let (module, caller, targets) = module_and_functions( + r#" + (module + (import "kernel" "kernel_fork" (func $fork (result i32))) + (func $caller (param $value externref) (param $condition i32) + (local $merged externref) + local.get $condition + if + local.get $value + local.set $merged + end + local.get $merged + ref.is_null + if + call $fork + drop + end)) + "#, + "caller", + &["fork"], + ); + let analysis = analyze_function_references(&module, caller, &targets).unwrap(); + let call = only_call(&analysis); + let merged = analysis + .reference_locals + .keys() + .copied() + .find(|local| { + !module + .funcs + .get(caller) + .kind + .unwrap_local() + .args + .contains(local) + }) + .unwrap(); + assert_eq!( + call.local_nullability_before_call[&merged], + ReferenceNullability::DefinitelyNull + ); + } + + #[test] + fn loop_backedge_participates_in_liveness() { + let (module, caller, targets) = module_and_functions( + r#" + (module + (import "kernel" "kernel_fork" (func $fork (result i32))) + (func $caller (param $value externref) + (loop $again + call $fork + drop + local.get $value + ref.is_null + br_if $again))) + "#, + "caller", + &["fork"], + ); + let analysis = analyze_function_references(&module, caller, &targets).unwrap(); + let call = only_call(&analysis); + let (&value, _) = analysis.reference_locals.iter().next().unwrap(); + assert!(call.live_ref_locals_on_normal_return.contains(&value)); + } + + #[test] + fn try_table_exception_edge_is_distinct_from_normal_return() { + let (module, caller, targets) = module_and_functions( + r#" + (module + (func $candidate) + (func $caller (param $value externref) + (block $done + (block $handler + (try_table (catch_all $handler) + call $candidate + br $done)) + local.get $value + drop))) + "#, + "caller", + &["candidate"], + ); + let analysis = analyze_function_references(&module, caller, &targets).unwrap(); + let call = only_call(&analysis); + let (&value, _) = analysis.reference_locals.iter().next().unwrap(); + assert!(!call.live_ref_locals_on_normal_return.contains(&value)); + assert!(call.live_ref_locals_on_any_successor.contains(&value)); + } +} diff --git a/crates/fork-instrument/src/runtime.rs b/crates/fork-instrument/src/runtime.rs index a9de7bd304..c646369f27 100644 --- a/crates/fork-instrument/src/runtime.rs +++ b/crates/fork-instrument/src/runtime.rs @@ -9,7 +9,7 @@ //! `wpk_fork_unwind_end`, `wpk_fork_rewind_begin`, //! `wpk_fork_rewind_end`, `wpk_fork_abort_begin`, //! `wpk_fork_abort_end`, and `wpk_fork_state`. -//! - In the ABI 42 linked format, three host imports that reserve, commit, and +//! - In the ABI 42+ linked format, three host imports that reserve, commit, and //! replay variable-sized frame nodes. //! //! ## Phase 4e additions: saved-globals area @@ -22,8 +22,10 @@ //! excluded: they are set explicitly by each begin function to the //! known transition values. //! -//! Ref-typed mutable globals (funcref/externref/exnref) require -//! auxiliary tables (Phase 4f); this phase skips them. +//! Mutable reference globals are deliberately absent from this scalar prefix. +//! ABI 43 emits activation-local module-state helpers that encode them into +//! the process reference graph during capture and reconstruct them in each +//! fresh child instance before continuation replay. //! //! Module-prefix layout (all offsets byte-exact; `P` is pointer width — //! 4 bytes on wasm32 and 8 on wasm64): @@ -41,7 +43,8 @@ //! chunks rather than directly after this prefix. use walrus::{ - ConstExpr, FunctionBuilder, FunctionId, GlobalId, InstrSeqBuilder, MemoryId, Module, ValType, + AbstractHeapType, ConstExpr, FunctionBuilder, FunctionId, GlobalId, HeapType, InstrSeqBuilder, + MemoryId, Module, RefType, TableId, TagId, ValType, ir::{BinaryOp, LoadKind, MemArg, StoreKind, Value}, }; @@ -70,6 +73,146 @@ pub mod names { pub const IMPORT_FRAME_RESERVE: &str = wasm_posix_shared::abi::WPK_FORK_FRAME_IMPORT_RESERVE; pub const IMPORT_FRAME_COMMIT: &str = wasm_posix_shared::abi::WPK_FORK_FRAME_IMPORT_COMMIT; pub const IMPORT_FRAME_NEXT: &str = wasm_posix_shared::abi::WPK_FORK_FRAME_IMPORT_NEXT; + pub const IMPORT_FRAME_PEEK: &str = wasm_posix_shared::abi::WPK_FORK_FRAME_IMPORT_PEEK; + pub const IMPORT_RESUME_PEEK: &str = wasm_posix_shared::abi::WPK_FORK_RESUME_IMPORT_PEEK; + pub const IMPORT_RESUME_TABLE: &str = wasm_posix_shared::abi::WPK_FORK_RESUME_IMPORT_TABLE; + pub const IMPORT_REFERENCE_VECTOR_BEGIN: &str = + wasm_posix_shared::abi::WPK_FORK_REFERENCE_IMPORT_VECTOR_BEGIN; + pub const IMPORT_REFERENCE_VECTOR_APPEND: &str = + wasm_posix_shared::abi::WPK_FORK_REFERENCE_IMPORT_VECTOR_APPEND; + pub const IMPORT_REFERENCE_VECTOR_FINISH: &str = + wasm_posix_shared::abi::WPK_FORK_REFERENCE_IMPORT_VECTOR_FINISH; + pub const IMPORT_REFERENCE_VECTOR_GET: &str = + wasm_posix_shared::abi::WPK_FORK_REFERENCE_IMPORT_VECTOR_GET; + + /// Process-owned zero-payload tag used only to transport the internal + /// unwind across arbitrary Wasm result types. + pub const IMPORT_UNWIND_TAG_MODULE: &str = "env"; + pub const IMPORT_UNWIND_TAG: &str = "__wpk_fork_unwind"; + + /// Activation-owned reference recipe codecs. Encoders return a numeric + /// recipe ID that can be stored in linear memory; decoders resolve that ID + /// against the fresh child's reconstruction arena. + pub const IMPORT_REFERENCE_CODEC_MODULE: &str = + wasm_posix_shared::abi::WPK_FORK_REFERENCE_CODEC_IMPORT_MODULE; + pub const IMPORT_REF_ENCODE_FUNCREF: &str = + wasm_posix_shared::abi::WPK_FORK_REFERENCE_IMPORT_ENCODE_FUNCREF; + pub const IMPORT_REF_DECODE_FUNCREF: &str = + wasm_posix_shared::abi::WPK_FORK_REFERENCE_IMPORT_DECODE_FUNCREF; + pub const IMPORT_REF_ENCODE_EXTERNREF: &str = + wasm_posix_shared::abi::WPK_FORK_REFERENCE_IMPORT_ENCODE_EXTERNREF; + pub const IMPORT_REF_DECODE_EXTERNREF: &str = + wasm_posix_shared::abi::WPK_FORK_REFERENCE_IMPORT_DECODE_EXTERNREF; + pub const IMPORT_REF_ENCODE_EXNREF: &str = + wasm_posix_shared::abi::WPK_FORK_REFERENCE_IMPORT_ENCODE_EXNREF; + pub const IMPORT_REF_DECODE_EXNREF: &str = + wasm_posix_shared::abi::WPK_FORK_REFERENCE_IMPORT_DECODE_EXNREF; + pub const IMPORT_REF_ENCODE_ANYREF: &str = + wasm_posix_shared::abi::WPK_FORK_REFERENCE_IMPORT_ENCODE_ANYREF; + pub const IMPORT_REF_DECODE_ANYREF: &str = + wasm_posix_shared::abi::WPK_FORK_REFERENCE_IMPORT_DECODE_ANYREF; +} + +/// The four disjoint WebAssembly reference hierarchies used by the recipe +/// provider. Concrete function references travel through `funcref`; concrete +/// struct/array references travel through `anyref` and are cast back to their +/// exact guest type after decoding. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ReferenceCodecClass { + Func, + Extern, + Exn, + Any, +} + +impl ReferenceCodecClass { + pub fn of(module: &Module, reference: RefType) -> Self { + match reference.heap_type { + HeapType::Abstract(AbstractHeapType::Func | AbstractHeapType::NoFunc) => Self::Func, + HeapType::Abstract(AbstractHeapType::Extern | AbstractHeapType::NoExtern) => { + Self::Extern + } + HeapType::Abstract(AbstractHeapType::Exn | AbstractHeapType::NoExn) => Self::Exn, + HeapType::Abstract( + AbstractHeapType::Any + | AbstractHeapType::None + | AbstractHeapType::Eq + | AbstractHeapType::Struct + | AbstractHeapType::Array + | AbstractHeapType::I31, + ) => Self::Any, + HeapType::Concrete(ty) | HeapType::Exact(ty) => { + if module.types.get(ty).is_function() { + Self::Func + } else { + Self::Any + } + } + // Walrus marks heap types non-exhaustive. New internal reference + // hierarchies must travel through the Wasm sidecar until they gain + // a more specific codec class. + _ => Self::Any, + } + } + + pub fn nullable_type(self) -> RefType { + match self { + Self::Func => RefType::FUNCREF, + Self::Extern => RefType::EXTERNREF, + Self::Exn => RefType::EXNREF, + Self::Any => RefType::ANYREF, + } + } + + pub fn encoder(self, codecs: ReferenceCodecs) -> FunctionId { + match self { + Self::Func => codecs.encode_funcref, + Self::Extern => codecs.encode_externref, + Self::Exn => codecs.encode_exnref, + Self::Any => codecs.encode_anyref, + } + } + + pub fn decoder(self, codecs: ReferenceCodecs) -> FunctionId { + match self { + Self::Func => codecs.decode_funcref, + Self::Extern => codecs.decode_externref, + Self::Exn => codecs.decode_exnref, + Self::Any => codecs.decode_anyref, + } + } +} + +/// Typed imported hooks used to turn instance-local references into +/// deterministic reconstruction recipe IDs and back again. A provider may be +/// a JavaScript host function for JS-compatible reference types or a Wasm +/// sidecar when the JS API cannot express the signature. +#[derive(Debug, Clone, Copy)] +pub struct ReferenceCodecs { + pub encode_funcref: FunctionId, + pub decode_funcref: FunctionId, + pub encode_externref: FunctionId, + pub decode_externref: FunctionId, + pub encode_exnref: FunctionId, + pub decode_exnref: FunctionId, + pub encode_anyref: FunctionId, + pub decode_anyref: FunctionId, +} + +/// Optional module-local providers for reference classes whose signatures +/// cannot cross the JavaScript API boundary. +/// +/// The default runtime imports every pair. Exact-tag exception and concrete +/// GC codecs override only the classes they own, while keeping the same +/// `ReferenceCodecs` call sites throughout frame and module-state emission. +#[derive(Debug, Clone, Copy, Default)] +pub struct ReferenceCodecOverrides { + pub funcref: Option<(FunctionId, FunctionId)>, + pub externref: Option<(FunctionId, FunctionId)>, + pub exnref: Option<(FunctionId, FunctionId)>, + pub anyref: Option<(FunctionId, FunctionId)>, + /// Clears provider-only alias/scratch roots at every completed transition. + pub cleanup: Option, } /// Metadata about a saved mutable global. @@ -103,6 +246,29 @@ pub struct Runtime { pub frame_reserve: Option, pub frame_commit: Option, pub frame_next: Option, + /// Non-consuming view of the next payload in this module activation's + /// continuation. Resume thunks use it to materialize their call operands; + /// the original function preamble consumes the same frame with + /// `frame_next`. + pub frame_peek: Option, + /// Process-wide replay router. Slot zero means the lexical callee must run; + /// nonzero slots select an activation resume thunk in `resume_table`. + pub resume_peek: Option, + pub resume_table: Option, + pub reference_vector_begin: Option, + pub reference_vector_append: Option, + pub reference_vector_finish: Option, + pub reference_vector_get: Option, + + /// Private process-owned unwind transport. Linked fork runtimes import + /// this tag so every activation can propagate unwind without fabricating + /// a value of its declared result type. + pub unwind_tag: Option, + + /// Present only in a linked fork runtime. Inert, no-seed modules must not + /// acquire host imports merely because they were passed through the + /// instrumenter. + pub reference_codecs: Option, /// Mutable scalar globals that `wpk_fork_unwind_begin` snapshots /// and `wpk_fork_rewind_begin` restores. Declaration order. @@ -162,14 +328,25 @@ fn zero_const(ptr_ty: ValType) -> ConstExpr { /// machinery in `wpk_fork_unwind_begin` / `wpk_fork_rewind_begin`. /// pub fn inject_runtime(module: &mut Module) -> Runtime { - inject_runtime_with_frame_storage(module, false) + inject_runtime_with_frame_storage(module, false, ReferenceCodecOverrides::default()) } pub fn inject_linked_runtime(module: &mut Module) -> Runtime { - inject_runtime_with_frame_storage(module, true) + inject_runtime_with_frame_storage(module, true, ReferenceCodecOverrides::default()) } -fn inject_runtime_with_frame_storage(module: &mut Module, linked_frames: bool) -> Runtime { +pub fn inject_linked_runtime_with_reference_overrides( + module: &mut Module, + overrides: ReferenceCodecOverrides, +) -> Runtime { + inject_runtime_with_frame_storage(module, true, overrides) +} + +fn inject_runtime_with_frame_storage( + module: &mut Module, + linked_frames: bool, + codec_overrides: ReferenceCodecOverrides, +) -> Runtime { let ptr_ty = ptr_type(module); let memory = module.memories.iter().next().map(|m| m.id()); @@ -190,16 +367,14 @@ fn inject_runtime_with_frame_storage(module: &mut Module, linked_frames: bool) - continue; } if matches!(g.ty, ValType::Ref(_)) { - // Ref-typed globals need auxiliary tables (Phase 4f). + // Reference globals are owned by the typed module-state helper, + // which is injected after this scalar prefix has been laid out. continue; } - if matches!(g.kind, walrus::GlobalKind::Import(_)) { - // Imported globals are host-managed and per-instance. The host - // creates a fresh `WebAssembly.Global` for each process (e.g. - // `env.__channel_base` gets the child's channel offset, not the - // parent's). Overwriting them from the parent's fork buffer - // would corrupt cross-process isolation — the child would end - // up making syscalls against the parent's channel region. + if imported_global_is_child_binding(module, g) { + // `env.__channel_base` is intentionally rebound to the child's + // syscall channel. Every other mutable scalar import has guest + // snapshot ownership and is restored below just like a local. continue; } saved_globals.push(SavedGlobal { @@ -233,19 +408,96 @@ fn inject_runtime_with_frame_storage(module: &mut Module, linked_frames: bool) - zero_const(ptr_ty), ); - let (frame_reserve, frame_commit, frame_next) = if linked_frames { + let ( + frame_reserve, + frame_commit, + frame_next, + frame_peek, + resume_peek, + resume_table, + reference_vector_begin, + reference_vector_append, + reference_vector_finish, + reference_vector_get, + unwind_tag, + reference_codecs, + ) = if linked_frames { let reserve_ty = module.types.add(&[ptr_ty], &[ptr_ty]); let commit_ty = module.types.add(&[ptr_ty], &[]); let next_ty = module.types.add(&[ptr_ty], &[ptr_ty]); + let resume_peek_ty = module.types.add(&[ValType::I32], &[ValType::I32]); + let reference_vector_begin_ty = module.types.add(&[ValType::I32], &[ValType::I32]); + let reference_vector_append_ty = module.types.add(&[ValType::I32, ValType::I32], &[]); + let reference_vector_finish_ty = module.types.add(&[ValType::I32], &[ValType::I32]); + let reference_vector_get_ty = module + .types + .add(&[ValType::I32, ValType::I32], &[ValType::I32]); + let unwind_ty = module.types.add(&[], &[]); let import_module = wasm_posix_shared::abi::WPK_FORK_FRAME_IMPORT_MODULE; let (reserve, _) = module.add_import_func(import_module, names::IMPORT_FRAME_RESERVE, reserve_ty); let (commit, _) = module.add_import_func(import_module, names::IMPORT_FRAME_COMMIT, commit_ty); let (next, _) = module.add_import_func(import_module, names::IMPORT_FRAME_NEXT, next_ty); - (Some(reserve), Some(commit), Some(next)) + let (peek, _) = module.add_import_func(import_module, names::IMPORT_FRAME_PEEK, next_ty); + let (resume_peek, _) = + module.add_import_func(import_module, names::IMPORT_RESUME_PEEK, resume_peek_ty); + let (reference_vector_begin, _) = module.add_import_func( + import_module, + names::IMPORT_REFERENCE_VECTOR_BEGIN, + reference_vector_begin_ty, + ); + let (reference_vector_append, _) = module.add_import_func( + import_module, + names::IMPORT_REFERENCE_VECTOR_APPEND, + reference_vector_append_ty, + ); + let (reference_vector_finish, _) = module.add_import_func( + import_module, + names::IMPORT_REFERENCE_VECTOR_FINISH, + reference_vector_finish_ty, + ); + let (reference_vector_get, _) = module.add_import_func( + import_module, + names::IMPORT_REFERENCE_VECTOR_GET, + reference_vector_get_ty, + ); + // WHY: this table is process-owned rather than module-owned. A replay + // edge may skip eliminated tail callers or cross a main/side-module + // boundary, so no individual module instance can be its registry. + // Slot zero remains null and is the explicit lexical-call sentinel. + let (resume_table, _) = module.add_import_table( + import_module, + names::IMPORT_RESUME_TABLE, + false, + 1, + None, + RefType::FUNCREF, + ); + let (unwind, _) = module.add_import_tag( + names::IMPORT_UNWIND_TAG_MODULE, + names::IMPORT_UNWIND_TAG, + unwind_ty, + ); + let reference_codecs = inject_reference_codecs(module, codec_overrides); + ( + Some(reserve), + Some(commit), + Some(next), + Some(peek), + Some(resume_peek), + Some(resume_table), + Some(reference_vector_begin), + Some(reference_vector_append), + Some(reference_vector_finish), + Some(reference_vector_get), + Some(unwind), + Some(reference_codecs), + ) } else { - (None, None, None) + ( + None, None, None, None, None, None, None, None, None, None, None, None, + ) }; // --- Control functions --- @@ -258,7 +510,13 @@ fn inject_runtime_with_frame_storage(module: &mut Module, linked_frames: bool) - &saved_globals, frames_start_offset, ); - let unwind_end = emit_end_fn(module, state_global, buf_global, ptr_ty); + let unwind_end = emit_end_fn( + module, + state_global, + buf_global, + ptr_ty, + codec_overrides.cleanup, + ); let rewind_begin = emit_rewind_begin( module, ptr_ty, @@ -268,7 +526,13 @@ fn inject_runtime_with_frame_storage(module: &mut Module, linked_frames: bool) - &saved_globals, STATE_REWINDING, ); - let rewind_end = emit_end_fn(module, state_global, buf_global, ptr_ty); + let rewind_end = emit_end_fn( + module, + state_global, + buf_global, + ptr_ty, + codec_overrides.cleanup, + ); let abort_begin = emit_rewind_begin( module, ptr_ty, @@ -278,7 +542,13 @@ fn inject_runtime_with_frame_storage(module: &mut Module, linked_frames: bool) - &saved_globals, STATE_ABORT_UNWINDING, ); - let abort_end = emit_end_fn(module, state_global, buf_global, ptr_ty); + let abort_end = emit_end_fn( + module, + state_global, + buf_global, + ptr_ty, + codec_overrides.cleanup, + ); let state = emit_state_fn(module, state_global); // --- Exports --- @@ -314,12 +584,93 @@ fn inject_runtime_with_frame_storage(module: &mut Module, linked_frames: bool) - frame_reserve, frame_commit, frame_next, + frame_peek, + resume_peek, + resume_table, + reference_vector_begin, + reference_vector_append, + reference_vector_finish, + reference_vector_get, + unwind_tag, + reference_codecs, saved_globals, frames_start_offset, fixed_prefix_size, } } +fn inject_reference_codecs( + module: &mut Module, + overrides: ReferenceCodecOverrides, +) -> ReferenceCodecs { + fn add_pair( + module: &mut Module, + reference: walrus::RefType, + encode_name: &str, + decode_name: &str, + ) -> (FunctionId, FunctionId) { + let reference = ValType::Ref(reference); + let encode_ty = module.types.add(&[reference], &[ValType::I32]); + let decode_ty = module.types.add(&[ValType::I32], &[reference]); + let (encode, _) = + module.add_import_func(names::IMPORT_REFERENCE_CODEC_MODULE, encode_name, encode_ty); + let (decode, _) = + module.add_import_func(names::IMPORT_REFERENCE_CODEC_MODULE, decode_name, decode_ty); + (encode, decode) + } + + let (encode_funcref, decode_funcref) = overrides.funcref.unwrap_or_else(|| { + add_pair( + module, + walrus::RefType::FUNCREF, + names::IMPORT_REF_ENCODE_FUNCREF, + names::IMPORT_REF_DECODE_FUNCREF, + ) + }); + let (encode_externref, decode_externref) = overrides.externref.unwrap_or_else(|| { + add_pair( + module, + walrus::RefType::EXTERNREF, + names::IMPORT_REF_ENCODE_EXTERNREF, + names::IMPORT_REF_DECODE_EXTERNREF, + ) + }); + let (encode_exnref, decode_exnref) = overrides.exnref.unwrap_or_else(|| { + add_pair( + module, + walrus::RefType::EXNREF, + names::IMPORT_REF_ENCODE_EXNREF, + names::IMPORT_REF_DECODE_EXNREF, + ) + }); + let (encode_anyref, decode_anyref) = overrides.anyref.unwrap_or_else(|| { + add_pair( + module, + walrus::RefType::ANYREF, + names::IMPORT_REF_ENCODE_ANYREF, + names::IMPORT_REF_DECODE_ANYREF, + ) + }); + ReferenceCodecs { + encode_funcref, + decode_funcref, + encode_externref, + decode_externref, + encode_exnref, + decode_exnref, + encode_anyref, + decode_anyref, + } +} + +fn imported_global_is_child_binding(module: &Module, global: &walrus::Global) -> bool { + let walrus::GlobalKind::Import(import_id) = global.kind else { + return false; + }; + let import = module.imports.get(import_id); + import.module == "env" && import.name == "__channel_base" +} + /// Emit `wpk_fork_unwind_begin(buf: ptr) -> ()`: /// 1. `_wpk_fork_state := UNWINDING` /// 2. `_wpk_fork_buf := buf` @@ -499,9 +850,16 @@ fn emit_end_fn( state_global: GlobalId, buf_global: GlobalId, ptr_ty: ValType, + codec_cleanup: Option, ) -> FunctionId { let mut builder = FunctionBuilder::new(&mut module.types, &[], &[]); let mut body = builder.func_body(); + if let Some(cleanup) = codec_cleanup { + // WHY: replay caches preserve alias identity only while a transition + // is active. Guest locals/globals now own every surviving reference, + // so retaining provider copies would create unbounded hidden GC roots. + body.call(cleanup); + } // WHY: a continuation mapping stops belonging to this module at every end // transition. Clear the alias to released/reusable storage before // publishing NORMAL; correctness still comes from having no NORMAL-state diff --git a/crates/fork-instrument/src/static_reference_catalog.rs b/crates/fork-instrument/src/static_reference_catalog.rs new file mode 100644 index 0000000000..b7b4221e78 --- /dev/null +++ b/crates/fork-instrument/src/static_reference_catalog.rs @@ -0,0 +1,284 @@ +//! Fresh-instance identities for statically initialized GC references. +//! +//! A continuation recipe must not structurally clone a reference that is also +//! recreated by module instantiation. Doing so would produce two objects in +//! the child and make `ref.eq` observe a fork-only identity split. +//! +//! The exported catalog is deliberately a *harvest buffer*, not permanent +//! module-instance storage. Immediately after instantiation the host calls the +//! generated harvest function, records weak object-to-ordinal mappings, and +//! clears every table entry. Immutable globals are read directly. Allocating +//! element expressions are copied one at a time from their still-live segment, +//! and a table initializer is read from its first initialized slot. Therefore +//! the pass neither evaluates an allocating expression twice nor hoists it into +//! a new immutable global that would retain a stale GC root forever. + +use std::collections::HashMap; + +use walrus::{ + AbstractHeapType, ConstExpr, ElementId, ElementItems, FunctionBuilder, GlobalId, GlobalKind, + HeapType, Module, RawCustomSection, RefType, TableId, ValType, + ir::{RefNull, TableFill, TableGet, TableInit, TableSet}, +}; + +pub const EXPORT: &str = "__wpk_fork_static_root_catalog"; +pub const HARVEST_EXPORT: &str = "__wpk_fork_static_root_harvest"; +pub const FORMAT_SECTION: &str = "kandelo.wpk_fork.static_root_catalog"; +pub const FORMAT_MAGIC: [u8; 4] = *b"KFSR"; +pub const FORMAT_VERSION: u16 = 1; +pub const FORMAT_HEADER_SIZE: u16 = 12; + +#[derive(Debug, Clone, Copy)] +enum RootSource { + Global(GlobalId), + TableFirst { table: TableId, table64: bool }, + ElementItem { element: ElementId, index: u32 }, +} + +#[derive(Debug, Default)] +pub struct StaticReferenceCatalogPlan { + roots: Vec, +} + +impl StaticReferenceCatalogPlan { + pub fn root_count(&self) -> usize { + self.roots.len() + } +} + +#[derive(Default)] +struct RootOrdinals { + roots: Vec, + by_global: HashMap, +} + +impl RootOrdinals { + fn intern_source(&mut self, source: RootSource) -> u32 { + let ordinal = u32::try_from(self.roots.len()) + .expect("static reference catalog exceeds the Wasm u32 index space"); + self.roots.push(source); + ordinal + } + + fn intern_global(&mut self, global: GlobalId) -> u32 { + if let Some(ordinal) = self.by_global.get(&global) { + return *ordinal; + } + let ordinal = self.intern_source(RootSource::Global(global)); + self.by_global.insert(global, ordinal); + ordinal + } + + fn alias(&mut self, alias: GlobalId, target: GlobalId) { + let ordinal = self.intern_global(target); + self.by_global.insert(alias, ordinal); + } +} + +/// Identify the template roots present in the source artifact. +/// +/// This must run before module-state planning. That pass converts active +/// element segments to passive segments, but preserves their IDs and +/// expressions; the harvest helper injected afterward can therefore copy the +/// exact already-instantiated object from each segment before bootstrap drops +/// it. +pub fn plan(module: &mut Module) -> StaticReferenceCatalogPlan { + let mut ordinals = RootOrdinals::default(); + + // Include immutable imports as well as locals. A local global.get alias + // folds onto its source coordinate, including a root supplied by another + // activation. + let globals: Vec<_> = module + .globals + .iter() + .filter_map(|global| { + let ValType::Ref(reference) = global.ty else { + return None; + }; + if global.mutable || !can_participate_in_ref_eq(module, reference) { + return None; + } + let source = match &global.kind { + GlobalKind::Local(ConstExpr::Global(target)) => Some(*target), + GlobalKind::Local(ConstExpr::RefNull(_) | ConstExpr::RefFunc(_)) => return None, + GlobalKind::Local(_) | GlobalKind::Import(_) => None, + }; + Some((global.id(), source)) + }) + .collect(); + for (global, source) in globals { + if let Some(target) = source { + ordinals.alias(global, target); + } else { + ordinals.intern_global(global); + } + } + + // A table declaration evaluates its initializer once and fills every + // initial slot with that one value. Reading slot zero after instantiation + // obtains the exact root without reevaluating the expression. A zero-sized + // table exposes no root and needs no identity coordinate. + let tables: Vec<_> = module + .tables + .iter() + .filter_map(|table| { + if table.import.is_some() + || table.initial == 0 + || !can_participate_in_ref_eq(module, table.element_ty) + { + return None; + } + table + .init + .as_ref() + .cloned() + .map(|initializer| (table.id(), table.table64, initializer)) + }) + .collect(); + for (table, table64, initializer) in tables { + match initializer { + ConstExpr::RefNull(_) | ConstExpr::RefFunc(_) => {} + ConstExpr::Global(global) => { + ordinals.intern_global(global); + } + _ => { + ordinals.intern_source(RootSource::TableFirst { table, table64 }); + } + } + } + + // Element expressions are instantiated once into their segment. Copy only + // allocating entries into the harvest table; global.get aliases reuse the + // global coordinate and null/function entries have other owners. + let elements: Vec<_> = module + .elements + .iter() + .filter_map(|element| { + let ElementItems::Expressions(reference, expressions) = &element.items else { + return None; + }; + if !can_participate_in_ref_eq(module, *reference) { + return None; + } + Some((element.id(), expressions.clone())) + }) + .collect(); + for (element, expressions) in elements { + for (index, initializer) in expressions.into_iter().enumerate() { + match initializer { + ConstExpr::RefNull(_) | ConstExpr::RefFunc(_) => {} + ConstExpr::Global(global) => { + ordinals.intern_global(global); + } + _ => { + ordinals.intern_source(RootSource::ElementItem { + element, + index: u32::try_from(index) + .expect("element segment exceeds the Wasm u32 index space"), + }); + } + } + } + } + + StaticReferenceCatalogPlan { + roots: ordinals.roots, + } +} + +/// Inject an initially-null fixed harvest table and its one-shot population +/// helper after guest module-state planning has completed. +pub fn inject(module: &mut Module, plan: StaticReferenceCatalogPlan) { + let count = u64::try_from(plan.roots.len()) + .expect("static reference catalog length exceeds the Wasm table index space"); + let table = module + .tables + .add_local(false, count, Some(count), RefType::ANYREF); + module.tables.get_mut(table).name = Some(EXPORT.into()); + module.exports.add(EXPORT, table); + + let mut builder = FunctionBuilder::new(&mut module.types, &[], &[]); + builder.name(HARVEST_EXPORT.into()); + { + let mut body = builder.func_body(); + if count != 0 { + // Make repeat invocation deterministic if registration failed + // after a partial host read. Successful registration clears the + // same table immediately and never calls harvest again. + body.i32_const(0) + .instr(RefNull { + ty: RefType::ANYREF, + }) + .i32_const(count as u32 as i32) + .instr(TableFill { table }); + } + for (ordinal, source) in plan.roots.into_iter().enumerate() { + let ordinal = u32::try_from(ordinal) + .expect("static reference catalog exceeds the Wasm u32 index space"); + match source { + RootSource::Global(global) => { + body.i32_const(ordinal as i32) + .global_get(global) + .instr(TableSet { table }); + } + RootSource::TableFirst { + table: source, + table64, + } => { + body.i32_const(ordinal as i32); + if table64 { + body.i64_const(0); + } else { + body.i32_const(0); + } + body.instr(TableGet { table: source }) + .instr(TableSet { table }); + } + RootSource::ElementItem { element, index } => { + body.i32_const(ordinal as i32) + .i32_const(index as i32) + .i32_const(1) + .instr(TableInit { + table, + elem: element, + }); + } + } + } + } + let harvest = builder.finish(Vec::new(), &mut module.funcs); + module.exports.add(HARVEST_EXPORT, harvest); + + let mut descriptor = Vec::with_capacity(usize::from(FORMAT_HEADER_SIZE)); + descriptor.extend_from_slice(&FORMAT_MAGIC); + descriptor.extend_from_slice(&FORMAT_VERSION.to_le_bytes()); + descriptor.extend_from_slice(&FORMAT_HEADER_SIZE.to_le_bytes()); + descriptor.extend_from_slice( + &u32::try_from(count) + .expect("static reference catalog length exceeds u32") + .to_le_bytes(), + ); + module.customs.add(RawCustomSection { + name: FORMAT_SECTION.into(), + data: descriptor, + }); +} + +fn can_participate_in_ref_eq(module: &Module, reference: RefType) -> bool { + match reference.heap_type { + HeapType::Abstract(kind) => matches!( + kind, + AbstractHeapType::Any + | AbstractHeapType::None + | AbstractHeapType::Eq + | AbstractHeapType::Struct + | AbstractHeapType::Array + | AbstractHeapType::I31 + ), + HeapType::Concrete(ty) | HeapType::Exact(ty) => { + let kind = module.types.get(ty).kind(); + kind.is_struct() || kind.is_array() + } + _ => false, + } +} diff --git a/crates/fork-instrument/tests/abort_restart_node.rs b/crates/fork-instrument/tests/abort_restart_node.rs new file mode 100644 index 0000000000..1c33ec5f7a --- /dev/null +++ b/crates/fork-instrument/tests/abort_restart_node.rs @@ -0,0 +1,175 @@ +use std::{ + fs, + process::Command, + time::{SystemTime, UNIX_EPOCH}, +}; + +use fork_instrument::{Options, instrument}; + +const MULTI_RESULT_REFERENCE_ABORT: &str = r#" + (module + (import "kernel" "kernel_fork" (func $fork (result i32))) + (memory (export "memory") 4) + + ;; The scalar below the fork result is a real operand-stack carryover. + ;; Returning externref as a second result also exercises a result-typed + ;; private-tag boundary without requiring a synthetic result local. + (func $callee (result i32 externref) + i32.const 100 + call $fork + i32.add + ref.null extern) + + (func (export "run") (result i32 externref) + call $callee)) +"#; + +#[test] +fn synchronous_frame_reserve_failure_restarts_live_activation_without_selector_local() { + let input = wat::parse_str(MULTI_RESULT_REFERENCE_ABORT).expect("parse abort fixture"); + let output = instrument(&input, &Options::default()).expect("instrument abort fixture"); + + let nonce = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("clock") + .as_nanos(); + let directory = std::env::temp_dir().join(format!( + "kandelo-fork-abort-restart-{}-{nonce}", + std::process::id(), + )); + fs::create_dir(&directory).expect("create abort fixture directory"); + fs::write(directory.join("fixture.wasm"), output).expect("write abort fixture"); + fs::write( + directory.join("test.mjs"), + r#" +import { readFileSync } from "node:fs"; + +const module = new WebAssembly.Module( + readFileSync(new URL("./fixture.wasm", import.meta.url)), +); +const imports = {}; +let instance; +let reserveCalls = 0; +let frameNextCalls = 0; +let frameCommitCalls = 0; +let forkCalls = 0; +const root = 0x10000; + +function hostCall(descriptor, args) { + if (descriptor.name === "__wpk_fork_frame_reserve") { + reserveCalls += 1; + if (instance.exports.wpk_fork_state() !== 1) { + throw new Error("frame reserve did not run during unwind"); + } + // This models the host's synchronous allocation-failure contract: replay + // cursors and ABORT_UNWINDING state are ready before zero is returned. + instance.exports.wpk_fork_abort_begin(root); + if (instance.exports.wpk_fork_state() !== 3) { + throw new Error("abort replay was not established synchronously"); + } + return 0; + } + if (descriptor.name === "__wpk_fork_frame_next") { + frameNextCalls += 1; + throw new Error("live abort restart must not consume a replay frame"); + } + if (descriptor.name === "__wpk_fork_frame_commit") { + frameCommitCalls += 1; + throw new Error("failed reservation must not commit a continuation frame"); + } + if (descriptor.name === "__wpk_fork_resume_peek") { + return 0; + } + if ( + descriptor.module === "kernel" + && descriptor.name === "kernel_fork" + ) { + forkCalls += 1; + const state = instance.exports.wpk_fork_state(); + if (state === 0) { + instance.exports.wpk_fork_unwind_begin(root); + return 0; + } + if (state === 3) { + instance.exports.wpk_fork_abort_end(); + return 7; + } + throw new Error(`kernel_fork observed unexpected state ${state}`); + } + // No reference/module-state codec import is live in this fixture. + return 0; +} + +for (const descriptor of WebAssembly.Module.imports(module)) { + const namespace = imports[descriptor.module] ??= {}; + switch (descriptor.kind) { + case "function": + namespace[descriptor.name] = (...args) => hostCall(descriptor, args); + break; + case "table": + namespace[descriptor.name] = new WebAssembly.Table({ + element: descriptor.name === "__wpk_fork_ref_gc_transit" + ? "anyref" + : "anyfunc", + initial: 64, + }); + break; + case "global": + namespace[descriptor.name] = + descriptor.name === "__wpk_fork_module_state_table_generation_addr" + ? new WebAssembly.Global({ value: "i64", mutable: false }, 0n) + : new WebAssembly.Global({ value: "i32", mutable: false }, 0); + break; + case "tag": + namespace[descriptor.name] = new WebAssembly.Tag({ parameters: [] }); + break; + default: + throw new Error( + `unexpected import ${descriptor.module}.${descriptor.name} (${descriptor.kind})`, + ); + } +} + +instance = new WebAssembly.Instance(module, imports); +const result = instance.exports.run(); +if (!Array.isArray(result) || result[0] !== 107 || result[1] !== null) { + throw new Error(`abort restart returned ${JSON.stringify(result)}`); +} +if (instance.exports.wpk_fork_state() !== 0) { + throw new Error("abort restart did not return the module to NORMAL"); +} +if (reserveCalls !== 1 || forkCalls !== 2) { + throw new Error( + `expected one failed reserve and two fork entries; got ${reserveCalls}/${forkCalls}`, + ); +} +if (frameNextCalls !== 0 || frameCommitCalls !== 0) { + throw new Error( + `failed reserve touched committed frames: next=${frameNextCalls}, commit=${frameCommitCalls}`, + ); +} + +// The helper selected the descriptor's header-sized abort scratch and wrote +// callee call-index zero there before restarting the live activation. +const callIndex = new DataView(instance.exports.memory.buffer) + .getUint32(root + 8 + 4, true); +if (callIndex !== 0) { + throw new Error(`abort scratch contains call index ${callIndex}, expected 0`); +} +"#, + ) + .expect("write Node abort test"); + + let result = Command::new("node") + .arg("--experimental-wasm-exnref") + .arg(directory.join("test.mjs")) + .output() + .expect("run Node abort test"); + let _ = fs::remove_dir_all(&directory); + assert!( + result.status.success(), + "Node synchronous-abort restart failed:\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&result.stdout), + String::from_utf8_lossy(&result.stderr), + ); +} diff --git a/crates/fork-instrument/tests/call_graph.rs b/crates/fork-instrument/tests/call_graph.rs index fefcb897b1..aa4595662c 100644 --- a/crates/fork-instrument/tests/call_graph.rs +++ b/crates/fork-instrument/tests/call_graph.rs @@ -1,10 +1,13 @@ -//! Tests for Phase 2: direct-call graph discovery. +//! Tests for semantic fork call-graph discovery. //! //! Each fixture is a small WAT module whose structure lets us assert -//! exactly which functions should be reported as reaching the -//! `kernel.kernel_fork` import through direct calls. +//! exactly which activations can survive while execution reaches the +//! `kernel.kernel_fork` import through direct, table, typed-ref, or tail edges. -use fork_instrument::{Options, analyze}; +use fork_instrument::{ + Options, analyze, + call_graph::{self, TailCallKind}, +}; use std::collections::HashSet; fn discover(wat_src: &str) -> HashSet { @@ -13,6 +16,35 @@ fn discover(wat_src: &str) -> HashSet { analysis.fork_path.iter().map(|e| e.name.clone()).collect() } +fn discover_semantic_activations(wat_src: &str) -> HashSet { + let bytes = wat::parse_str(wat_src).expect("wat parse"); + let module = walrus::Module::from_buffer(&bytes).expect("walrus parse"); + let seed = + call_graph::find_import_func(&module, "kernel.kernel_fork").expect("fork seed import"); + call_graph::analyze_reaching_closure(&module, seed) + .activations + .into_iter() + .map(|id| call_graph::func_display_name(&module, id)) + .collect() +} + +fn discover_tail_landings(wat_src: &str) -> HashSet<(String, TailCallKind)> { + let bytes = wat::parse_str(wat_src).expect("wat parse"); + let module = walrus::Module::from_buffer(&bytes).expect("walrus parse"); + let seed = + call_graph::find_import_func(&module, "kernel.kernel_fork").expect("fork seed import"); + call_graph::analyze_reaching_closure(&module, seed) + .tail_call_landings + .into_iter() + .map(|site| { + ( + call_graph::func_display_name(&module, site.caller), + site.kind, + ) + }) + .collect() +} + #[test] fn seed_alone_when_nothing_calls_fork() { // No function in the module calls $fork. The result should just be @@ -126,6 +158,30 @@ fn missing_entry_import_is_an_error() { ); } +#[test] +fn lowered_legacy_loader_is_a_boundary_without_a_direct_fork_import() { + let wat = r#" + (module + (import "env" "__wasm_dlopen" + (func $legacy (param i32 i32) (result i32))) + (memory 1) + (table (export "__indirect_function_table") 1 funcref) + (func $open_side (export "open_side") (result i32) + i32.const 100 + i32.const 20 + call $legacy)) + "#; + let found = discover(wat); + assert!( + found.contains("open_side"), + "the caller remains live while the generated driver invokes a side initializer: {found:?}", + ); + assert!( + found.contains("__wpk_fork_legacy_dlopen_driver"), + "the staged driver's external call_indirect is the suspension boundary: {found:?}", + ); +} + #[test] fn custom_entry_import_name() { // The entry import is configurable; verify. @@ -144,6 +200,97 @@ fn custom_entry_import_name() { assert!(analysis.fork_path.iter().any(|e| e.name == "a")); } +#[test] +fn duplicate_entry_import_declarations_are_all_roots() { + // A module/name pair is not a unique function identity in Wasm. Both + // declarations can be called by different live activations and therefore + // must seed the configured main-module closure. + let wat = r#" + (module + (import "kernel" "kernel_fork" (func $fork0 (result i32))) + (import "kernel" "kernel_fork" (func $fork1 (param i32) (result i32))) + (func $calls_first (export "calls_first") (result i32) + call $fork0) + (func $calls_second (export "calls_second") (result i32) + i32.const 7 + call $fork1)) + "#; + let found = discover(wat); + assert!( + found.contains("calls_first"), + "first declaration caller: {found:?}" + ); + assert!( + found.contains("calls_second"), + "second declaration caller: {found:?}" + ); + assert_eq!( + found.len(), + 4, + "both imports and both callers must be present: {found:?}" + ); +} + +#[test] +fn dylink_side_module_covers_every_cross_module_boundary() { + // Side A can remain live while an imported function in side B forks even + // though A has no env.fork import. An unresolved table/reference dispatch + // can cross the same boundary. Tail callers remain transparent, but their + // older ordinary callers still own resumable activation frames. + let wat = r#" + (module + (@custom "dylink.0" (before first) "side") + (type $ft (func (result i32))) + (import "env" "side_b" (func $side_b (type $ft))) + (table $dispatch 1 funcref) + (func $direct (export "direct") (result i32) + call $side_b) + (func $direct_parent (export "direct_parent") (result i32) + call $direct) + (func $indirect (export "indirect") (result i32) + i32.const 0 + call_indirect $dispatch (type $ft)) + (func $reference (export "reference") + (param $callee (ref null $ft)) (result i32) + local.get $callee + call_ref $ft) + (func $tail_import (export "tail_import") (result i32) + return_call $side_b) + (func $tail_parent (export "tail_parent") (result i32) + call $tail_import) + (func $unrelated (export "unrelated") (result i32) + i32.const 42)) + "#; + let bytes = wat::parse_str(wat).expect("wat parse"); + let analysis = analyze(&bytes, &Options::default()).expect("side-boundary analysis"); + let found: HashSet<_> = analysis + .fork_path + .iter() + .map(|entry| entry.name.as_str()) + .collect(); + + for expected in [ + "direct", + "direct_parent", + "indirect", + "reference", + "tail_parent", + ] { + assert!( + found.contains(expected), + "{expected} must be activation-owned above a cross-module boundary: {found:?}" + ); + } + assert!( + !found.contains("tail_import"), + "a true tail caller has no surviving activation: {found:?}" + ); + assert!( + !found.contains("unrelated"), + "a static local leaf must retain zero instrumentation overhead: {found:?}" + ); +} + #[test] fn cycle_terminates() { // $a calls $b, $b calls $a, $b calls $fork. Cycle must not loop. @@ -417,6 +564,42 @@ fn dynamic_linker_indirect_call_is_conservative_fork_boundary() { ); } +#[test] +fn dynamic_linker_host_calls_are_direct_fork_boundaries() { + // dlopen can synchronously enter a side module's deferred start and + // constructors. The main activation at the host import therefore survives + // a downstream side-module fork even when it performs no table dispatch. + let wat = r#" + (module + (import "kernel" "kernel_fork" (func $fork (result i32))) + (import "env" "__wasm_dlopen" + (func $dlopen (param i32 i32 i32 i32) (result i32))) + (func $open_side (export "open_side") (result i32) + i32.const 100 + i32.const 20 + i32.const 200 + i32.const 10 + call $dlopen) + (func $open_parent (export "open_parent") (result i32) + call $open_side) + (func $ordinary (export "ordinary") (result i32) + i32.const 7)) + "#; + let found = discover(wat); + assert!( + found.contains("open_side"), + "dlopen caller must survive: {found:?}" + ); + assert!( + found.contains("open_parent"), + "ordinary callers above dlopen must survive: {found:?}" + ); + assert!( + !found.contains("ordinary"), + "unrelated local work must stay outside the closure: {found:?}" + ); +} + #[test] fn constant_slot_pointing_to_safe_target_excludes_indirect_caller() { // Both functions have the same signature and inhabit the same table. @@ -557,8 +740,123 @@ fn dynamic_table_write_preserves_conservative_indirect_inclusion() { } #[test] -fn return_call_and_return_call_indirect_follow_reachability_rules() { - // Tail-call variants must be graph-equivalent to ordinary calls. +fn call_ref_uses_precise_ref_func_provenance_when_available() { + // Both possible targets have the same type. Immediate ref.func provenance + // proves that one call is safe and the other reaches fork, avoiding the + // all-compatible fallback for these statically named callees. + let wat = r#" + (module + (import "kernel" "kernel_fork" (func $fork (result i32))) + (type $ft (func (result i32))) + (elem declare func $safe_target $fork_target) + (func $safe_target (result i32) + i32.const 7) + (func $fork_target (result i32) + call $fork) + (func $calls_safe_ref (export "calls_safe_ref") (result i32) + ref.func $safe_target + call_ref $ft) + (func $calls_fork_ref (export "calls_fork_ref") (result i32) + ref.func $fork_target + call_ref $ft)) + "#; + let found = discover(wat); + assert!(found.contains("calls_fork_ref"), "{found:?}"); + assert!( + !found.contains("calls_safe_ref"), + "precise ref.func must not become an all-signature edge: {found:?}" + ); +} + +#[test] +fn call_ref_with_unknown_provenance_includes_all_type_compatible_targets() { + // A reference parameter can name any compatible function. Once a + // compatible target reaches fork, the caller must be in the closure. + let wat = r#" + (module + (import "kernel" "kernel_fork" (func $fork (result i32))) + (type $ft (func (result i32))) + (func $fork_target (type $ft) (result i32) + call $fork) + (func $calls_unknown_ref + (export "calls_unknown_ref") + (param (ref null $ft)) + (result i32) + local.get 0 + call_ref $ft)) + "#; + let found = discover(wat); + assert!( + found.contains("calls_unknown_ref"), + "unknown call_ref provenance must cover compatible fork targets: {found:?}" + ); +} + +#[test] +fn call_ref_unknown_provenance_honors_declared_function_subtyping() { + // call_ref $base accepts a ref to a declared subtype. Exact TypeId or + // structural-equality-only matching would miss this valid dispatch edge. + let wat = r#" + (module + (import "kernel" "kernel_fork" (func $fork (result i32))) + (type $base (sub (func (result i32)))) + (type $derived (sub $base (func (result i32)))) + (func $derived_target (type $derived) (result i32) + call $fork) + (func $calls_base_ref + (export "calls_base_ref") + (param (ref null $base)) + (result i32) + local.get 0 + call_ref $base)) + "#; + let found = discover(wat); + assert!( + found.contains("calls_base_ref"), + "declared function subtype must satisfy the call_ref edge: {found:?}" + ); +} + +#[test] +fn direct_indirect_and_call_ref_edges_share_one_fixed_point() { + // target -> call_ref caller -> call_indirect caller -> direct caller. + // Computing each edge class only once, in phases, would miss the outer + // activations after a later edge class discovers a new inner function. + let wat = r#" + (module + (import "kernel" "kernel_fork" (func $fork (result i32))) + (type $target_ty (func (result i32))) + (type $ref_caller_ty (func (result i64))) + (table 1 funcref) + (elem (i32.const 0) $ref_caller) + (elem declare func $target) + (func $target (type $target_ty) (result i32) + call $fork) + (func $ref_caller (type $ref_caller_ty) (result i64) + ref.func $target + call_ref $target_ty + drop + i64.const 1) + (func $indirect_caller (export "indirect_caller") (result i64) + i32.const 0 + call_indirect (type $ref_caller_ty)) + (func $outer (export "outer") (result i64) + call $indirect_caller)) + "#; + let found = discover(wat); + for name in ["target", "ref_caller", "indirect_caller", "outer"] { + assert!( + found.contains(name), + "mixed-edge fixed point missed {name}: {found:?}" + ); + } +} + +#[test] +fn tail_calls_are_transparent_to_the_activation_closure() { + // A true tail call replaces its caller activation. The tail caller must + // still be traversed so an older ordinary caller is found, but the + // eliminated frame itself must not be serialized. let wat = r#" (module (import "kernel" "kernel_fork" (func $fork (result i32))) @@ -567,30 +865,151 @@ fn return_call_and_return_call_indirect_follow_reachability_rules() { (elem (i32.const 0) $tail_indirect_target) (func $tail_direct (export "tail_direct") (result i32) return_call $fork) + (func $above_tail_direct (export "above_tail_direct") (result i32) + call $tail_direct) (func $tail_indirect_target (export "tail_indirect_target") (result i32) call $fork) (func $calls_tail_indirect (export "calls_tail_indirect") (result i32) i32.const 0 - return_call_indirect (type $ft))) + return_call_indirect (type $ft)) + (func $above_tail_indirect (export "above_tail_indirect") (result i32) + call $calls_tail_indirect)) "#; - let found = discover(wat); - for name in ["tail_direct", "tail_indirect_target", "calls_tail_indirect"] { + let found = discover_semantic_activations(wat); + for name in [ + "above_tail_direct", + "tail_indirect_target", + "above_tail_indirect", + ] { assert!(found.iter().any(|n| n == name), "missing {name}: {found:?}"); } + for name in ["tail_direct", "calls_tail_indirect"] { + assert!( + !found.iter().any(|n| n == name), + "tail-eliminated activation {name} must not be serialized: {found:?}" + ); + } +} + +#[test] +fn suspension_capable_tail_sites_are_reported_without_becoming_activations() { + // The activation graph stays semantically truthful. Exact tail sites + // remain useful diagnostics, but replay no longer lowers them into frames. + let wat = r#" + (module + (import "kernel" "kernel_fork" (func $fork (result i32))) + (type $ft (func (result i32))) + (table 1 funcref) + (elem (i32.const 0) $fork_target) + (elem declare func $fork_target) + (func $fork_target (result i32) + call $fork) + (func $tail_direct (export "tail_direct") (result i32) + return_call $fork) + (func $tail_indirect (export "tail_indirect") (result i32) + i32.const 0 + return_call_indirect (type $ft)) + (func $tail_ref (export "tail_ref") (result i32) + ref.func $fork_target + return_call_ref $ft)) + "#; + let found = discover_tail_landings(wat); + for expected in [ + ("tail_direct".to_string(), TailCallKind::Direct), + ("tail_indirect".to_string(), TailCallKind::Indirect), + ("tail_ref".to_string(), TailCallKind::Ref), + ] { + assert!( + found.contains(&expected), + "missing tail landing {expected:?}: {found:?}" + ); + } +} + +#[test] +fn public_analysis_does_not_report_eliminated_tail_callers() { + let wat = r#" + (module + (import "kernel" "kernel_fork" (func $fork (result i32))) + (func $tail_direct (export "tail_direct") (result i32) + return_call $fork)) + "#; + let found = discover(wat); + assert!( + !found.contains("tail_direct"), + "the public transform analysis must not invent an activation for a \ + suspension-capable tail edge: {found:?}" + ); } #[test] -fn indirect_closure_allows_two_hops_but_does_not_cascade_forever() { - // Models trampoline-shaped runtimes without allowing unbounded - // same-table callback closure: +fn semantic_control_closure_retains_tail_nodes_without_materializing_frames() { + let wat = r#" + (module + (import "kernel" "kernel_fork" (func $fork (result i32))) + (type $ft (func (result i32))) + (table 1 funcref) + (elem (i32.const 0) $fork_target) + (elem declare func $fork_target) + (func $fork_target (result i32) + call $fork) + (func $ordinary_target (result i32) + i32.const 9) + (func $ordinary_tail (export "ordinary_tail") (result i32) + return_call $ordinary_target) + (func $tail_direct (export "tail_direct") (result i32) + return_call $fork) + (func $tail_indirect (export "tail_indirect") (result i32) + i32.const 0 + return_call_indirect (type $ft)) + (func $tail_ref (export "tail_ref") (result i32) + ref.func $fork_target + return_call_ref $ft)) + "#; + let bytes = wat::parse_str(wat).expect("wat parse"); + let module = walrus::Module::from_buffer(&bytes).expect("walrus parse"); + let seed = + call_graph::find_import_func(&module, "kernel.kernel_fork").expect("fork seed import"); + let after = call_graph::analyze_reaching_closure(&module, seed); + assert!( + after.tail_call_landings.len() == 3, + "all fork-reaching tail sites remain semantic tail edges: {:?}", + after.tail_call_landings + ); + let activation_names: HashSet<_> = after + .activations + .iter() + .map(|&id| call_graph::func_display_name(&module, id)) + .collect(); + for name in ["tail_direct", "tail_indirect", "tail_ref"] { + assert!( + !activation_names.contains(name), + "tail traversal must not materialize {name} as a real activation: \ + {activation_names:?}" + ); + let function = module + .funcs + .iter() + .find(|function| function.name.as_deref() == Some(name)) + .expect("named tail function"); + assert!( + after.control_reachable.contains(&function.id()), + "{name} must remain in the semantic call-target closure" + ); + } + assert!(!activation_names.contains("ordinary_tail")); +} + +#[test] +fn indirect_closure_reaches_a_fixed_point_beyond_two_hops() { + // Models a three-dispatch trampoline chain: // // $hop1 call_indirect -> $fork_target (depth 1) // $hop2 call_indirect -> $hop1 (depth 2) - // $false_positive call_indirect -> $hop2 (depth 3; excluded) + // $third_hop call_indirect -> $hop2 (depth 3) // - // The third edge uses a dynamic index and could dispatch to $hop2. - // The unchanged depth bound is the resource-safety guard that stops - // this kind of cascade. + // Every edge is a real possible fork path. A package-specific depth cap + // would miss the third activation and corrupt replay. let wat = r#" (module (import "kernel" "kernel_fork" (func $fork (result i32))) @@ -618,18 +1037,14 @@ fn indirect_closure_allows_two_hops_but_does_not_cascade_forever() { (func $safe_hop2_target (result f32) f32.const 0) - (func $false_positive (export "false_positive") (param i32) (result f32) + (func $third_hop (export "third_hop") (param i32) (result f32) local.get 0 call_indirect (type $hop2_ty))) "#; let found = discover(wat); - for name in ["fork_target", "hop1", "hop2"] { + for name in ["fork_target", "hop1", "hop2", "third_hop"] { assert!(found.iter().any(|n| n == name), "missing {name}: {found:?}"); } - assert!( - !found.iter().any(|n| n == "false_positive"), - "indirect closure should not cascade beyond two dispatch hops; got {found:?}" - ); assert!( !found.iter().any(|n| n == "safe_hop2_target"), "safe table target should not be pulled into the fork path; got {found:?}" diff --git a/crates/fork-instrument/tests/catch_selector_lifetime_node.rs b/crates/fork-instrument/tests/catch_selector_lifetime_node.rs new file mode 100644 index 0000000000..997bd194cd --- /dev/null +++ b/crates/fork-instrument/tests/catch_selector_lifetime_node.rs @@ -0,0 +1,709 @@ +use std::{ + fs, + process::Command, + time::{SystemTime, UNIX_EPOCH}, +}; + +use fork_instrument::{Options, instrument}; + +const SEQUENTIAL_REGIONS: &str = r#" + (module + (import "kernel" "kernel_fork" (func $fork (result i32))) + (tag $a (param i32)) + (tag $b (param i32)) + (memory (export "memory") 8) + + (func (export "run") + (local $phase i32) + (local $caught i32) + (loop $again + (block $skip_a + (block $caught_a (result i32) + (try_table (catch $a $caught_a) + local.get $phase + if + i32.const 101 + throw $a + else + br $skip_a + end + unreachable) + unreachable) + local.set $caught + i32.const 4096 + call $fork + local.get $caught + i32.add + i32.store + return) + + (block $skip_b + (block $caught_b (result i32) + (try_table (catch $b $caught_b) + local.get $phase + i32.eqz + if + i32.const 202 + throw $b + else + br $skip_b + end + unreachable) + unreachable) + drop) + + i32.const 1 + local.set $phase + br $again))) +"#; + +const NESTED_REGIONS: &str = r#" + (module + (import "kernel" "kernel_fork" (func $fork (result i32))) + (tag $outer (param i32)) + (tag $inner (param i32)) + (memory (export "memory") 8) + + (func (export "run") + (local $phase i32) + (local $caught i32) + (loop $again + (block $outer_handler + (block $caught_outer (result i32) + (try_table (catch $outer $caught_outer) + i32.const 301 + throw $outer + unreachable) + unreachable) + local.set $caught + + local.get $phase + if + i32.const 4096 + call $fork + local.get $caught + i32.add + i32.store + return + end + + (block $caught_inner (result i32) + (try_table (catch $inner $caught_inner) + i32.const 302 + throw $inner + unreachable) + unreachable) + drop) + + i32.const 1 + local.set $phase + br $again))) +"#; + +const LOOP_REENTERED_ARMS: &str = r#" + (module + (import "kernel" "kernel_fork" (func $fork (result i32))) + (tag $a (param i32)) + (tag $b (param i32)) + (memory (export "memory") 8) + + (func (export "run") + (local $phase i32) + (local $caught i32) + (loop $again + (block $caught (result i32) + (try_table (catch $a $caught) (catch $b $caught) + local.get $phase + if + i32.const 401 + throw $a + else + i32.const 402 + throw $b + end + unreachable) + unreachable) + local.set $caught + + local.get $phase + if + i32.const 4096 + call $fork + local.get $caught + i32.add + i32.store + return + end + + i32.const 1 + local.set $phase + br $again))) +"#; + +const NESTED_INNER_LATEST: &str = r#" + (module + (import "kernel" "kernel_fork" (func $fork (result i32))) + (tag $outer (param i32)) + (tag $inner (param i32)) + (memory (export "memory") 8) + + (func (export "run") + (local $outer_value i32) + (local $inner_value i32) + (block $caught_outer (result i32) + (try_table (catch $outer $caught_outer) + i32.const 501 + throw $outer + unreachable) + unreachable) + local.set $outer_value + (block $caught_inner (result i32) + (try_table (catch $inner $caught_inner) + i32.const 502 + throw $inner + unreachable) + unreachable) + local.set $inner_value + i32.const 4096 + call $fork + local.get $outer_value + i32.add + local.get $inner_value + i32.add + i32.store)) +"#; + +const NESTED_RECIPE_REGIONS: &str = r#" + (module + (import "kernel" "kernel_fork" (func $fork (result i32))) + (tag $outer (param v128)) + (tag $inner (param v128)) + (memory (export "memory") 8) + + (func (export "run") + (local $outer_value i32) + (local $inner_value i32) + (block $outer_handler_scope + (block $caught_outer (result v128 exnref) + (try_table (catch_ref $outer $caught_outer) + v128.const i32x4 701 0 0 0 + throw $outer + unreachable) + unreachable) + drop + i32x4.extract_lane 0 + local.set $outer_value + + ;; This second recipe-backed catch executes in the continuation of + ;; the outer handler. Its selector and exception supersede the outer + ;; synthetic replay state, while the ordinary scalar value remains + ;; independently activation-owned. + (block $caught_inner (result v128 exnref) + (try_table (catch_ref $inner $caught_inner) + v128.const i32x4 702 0 0 0 + throw $inner + unreachable) + unreachable) + drop + i32x4.extract_lane 0 + local.set $inner_value + + i32.const 4096 + call $fork + local.get $outer_value + i32.add + local.get $inner_value + i32.add + i32.store)) + ) +"#; + +const SCALAR_SUPERSEDES_RECIPE: &str = r#" + (module + (import "kernel" "kernel_fork" (func $fork (result i32))) + (tag $recipe (param v128)) + (tag $scalar (param i32)) + (memory (export "memory") 8) + + (func (export "run") + (local $recipe_value i32) + (local $scalar_value i32) + (block $caught_recipe (result v128 exnref) + (try_table (catch_ref $recipe $caught_recipe) + v128.const i32x4 801 0 0 0 + throw $recipe + unreachable) + unreachable) + drop + i32x4.extract_lane 0 + local.set $recipe_value + + (block $caught_scalar (result i32) + (try_table (catch $scalar $caught_scalar) + i32.const 802 + throw $scalar + unreachable) + unreachable) + local.set $scalar_value + + i32.const 4096 + call $fork + local.get $recipe_value + i32.add + local.get $scalar_value + i32.add + i32.store)) +"#; + +const MERGED_NORMAL_AFTER_CATCH: &str = r#" + (module + (import "kernel" "kernel_fork" (func $fork (result i32))) + (tag $caught (param i32)) + (memory (export "memory") 8) + + (func (export "run") + (local $caught_value i32) + (block $handler (result i32) + (try_table (catch $caught $handler) + i32.const 601 + throw $caught + unreachable) + unreachable) + local.set $caught_value + + ;; Both the catch and normal predecessor have left their structured + ;; region before this ordinary merged suffix. Replay must dispatch + ;; directly to fork without executing the obsolete throw stub. + (block $merged + nop) + i32.const 4096 + call $fork + local.get $caught_value + i32.add + i32.store)) +"#; + +fn instrument_fixture(source: &str) -> Vec { + let input = wat::parse_str(source).expect("parse catch lifetime fixture"); + instrument(&input, &Options::default()).expect("instrument catch lifetime fixture") +} + +#[test] +fn fresh_replay_uses_dynamic_selector_without_reentering_obsolete_catches() { + let nonce = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("clock") + .as_nanos(); + let directory = std::env::temp_dir().join(format!( + "kandelo-catch-selector-lifetime-{}-{nonce}", + std::process::id(), + )); + fs::create_dir(&directory).expect("create fixture directory"); + for (name, source, expected_selector, expected_result, expected_recipe) in [ + ("sequential", SEQUENTIAL_REGIONS, 1, 108, -1), + ("nested", NESTED_REGIONS, 1, 308, -1), + ("reentered", LOOP_REENTERED_ARMS, 1, 408, -1), + ("inner-latest", NESTED_INNER_LATEST, 2, 1010, -1), + ("nested-recipe", NESTED_RECIPE_REGIONS, 2, 1410, 1), + ( + "scalar-supersedes-recipe", + SCALAR_SUPERSEDES_RECIPE, + 2, + 1610, + 0, + ), + ("merged-normal", MERGED_NORMAL_AFTER_CATCH, 1, 608, -1), + ] { + fs::write( + directory.join(format!("{name}.wasm")), + instrument_fixture(source), + ) + .unwrap_or_else(|error| panic!("write {name} fixture: {error}")); + fs::write( + directory.join(format!("{name}.expect")), + format!("{expected_selector} {expected_result} {expected_recipe}\n"), + ) + .unwrap_or_else(|error| panic!("write {name} expectation: {error}")); + } + fs::write( + directory.join("test.mjs"), + r#" +import { readFileSync } from "node:fs"; + +function importsFor(module, role) { + const imports = {}; + for (const descriptor of WebAssembly.Module.imports(module)) { + const namespace = imports[descriptor.module] ??= {}; + switch (descriptor.kind) { + case "table": + namespace[descriptor.name] = new WebAssembly.Table({ + element: descriptor.name === "__wpk_fork_ref_gc_transit" + ? "anyref" + : "anyfunc", + initial: 1024, + }); + break; + case "global": + if (descriptor.name === "__wpk_fork_module_state_table_generation_addr") { + namespace[descriptor.name] = new WebAssembly.Global( + { value: "i64", mutable: false }, + 0n, + ); + break; + } + namespace[descriptor.name] = new WebAssembly.Global( + { value: "i32", mutable: false }, + 0, + ); + break; + case "tag": + namespace[descriptor.name] = new WebAssembly.Tag({ parameters: [] }); + break; + case "memory": + throw new Error(`unexpected memory import ${descriptor.module}.${descriptor.name}`); + case "function": + namespace[descriptor.name] = (...args) => role.call(descriptor, args); + break; + default: + throw new Error(`unexpected import kind ${descriptor.kind}`); + } + } + return imports; +} + +function captureAndReplay( + path, + expectedSelector, + expectedResult, + expectedRecipe, +) { + const module = new WebAssembly.Module(readFileSync(path)); + const frames = []; + const exceptionRecipes = new Map(); + const referenceVectors = new Map(); + const unhandled = Symbol("unhandled reference import"); + const root = 0x10000; + let nextPayload = 0x30000; + let nextExceptionRecipe = 1; + let nextReferenceVector = 1; + + function referenceCalls(getInstance) { + const exceptionIds = new WeakMap(); + const scratch = []; + let scratchTop = 0x20000; + const memory = () => getInstance().exports.memory; + const thrownFromSlot = (slot) => { + try { + getInstance().exports.__wpk_fork_ref_exn_throw_slot(slot); + } catch (value) { + if (!(value instanceof WebAssembly.Exception)) throw value; + return value; + } + throw new Error("exception scratch slot returned"); + }; + return (descriptor, args) => { + switch (descriptor.name) { + case "__wpk_fork_ref_vector_begin": { + const id = nextReferenceVector++; + referenceVectors.set(id, { + expected: Number(args[0]), + recipes: [], + }); + return id; + } + case "__wpk_fork_ref_vector_append": { + const vector = referenceVectors.get(Number(args[0])); + if (!vector || vector.recipes.length >= vector.expected) { + throw new Error("invalid reference-vector append"); + } + vector.recipes.push(Number(args[1])); + return 0; + } + case "__wpk_fork_ref_vector_finish": { + const id = Number(args[0]); + const vector = referenceVectors.get(id); + if (!vector || vector.recipes.length !== vector.expected) { + throw new Error("incomplete reference vector"); + } + return id; + } + case "__wpk_fork_ref_vector_get": { + const vector = referenceVectors.get(Number(args[0])); + const index = Number(args[1]); + if (!vector || vector.recipes.length !== vector.expected) { + throw new Error("incomplete reference vector"); + } + if (index < 0 || index >= vector.recipes.length) { + throw new Error("reference-vector index out of range"); + } + return vector.recipes[index]; + } + case "__wpk_fork_ref_exn_lookup": + return exceptionIds.get(thrownFromSlot(Number(args[0]))) ?? 0; + case "__wpk_fork_ref_exn_claim": { + const exception = thrownFromSlot(Number(args[0])); + let id = exceptionIds.get(exception); + if (id === undefined) { + id = nextExceptionRecipe++; + exceptionIds.set(exception, id); + } + return id; + } + case "__wpk_fork_ref_exn_define": { + const [ + id, activation, tag, layout, + scalarPointer, scalarLength, refsPointer, refCount, + ] = args.map(Number); + exceptionRecipes.set(id, { + activation, + tag, + layout, + scalars: new Uint8Array( + memory().buffer, + scalarPointer, + scalarLength, + ).slice(), + refs: new Uint32Array( + memory().buffer, + refsPointer, + refCount, + ).slice(), + }); + return 0; + } + case "__wpk_fork_ref_exn_load": { + const [ + id, activation, tag, layout, + scalarPointer, scalarLength, refsPointer, refCount, + ] = args.map(Number); + const recipe = exceptionRecipes.get(id); + if ( + !recipe + || recipe.activation !== activation + || recipe.tag !== tag + || recipe.layout !== layout + || recipe.scalars.length !== scalarLength + || recipe.refs.length !== refCount + ) return 0; + new Uint8Array(memory().buffer, scalarPointer, scalarLength) + .set(recipe.scalars); + new Uint32Array(memory().buffer, refsPointer, refCount) + .set(recipe.refs); + return 1; + } + case "__wpk_fork_ref_exn_route": { + const recipe = exceptionRecipes.get(Number(args[0])); + return recipe?.activation === Number(args[1]) + ? recipe.layout + : -1; + } + case "__wpk_fork_ref_exn_cache_index": + return Number(args[0]); + case "__wpk_fork_ref_exn_broker_encode": + case "__wpk_fork_ref_exn_broker_throw_recipe": + case "__wpk_fork_ref_exn_ingress_throw": + throw new Error("known local exception unexpectedly used broker routing"); + case "__wpk_fork_ref_scratch_reserve": { + const size = Number(args[0]); + const aligned = (size + 15) & ~15; + const address = scratchTop; + scratchTop += aligned; + scratch.push({ address, size, aligned }); + new Uint8Array(memory().buffer, address, aligned).fill(0); + return address; + } + case "__wpk_fork_ref_scratch_release": { + const address = Number(args[0]); + const size = Number(args[1]); + const reservation = scratch.pop(); + if ( + !reservation + || reservation.address !== address + || reservation.size !== size + ) throw new Error("non-LIFO exception scratch release"); + new Uint8Array(memory().buffer, address, reservation.aligned).fill(0); + scratchTop = address; + return 0; + } + default: + return unhandled; + } + }; + } + + let parent; + const parentReferenceCall = referenceCalls(() => parent); + const parentRole = { + call(descriptor, args) { + if (descriptor.name === "__wpk_fork_frame_reserve") { + const payload = nextPayload; + nextPayload += (Number(args[0]) + 15) & ~15; + frames.push({ payload, size: Number(args[0]) }); + return payload; + } + if (descriptor.name === "__wpk_fork_frame_commit") return; + if (descriptor.name === "__wpk_fork_frame_next") { + throw new Error("parent capture must not enter replay"); + } + if ( + descriptor.module === "kernel" + && descriptor.name === "kernel_fork" + ) { + parent.exports.wpk_fork_unwind_begin(root); + return 0; + } + const referenceResult = parentReferenceCall(descriptor, args); + if (referenceResult !== unhandled) return referenceResult; + return 0; + } + }; + parent = new WebAssembly.Instance(module, importsFor(module, parentRole)); + try { + parent.exports.run(); + } catch (error) { + if (!(error instanceof WebAssembly.Exception)) throw error; + } + if (parent.exports.wpk_fork_state() !== 1) { + throw new Error("fixture did not unwind from fork"); + } + if (frames.length !== 1) { + throw new Error(`expected one activation frame, got ${frames.length}`); + } + const parentMemory = parent.exports.memory; + if (!(parentMemory instanceof WebAssembly.Memory)) { + throw new Error("instrumented fixture did not export its staging memory"); + } + const selector = new DataView(parentMemory.buffer) + .getUint32(frames[0].payload + 8, true); + if (selector !== expectedSelector) { + throw new Error( + `expected dynamically latest selector ${expectedSelector}, got ${selector}`, + ); + } + const vectorId = new DataView(parentMemory.buffer) + .getUint32(frames[0].payload + 12, true); + if (expectedRecipe < 0) { + if (vectorId !== 0) { + throw new Error(`unexpected reference vector ${vectorId}`); + } + } else { + const vector = referenceVectors.get(vectorId); + if ( + !vector + || vector.expected !== 1 + || vector.recipes.length !== 1 + ) { + throw new Error( + `expected one pooled exception recipe, got ${ + JSON.stringify(vector ?? null) + }`, + ); + } + const recipe = vector.recipes[0]; + if (expectedRecipe === 0 && recipe !== 0) { + throw new Error( + `scalar catch retained superseded exception recipe ${recipe}`, + ); + } + if (expectedRecipe > 0 && recipe === 0) { + throw new Error("selected recipe catch encoded a null exception"); + } + } + parent.exports.wpk_fork_unwind_end(); + + let child; + const childReferenceCall = referenceCalls(() => child); + let nextFrame = 0; + const childRole = { + call(descriptor, args) { + if (descriptor.name === "__wpk_fork_frame_next") { + const frame = frames[nextFrame++]; + if (!frame) throw new Error("child requested an unexpected frame"); + return frame.payload; + } + if (descriptor.name === "__wpk_fork_frame_reserve") { + throw new Error("child replay must not reserve a continuation frame"); + } + if (descriptor.name === "__wpk_fork_frame_commit") { + throw new Error("child replay must not commit a continuation frame"); + } + if ( + descriptor.module === "kernel" + && descriptor.name === "kernel_fork" + ) { + if (child.exports.wpk_fork_state() !== 2) { + throw new Error("child did not reach fork while replaying"); + } + child.exports.wpk_fork_rewind_end(); + return 7; + } + const referenceResult = childReferenceCall(descriptor, args); + if (referenceResult !== unhandled) return referenceResult; + // Unused codecs and module-state helpers have no state in these + // focused fixtures. + return 0; + } + }; + child = new WebAssembly.Instance(module, importsFor(module, childRole)); + const childMemory = child.exports.memory; + if (!(childMemory instanceof WebAssembly.Memory)) { + throw new Error("fresh child did not export memory"); + } + new Uint8Array(childMemory.buffer).set(new Uint8Array(parentMemory.buffer)); + child.exports.wpk_fork_rewind_begin(root); + child.exports.run(); + if (nextFrame !== frames.length) { + throw new Error(`child consumed ${nextFrame}/${frames.length} frames`); + } + const replayResult = new DataView(childMemory.buffer).getInt32(4096, true); + if (replayResult !== expectedResult) { + throw new Error( + `fresh child replay produced ${replayResult}, expected ${expectedResult}`, + ); + } +} + +for (const name of [ + "sequential", + "nested", + "reentered", + // The selector deliberately remains nonzero in these two cases. Their fork + // call is after the selected try_table has completed, so the structured + // switch dispatcher must skip that obsolete throw stub in the fresh child. + "inner-latest", + "nested-recipe", + "scalar-supersedes-recipe", + "merged-normal", +]) { + const [selector, result, recipe] = readFileSync( + new URL(`./${name}.expect`, import.meta.url), + "utf8", + ).trim().split(/\s+/).map(Number); + try { + captureAndReplay( + new URL(`./${name}.wasm`, import.meta.url), + selector, + result, + recipe, + ); + } catch (error) { + error.message = `${name}: ${error.message}`; + throw error; + } +} +"#, + ) + .expect("write Node test"); + + let output = Command::new("node") + .arg(directory.join("test.mjs")) + .output() + .expect("run Node"); + let _ = fs::remove_dir_all(&directory); + assert!( + output.status.success(), + "Node catch-selector lifetime test failed:\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + ); +} diff --git a/crates/fork-instrument/tests/contract_inventory.rs b/crates/fork-instrument/tests/contract_inventory.rs new file mode 100644 index 0000000000..13ba7eabe0 --- /dev/null +++ b/crates/fork-instrument/tests/contract_inventory.rs @@ -0,0 +1,299 @@ +use fork_instrument::contract_inventory::{ + ArtifactAbiVersion, ForkContractInventory, artifact_identity, fork_capability_section_hex, + fork_contract_inventory, linked_frame_descriptor_section_hex, +}; +use std::fs; +use std::path::PathBuf; +use std::process::Command; +use std::sync::atomic::{AtomicU64, Ordering}; + +static NEXT_TEMP_ID: AtomicU64 = AtomicU64::new(0); + +fn contract_wat(pointer: &str, memory: &str) -> String { + format!( + r#" + (module + (@custom "kandelo.wpk_fork.linked_frames" "descriptor") + (@custom "kandelo.wpk_fork.capabilities" "\01\04") + ;; Keep non-function GC and exception types ahead of ABI function + ;; types. Type indices must be resolved structurally, not by assuming + ;; every type-section entry is a function. + (type $cell (struct (field (mut i32)))) + (type $exception (func (param i32))) + (tag $exception_tag (type $exception)) + (import "kernel" "kernel_fork" (func $kernel_fork)) + (import "env" "__wpk_fork_frame_reserve" + (func $frame_reserve (param {pointer}) (result {pointer}))) + (import "env" "__wpk_fork_frame_commit" + (func $frame_commit (param {pointer}))) + (import "env" "__wpk_fork_frame_next" + (func $frame_next (param {pointer}) (result {pointer}))) + {memory} + (func (export "wpk_fork_abort_begin") (param {pointer})) + (func (export "wpk_fork_abort_end")) + (func (export "wpk_fork_rewind_begin") (param {pointer})) + (func (export "wpk_fork_rewind_end")) + (func (export "wpk_fork_state") (result i32) + i32.const 0) + (func (export "wpk_fork_unwind_begin") (param {pointer})) + (func (export "wpk_fork_unwind_end"))) + "# + ) +} + +fn parse_contract(pointer: &str, memory: &str) -> ForkContractInventory { + let bytes = wat::parse_str(contract_wat(pointer, memory)).expect("compile contract WAT"); + fork_contract_inventory(&bytes).expect("inventory contract") +} + +#[test] +fn inventories_gc_and_exception_modules_without_decoding_code_bodies() { + let inventory = parse_contract("i32", "(memory 1)"); + assert_eq!( + inventory.to_string(), + "0\t1\t1\t1\t1\t1\t1\t1\t1\t1\t1\t1\t1\t1\t1\t0\t0\t0\t0" + ); +} + +#[test] +fn imported_memory64_selects_i64_pointer_signatures() { + let inventory = parse_contract("i64", r#"(import "env" "memory" (memory i64 1))"#); + assert_eq!(inventory.memory_count, 1); + assert_eq!(inventory.memory64_count, 1); + assert_eq!(inventory.signature_mismatch, 0); +} + +#[test] +fn reports_each_signature_that_disagrees_with_memory_width() { + let bytes = wat::parse_str( + contract_wat("i64", "(memory 1)") + .replace( + r#"(func (export "wpk_fork_state") (result i32)"#, + r#"(func (export "wpk_fork_state") (result i64)"#, + ) + .replace("i32.const 0", "i64.const 0"), + ) + .expect("compile mismatched contract WAT"); + let inventory = fork_contract_inventory(&bytes).expect("inventory contract"); + + // Six pointer-bearing imports/exports use i64 in a memory32 module, and + // wpk_fork_state independently has the wrong result type. + assert_eq!(inventory.signature_mismatch, 7); +} + +#[test] +fn counts_duplicate_contract_sections_and_function_exports() { + let bytes = wat::parse_str( + contract_wat("i32", "(memory 1)") + .replace( + r#"(@custom "kandelo.wpk_fork.capabilities" "\01\04")"#, + r#"(@custom "kandelo.wpk_fork.capabilities" "\01\04") + (@custom "kandelo.wpk_fork.capabilities" "\01\04")"#, + ) + .replace( + r#"(func (export "wpk_fork_abort_end"))"#, + r#"(func (export "wpk_fork_abort_end") + (export "wpk_fork_abort_end"))"#, + ), + ) + .expect("compile duplicate contract WAT"); + let inventory = fork_contract_inventory(&bytes).expect("inventory contract"); + + assert_eq!(inventory.fork_capability, 2); + assert_eq!(inventory.abort_end, 2); + assert_eq!(inventory.signature_mismatch, 0); +} + +#[test] +fn artifact_identity_reads_abi_thunks_without_decoding_modern_helpers_as_text() { + let bytes = wat::parse_str( + r#" + (module + (type $cell (struct (field (mut i32)))) + (memory i64 1) + (func $modern_helper (result (ref null $cell)) + ref.null $cell) + (func $ctors) + (func $abi_actual (result i32) + i32.const 43) + (func (export "__abi_version") (result i32) + call $ctors + call $abi_actual)) + "#, + ) + .expect("compile artifact identity WAT"); + let identity = artifact_identity(&bytes).expect("inspect artifact identity"); + + assert_eq!(identity.relocatable, 0); + assert_eq!(identity.memory_count, 1); + assert_eq!(identity.memory64_count, 1); + assert_eq!(identity.abi_version, ArtifactAbiVersion::Present(43)); + assert_eq!(identity.imports_kernel_fork, 0); + assert_eq!(identity.has_fork_exports, 0); + assert_eq!(identity.to_string(), "0\t1\t1\tpresent\t43\t0\t0"); +} + +#[test] +fn artifact_identity_distinguishes_missing_and_invalid_abi_exports() { + let missing = wat::parse_str(r#"(module (memory 1))"#).expect("compile missing ABI WAT"); + assert_eq!( + artifact_identity(&missing) + .expect("inspect missing ABI") + .abi_version, + ArtifactAbiVersion::Missing, + ); + + let dynamic = wat::parse_str( + r#" + (module + (memory 1) + (global $version i32 (i32.const 43)) + (func (export "__abi_version") (result i32) + global.get $version)) + "#, + ) + .expect("compile dynamic ABI WAT"); + assert_eq!( + artifact_identity(&dynamic) + .expect("inspect invalid ABI") + .abi_version, + ArtifactAbiVersion::Invalid, + ); +} + +#[test] +fn cli_contract_inventory_emits_only_the_stable_tsv_row() { + let id = NEXT_TEMP_ID.fetch_add(1, Ordering::Relaxed); + let path: PathBuf = std::env::temp_dir().join(format!( + "kandelo-contract-inventory-{}-{id}.wasm", + std::process::id() + )); + fs::write( + &path, + wat::parse_str(contract_wat("i32", "(memory 1)")).expect("compile contract WAT"), + ) + .expect("write contract module"); + + let output = Command::new(env!("CARGO_BIN_EXE_wasm-fork-instrument")) + .arg("--contract-inventory") + .arg(&path) + .output() + .expect("run contract inventory CLI"); + + assert!( + output.status.success(), + "inventory CLI failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!( + String::from_utf8(output.stdout).expect("UTF-8 inventory"), + "0\t1\t1\t1\t1\t1\t1\t1\t1\t1\t1\t1\t1\t1\t1\t0\t0\t0\t0\n" + ); + assert!(output.stderr.is_empty()); + + let capability = Command::new(env!("CARGO_BIN_EXE_wasm-fork-instrument")) + .arg("--fork-capability-hex") + .arg(&path) + .output() + .expect("run capability inventory CLI"); + assert!( + capability.status.success(), + "capability CLI failed: {}", + String::from_utf8_lossy(&capability.stderr) + ); + assert_eq!( + String::from_utf8(capability.stdout).expect("UTF-8 capability"), + "1d6b616e64656c6f2e77706b5f666f726b2e6361706162696c69746965730104\n" + ); + + let identity = Command::new(env!("CARGO_BIN_EXE_wasm-fork-instrument")) + .arg("--artifact-identity") + .arg(&path) + .output() + .expect("run artifact identity CLI"); + assert!( + identity.status.success(), + "artifact identity CLI failed: {}", + String::from_utf8_lossy(&identity.stderr) + ); + assert_eq!( + String::from_utf8(identity.stdout).expect("UTF-8 artifact identity"), + "0\t1\t0\tmissing\t-\t1\t1\n" + ); + + let descriptor = Command::new(env!("CARGO_BIN_EXE_wasm-fork-instrument")) + .arg("--linked-frame-descriptor-hex") + .arg(&path) + .output() + .expect("run descriptor inventory CLI"); + let _ = fs::remove_file(path); + assert!( + descriptor.status.success(), + "descriptor CLI failed: {}", + String::from_utf8_lossy(&descriptor.stderr) + ); + assert_eq!( + String::from_utf8(descriptor.stdout).expect("UTF-8 descriptor"), + concat!( + "1e6b616e64656c6f2e77706b5f666f726b2e6c696e6b65645f6672616d6573", + "64657363726970746f72\n", + ) + ); +} + +#[test] +fn inventories_the_reentrant_legacy_loader_import() { + let bytes = wat::parse_str( + contract_wat("i32", "(memory 1)").replace( + r#"(import "kernel" "kernel_fork" (func $kernel_fork))"#, + r#"(import "kernel" "kernel_fork" (func $kernel_fork)) + (import "env" "__wasm_dlopen" + (func (param i32 i32 i32 i32 i32) (result i32)))"#, + ), + ) + .expect("compile legacy loader inventory WAT"); + let inventory = fork_contract_inventory(&bytes).expect("inventory legacy loader"); + assert_eq!(inventory.legacy_dlopen, 1); +} + +#[test] +fn inventories_a_native_start_section() { + let bytes = wat::parse_str( + contract_wat("i32", "(memory 1)").replace( + r#"(func (export "wpk_fork_abort_end"))"#, + r#"(func $native_start (export "wpk_fork_abort_end")) + (start $native_start)"#, + ), + ) + .expect("compile native-start inventory WAT"); + let inventory = fork_contract_inventory(&bytes).expect("inventory native start"); + assert_eq!(inventory.native_start, 1); +} + +#[test] +fn custom_section_modes_require_one_exact_section() { + let complete = wat::parse_str(contract_wat("i32", "(memory 1)")).expect("compile contract WAT"); + assert_eq!( + fork_capability_section_hex(&complete).expect("capability hex"), + "1d6b616e64656c6f2e77706b5f666f726b2e6361706162696c69746965730104" + ); + assert_eq!( + linked_frame_descriptor_section_hex(&complete).expect("descriptor hex"), + concat!( + "1e6b616e64656c6f2e77706b5f666f726b2e6c696e6b65645f6672616d6573", + "64657363726970746f72", + ) + ); + + let duplicate = wat::parse_str(contract_wat("i32", "(memory 1)").replace( + r#"(@custom "kandelo.wpk_fork.capabilities" "\01\04")"#, + r#"(@custom "kandelo.wpk_fork.capabilities" "\01\04") + (@custom "kandelo.wpk_fork.capabilities" "\01\04")"#, + )) + .expect("compile duplicate capability WAT"); + assert!(fork_capability_section_hex(&duplicate).is_err()); + + let missing = wat::parse_str(r#"(module (memory 1))"#).expect("compile missing sections WAT"); + assert!(fork_capability_section_hex(&missing).is_err()); + assert!(linked_frame_descriptor_section_hex(&missing).is_err()); +} diff --git a/crates/fork-instrument/tests/coverage_wat.rs b/crates/fork-instrument/tests/coverage_wat.rs index 4d4d45d9da..d513033c68 100644 --- a/crates/fork-instrument/tests/coverage_wat.rs +++ b/crates/fork-instrument/tests/coverage_wat.rs @@ -1,16 +1,15 @@ //! WAT-fixture coverage for fork-instrument patterns that don't have a //! direct C/C++ source surface: //! -//! - **S-04..S-07**: side-effect operations (table.fill, table.copy, -//! table.grow, non-nullable funcref Call result) before fork. -//! Switch-dispatch's body-skip-on-REWIND construction means these -//! ops run exactly once on NORMAL; the test verifies fork-instrument -//! produces validating wasm for these shapes. -//! - **F-03/F-04**: wasm-GC accepted limits. fork-instrument must -//! panic with a clear error rather than silently miscompile. -//! - **C-08/C-09**: ref-typed catch operands. Currently A4 -//! territory — fork-instrument either supports via aux-table -//! spilling (future) or panics with a clear error today. +//! - **S-04..S-06**: mutable table operations are captured by the +//! module-state owner and remain valid across fresh-instance replay. +//! - **S-07**: a dead reference-typed call result needs no continuation +//! recipe and remains accepted. +//! - **F-03/F-04**: abstract and concrete wasm-GC references are encoded as +//! activation-owned recipe IDs and reconstructed through the generated +//! anyref codec. +//! - **C-08/C-09**: ref-typed catch operands are captured as complete +//! exception recipes and reconstructed with `throw_ref`. //! //! These complement `host/test/fork-instrument-coverage.test.ts` //! by covering patterns whose validation can be done at the @@ -18,47 +17,23 @@ use fork_instrument::{Options, instrument}; -fn assert_instruments_and_validates(wat: &str, label: &str) { +fn instrument_and_validate(wat: &str, label: &str) -> Vec { let input = wat::parse_str(wat).unwrap_or_else(|e| panic!("{label}: wat parse: {e}")); let output = instrument(&input, &Options::default()) - .unwrap_or_else(|e| panic!("{label}: instrument: {e}")); - let mut validator = - wasmparser::Validator::new_with_features(wasmparser::WasmFeatures::default()); - validator + .unwrap_or_else(|e| panic!("{label}: fork-instrument rejected supported wasm: {e:#}")); + wasmparser::Validator::new_with_features(wasmparser::WasmFeatures::all()) .validate_all(&output) - .unwrap_or_else(|e| panic!("{label}: wasmparser validation: {e}")); -} - -fn assert_instrument_rejects(wat: &str, label: &str, expected: &[&str]) { - let input = wat::parse_str(wat).unwrap_or_else(|e| panic!("{label}: wat parse: {e}")); - let result = std::panic::catch_unwind(|| instrument(&input, &Options::default())); - let msg = match result { - Ok(Ok(_)) => panic!("{label}: fork-instrument unexpectedly accepted accepted-limit wasm"), - Ok(Err(e)) => e.to_string(), - Err(p) => p - .downcast::() - .map(|s| *s) - .or_else(|p| p.downcast::<&'static str>().map(|s| (*s).to_string())) - .unwrap_or_else(|_| "".into()), - }; - for needle in expected { - assert!( - msg.contains(needle), - "{label}: rejection diagnostic did not contain `{needle}`; got: {msg}", - ); - } + .unwrap_or_else(|e| panic!("{label}: instrumented wasm did not validate: {e}")); + output } // --------------------------------------------------------------------- -// S-04..S-07: side effects before fork +// S-04..S-07: state before fork // --------------------------------------------------------------------- // -// Switch-dispatch's body-skip-on-REWIND construction (sub-commits -// 2.4c/2.5c/2.6c) means the body chunks BEFORE the chosen POST_K -// never re-execute on REWIND — non-fork-path calls and side-effect -// ops in those chunks run exactly once on NORMAL. These tests -// verify fork-instrument produces validating wasm for each -// side-effect op pattern. +// The child starts with a freshly instantiated table. The module-state +// transaction must therefore capture each mutation rather than relying on the +// child instance's static element initialization. #[test] fn s_04_table_fill_before_fork() { @@ -77,7 +52,7 @@ fn s_04_table_fill_before_fork() { (drop (call $fork)) (i32.const 0))) "#; - assert_instruments_and_validates(wat, "S-04 table.fill"); + instrument_and_validate(wat, "S-04 table.fill"); } #[test] @@ -97,7 +72,7 @@ fn s_05_table_copy_before_fork() { (drop (call $fork)) (i32.const 0))) "#; - assert_instruments_and_validates(wat, "S-05 table.copy"); + instrument_and_validate(wat, "S-05 table.copy"); } #[test] @@ -117,28 +92,13 @@ fn s_06_table_grow_before_fork() { (drop (call $fork)) (i32.const 0))) "#; - assert_instruments_and_validates(wat, "S-06 table.grow"); + instrument_and_validate(wat, "S-06 table.grow"); } #[test] fn s_07_non_nullable_funcref_call_result_before_fork() { - // S-07 originally targets the case where a direct call returns - // a non-nullable Ref and that result is consumed AFTER fork - // (the result would need to be saved across the fork boundary, - // but ref-typed values can't be stored in scalar frame slots). - // - // For switch-dispatch's body-skip path, the call's result lives - // in the chunk BEFORE the fork; on REWIND that chunk is skipped - // and the result is never produced. As long as no instruction - // between the call and fork consumes the ref, this validates. - // - // Today fork-instrument REJECTS fork-path functions with ref- - // typed argument types via a panic ("ref-typed argument - // ... needs aux-table spilling, which the MVP switch-dispatch - // transform does not yet support"). To exercise this case - // cleanly we use a non-nullable funcref CALLED-RESULT (not - // arg). The non-nullable funcref result of a direct call is - // dropped immediately so no spilling is needed. + // The original-IR liveness pass proves the result is dropped before fork, + // so this shape must not pay for or be rejected by reference replay. let wat = r#" (module (import "kernel" "kernel_fork" (func $fork (result i32))) @@ -157,44 +117,50 @@ fn s_07_non_nullable_funcref_call_result_before_fork() { (drop (call $fork)) (i32.const 0))) "#; - assert_instruments_and_validates(wat, "S-07 non-nullable funcref call result"); + let input = wat::parse_str(wat).unwrap(); + let output = instrument(&input, &Options::default()) + .expect("dead funcref result should remain instrumentable"); + wasmparser::Validator::new_with_features(wasmparser::WasmFeatures::all()) + .validate_all(&output) + .expect("instrumented dead-funcref fixture should validate"); } // --------------------------------------------------------------------- -// F-03 / F-04: wasm-GC accepted limits — must panic loudly +// F-03 / F-04: wasm-GC activation state uses generated codecs // --------------------------------------------------------------------- // -// Per docs/fork-instrumentation.md §Not guaranteed, abstract and -// concrete wasm-GC reference types on the fork path are explicitly -// out of scope. fork-instrument must reject them at the -// `classify_ref` step rather than silently miscompile. +// JavaScript cannot directly implement anyref-typed imports. The artifact +// therefore calls the generated Wasm codec, which converts each live reference +// to an activation-owned scalar recipe ID and narrows the decoded anyref back +// to the statically expected type in the fresh child. #[test] -fn f_03_anyref_on_fork_path_rejects_with_diagnostic() { - // Use anyref as a function-local on a fork-path function. +fn f_03_anyref_on_fork_path_uses_generated_codec() { + // A non-null i31 value widened to anyref is live across fork, so this + // exercises the codec rather than the definitely-null fast path. let wat = r#" (module (import "kernel" "kernel_fork" (func $fork (result i32))) (memory 1) (func $main (export "_start") (result i32) (local $r anyref) - ref.null any + i32.const 17 + ref.i31 local.set $r (drop (call $fork)) local.get $r drop (i32.const 0))) "#; - assert_instrument_rejects(wat, "F-03 anyref", &["fork-instrument 4f", "not yet supported"]); + instrument_and_validate(wat, "F-03 anyref"); } #[test] -fn f_04_struct_ref_on_fork_path_rejects_with_diagnostic() { - // wasm-GC struct.new isn't produced by our LLVM toolchain, - // but concrete GC references on a fork-path must not silently - // miscompile. A local of `(ref null $pair)` is enough to exercise - // the same accepted-limit rejection path that a `struct.new` - // producer would need before its value could survive fork. +fn f_04_struct_ref_on_fork_path_uses_generated_codec() { + // Concrete GC references encode through the broad anyref codec and are + // ref.cast back to `$pair` on replay. Keep an inline allocation live across + // fork so the original-IR stack analysis must preserve the producer's + // precise concrete type rather than relying on a typed helper call. let wat = r#" (module (import "kernel" "kernel_fork" (func $fork (result i32))) @@ -202,83 +168,71 @@ fn f_04_struct_ref_on_fork_path_rejects_with_diagnostic() { (type $pair (struct (field i32) (field i32))) (func $main (export "_start") (result i32) (local $r (ref null $pair)) - ref.null $pair + i32.const 23 + i32.const 42 + struct.new $pair local.set $r (drop (call $fork)) local.get $r drop (i32.const 0))) "#; - assert_instrument_rejects(wat, "F-04 struct ref", &["fork-instrument 4f", "not yet supported"]); + instrument_and_validate(wat, "F-04 struct ref"); } // --------------------------------------------------------------------- -// C-08 / C-09: ref-typed catch operands (A4 territory) +// C-08 / C-09: ref-typed catch operands // --------------------------------------------------------------------- // -// Per the unsupported-cases review doc, a function whose plain-catch -// arms carry ref-typed operands (funcref / externref) is excluded from -// plain-catch replay support via `PlainCatchPlan::b2_carveout`. The -// function can still be instrumented for other fork sites; a fork reached -// from the affected handler remains explicitly unsupported. A future A4 -// implementation would extend per-arm auxiliary storage to support these -// operands. The current coverage proves the tool accepts the Wasm shape -// without trying to serialize references as scalars. +// A reference-bearing tag payload cannot be serialized independently without +// losing exception identity. The transformed catch therefore captures the +// complete exception as an exnref recipe and replay reconstructs that exception +// before `throw_ref` re-enters the original clause. #[test] -fn c_08_funcref_catch_operand_does_not_panic() { - // Try_table with a `catch` clause whose tag has a funcref - // operand. Since the wat crate may not parse arbitrary tag - // signatures with ref types, this test gracefully skips on - // parse failure. +fn c_08_funcref_catch_operand_uses_exception_recipe() { let wat = r#" (module (import "kernel" "kernel_fork" (func $fork (result i32))) (memory 1) (tag $func_tag (param funcref)) + (func $target) + (elem declare func $target) (func $main (export "_start") (result i32) + (local $caught funcref) (block $h (result funcref) (try_table (result funcref) (catch $func_tag $h) - ref.null func)) - drop + ref.func $target + throw $func_tag + unreachable)) + local.set $caught (drop (call $fork)) + local.get $caught + drop (i32.const 0))) "#; - match wat::parse_str(wat) { - Ok(input) => { - // Should NOT panic. The instrumenter excludes this function - // from plain-catch replay without serializing the reference. - let _ = instrument(&input, &Options::default()) - .expect("fork-instrument should not error on funcref catch arm"); - } - Err(e) => { - eprintln!("skip: wat crate did not parse funcref tag: {e}"); - } - } + instrument_and_validate(wat, "C-08 funcref catch payload"); } #[test] -fn c_09_externref_catch_operand_does_not_panic() { +fn c_09_externref_catch_operand_uses_exception_recipe() { let wat = r#" (module (import "kernel" "kernel_fork" (func $fork (result i32))) (memory 1) (tag $ext_tag (param externref)) (func $main (export "_start") (result i32) + (local $caught externref) (block $h (result externref) (try_table (result externref) (catch $ext_tag $h) - ref.null extern)) - drop + ref.null extern + throw $ext_tag + unreachable)) + local.set $caught (drop (call $fork)) + local.get $caught + drop (i32.const 0))) "#; - match wat::parse_str(wat) { - Ok(input) => { - let _ = instrument(&input, &Options::default()) - .expect("fork-instrument should not error on externref catch arm"); - } - Err(e) => { - eprintln!("skip: wat crate did not parse externref tag: {e}"); - } - } + instrument_and_validate(wat, "C-09 externref catch payload"); } diff --git a/crates/fork-instrument/tests/determinism.rs b/crates/fork-instrument/tests/determinism.rs index 3e5f3d8c4a..afc2930062 100644 --- a/crates/fork-instrument/tests/determinism.rs +++ b/crates/fork-instrument/tests/determinism.rs @@ -71,7 +71,10 @@ fn cli_output_is_byte_reproducible_across_processes() { ); } } else { - assert_ne!(bytes, input, "instrumentation unexpectedly changed no bytes"); + assert_ne!( + bytes, input, + "instrumentation unexpectedly changed no bytes" + ); Validator::new() .validate_all(&bytes) .expect("instrumented baseline validates"); diff --git a/crates/fork-instrument/tests/dispatch_tree.rs b/crates/fork-instrument/tests/dispatch_tree.rs index d658b19d22..adeb25bced 100644 --- a/crates/fork-instrument/tests/dispatch_tree.rs +++ b/crates/fork-instrument/tests/dispatch_tree.rs @@ -144,7 +144,13 @@ fn build_dispatch_tree_just_over_two_levels_promotes_to_three() { assert_eq!(root_children.len(), 2); assert_eq!(root_children[0].start(), 0); assert_eq!(root_children[0].end(), 1024); - assert_eq!(root_children[1], DispatchTree::Leaf { start: 1024, end: 1025 }); + assert_eq!( + root_children[1], + DispatchTree::Leaf { + start: 1024, + end: 1025 + } + ); // Left child must itself be the full two-level shape. let DispatchTree::Internal { @@ -172,10 +178,7 @@ fn build_dispatch_tree_partition_covers_every_index_disjointly() { for (start, end) in &leaves { assert_eq!(*start, cursor, "N={n}: leaf gap at {cursor} → {start}"); assert!(start < end, "N={n}: empty leaf [{start}, {end})"); - assert!( - end - start <= 32, - "N={n}: oversize leaf [{start}, {end})", - ); + assert!(end - start <= 32, "N={n}: oversize leaf [{start}, {end})",); cursor = *end; } assert_eq!(cursor, n, "N={n}: leaves cover only {cursor}/{n}"); @@ -228,8 +231,8 @@ fn max_depth_bounded_by_log_of_n_times_bucket_size() { const LEAF_EXTRA: usize = 3; let bucket_size = 32usize; for &n in &[ - 1usize, 8, 32, 33, 64, 100, 1024, 1025, 2_000, 5_000, 32_768, 32_769, - 100_000, 1_000_000, 10_000_000, + 1usize, 8, 32, 33, 64, 100, 1024, 1025, 2_000, 5_000, 32_768, 32_769, 100_000, 1_000_000, + 10_000_000, ] { let tree = build_dispatch_tree(n, bucket_size); let levels = (ceil_log(n, bucket_size) as usize).max(1); @@ -270,7 +273,9 @@ fn simulate_decode(tree: &DispatchTree, call_idx: usize) -> Option<(usize, usize span_per_child, } => { let child_idx = (call_idx - tree.start()) / span_per_child; - children.get(child_idx).and_then(|c| simulate_decode(c, call_idx)) + children + .get(child_idx) + .and_then(|c| simulate_decode(c, call_idx)) } } } @@ -280,8 +285,8 @@ fn decode_every_k_lands_in_correct_leaf() { for &n in &[33usize, 64, 100, 200, 1024, 1025, 2_000, 5_000] { let tree = build_dispatch_tree(n, BUCKET_SIZE); for k in 0..n { - let (start, end) = simulate_decode(&tree, k) - .unwrap_or_else(|| panic!("N={n}: K={k} → None")); + let (start, end) = + simulate_decode(&tree, k).unwrap_or_else(|| panic!("N={n}: K={k} → None")); assert!( start <= k && k < end, "N={n}: K={k} landed in [{start}, {end})", @@ -302,17 +307,13 @@ fn max_depth_property_holds_for_power_of_bucket_size_progression() { let bucket_size = 32usize; let m = bucket_size; let expected: [(usize, usize); 4] = [ - (32, 35), // 1 level (leaf only) - (1024, 67), // 2 levels - (32_768, 99), // 3 levels + (32, 35), // 1 level (leaf only) + (1024, 67), // 2 levels + (32_768, 99), // 3 levels (1_048_576, 131), // 4 levels ]; for (n, want) in expected { let tree = build_dispatch_tree(n, m); - assert_eq!( - tree.max_depth(), - want, - "N={n}: expected depth {want}", - ); + assert_eq!(tree.max_depth(), want, "N={n}: expected depth {want}",); } } diff --git a/crates/fork-instrument/tests/fixtures/trampoline/legacy_catch_fork.wat b/crates/fork-instrument/tests/fixtures/trampoline/legacy_catch_fork.wat new file mode 100644 index 0000000000..715a25ed66 --- /dev/null +++ b/crates/fork-instrument/tests/fixtures/trampoline/legacy_catch_fork.wat @@ -0,0 +1,19 @@ +;; A fork from a legacy catch handler. The implicit legacy exception context +;; must be made activation-owned before nested switch replay can resume here. +(module + (import "kernel" "kernel_fork" (func $kernel_fork (result i32))) + + (tag $number (param i32)) + + (memory (export "memory") 1) + + (func $main (export "_start") (result i32) + (try (result i32) + (do + i32.const 37 + throw $number) + (catch $number + ;; Keep the payload below the fork result to exercise the handler's + ;; typed operand stack as well as its implicit exception context. + call $kernel_fork + drop)))) diff --git a/crates/fork-instrument/tests/instrument.rs b/crates/fork-instrument/tests/instrument.rs index 182153b96a..f41ecaa5f5 100644 --- a/crates/fork-instrument/tests/instrument.rs +++ b/crates/fork-instrument/tests/instrument.rs @@ -10,7 +10,8 @@ //! - **guard-dispatch**: used when any fork-path call is nested inside //! a block/loop/if/try_table. Each call site carries an in-place //! if-else guard that fires on `(NORMAL) || (REWIND && call_idx == -//! N)`; Phase 4g gates state-mutating ops during REWIND replay. +//! N)`; replay restores activation-owned frame state before entering +//! the selected continuation. //! //! Both schemes share the same frame layout and a result-typed restart loop //! containing `[preamble-ifelse, Block($unwind_save), postamble]`. @@ -35,6 +36,32 @@ fn instrument_wat(wat_src: &str) -> Vec { instrument(&bytes, &Options::default()).expect("instrument") } +/// Exercise the rewrite itself without the artifact-level activation-state +/// policy. This keeps transport tests focused on emitted control flow while +/// reference reconstruction support is expanded independently. +fn instrument_wat_unchecked(wat_src: &str) -> Vec { + let bytes = parse_wat(wat_src); + let mut module = Module::from_buffer(&bytes).expect("walrus parse"); + let seed = fork_instrument::call_graph::find_import_func(&module, "kernel.kernel_fork") + .expect("fork import"); + let fork_path = fork_instrument::call_graph::reaching_closure(&module, seed); + let mut targets: Vec<_> = fork_path + .iter() + .copied() + .filter(|id| matches!(module.funcs.get(*id).kind, FunctionKind::Local(_))) + .collect(); + targets.sort(); + let catch_plan = fork_instrument::instrument::plan_plain_catches(&module, &targets); + let runtime = fork_instrument::runtime::inject_linked_runtime(&mut module); + fork_instrument::instrument::instrument_functions( + &mut module, + &runtime, + &fork_path, + &catch_plan, + ); + module.emit_wasm() +} + fn validate(bytes: &[u8]) { let mut validator = wasmparser::Validator::new_with_features(wasmparser::WasmFeatures::default()); @@ -59,11 +86,14 @@ fn local_func(module: &Module, id: FunctionId) -> &LocalFunction { fn logical_entry_seq(f: &LocalFunction) -> InstrSeqId { let entry = f.block(f.entry_block()); - if let [(Instr::Loop(ir::Loop { seq }), _)] = entry.instrs.as_slice() { - *seq - } else { - f.entry_block() - } + entry + .instrs + .last() + .and_then(|(instruction, _)| match instruction { + Instr::Loop(ir::Loop { seq }) => Some(*seq), + _ => None, + }) + .unwrap_or_else(|| f.entry_block()) } fn entry_instr_kinds(module: &Module, id: FunctionId) -> Vec { @@ -84,8 +114,11 @@ fn seq_kinds(module: &Module, func_id: FunctionId, seq_id: InstrSeqId) -> Vec InstrSeqId { +/// Return the dispatch body inside the live-restart loop. +/// +/// Each fork-reaching call now owns its own result-typed private unwind catch, +/// so there is no function-wide catch wrapper or activation-local selector. +fn protected_unwind_body_seq(module: &Module, id: FunctionId) -> InstrSeqId { let f = local_func(module, id); let blocks: Vec = f .block(logical_entry_seq(f)) @@ -121,6 +154,7 @@ enum InstrKind { IfElse, BrIf, BrTable, + Throw, Other, } @@ -142,6 +176,7 @@ impl InstrKind { Instr::IfElse(_) => InstrKind::IfElse, Instr::BrIf(_) => InstrKind::BrIf, Instr::BrTable(_) => InstrKind::BrTable, + Instr::Throw(_) => InstrKind::Throw, _ => InstrKind::Other, } } @@ -170,6 +205,197 @@ fn walk_all(f: &LocalFunction, seq: InstrSeqId, vi } } +fn reference_codec_function(module: &Module, name: &str) -> FunctionId { + if let Some(function) = module.imports.iter().find_map(|import| { + if import.module != runtime_names::IMPORT_REFERENCE_CODEC_MODULE || import.name != name { + return None; + } + match import.kind { + walrus::ImportKind::Function(function) => Some(function), + _ => None, + } + }) { + return function; + } + + module + .exports + .iter() + .find_map(|export| { + (export.name == name) + .then_some(export.item) + .and_then(|item| { + if let ExportItem::Function(function) = item { + Some(function) + } else { + None + } + }) + }) + .or_else(|| { + module + .funcs + .iter() + .find(|function| function.name.as_deref() == Some(name)) + .map(|function| function.id()) + }) + .unwrap_or_else(|| panic!("missing reference-codec function `{name}`")) +} + +fn assert_function_calls_codec_pair( + module: &Module, + function_name: &str, + encode_name: &str, + decode_name: &str, +) { + let encode = reference_codec_function(module, encode_name); + let decode = reference_codec_function(module, decode_name); + let function = local_func(module, func_by_name(module, function_name)); + let mut calls = HashSet::new(); + walk_all(function, function.entry_block(), &mut |_, instruction| { + if let Instr::Call(call) = instruction { + calls.insert(call.func); + } + }); + assert!( + calls.contains(&encode), + "`{function_name}` must encode its live reference through `{encode_name}`" + ); + assert!( + calls.contains(&decode), + "`{function_name}` must decode its live reference through `{decode_name}`" + ); +} + +fn assert_function_uses_exception_recipe(module: &Module, function_name: &str) { + assert_function_calls_codec_pair( + module, + function_name, + runtime_names::IMPORT_REF_ENCODE_EXNREF, + runtime_names::IMPORT_REF_DECODE_EXNREF, + ); + let function = local_func(module, func_by_name(module, function_name)); + let mut has_throw_ref = false; + walk_all(function, function.entry_block(), &mut |_, instruction| { + has_throw_ref |= matches!(instruction, Instr::ThrowRef(_)); + }); + assert!( + has_throw_ref, + "`{function_name}` must replay a codec-owned exception with throw_ref" + ); +} + +fn sequences_with_direct_call( + module: &Module, + owner_name: &str, + target_name: &str, +) -> Vec> { + let owner = local_func(module, func_by_name(module, owner_name)); + let target = func_by_name(module, target_name); + let mut sequences = HashSet::new(); + walk_all(owner, owner.entry_block(), &mut |sequence, instruction| { + if matches!(instruction, Instr::Call(call) if call.func == target) { + sequences.insert(sequence); + } + }); + sequences + .into_iter() + .map(|sequence| { + owner + .block(sequence) + .instrs + .iter() + .map(|(instruction, _)| InstrKind::of(instruction)) + .collect() + }) + .collect() +} + +fn assert_resume_routing(module: &Module, owner_name: &str) { + let resume_peek = module + .imports + .iter() + .find_map(|import| match &import.kind { + walrus::ImportKind::Function(function) if import.name == "__wpk_fork_resume_peek" => { + Some(*function) + } + _ => None, + }) + .expect("resume peek import"); + let resume_table = module + .imports + .iter() + .find_map(|import| match &import.kind { + walrus::ImportKind::Table(table) if import.name == "__wpk_fork_resume_table" => { + Some(*table) + } + _ => None, + }) + .expect("resume table import"); + let owner = local_func(module, func_by_name(module, owner_name)); + let mut peeks = 0; + let mut dispatches = 0; + walk_all( + owner, + owner.entry_block(), + &mut |_, instruction| match instruction { + Instr::Call(call) if call.func == resume_peek => peeks += 1, + Instr::CallIndirect(call) if call.table == resume_table => dispatches += 1, + _ => {} + }, + ); + assert!(peeks > 0, "{owner_name} must peek the next replay event"); + assert!( + dispatches > 0, + "{owner_name} must dispatch a committed activation through the shared table" + ); +} + +fn assert_direct_activation_replay_is_lexical(module: &Module, owner_name: &str) { + let resume_peek = module + .imports + .iter() + .find_map(|import| match &import.kind { + walrus::ImportKind::Function(function) + if import.name == "__wpk_fork_resume_peek" => + { + Some(*function) + } + _ => None, + }) + .expect("resume peek import"); + let resume_table = module + .imports + .iter() + .find_map(|import| match &import.kind { + walrus::ImportKind::Table(table) + if import.name == "__wpk_fork_resume_table" => + { + Some(*table) + } + _ => None, + }) + .expect("resume table import"); + let owner = local_func(module, func_by_name(module, owner_name)); + let mut peeks = 0; + let mut dispatches = 0; + walk_all( + owner, + owner.entry_block(), + &mut |_, instruction| match instruction { + Instr::Call(call) if call.func == resume_peek => peeks += 1, + Instr::CallIndirect(call) if call.table == resume_table => dispatches += 1, + _ => {} + }, + ); + assert_eq!( + (peeks, dispatches), + (0, 0), + "{owner_name} must enter its exact direct activation without adding a \ + resume-thunk frame" + ); +} + fn count_br_tables(f: &LocalFunction) -> usize { let mut n = 0usize; walk_all(f, f.entry_block(), &mut |_, instr| { @@ -185,28 +411,31 @@ fn entry_preamble_and_postamble( func_id: FunctionId, ) -> (InstrSeqId, InstrSeqId, usize) { let f = local_func(module, func_id); - let entry = f.block(logical_entry_seq(f)); + let physical_entry = f.block(f.entry_block()); + let logical_entry = f.block(logical_entry_seq(f)); - let mut preamble_then: Option = None; - let mut wrapper: Option = None; - let mut postamble_start = 0usize; + let preamble_then = + physical_entry + .instrs + .iter() + .find_map(|(instruction, _)| match instruction { + Instr::IfElse(ie) => Some(ie.consequent), + _ => None, + }); - for (idx, (instr, _)) in entry.instrs.iter().enumerate() { - match instr { - Instr::IfElse(ie) if preamble_then.is_none() => { - preamble_then = Some(ie.consequent); - } - Instr::Block(b) if wrapper.is_none() => { - wrapper = Some(b.seq); - postamble_start = idx + 1; - } - _ => {} - } - } + let (wrapper, postamble_start) = logical_entry + .instrs + .iter() + .enumerate() + .find_map(|(index, (instruction, _))| match instruction { + Instr::Block(block) => Some((block.seq, index + 1)), + _ => None, + }) + .expect("unwind-save Block missing from live-restart loop"); ( preamble_then.expect("preamble IfElse missing"), - wrapper.expect("wrapper Block missing"), + wrapper, postamble_start, ) } @@ -394,24 +623,41 @@ fn direct_caller_entry_shape_is_preamble_wrapper_postamble() { let bytes = instrument_wat(FIXTURE_DIRECT_CALLER); let module = Module::from_buffer(&bytes).unwrap(); let caller = func_by_name(&module, "caller"); - let kinds = entry_instr_kinds(&module, caller); + let function = local_func(&module, caller); + let physical_kinds = function + .block(function.entry_block()) + .instrs + .iter() + .map(|(instruction, _)| InstrKind::of(instruction)) + .collect::>(); + let restart_kinds = entry_instr_kinds(&module, caller); - // The restart-loop body opens with the replay-state preamble check. - assert!( - matches!(kinds.first(), Some(InstrKind::GlobalGet)), - "restart loop should start with GlobalGet (state) for replay check: {kinds:?}", + // Replay restoration is deliberately outside the live-restart loop so a + // synchronous reserve failure can restart without an activation-local flag. + assert_eq!( + &physical_kinds[..4], + &[ + InstrKind::GlobalGet, + InstrKind::Const, + InstrKind::Binop, + InstrKind::IfElse, + ], + "physical entry should perform the replay-state preamble before the \ + live-restart loop: {physical_kinds:?}", ); - // Exactly one wrapper Block ($unwind_save) inside the restart loop. + // Exactly one $unwind_save Block is inside the restart loop. assert_eq!( - kinds.iter().filter(|k| **k == InstrKind::Block).count(), + restart_kinds + .iter() + .filter(|kind| **kind == InstrKind::Block) + .count(), 1, - "entry should contain exactly one wrapper Block: {kinds:?}", + "restart loop should contain exactly one unwind-save Block: {restart_kinds:?}", ); - // Must not terminate with Unreachable (postamble pushes real - // default return values). + // The postamble terminates by transporting the private unwind tag. assert!( - !matches!(kinds.last(), Some(InstrKind::Unreachable)), - "entry must not end in an Unreachable placeholder: {kinds:?}", + matches!(restart_kinds.last(), Some(InstrKind::Throw)), + "restart loop should end in the private unwind throw: {restart_kinds:?}", ); } @@ -514,14 +760,21 @@ fn multivalue_return_wraps_and_validates() { validate(&bytes); let module = Module::from_buffer(&bytes).unwrap(); let mv = func_by_name(&module, "mv"); + let function = local_func(&module, mv); let kinds = entry_instr_kinds(&module, mv); + let physical_kinds = function + .block(function.entry_block()) + .instrs + .iter() + .map(|(instruction, _)| InstrKind::of(instruction)) + .collect::>(); assert!( kinds.iter().any(|k| *k == InstrKind::Block), - "mv entry missing wrapper Block: {kinds:?}", + "mv restart loop missing unwind-save Block: {kinds:?}", ); assert!( - kinds.iter().any(|k| *k == InstrKind::IfElse), - "mv entry missing preamble IfElse: {kinds:?}", + physical_kinds.iter().any(|kind| *kind == InstrKind::IfElse), + "mv physical entry missing replay preamble IfElse: {physical_kinds:?}", ); } @@ -529,7 +782,7 @@ fn multivalue_return_wraps_and_validates() { fn instrument_functions_returns_rewritten_set() { use fork_instrument::call_graph; use fork_instrument::instrument::{PlainCatchPlan, instrument_functions}; - use fork_instrument::runtime::inject_runtime; + use fork_instrument::runtime::inject_linked_runtime; let bytes = wat::parse_str(FIXTURE_TRANSITIVE).unwrap(); let mut module = Module::from_buffer(&bytes).unwrap(); @@ -537,7 +790,7 @@ fn instrument_functions_returns_rewritten_set() { let seed = call_graph::find_import_func(&module, "kernel.kernel_fork").expect("seed import present"); let fork_path = call_graph::reaching_closure(&module, seed); - let runtime = inject_runtime(&mut module); + let runtime = inject_linked_runtime(&mut module); let b1_plan = PlainCatchPlan::default(); let rewritten = instrument_functions(&mut module, &runtime, &fork_path, &b1_plan); @@ -632,7 +885,7 @@ fn non_fork_call_remains_bare_in_chunk_0() { let module = Module::from_buffer(&bytes).unwrap(); let caller = func_by_name(&module, "caller"); - let unwind_save = entry_wrapper_seq(&module, caller); + let unwind_save = protected_unwind_body_seq(&module, caller); // Walk the whole $unwind_save body and count direct `Call`s to // `$helper`. There should be exactly one (chunk 0's helper call @@ -653,31 +906,87 @@ fn non_fork_call_remains_bare_in_chunk_0() { } #[test] -fn call_site_post_sequence_sets_call_idx_and_checks_unwinding() { - // For each fork-path call site, the post-call sequence is: - // , GlobalGet(state), Const(UNWINDING), Binop(eq), - // IfElse(then: frame.call_index = K; br $unwind_save). +fn source_call_results_do_not_cross_an_unwinding_state_probe() { + // The imported fork result is held across STATE_UNWINDING only inside a + // short generated helper. The source activation calls that helper through + // a statically indexed private-tag boundary, avoiding per-recursion + // result-spill or call-selector scratch. let bytes = instrument_wat(FIXTURE_DIRECT_CALLER); validate(&bytes); let module = Module::from_buffer(&bytes).unwrap(); let caller = func_by_name(&module, "caller"); - let unwind_save = entry_wrapper_seq(&module, caller); + let unwind_save = protected_unwind_body_seq(&module, caller); + let transport_id = module + .funcs + .iter() + .find(|function| { + function + .name + .as_deref() + .is_some_and(|name| name.starts_with("__wpk_fork_unwind_transport_direct_")) + }) + .expect("direct imported-call transport helper") + .id(); - // $unwind_save body (one call case): - // Block($POST_0), Call($fork), GlobalGet, Const, Binop, IfElse, Return + // The lexical call now lives in NORMAL and zero-sentinel branches, while + // REWIND with another committed frame uses the shared resume table. let kinds = seq_kinds(&module, caller, unwind_save); + assert_eq!(kinds.first(), Some(&InstrKind::Block)); assert_eq!( - kinds, - vec![ - InstrKind::Block, // $POST_0 - InstrKind::Call, // the fork call - InstrKind::GlobalGet, // state - InstrKind::Const, // UNWINDING - InstrKind::Binop, // i32.eq - InstrKind::IfElse, // then stores frame.call_index and branches - InstrKind::Return, // normal-path exit - ], + kinds.get(1), + Some(&InstrKind::Block), + "the lexical call should be followed by a per-site result-typed \ + private-tag boundary, not a selector LocalSet: {kinds:?}", + ); + assert!( + !kinds.contains(&InstrKind::LocalSet), + "the source activation must not carry an active-call selector: {kinds:?}", + ); + let caller_local = local_func(&module, caller); + let mut transport_calls = 0usize; + let mut post_result_state_probes = 0usize; + walk_all( + caller_local, + caller_local.entry_block(), + &mut |_, instruction| { + if matches!(instruction, Instr::Call(call) if call.func == transport_id) { + transport_calls += 1; + } + }, + ); + fn count_post_result_probes(function: &LocalFunction, sequence: InstrSeqId) -> usize { + let mut count = function + .block(sequence) + .instrs + .windows(5) + .filter(|window| { + matches!(window[0].0, Instr::IfElse(_)) + && matches!(window[1].0, Instr::GlobalGet(_)) + && matches!( + window[2].0, + Instr::Const(ir::Const { + value: ir::Value::I32(fork_instrument::runtime::STATE_UNWINDING), + }) + ) + && matches!(window[3].0, Instr::Binop(_)) + && matches!(window[4].0, Instr::IfElse(_)) + }) + .count(); + for (instruction, _) in &function.block(sequence).instrs { + for child in nested_of(instruction) { + count += count_post_result_probes(function, child); + } + } + count + } + post_result_state_probes += count_post_result_probes(caller_local, caller_local.entry_block()); + assert_eq!(transport_calls, 2, "NORMAL and zero-sentinel helper calls"); + assert_eq!( + post_result_state_probes, 0, + "a replay-selection IfElse result must not cross a following \ + UNWINDING state probe in the source activation" ); + assert_resume_routing(&module, "caller"); } #[test] @@ -748,7 +1057,7 @@ fn call_with_pure_args_replays_tail_without_spill_locals() { let module = Module::from_buffer(&bytes).unwrap(); let caller = func_by_name(&module, "caller_with_args"); - let unwind_save = entry_wrapper_seq(&module, caller); + let unwind_save = protected_unwind_body_seq(&module, caller); // Structure after rewrite: // $unwind_save: @@ -763,11 +1072,14 @@ fn call_with_pure_args_replays_tail_without_spill_locals() { // NORMAL and REWIND both reach the same post-call sequence, so // replaying the pure tail here preserves the call arguments without // adding frame-backed arg locals. - let unwind_kinds = seq_kinds(&module, caller, unwind_save); - assert_eq!(unwind_kinds[0], InstrKind::Block); - assert_eq!(unwind_kinds[1], InstrKind::Const, "replay arg 0"); - assert_eq!(unwind_kinds[2], InstrKind::Const, "replay arg 1"); - assert_eq!(unwind_kinds[3], InstrKind::Call); + let lexical = sequences_with_direct_call(&module, "caller_with_args", "leaf"); + assert_eq!(lexical.len(), 2, "NORMAL and direct-replay lexical calls"); + assert!( + lexical + .iter() + .all(|kinds| { kinds == &vec![InstrKind::Const, InstrKind::Const, InstrKind::Call] }) + ); + assert_direct_activation_replay_is_lexical(&module, "caller_with_args"); // Find $POST_0 — it's the inner Block of $unwind_save. let f = local_func(&module, caller); @@ -790,15 +1102,15 @@ fn call_with_non_pure_arg_falls_back_to_spill_local() { let module = Module::from_buffer(&bytes).unwrap(); let caller = func_by_name(&module, "caller_with_load_arg"); - let unwind_save = entry_wrapper_seq(&module, caller); - let unwind_kinds = seq_kinds(&module, caller, unwind_save); - assert_eq!(unwind_kinds[0], InstrKind::Block); - assert_eq!( - unwind_kinds[1], - InstrKind::LocalGet, - "load-produced arg must reload from fallback spill local", + let unwind_save = protected_unwind_body_seq(&module, caller); + let lexical = sequences_with_direct_call(&module, "caller_with_load_arg", "leaf"); + assert_eq!(lexical.len(), 2, "NORMAL and direct-replay lexical calls"); + assert!( + lexical + .iter() + .all(|kinds| { kinds == &vec![InstrKind::LocalGet, InstrKind::Call] }) ); - assert_eq!(unwind_kinds[2], InstrKind::Call); + assert_direct_activation_replay_is_lexical(&module, "caller_with_load_arg"); let f = local_func(&module, caller); let post_0 = match f.block(unwind_save).instrs[0].0 { @@ -820,13 +1132,22 @@ fn call_with_i64_shift_arg_replays_shift_tail() { let module = Module::from_buffer(&bytes).unwrap(); let caller = func_by_name(&module, "caller_with_i64_shift_arg"); - let unwind_save = entry_wrapper_seq(&module, caller); - let unwind_kinds = seq_kinds(&module, caller, unwind_save); - assert_eq!(unwind_kinds[0], InstrKind::Block); - assert_eq!(unwind_kinds[1], InstrKind::Const); - assert_eq!(unwind_kinds[2], InstrKind::Const); - assert_eq!(unwind_kinds[3], InstrKind::Binop); - assert_eq!(unwind_kinds[4], InstrKind::Call); + let unwind_save = protected_unwind_body_seq(&module, caller); + let lexical = sequences_with_direct_call(&module, "caller_with_i64_shift_arg", "leaf"); + assert_eq!(lexical.len(), 2, "NORMAL and direct-replay lexical calls"); + assert!(lexical.iter().all(|kinds| { + kinds + == &vec![ + InstrKind::Const, + InstrKind::Const, + InstrKind::Binop, + InstrKind::Call, + ] + })); + assert_direct_activation_replay_is_lexical( + &module, + "caller_with_i64_shift_arg", + ); let f = local_func(&module, caller); let post_0 = match f.block(unwind_save).instrs[0].0 { @@ -846,7 +1167,7 @@ fn two_calls_assign_sequential_call_idx() { validate(&bytes); let module = Module::from_buffer(&bytes).unwrap(); let caller = func_by_name(&module, "caller"); - let _unwind_save = entry_wrapper_seq(&module, caller); + let _unwind_save = protected_unwind_body_seq(&module, caller); let f = local_func(&module, caller); let reserve = module .imports @@ -857,6 +1178,12 @@ fn two_calls_assign_sequential_call_idx() { _ => None, }) .expect("linked frame reserve import"); + let frame_select = module + .funcs + .iter() + .find(|function| function.name.as_deref() == Some("__wpk_fork_select_unwind_frame")) + .expect("generated unwind-frame selector") + .id(); // Count Const values immediately preceding stores to frame.call_index. fn walk_seqs(f: &LocalFunction, seq: InstrSeqId, visit: &mut F) { @@ -869,38 +1196,97 @@ fn two_calls_assign_sequential_call_idx() { } let mut idxs: Vec = Vec::new(); - let mut reserve_calls = 0usize; + let mut frame_sizes = Vec::new(); + let mut frame_select_calls = 0usize; walk_seqs(f, f.entry_block(), &mut |seq| { let instrs = &f.block(seq).instrs; - reserve_calls += instrs - .iter() - .filter( - |(instr, _)| matches!(instr, Instr::Call(ir::Call { func }) if *func == reserve), - ) - .count(); - for i in 1..instrs.len() { - if let Instr::Store(store) = &instrs[i].0 { - if store.arg.offset == 4 { - if let Instr::Const(c) = &instrs[i - 1].0 { - if let ir::Value::I32(v) = c.value { - idxs.push(v); - } - } - } + for index in 2..instrs.len() { + if matches!( + instrs[index].0, + Instr::Call(ir::Call { func }) if func == frame_select + ) { + let ( + Instr::Const(ir::Const { + value: ir::Value::I32(size), + }), + Instr::Const(ir::Const { + value: ir::Value::I32(call_index), + }), + ) = (&instrs[index - 2].0, &instrs[index - 1].0) + else { + panic!( + "unwind-frame selector must receive static size and \ + call-index constants" + ); + }; + frame_select_calls += 1; + frame_sizes.push(*size); + idxs.push(*call_index); } } }); - // The structure yields the sites in reverse-nesting order: the - // outermost $unwind_save body has call 1's post-sequence, the - // inner $POST_1 body has call 0's post-sequence. Sort before - // asserting the set of assigned indices. + let selector_function = local_func(&module, frame_select); + let mut helper_reserve_calls = 0usize; + walk_seqs( + selector_function, + selector_function.entry_block(), + &mut |sequence| { + helper_reserve_calls += selector_function + .block(sequence) + .instrs + .iter() + .filter(|(instruction, _)| { + matches!(instruction, Instr::Call(ir::Call { func }) if *func == reserve) + }) + .count(); + }, + ); + + let mut active_selectors = Vec::new(); + walk_seqs(f, f.entry_block(), &mut |seq| { + let instrs = &f.block(seq).instrs; + for pair in instrs.windows(2) { + if let ( + Instr::Const(ir::Const { + value: ir::Value::I32(value), + }), + Instr::LocalSet(_), + ) = (&pair[0].0, &pair[1].0) + { + if matches!(*value, 1 | 2) { + active_selectors.push(*value); + } + } + } + }); + active_selectors.sort(); idxs.sort(); - assert_eq!(reserve_calls, 2, "each call site should reserve one frame"); + assert_eq!( + frame_select_calls, 2, + "each statically indexed private-tag call boundary should call the \ + shared unwind-frame selector once", + ); + assert_eq!( + helper_reserve_calls, 1, + "the module helper should own exactly one cold frame-reservation \ + sequence regardless of lexical call-site count", + ); + assert_eq!( + frame_sizes, + vec![16, 16], + "both call sites should pass this function's exact static frame size", + ); + assert_eq!( + active_selectors, + Vec::::new(), + "static call boundaries must not install an activation-local selector", + ); assert_eq!( idxs, - vec![0, 0, 1, 1], - "each call_idx should appear in its committed frame and abort scratch selector", + vec![0, 1], + "each call should pass its static zero-based index directly to the \ + shared frame selector", ); } @@ -911,28 +1297,45 @@ fn call_indirect_replays_pure_table_index_arg() { let module = Module::from_buffer(&bytes).unwrap(); let caller = func_by_name(&module, "caller"); - let unwind_save = entry_wrapper_seq(&module, caller); + let unwind_save = protected_unwind_body_seq(&module, caller); let f = local_func(&module, caller); - // $unwind_save: - // Block($POST_0), - // , CallIndirect, - // GlobalGet, Const, Binop, IfElse, - // Return - let kinds = seq_kinds(&module, caller, unwind_save); + let original_table = module + .tables + .iter() + .find(|table| table.import.is_none() && table.initial == 1) + .expect("fixture indirect table") + .id(); + let transport_id = module + .funcs + .iter() + .find(|function| { + function + .name + .as_deref() + .is_some_and(|name| name.starts_with("__wpk_fork_unwind_transport_indirect_")) + }) + .expect("indirect transport helper") + .id(); + let mut lexical_calls = 0; + walk_all(f, f.entry_block(), &mut |_, instruction| { + if matches!(instruction, Instr::Call(call) if call.func == transport_id) { + lexical_calls += 1; + } + }); + assert_eq!(lexical_calls, 2, "NORMAL and zero-sentinel helper calls"); + let helper = local_func(&module, transport_id); + let mut helper_indirect_calls = 0; + walk_all(helper, helper.entry_block(), &mut |_, instruction| { + if matches!(instruction, Instr::CallIndirect(call) if call.table == original_table) { + helper_indirect_calls += 1; + } + }); assert_eq!( - kinds, - vec![ - InstrKind::Block, - InstrKind::Const, // replay i32 table index - InstrKind::CallIndirect, // indirect call - InstrKind::GlobalGet, - InstrKind::Const, - InstrKind::Binop, - InstrKind::IfElse, - InstrKind::Return, - ], + helper_indirect_calls, 1, + "the shared helper must own exactly one guest-table dispatch" ); + assert_resume_routing(&module, "caller"); // The pure table-index tail is removed from $POST_0 rather than // spilled into a frame-backed local. @@ -955,23 +1358,24 @@ fn preamble_starts_with_rewinding_state_check() { let bytes = instrument_wat(FIXTURE_DIRECT_CALLER); let module = Module::from_buffer(&bytes).unwrap(); let caller = func_by_name(&module, "caller"); - let kinds = entry_instr_kinds(&module, caller); + let f = local_func(&module, caller); + let entry = f.block(f.entry_block()); + let kinds = entry + .instrs + .iter() + .map(|(instruction, _)| InstrKind::of(instruction)) + .collect::>(); assert_eq!( - &kinds[..7], + &kinds[..4], &[ InstrKind::GlobalGet, InstrKind::Const, InstrKind::Binop, - InstrKind::LocalGet, - InstrKind::Unop, - InstrKind::Binop, InstrKind::IfElse, ], ); - let f = local_func(&module, caller); - let entry = f.block(logical_entry_seq(f)); let rewinding_const = match &entry.instrs[1].0 { Instr::Const(c) => c.value, other => panic!("expected Const at entry[1], got {other:?}"), @@ -1019,30 +1423,34 @@ fn postamble_writes_and_commits_the_reserved_linked_frame() { InstrKind::GlobalGet, InstrKind::Other, // Load current frame InstrKind::Const, - InstrKind::Other, // Store packed zero catch_region_id + exnref_slot + InstrKind::Other, // Store zero catch_region_id + InstrKind::GlobalGet, + InstrKind::Other, // Load current frame + InstrKind::Const, + InstrKind::Other, // Store reserved zero catch metadata InstrKind::GlobalGet, InstrKind::Other, // Load current frame InstrKind::Call, // __wpk_fork_frame_commit - InstrKind::Const, // default return value + InstrKind::Throw, // process-owned unwind transport ]; assert_eq!(postamble, expected); } #[test] -fn no_catch_postamble_packs_zero_catch_header_fields() { +fn no_catch_postamble_writes_deterministic_zero_catch_header_fields() { let bytes = instrument_wat(FIXTURE_DIRECT_CALLER); validate(&bytes); let printed = wasmprinter::print_bytes(&bytes).expect("wasmprinter"); let caller_section = extract_function_text(&printed, "caller"); assert!( - caller_section.contains("i64.store offset=8"), - "no-catch postamble should pack catch_region_id/exnref_slot zeroes:\n{caller_section}", + caller_section.contains("i32.store offset=8") + && caller_section.contains("i32.store offset=12"), + "no-catch postamble should zero the catch region and reserved field:\n{caller_section}", ); assert!( - !(caller_section.contains("i32.store offset=8") - && caller_section.contains("i32.store offset=12")), - "no-catch postamble should not emit separate zero stores:\n{caller_section}", + !caller_section.contains("i64.store offset=8"), + "ABI 43 uses explicit versioned header fields:\n{caller_section}", ); } @@ -1059,7 +1467,7 @@ fn catch_capable_postamble_keeps_dynamic_catch_header_stores() { ); assert!( caller_section.contains("i32.store offset=12"), - "catch-capable postamble must store dynamic exnref_slot:\n{caller_section}", + "catch-capable postamble must zero the reserved former exnref slot:\n{caller_section}", ); assert!( !caller_section.contains("i64.store offset=8"), @@ -1106,182 +1514,1119 @@ fn postamble_serializes_user_scalar_locals() { let postamble = &kinds[postamble_start..]; // Postamble with one user local: - // 4 current-frame pointer loads/stores plus three payload stores - // (func_index, packed zero catch fields, user_x) = 7 Others. The linked - // commit replaces the legacy current_pos bump. + // 4 current-frame pointer loads/stores plus four payload stores + // (func_index, catch_region_id, reserved zero, user_x) = 8 stores/loads, + // plus the linked-frame reservation result = 9 Others. The catch fields + // remain separate i32 slots so a catch-capable function can store its + // dynamic region identifier without changing the frame shape. The final + // private Throw has its own instruction kind and is not counted here. let other_count = postamble .iter() .filter(|k| matches!(k, InstrKind::Other)) .count(); assert_eq!( - other_count, 7, + other_count, 9, "postamble should load/store the active payload and serialize its fields: {postamble:?}", ); } #[test] -fn postamble_emits_defaults_for_each_result_type() { +fn postamble_throws_without_fabricating_result_values() { let bytes = instrument_wat(FIXTURE_COMPLEX_RETURN); validate(&bytes); let module = Module::from_buffer(&bytes).unwrap(); let caller = func_by_name(&module, "caller"); let kinds = entry_instr_kinds(&module, caller); - let trailing_consts = kinds - .iter() - .rev() - .take_while(|k| **k == InstrKind::Const) - .count(); assert_eq!( - trailing_consts, 2, - "postamble should emit one Const per result type: {kinds:?}", + kinds.last(), + Some(&InstrKind::Throw), + "postamble must transport unwind independently of result types: {kinds:?}", + ); + assert!( + !matches!( + kinds.last(), + Some(InstrKind::Const | InstrKind::Unreachable) + ), + "postamble must not fabricate typed defaults or trap on a result type: {kinds:?}", ); } -// --- Aux-table (Phase 4f) tests -------------------------------------- - #[test] -fn funcref_local_triggers_aux_table_injection() { - let bytes = instrument_wat(FIXTURE_FUNCREF_LOCAL); +fn nonnullable_reference_result_unwinds_via_private_tag() { + let bytes = instrument_wat_unchecked( + r#" + (module + (import "kernel" "kernel_fork" (func $fork (result i32))) + (elem declare func $target) + (func $target) + (func $caller (export "caller") (result (ref func)) + call $fork + drop + ref.func $target) + (memory 1)) + "#, + ); validate(&bytes); - let module = Module::from_buffer(&bytes).unwrap(); - - let stash_count = module - .tables - .iter() - .filter(|t| t.name.as_deref() == Some("_wpk_fork_funcref_stash")) - .count(); - assert_eq!(stash_count, 1, "expected exactly one funcref stash table"); - - let stash = module - .tables - .iter() - .find(|t| t.name.as_deref() == Some("_wpk_fork_funcref_stash")) - .unwrap(); - assert_eq!(stash.initial, 1); + let module = Module::from_buffer(&bytes).expect("parse rewritten module"); + let caller = func_by_name(&module, "caller"); + let (_, _, postamble_start) = entry_preamble_and_postamble(&module, caller); + let postamble = &entry_instr_kinds(&module, caller)[postamble_start..]; + assert_eq!( + postamble.last(), + Some(&InstrKind::Throw), + "non-nullable result needs no fake default when unwind is exceptional: {postamble:?}", + ); + assert!( + !postamble.contains(&InstrKind::Unreachable), + "result typing must not turn a valid unwind into a trap: {postamble:?}", + ); } +// --- Fresh-instance reference-state validation ----------------------- + #[test] -fn funcref_local_is_spilled_to_table_and_reloaded() { +fn definitely_null_funcref_local_needs_no_recipe() { let bytes = instrument_wat(FIXTURE_FUNCREF_LOCAL); - let module = Module::from_buffer(&bytes).unwrap(); - let caller = func_by_name(&module, "caller"); - let f = local_func(&module, caller); - - // Count TableSet and TableGet anywhere in the function. - let mut table_sets = 0usize; - let mut table_gets = 0usize; - walk_all(f, f.entry_block(), &mut |_, instr| match instr { - Instr::TableSet(_) => table_sets += 1, - Instr::TableGet(_) => table_gets += 1, - _ => {} - }); + validate(&bytes); +} - assert_eq!(table_sets, 1, "postamble must spill the one funcref local"); - assert_eq!( - table_gets, 1, - "preamble-then must reload the one funcref local", +#[test] +fn dead_reference_parameter_in_fork_closure_remains_legal() { + let bytes = instrument_wat( + r#" + (module + (import "kernel" "kernel_fork" (func $fork (result i32))) + (func $caller (param funcref) (result i32) + call $fork) + (memory 1)) + "#, ); + validate(&bytes); } #[test] -fn functions_without_ref_locals_inject_no_aux_tables() { +fn instrumented_modules_never_emit_legacy_reference_tables() { let bytes = instrument_wat(FIXTURE_DIRECT_CALLER); let module = Module::from_buffer(&bytes).unwrap(); - let stash_names = [ + let legacy_reference_table_names = [ "_wpk_fork_funcref_stash", "_wpk_fork_externref_stash", "_wpk_fork_exnref_stash", ]; - for name in stash_names { + for name in legacy_reference_table_names { assert!( !module .tables .iter() .any(|t| t.name.as_deref() == Some(name)), - "module without ref locals should not have `{name}`", + "ABI 43 must not emit retired module-instance table `{name}`", + ); + } +} + +#[test] +fn nested_reference_activations_validate_without_static_slots() { + let bytes = instrument_wat(FIXTURE_TWO_FUNCREF_CALLERS); + validate(&bytes); +} + +#[test] +fn call_specific_reference_vectors_do_not_enlarge_activation_frames() { + let bytes = instrument_wat( + r#" + (module + (import "kernel" "kernel_fork" (func $fork (result i32))) + (import "env" "make_first" (func $make_first (result externref))) + (import "env" "make_second" (func $make_second (result externref))) + (func $caller (result i32) + (local $first externref) + (local $second externref) + call $make_first + local.set $first + call $fork + drop + local.get $first + drop + call $make_second + local.set $second + call $fork + drop + local.get $second + drop + i32.const 0) + (memory 1)) + "#, + ); + validate(&bytes); + let module = Module::from_buffer(&bytes).expect("parse reference-vector fixture"); + let caller = local_func(&module, func_by_name(&module, "caller")); + let imported = |name: &str| { + module + .imports + .iter() + .find_map(|import| { + (import.name == name).then(|| match import.kind { + walrus::ImportKind::Function(function) => Some(function), + _ => None, + })? + }) + .unwrap_or_else(|| panic!("missing import {name}")) + }; + let frame_select = module + .funcs + .iter() + .find(|function| function.name.as_deref() == Some("__wpk_fork_select_unwind_frame")) + .expect("generated unwind-frame selector") + .id(); + let vector_begin = imported(runtime_names::IMPORT_REFERENCE_VECTOR_BEGIN); + let vector_append = imported(runtime_names::IMPORT_REFERENCE_VECTOR_APPEND); + let vector_finish = imported(runtime_names::IMPORT_REFERENCE_VECTOR_FINISH); + let vector_get = imported(runtime_names::IMPORT_REFERENCE_VECTOR_GET); + + let mut reserve_sizes = Vec::new(); + let mut vector_calls = [0usize; 4]; + fn visit_sequences( + function: &LocalFunction, + sequence: InstrSeqId, + frame_select: FunctionId, + vector_functions: [FunctionId; 4], + reserve_sizes: &mut Vec, + vector_calls: &mut [usize; 4], + ) { + let instructions = &function.block(sequence).instrs; + for (index, (instruction, _)) in instructions.iter().enumerate() { + if let Instr::Call(call) = instruction { + if call.func == frame_select { + let Some(( + Instr::Const(ir::Const { + value: ir::Value::I32(size), + }), + _, + )) = index.checked_sub(2).and_then(|i| instructions.get(i)) + else { + panic!( + "unwind-frame selector is not preceded by its \ + constant size and call index" + ); + }; + reserve_sizes.push(*size); + } + for (slot, function) in vector_functions.iter().enumerate() { + if call.func == *function { + vector_calls[slot] += 1; + } + } + } + for child in nested_of(instruction) { + visit_sequences( + function, + child, + frame_select, + vector_functions, + reserve_sizes, + vector_calls, + ); + } + } + } + visit_sequences( + caller, + caller.entry_block(), + frame_select, + [vector_begin, vector_append, vector_finish, vector_get], + &mut reserve_sizes, + &mut vector_calls, + ); + + assert!(!reserve_sizes.is_empty()); + assert!( + reserve_sizes.iter().all(|size| *size == 16), + "this fixture has no scalar activation state, so its total frame is \ + the 16-byte header: two references live at disjoint call landings \ + must add zero frame bytes, not the old function-wide 8-byte slot \ + union: {reserve_sizes:?}", + ); + assert!(vector_calls[0] > 0, "save path must allocate a call vector"); + assert!(vector_calls[1] >= 2, "each live recipe must be appended"); + assert!( + vector_calls[2] > 0, + "save path must replace its transient builder handle with a canonical ordinal" + ); + assert!( + vector_calls[3] >= 2, + "rewind must perform indexed vector lookup" + ); +} + +#[test] +fn externref_local_is_activation_owned() { + let wat = r#" + (module + (import "kernel" "kernel_fork" (func $fork (result i32))) + (func $caller (export "caller") (result i32) + (local $x externref) + ref.null extern + local.set $x + call $fork + local.get $x + drop) + (memory 1)) + "#; + let bytes = instrument_wat(wat); + validate(&bytes); +} + +#[test] +fn live_reference_call_argument_is_activation_owned() { + let wat = r#" + (module + (import "kernel" "kernel_fork" (func $fork (result i32))) + (func $target (param $value externref) (result i32) + call $fork) + (func $caller (param $value externref) (result i32) + local.get $value + call $target) + (memory 1)) + "#; + let bytes = instrument_wat(wat); + validate(&bytes); + let module = + Module::from_buffer(&bytes).expect("parse instrumented reference-argument fixture"); + assert_function_calls_codec_pair( + &module, + "caller", + runtime_names::IMPORT_REF_ENCODE_EXTERNREF, + runtime_names::IMPORT_REF_DECODE_EXTERNREF, + ); +} + +#[test] +fn call_ref_callee_is_activation_owned_and_narrowed_for_replay() { + let wat = r#" + (module + (type $fork_ty (func (result i32))) + (import "kernel" "kernel_fork" (func $fork (type $fork_ty))) + (elem declare func $fork) + (func $caller (result i32) + ref.func $fork + call_ref $fork_ty) + (memory 1)) + "#; + let bytes = instrument_wat(wat); + validate(&bytes); + let module = Module::from_buffer(&bytes).expect("parse instrumented call_ref fixture"); + assert_function_calls_codec_pair( + &module, + "caller", + runtime_names::IMPORT_REF_ENCODE_FUNCREF, + runtime_names::IMPORT_REF_DECODE_FUNCREF, + ); + let caller = local_func(&module, func_by_name(&module, "caller")); + let transport_id = module + .funcs + .iter() + .find(|function| { + function + .name + .as_deref() + .is_some_and(|name| name.starts_with("__wpk_fork_unwind_transport_ref_")) + }) + .expect("call_ref transport helper") + .id(); + let mut has_callee_cast = false; + let transport = local_func(&module, transport_id); + walk_all(transport, transport.entry_block(), &mut |_, instruction| { + has_callee_cast |= matches!(instruction, Instr::RefCast(_)); + }); + assert!( + has_callee_cast, + "the call_ref helper must restore the declared concrete function type" + ); + let mut source_calls_transport = false; + walk_all(caller, caller.entry_block(), &mut |_, instruction| { + source_calls_transport |= + matches!(instruction, Instr::Call(call) if call.func == transport_id); + }); + assert!( + source_calls_transport, + "source call_ref must use its helper" + ); +} + +#[test] +fn gc_ref_local_uses_anyref_recipe_codec() { + let wat = r#" + (module + (import "kernel" "kernel_fork" (func $fork (result i32))) + (func $caller (result i32) + (local $r (ref null any)) + i32.const 17 + ref.i31 + local.set $r + call $fork + local.get $r + drop) + (memory 1)) + "#; + let bytes = instrument_wat(wat); + validate(&bytes); + let module = Module::from_buffer(&bytes).expect("parse instrumented anyref fixture"); + assert_function_calls_codec_pair( + &module, + "caller", + runtime_names::IMPORT_REF_ENCODE_ANYREF, + runtime_names::IMPORT_REF_DECODE_ANYREF, + ); +} + +#[test] +fn nullable_reference_operand_stack_carryover_is_activation_owned() { + let bytes = instrument_wat( + r#" + (module + (import "kernel" "kernel_fork" (func $fork (result i32))) + (func $caller (result i32) + ref.null extern + call $fork + drop + drop + i32.const 0) + (memory 1)) + "#, + ); + validate(&bytes); +} + +#[test] +fn non_null_reference_operand_stack_carryover_uses_recipe() { + let bytes = instrument_wat( + r#" + (module + (import "kernel" "kernel_fork" (func $fork (result i32))) + (func $caller (param $value externref) (result i32) + local.get $value + call $fork + drop + drop + i32.const 0) + (memory 1)) + "#, + ); + validate(&bytes); + let module = Module::from_buffer(&bytes).expect("parse reference-carryover fixture"); + assert_function_calls_codec_pair( + &module, + "caller", + runtime_names::IMPORT_REF_ENCODE_EXTERNREF, + runtime_names::IMPORT_REF_DECODE_EXTERNREF, + ); +} + +#[test] +fn nested_call_reference_carryover_uses_recipe() { + let bytes = instrument_wat( + r#" + (module + (import "kernel" "kernel_fork" (func $fork (result i32))) + (func $caller (param $value externref) (result i32) + (block + local.get $value + call $fork + drop + drop) + i32.const 0) + (memory 1)) + "#, + ); + validate(&bytes); + let module = Module::from_buffer(&bytes).expect("parse nested reference-carryover fixture"); + assert_function_calls_codec_pair( + &module, + "caller", + runtime_names::IMPORT_REF_ENCODE_EXTERNREF, + runtime_names::IMPORT_REF_DECODE_EXTERNREF, + ); +} + +#[test] +fn parent_stack_reference_across_fork_bearing_subregion_uses_recipe() { + let bytes = instrument_wat( + r#" + (module + (import "kernel" "kernel_fork" (func $fork (result i32))) + (func $caller (param $value externref) (result i32) + local.get $value + (block + call $fork + drop) + drop + i32.const 0) + (memory 1)) + "#, + ); + validate(&bytes); + let module = Module::from_buffer(&bytes).expect("parse subregion reference-carryover fixture"); + assert_function_calls_codec_pair( + &module, + "caller", + runtime_names::IMPORT_REF_ENCODE_EXTERNREF, + runtime_names::IMPORT_REF_DECODE_EXTERNREF, + ); +} + +#[test] +fn dead_polymorphic_reference_subregion_does_not_create_an_analysis_gap() { + // A valid Wasm sequence remains stack-polymorphic after `return`. Static + // call-graph discovery may still conservatively find a fork edge in that + // dead suffix, so the structural rewrite must keep it validator-clean + // rather than treating absent lexical operands as an analysis failure. + // The block parameter is deliberately a reference: replay + // spill storage must use its nullable representation. + let bytes = instrument_wat( + r#" + (module + (import "kernel" "kernel_fork" (func $fork (result i32))) + (func $caller + return + (block (param externref) + call $fork + drop + drop)) + (memory 1)) + "#, + ); + validate(&bytes); +} + +#[test] +fn dead_polymorphic_reference_call_does_not_create_an_analysis_gap() { + // Call discovery intentionally remains conservative in dead suffixes. + // The nested carryover walk must therefore account for the call ordinal + // even though `return` made the operand stack polymorphic. + let bytes = instrument_wat( + r#" + (module + (import "kernel" "kernel_fork" (func $fork (result i32))) + (func $target (param externref) (result i32) + call $fork) + (func $caller + (block + return + ref.null extern + call $target + drop)) + (memory 1)) + "#, + ); + validate(&bytes); +} + +#[test] +fn escaped_catch_ref_operand_stack_value_uses_exnref_recipe_codec() { + let bytes = instrument_wat( + r#" + (module + (import "kernel" "kernel_fork" (func $fork (result i32))) + (tag $exn) + (func $caller (result i32) + (block $handler (result exnref) + (try_table (result exnref) (catch_ref $exn $handler) + throw $exn)) + call $fork + drop + drop + i32.const 0) + (memory 1)) + "#, + ); + validate(&bytes); +} + +#[test] +fn static_table_reference_operand_stack_carryover_uses_recipe() { + let bytes = instrument_wat( + r#" + (module + (import "kernel" "kernel_fork" (func $fork (result i32))) + (table 1 funcref) + (func $target) + (elem (i32.const 0) func $target) + (func $caller (result i32) + i32.const 0 + table.get + call $fork + drop + drop + i32.const 0) + (memory 1)) + "#, + ); + validate(&bytes); +} + +#[test] +fn dead_nonnullable_ref_func_instruction_remains_legal() { + let bytes = instrument_wat( + r#" + (module + (import "kernel" "kernel_fork" (func $fork (result i32))) + (elem declare func $target) + (func $target) + (func $caller (result i32) + ref.func $target + drop + call $fork) + (memory 1)) + "#, + ); + validate(&bytes); +} + +#[test] +fn concrete_gc_reference_uses_anyref_recipe_codec_and_narrowing() { + let bytes = instrument_wat( + r#" + (module + (import "kernel" "kernel_fork" (func $fork (result i32))) + (type $pair (struct (field i32))) + (func $make_pair (result (ref $pair)) + i32.const 7 + struct.new $pair) + (func $caller (result i32) + (local $value (ref null $pair)) + call $make_pair + local.set $value + call $fork + local.get $value + drop) + (memory 1)) + "#, + ); + validate(&bytes); + let module = Module::from_buffer(&bytes).expect("parse instrumented concrete-ref fixture"); + assert_function_calls_codec_pair( + &module, + "caller", + runtime_names::IMPORT_REF_ENCODE_ANYREF, + runtime_names::IMPORT_REF_DECODE_ANYREF, + ); + let caller = local_func(&module, func_by_name(&module, "caller")); + let mut has_narrowing_cast = false; + walk_all(caller, caller.entry_block(), &mut |_, instruction| { + has_narrowing_cast |= matches!(instruction, Instr::RefCast(_)); + }); + assert!( + has_narrowing_cast, + "decoded anyref must be narrowed back to the concrete `$pair` type", + ); +} + +#[test] +fn module_without_try_tables_has_no_legacy_reference_storage() { + let bytes = instrument_wat(FIXTURE_DIRECT_CALLER); + let module = Module::from_buffer(&bytes).unwrap(); + assert!( + !module + .tables + .iter() + .any(|t| t.name.as_deref() == Some("_wpk_fork_exnref_stash")), + "module with no try_tables must not inject retired exnref storage", + ); +} + +#[test] +fn mutable_reference_global_has_module_state_owner() { + let bytes = instrument_wat( + r#" + (module + (import "kernel" "kernel_fork" (func $fork (result i32))) + (global $callback (mut funcref) (ref.null func)) + (func $caller (result i32) call $fork) + (memory 1)) + "#, + ); + validate(&bytes); +} + +#[test] +fn immutable_reference_global_read_before_fork_remains_legal() { + let bytes = instrument_wat( + r#" + (module + (import "kernel" "kernel_fork" (func $fork (result i32))) + (global $callback funcref (ref.null func)) + (func $caller (result i32) + global.get $callback + drop + call $fork) + (memory 1)) + "#, + ); + validate(&bytes); +} + +#[test] +fn immutable_reference_global_outside_fork_closure_remains_legal() { + let bytes = instrument_wat( + r#" + (module + (import "kernel" "kernel_fork" (func $fork (result i32))) + (global $callback funcref (ref.null func)) + (func $unrelated (export "unrelated") + global.get $callback + drop) + (func $caller (result i32) call $fork) + (memory 1)) + "#, + ); + validate(&bytes); +} + +#[test] +fn dead_reference_typed_call_before_fork_remains_legal() { + let bytes = instrument_wat( + r#" + (module + (import "kernel" "kernel_fork" (func $fork (result i32))) + (import "env" "consume" (func $consume (param externref))) + (func $caller (result i32) + ref.null extern + call $consume + call $fork) + (memory 1)) + "#, + ); + validate(&bytes); +} + +#[test] +fn non_fork_reaching_return_call_ref_in_fork_activation_remains_tail() { + let bytes = instrument_wat( + r#" + (module + (type $ft (func (result i32))) + (import "kernel" "kernel_fork" (func $fork (type $ft))) + (func $safe (type $ft) + i32.const 7) + (elem declare func $safe) + (func $caller (param $take_tail i32) (result i32) + local.get $take_tail + if (result i32) + ref.func $safe + return_call_ref $ft + else + call $fork + end) + (memory 1)) + "#, + ); + validate(&bytes); + let printed = wasmprinter::print_bytes(&bytes).expect("print instrumented tail-call fixture"); + assert!( + printed.contains("return_call_ref"), + "a tail call that cannot reach fork must retain bounded-stack semantics" + ); +} + +#[test] +fn fork_reaching_tail_calls_remain_bounded_and_route_to_resume_thunks() { + let wat = r#" + (module + (import "kernel" "kernel_fork" (func $fork (result i32))) + (type $ft (func (result i32))) + (table 1 funcref) + (elem (i32.const 0) $deep) + (elem declare func $deep) + (func $deep (type $ft) (result i32) + call $fork) + (func $tail_direct (type $ft) (result i32) + return_call $deep) + (func $tail_indirect (type $ft) (result i32) + i32.const 0 + return_call_indirect (type $ft)) + (func $tail_ref (type $ft) (result i32) + ref.func $deep + return_call_ref $ft) + (func $root_direct (export "root_direct") (result i32) + call $tail_direct) + (func $root_indirect (export "root_indirect") (result i32) + call $tail_indirect) + (func $root_ref (export "root_ref") (result i32) + call $tail_ref) + (memory 1)) + "#; + let bytes = instrument_wat(wat); + validate(&bytes); + let module = Module::from_buffer(&bytes).expect("instrumented module"); + + for name in ["tail_direct", "tail_indirect", "tail_ref"] { + let function = local_func(&module, func_by_name(&module, name)); + assert!( + function + .block(function.entry_block()) + .instrs + .iter() + .any(|(instruction, _)| matches!(instruction, Instr::ReturnCall(_))), + "{name} must retain a bounded direct tail call, either to the \ + original local target or to a generated transport helper" ); } -} - -#[test] -fn slot_counts_aggregate_across_functions() { - let bytes = instrument_wat(FIXTURE_TWO_FUNCREF_CALLERS); - validate(&bytes); - let module = Module::from_buffer(&bytes).unwrap(); + assert!( + module.funcs.iter().any(|function| function + .name + .as_deref() + .is_some_and(|name| name.starts_with("__wpk_fork_unwind_transport_indirect_"))), + "fork-reaching return_call_indirect must tail-call a transport helper" + ); + assert!( + module.funcs.iter().any(|function| function + .name + .as_deref() + .is_some_and(|name| name.starts_with("__wpk_fork_unwind_transport_ref_"))), + "fork-reaching return_call_ref must tail-call a transport helper" + ); + for root in ["root_direct", "root_indirect", "root_ref"] { + assert_resume_routing(&module, root); + } - let stash = module - .tables + let catalog = module + .exports .iter() - .find(|t| t.name.as_deref() == Some("_wpk_fork_funcref_stash")) - .expect("funcref stash should be injected"); - assert_eq!(stash.initial, 2); + .find(|export| export.name == "__wpk_fork_resume_catalog") + .and_then(|export| match export.item { + ExportItem::Table(table) => Some(module.tables.get(table)), + _ => None, + }) + .expect("resume catalog export"); + assert_eq!( + catalog.initial, 4, + "only the deep and three root activations receive resume thunks" + ); + assert!( + module + .customs + .iter() + .any(|(_, section)| section.name() == "kandelo.wpk_fork.resume_catalog"), + "resume catalog metadata must bind function ordinals to local slots" + ); } #[test] -fn externref_local_routes_through_externref_stash() { +fn fixed_main_and_pthread_resume_boundaries_dispatch_inside_wasm() { let wat = r#" (module (import "kernel" "kernel_fork" (func $fork (result i32))) - (func $caller (export "caller") (result i32) - (local $x externref) - ref.null extern - local.set $x + (type $thread_ty (func (param i32) (result i32))) + (table $functions (export "__indirect_function_table") 1 1 funcref) + (elem (i32.const 0) $thread) + (func $thread (type $thread_ty) (param $arg i32) (result i32) + local.get $arg) + (func $_start (export "_start") call $fork - local.get $x drop) (memory 1)) "#; let bytes = instrument_wat(wat); validate(&bytes); - let module = Module::from_buffer(&bytes).unwrap(); + let module = Module::from_buffer(&bytes).expect("instrumented module"); - assert!( - module - .tables - .iter() - .any(|t| t.name.as_deref() == Some("_wpk_fork_externref_stash")), - "externref local should trigger externref stash injection", + let start_wrapper = func_by_name(&module, "wpk_fork_resume_start"); + let start_ty = module.types.get(module.funcs.get(start_wrapper).ty()); + assert!(start_ty.params().is_empty()); + assert!(start_ty.results().is_empty()); + assert_resume_routing(&module, "wpk_fork_resume_start"); + assert_eq!( + sequences_with_direct_call(&module, "wpk_fork_resume_start", "_start").len(), + 1, + "zero-sentinel start replay must retain the lexical _start path" ); - assert!( - !module - .tables - .iter() - .any(|t| t.name.as_deref() == Some("_wpk_fork_funcref_stash")), - "externref-only module should not inject funcref stash", + + let thread_wrapper = func_by_name(&module, "wpk_fork_resume_thread"); + let thread_ty = module.types.get(module.funcs.get(thread_wrapper).ty()); + assert_eq!(thread_ty.params(), &[ValType::I32, ValType::I32]); + assert_eq!(thread_ty.results(), &[ValType::I32]); + assert_resume_routing(&module, "wpk_fork_resume_thread"); + let original_table = module + .exports + .iter() + .find(|export| export.name == "__indirect_function_table") + .and_then(|export| match export.item { + ExportItem::Table(table) => Some(table), + _ => None, + }) + .expect("original pthread function table"); + let wrapper = local_func(&module, thread_wrapper); + let mut lexical_thread_calls = 0; + walk_all(wrapper, wrapper.entry_block(), &mut |_, instruction| { + if matches!(instruction, Instr::CallIndirect(call) if call.table == original_table) { + lexical_thread_calls += 1; + } + }); + assert_eq!( + lexical_thread_calls, 1, + "zero-sentinel pthread replay must retain the lexical table dispatch" ); } #[test] -#[should_panic(expected = "fork-instrument 4f")] -fn unsupported_ref_type_panics_with_diagnostic() { - let wat = r#" +fn references_outside_the_fork_closure_remain_legal() { + let bytes = instrument_wat( + r#" (module (import "kernel" "kernel_fork" (func $fork (result i32))) - (func $caller (result i32) - (local $r (ref null any)) - ref.null any - local.set $r - call $fork - local.get $r + (elem declare func $target) + (func $target) + (func $unrelated (export "unrelated") + (local $value externref) + ref.null extern + local.set $value + local.get $value + ref.func $target + drop drop) + (func $caller (result i32) call $fork) (memory 1)) - "#; - let _ = instrument_wat(wat); + "#, + ); + validate(&bytes); } #[test] -fn module_without_try_tables_skips_exnref_stash() { - let bytes = instrument_wat(FIXTURE_DIRECT_CALLER); - let module = Module::from_buffer(&bytes).unwrap(); - assert!( - !module - .tables - .iter() - .any(|t| t.name.as_deref() == Some("_wpk_fork_exnref_stash")), - "module with no try_tables should not inject the exnref stash", +fn dead_catch_all_ref_value_before_fork_remains_legal() { + let bytes = instrument_wat( + r#" + (module + (import "kernel" "kernel_fork" (func $fork (result i32))) + (tag $exn) + (func $caller (result i32) + (block $handler (result exnref) + (try_table (result exnref) (catch_all_ref $handler) + throw $exn)) + drop + call $fork) + (memory 1)) + "#, + ); + validate(&bytes); +} + +#[test] +fn catch_all_without_live_reference_state_remains_legal() { + let bytes = instrument_wat( + r#" + (module + (import "kernel" "kernel_fork" (func $fork (result i32))) + (tag $exn) + (func $caller (result i32) + (block $handler + (try_table (catch_all $handler) + throw $exn)) + call $fork) + (memory 1)) + "#, + ); + validate(&bytes); +} + +#[test] +fn private_unwind_precedes_user_catch_all_and_catch_all_ref() { + let fixtures = [ + ( + "catch_all", + r#" + (module + (import "kernel" "kernel_fork" (func $fork (result i32))) + (func $leaf (result i32) call $fork) + (func $caller (export "caller") (result i32) + (block $handler + (try_table (catch_all $handler) + call $leaf + drop)) + i32.const 0) + (memory 1)) + "#, + ), + ( + "catch_all_ref", + r#" + (module + (import "kernel" "kernel_fork" (func $fork (result i32))) + (func $leaf (result i32) call $fork) + (func $caller (export "caller") (result i32) + (block $handler (result exnref) + (try_table (result exnref) (catch_all_ref $handler) + call $leaf + drop + ref.null exn)) + drop + i32.const 0) + (memory 1)) + "#, + ), + ]; + + for (label, wat) in fixtures { + let bytes = instrument_wat_unchecked(wat); + validate(&bytes); + let module = Module::from_buffer(&bytes).expect("parse rewritten module"); + let unwind_tag = private_unwind_tag(&module); + let caller = func_by_name(&module, "caller"); + let mut shielded = 0usize; + walk_all( + local_func(&module, caller), + local_func(&module, caller).entry_block(), + &mut |_, instr| { + let Instr::TryTable(table) = instr else { + return; + }; + let Some(catch_all_index) = table.catches.iter().position(|catch| { + matches!( + catch, + ir::TryTableCatch::CatchAll { .. } | ir::TryTableCatch::CatchAllRef { .. } + ) + }) else { + return; + }; + assert!(catch_all_index > 0, "{label}: catch-all cannot be first"); + assert!( + matches!( + table.catches[catch_all_index - 1], + ir::TryTableCatch::Catch { tag, .. } if tag == unwind_tag + ), + "{label}: private transport must be intercepted and rethrown before user catch-all: {:?}", + table.catches, + ); + shielded += 1; + }, + ); + assert_eq!(shielded, 1, "{label}: expected one shielded user try_table"); + } +} + +#[test] +fn catch_all_and_catch_all_ref_live_across_fork_use_complete_exception_recipes() { + let fixtures = [ + ( + "catch_all", + r#" + (module + (import "kernel" "kernel_fork" (func $fork (result i32))) + (tag $failure) + (func $caller (export "caller") (result i32) + (block $done (result i32) + (block $handler + (try_table (catch_all $handler) + throw $failure) + unreachable) + call $fork + drop + i32.const 17 + br $done)) + (memory 1)) + "#, + ), + ( + "catch_all_ref", + r#" + (module + (import "kernel" "kernel_fork" (func $fork (result i32))) + (tag $failure) + (func $caller (export "caller") (result i32) + (block $done (result i32) + (block $handler (result exnref) + (try_table (result exnref) (catch_all_ref $handler) + throw $failure)) + drop + call $fork + drop + i32.const 23 + br $done)) + (memory 1)) + "#, + ), + ]; + + for (label, wat) in fixtures { + let bytes = instrument_wat(wat); + validate(&bytes); + let module = Module::from_buffer(&bytes).expect("parse catch-all replay fixture"); + assert_function_calls_codec_pair( + &module, + "caller", + runtime_names::IMPORT_REF_ENCODE_EXNREF, + runtime_names::IMPORT_REF_DECODE_EXNREF, + ); + let printed = wasmprinter::print_bytes(&bytes).expect("print catch-all replay fixture"); + let caller = extract_function_text(&printed, "caller"); + assert!( + caller.contains("catch_all_ref"), + "{label}: the capture path must bind an instance-local exnref:\n{caller}", + ); + assert!( + caller.contains("throw_ref"), + "{label}: rewind must replay the complete exception recipe:\n{caller}", + ); + } +} + +#[test] +fn every_wasm_table_mutation_has_module_state_owner() { + let cases = [ + ("table.set", "i32.const 0 ref.null func table.set", ""), + ( + "table.fill", + "i32.const 0 ref.null func i32.const 1 table.fill", + "", + ), + ( + "table.copy", + "i32.const 0 i32.const 0 i32.const 1 table.copy", + "", + ), + ( + "table.init", + "i32.const 0 i32.const 0 i32.const 1 table.init $elements", + "(elem $elements funcref (ref.null func))", + ), + ( + "table.grow", + "ref.null func i32.const 1 table.grow drop", + "", + ), + ]; + for (_name, operation, element) in cases { + let wat = format!( + r#" + (module + (import "kernel" "kernel_fork" (func $fork (result i32))) + (table 1 funcref) + {element} + (func $unrelated {operation}) + (func $caller (result i32) call $fork) + (memory 1)) + "#, + ); + let bytes = instrument_wat(&wat); + validate(&bytes); + } +} + +#[test] +fn static_table_initialization_is_recreated_and_remains_legal() { + let bytes = instrument_wat( + r#" + (module + (import "kernel" "kernel_fork" (func $fork (result i32))) + (table 1 funcref) + (func $target) + (elem (i32.const 0) func $target) + (func $caller (result i32) call $fork) + (memory 1)) + "#, ); + validate(&bytes); } // --- Non-fork-path try_tables ---------------------------------------- @@ -1328,7 +2673,7 @@ fn try_table_on_non_fork_path_is_not_instrumented() { .tables .iter() .any(|t| t.name.as_deref() == Some("_wpk_fork_exnref_stash")), - "non-fork-path try_tables should not force exnref stash injection", + "non-fork-path references must not cause legacy reference storage", ); } @@ -1418,16 +2763,12 @@ fn fork_inside_try_body_uses_per_block_switch_dispatch() { (br_table emitted), not guard-dispatch's body-replay", ); - // The exnref stash and Phase 6a/6c/6d plumbing are still injected - // for try_tables — the per-block dispatch overlays on top of the - // existing catch-handler scaffolding (used by fork-from-catch in - // the B1 follow-up). assert!( - module + !module .tables .iter() .any(|t| t.name.as_deref() == Some("_wpk_fork_exnref_stash")), - "Phase 6a must inject exnref stash for a fork-path try_table", + "fork-path try_tables must not inject module-instance exnref storage", ); } @@ -1498,7 +2839,7 @@ fn fork_in_both_top_level_and_nested_uses_per_block_switch_dispatch() { ); } -// --- Phase 6 (guard-dispatch only) tests ------------------------------------- +// --- Tagged-catch reconstruction (guard-dispatch) tests ---------------------- // // These pin down the Phase 6 plumbing that guard-dispatch uses for // `try_table` catch-handler reconstruction. The fixtures all have @@ -1549,24 +2890,15 @@ fn distinct_try_tables_get_sequential_region_ids() { let caller = func_by_name(&module, "caller"); let f = local_func(&module, caller); - let mut bodies = Vec::new(); - collect_try_table_bodies(f, f.entry_block(), &mut bodies); - assert_eq!(bodies.len(), 2, "fixture has two try_tables"); - - // After per-block switch-dispatch lands on a try_table body's - // seq, the body is rebuilt as [Block(POST_{n-1}), post-call, - // chunks[n], ...]. Phase 6c stubs (which run before the rebuild) - // are folded into the cascade — they live somewhere in the - // chunks but are no longer at fixed positions. Just verify the - // exnref stash is injected with one slot per try_table. - let stash = module - .tables - .iter() - .find(|t| t.name.as_deref() == Some("_wpk_fork_exnref_stash")) - .expect("stash must be injected"); - assert_eq!( - stash.initial, 2, - "two try_tables → two exnref stash slots (one region_id each)", + let try_tables = collect_user_try_tables(&module, f); + assert_eq!(try_tables.len(), 2, "fixture has two user try_tables"); + + assert!( + !module + .tables + .iter() + .any(|t| t.name.as_deref() == Some("_wpk_fork_exnref_stash")), + "region identity must live in activation frames, not module tables", ); } @@ -1583,8 +2915,12 @@ fn catch_ref_clause_is_rewritten_with_capture_block() { // $capture block (not the original $handler). let mut try_table: Option = None; walk_all(f, f.entry_block(), &mut |_, instr| { - if try_table.is_none() { - if let Instr::TryTable(tt) = instr { + if let Instr::TryTable(tt) = instr { + if tt + .catches + .iter() + .any(|catch| matches!(catch, ir::TryTableCatch::CatchRef { .. })) + { try_table = Some(tt.clone()); } } @@ -1603,11 +2939,10 @@ fn catch_ref_clause_is_rewritten_with_capture_block() { } #[test] -fn plain_catch_only_try_table_is_not_6d_rewritten() { - // Plain `catch` clauses (no exnref) are not redirected by Phase - // 6d — fork-from-catch-without-exnref is unsupported. The - // try_table still receives a 6c rewind-throw stub at its body, - // but its catch clause remains pointing at the original handler. +fn plain_catch_capture_preserves_plain_clause_kind() { + // Scalar plain catches use the activation-owned selector/payload path. + // Their generated capture still uses a plain Catch (not CatchRef), + // because replay can reconstruct the exact tag and scalar payload. let wat = r#" (module (import "kernel" "kernel_fork" (func $fork (result i32))) @@ -1642,7 +2977,7 @@ fn plain_catch_only_try_table_is_not_6d_rewritten() { .catches .iter() .all(|c| matches!(c, ir::TryTableCatch::Catch { .. })), - "plain-catch-only try_tables should not be retargeted by Phase 6d", + "scalar plain catches should preserve their Catch clause kind", ); } @@ -1720,7 +3055,8 @@ fn discover_plain_catch_arms_returns_one_arm_for_single_catch() { .expect("module declares one tag") .id(); assert_eq!( - arm.tag, module_tag_id, + arm.tag, + Some(module_tag_id), "arm.tag should equal the module's declared tag id", ); @@ -1943,17 +3279,10 @@ fn plain_catch_plan_preserves_f32_f64_operand_types() { assert_eq!(arms[0].operand_tys, vec![ValType::F32, ValType::F64]); } -// --- B1 Stage 2 Task 2.1 — operand-type carve-out tests ---------------- +// --- Exception-recipe tag payloads and multi-target support ------------ #[test] -fn plain_catch_plan_ref_operand_function_is_carved_out() { - // A try_table with a tag whose payload includes externref. - // The function should land in b2_carveout, NOT in per_function. - // - // Catch label semantics mirror the existing scalar tests: the - // block's RESULT type matches the tag's payload arity, and the - // body drops the value before falling through to a synthesized - // ref to keep stack arity consistent. +fn reference_typed_catch_payload_uses_complete_exception_recipe() { let wat = r#" (module (import "kernel" "kernel_fork" (func $fork (result i32))) @@ -1968,26 +3297,14 @@ fn plain_catch_plan_ref_operand_function_is_carved_out() { call $fork) (memory 1)) "#; - let bytes = parse_wat(wat); - let module = walrus::Module::from_buffer(&bytes).unwrap(); - let caller = func_by_name(&module, "caller"); - let plan = fork_instrument::instrument::plan_plain_catches(&module, &[caller]); - assert!( - !plan.per_function.contains_key(&caller), - "carved-out function must not appear in per_function" - ); - assert!( - plan.b2_carveout.contains(&caller), - "carved-out function must be in b2_carveout" - ); + let bytes = instrument_wat(wat); + validate(&bytes); + let module = Module::from_buffer(&bytes).unwrap(); + assert_function_uses_exception_recipe(&module, "caller"); } #[test] -fn plain_catch_plan_mixed_ref_and_scalar_arms_carves_whole_function() { - // A function with two try_tables: one with i32 payload (supported), - // one with externref (unsupported). The whole function gets carved - // out because we don't selectively drop arms — Task 2.3's rewind - // dispatcher needs the whole function's regions or none. +fn mixed_scalar_and_reference_typed_arms_use_their_matching_replay_form() { let wat = r#" (module (import "kernel" "kernel_fork" (func $fork (result i32))) @@ -2009,27 +3326,30 @@ fn plain_catch_plan_mixed_ref_and_scalar_arms_carves_whole_function() { call $fork) (memory 1)) "#; - let bytes = parse_wat(wat); - let module = walrus::Module::from_buffer(&bytes).unwrap(); - let caller = func_by_name(&module, "caller"); - let plan = fork_instrument::instrument::plan_plain_catches(&module, &[caller]); + let source = Module::from_buffer(&parse_wat(wat)).unwrap(); + let caller = func_by_name(&source, "caller"); + let plan = fork_instrument::instrument::plan_plain_catches(&source, &[caller]); + let arms: Vec<_> = plan.per_function[&caller] + .iter() + .flat_map(|(_, arms)| arms) + .collect(); assert!( - plan.b2_carveout.contains(&caller), - "carve-out must include functions with mixed scalar+ref arms" + arms.iter().any(|arm| arm.uses_exception_recipe), + "the reference-bearing arm must own a complete exception recipe" ); assert!( - !plan.per_function.contains_key(&caller), - "carved-out function must not appear in per_function even \ - though one arm is otherwise supported" + arms.iter().any(|arm| !arm.uses_exception_recipe), + "the scalar arm should retain compact tag-and-payload replay" ); + + let bytes = instrument_wat(wat); + validate(&bytes); + let module = Module::from_buffer(&bytes).unwrap(); + assert_function_uses_exception_recipe(&module, "caller"); } #[test] -fn plain_catch_plan_scalar_only_function_is_not_carved_out() { - // Sanity: the existing scalar-only fixture must NOT be carved out. - // Mirrors the scalar tests above to ensure carve-out is gated - // strictly on ref-typed operands and doesn't accidentally trip - // for the supported case. +fn plain_catch_plan_scalar_only_function_is_supported() { let wat = r#" (module (import "kernel" "kernel_fork" (func $fork (result i32))) @@ -2045,10 +3365,6 @@ fn plain_catch_plan_scalar_only_function_is_not_carved_out() { let module = walrus::Module::from_buffer(&bytes).unwrap(); let caller = func_by_name(&module, "caller"); let plan = fork_instrument::instrument::plan_plain_catches(&module, &[caller]); - assert!( - !plan.b2_carveout.contains(&caller), - "scalar-only function must not be in b2_carveout" - ); assert!( plan.per_function.contains_key(&caller), "scalar-only function must have a per_function entry" @@ -2056,10 +3372,7 @@ fn plain_catch_plan_scalar_only_function_is_not_carved_out() { } #[test] -fn plain_catch_plan_multi_target_plain_catch_carved_out() { - // Two arms in one try_table, pointing at different labels. - // Should be carved out (Task 2.4 conservative guard: multi-target - // plain-catch fork has not been verified end-to-end). +fn plain_catch_plan_multi_target_plain_catch_is_supported() { let wat = r#" (module (import "kernel" "kernel_fork" (func $fork (result i32))) @@ -2077,21 +3390,14 @@ fn plain_catch_plan_multi_target_plain_catch_carved_out() { let module = walrus::Module::from_buffer(&bytes).unwrap(); let caller = func_by_name(&module, "caller"); let plan = fork_instrument::instrument::plan_plain_catches(&module, &[caller]); - assert!( - plan.b2_carveout.contains(&caller), - "multi-target try_table should be carved out" - ); - assert!( - !plan.per_function.contains_key(&caller), - "carved-out function should not have a slot plan" - ); + let regions = &plan.per_function[&caller]; + assert_eq!(regions.len(), 1); + assert_eq!(regions[0].1.len(), 2); + assert_ne!(regions[0].1[0].label, regions[0].1[1].label); } #[test] fn plain_catch_plan_single_target_multi_arm_is_supported() { - // Two arms in one try_table, both pointing at the SAME label. - // Should NOT be carved out (this is the supported multi-arm case - // — Task 2.4's guard only triggers when arms diverge). let wat = r#" (module (import "kernel" "kernel_fork" (func $fork (result i32))) @@ -2108,10 +3414,6 @@ fn plain_catch_plan_single_target_multi_arm_is_supported() { let module = walrus::Module::from_buffer(&bytes).unwrap(); let caller = func_by_name(&module, "caller"); let plan = fork_instrument::instrument::plan_plain_catches(&module, &[caller]); - assert!( - !plan.b2_carveout.contains(&caller), - "single-target multi-arm should be supported" - ); assert!(plan.per_function.contains_key(&caller)); let per_func = &plan.per_function[&caller]; assert_eq!(per_func.len(), 1, "one try_table"); @@ -2184,8 +3486,42 @@ fn collect_try_tables(f: &LocalFunction) -> Vec { out } +fn private_unwind_tag(module: &Module) -> walrus::TagId { + module + .imports + .iter() + .find_map(|import| { + if import.module == runtime_names::IMPORT_UNWIND_TAG_MODULE + && import.name == runtime_names::IMPORT_UNWIND_TAG + { + match import.kind { + walrus::ImportKind::Tag(tag) => Some(tag), + _ => None, + } + } else { + None + } + }) + .expect("private unwind tag import") +} + +fn is_function_unwind_boundary(table: &ir::TryTable, unwind_tag: walrus::TagId) -> bool { + matches!( + table.catches.as_slice(), + [ir::TryTableCatch::Catch { tag, .. }] if *tag == unwind_tag + ) +} + +fn collect_user_try_tables(module: &Module, f: &LocalFunction) -> Vec { + let unwind_tag = private_unwind_tag(module); + collect_try_tables(f) + .into_iter() + .filter(|table| !is_function_unwind_boundary(table, unwind_tag)) + .collect() +} + #[test] -fn b1_stage_2_plain_catch_arm_uses_frame_backed_state() { +fn catch_arm_uses_header_selector_without_an_active_arm_frame_local() { // After instrumentation, the original try_table's plain Catch // clause should point at an injected capture block, not at the // original handler label `$h`. The capture block contains the @@ -2217,7 +3553,7 @@ fn b1_stage_2_plain_catch_arm_uses_frame_backed_state() { let module = Module::from_buffer(&bytes).unwrap(); let caller = func_by_name(&module, "caller"); let f = local_func(&module, caller); - let try_tables = collect_try_tables(f); + let try_tables = collect_user_try_tables(&module, f); assert_eq!( try_tables.len(), 1, @@ -2230,9 +3566,8 @@ fn b1_stage_2_plain_catch_arm_uses_frame_backed_state() { "should be a plain Catch clause" ); - // 2. Byte-level (wasmprinter): active-arm is an ordinary scalar - // local and therefore round-trips through the function frame at - // the first offset after its 16-byte header. + // 2. Byte-level (wasmprinter): the exact region/arm selector reuses + // header word +8. An empty-payload arm adds no scalar frame word. let printed = wasmprinter::print_bytes(&bytes).expect("wasmprinter"); let caller_section = extract_function_text(&printed, "caller"); assert!( @@ -2240,13 +3575,136 @@ fn b1_stage_2_plain_catch_arm_uses_frame_backed_state() { "caller must still have a try_table:\n{caller_section}" ); assert!( - caller_section.contains("i32.store offset=16"), - "active-arm local must be serialized after the frame header:\n\ - {caller_section}" + caller_section.contains("i32.store offset=8") + && caller_section.contains("i32.load offset=8"), + "the exact catch selector must round-trip through header word +8:\n\ + {caller_section}", + ); + assert!( + !caller_section.contains("store offset=16") && !caller_section.contains("load offset=16"), + "an empty catch payload must not allocate the former active-arm frame \ + word:\n{caller_section}", + ); + + let mut locals = HashSet::new(); + walk_all( + f, + f.entry_block(), + &mut |_, instruction| match instruction { + Instr::LocalGet(local) => { + locals.insert(local.local); + } + Instr::LocalSet(local) => { + locals.insert(local.local); + } + _ => {} + }, + ); + assert_eq!( + locals.len(), + 1, + "only the activation-local catch selector is needed; static call \ + boundaries add no abort/live-frame selector, and no per-region marker \ + or native active-arm local should exist: {locals:?}", + ); +} + +#[test] +fn catch_payload_frame_overlays_arms_at_the_maximum_arm_size() { + let wat = r#" + (module + (import "kernel" "kernel_fork" (func $fork (result i32))) + (tag $small (param i32)) + (tag $wide (param i64 i32)) + (tag $medium (param f64)) + (func $caller (export "caller") (result i32) + (block $small_handler (result i32) + (block $wide_handler (result i64 i32) + (block $medium_handler (result f64) + (try_table + (catch $small $small_handler) + (catch $wide $wide_handler) + (catch $medium $medium_handler) + nop) + f64.const 0) + drop + i64.const 0 + i32.const 0) + drop + drop + i32.const 0) + drop + call $fork) + (memory 1)) + "#; + let bytes = instrument_wat(wat); + validate(&bytes); + let module = Module::from_buffer(&bytes).expect("instrumented catch payload module"); + let caller = local_func(&module, func_by_name(&module, "caller")); + let frame_select = module + .funcs + .iter() + .find(|function| function.name.as_deref() == Some("__wpk_fork_select_unwind_frame")) + .expect("generated unwind-frame selector") + .id(); + + fn collect_reserve_sizes( + function: &LocalFunction, + sequence: InstrSeqId, + frame_select: FunctionId, + sizes: &mut Vec, + ) { + let instructions = &function.block(sequence).instrs; + for (index, (instruction, _)) in instructions.iter().enumerate() { + if matches!(instruction, Instr::Call(call) if call.func == frame_select) { + let Some(( + Instr::Const(ir::Const { + value: ir::Value::I32(size), + }), + _, + )) = index + .checked_sub(2) + .and_then(|previous| instructions.get(previous)) + else { + panic!( + "unwind-frame selector must be preceded by its exact \ + static size and call index" + ); + }; + sizes.push(*size); + } + for child in nested_of(instruction) { + collect_reserve_sizes(function, child, frame_select, sizes); + } + } + } + + let mut sizes = Vec::new(); + collect_reserve_sizes(caller, caller.entry_block(), frame_select, &mut sizes); + assert!( + !sizes.is_empty(), + "caller must reserve at least one unwind frame" + ); + assert!( + sizes.iter().all(|size| *size == 28), + "new frame = 16-byte header + max(4, 12, 8) payload = 28 bytes; \ + the former sum layout was 16 + 4-byte active-arm + 4 + 12 + 8 = \ + 44 bytes: {sizes:?}", + ); + + let printed = wasmprinter::print_bytes(&bytes).expect("wasmprinter"); + let caller_section = extract_function_text(&printed, "caller"); + assert!( + caller_section.contains("i64.store offset=16") + && caller_section.contains("i32.store offset=24") + && caller_section.contains("f64.store offset=16"), + "each selected arm must use the shared payload range, with only the \ + widest arm extending to +24:\n{caller_section}", ); assert!( - caller_section.contains("i32.load offset=16"), - "active-arm local must be restored from its frame:\n{caller_section}" + !caller_section.contains("store offset=28"), + "no catch payload may be appended after the 12-byte union:\n\ + {caller_section}", ); } @@ -2276,21 +3734,7 @@ fn extract_function_text<'a>(printed: &'a str, name: &str) -> String { } #[test] -fn b1_stage_2_b2_carveout_function_is_not_transformed() { - // A function whose plain-catch arm has a ref-typed payload is in - // b2_carveout (per Task 2.1). For these functions, B1 emission - // is skipped, so the byte output must NOT contain the B1 capture - // block's save-to-scratch pattern. The Catch clause must still - // be present (Phase 6 doesn't intercept plain catch). - // - // Note: ref-typed catch payloads are not yet supported by the - // existing Phase-6 ref-local pipeline (function would panic with - // a "non-nullable or non-abstract ref" or fail wasm validation in - // some shapes). We use a fork-bearing function that *contains* - // a try_table whose tag has a ref operand, but the catch handler - // itself stays simple. To avoid type-mismatch errors during wat - // parse, we feed the catch via `throw`-then-`drop` inside a block - // typed `(result externref)`. +fn thrown_reference_payload_replays_from_complete_exception_recipe() { let wat = r#" (module (import "kernel" "kernel_fork" (func $fork (result i32))) @@ -2307,49 +3751,15 @@ fn b1_stage_2_b2_carveout_function_is_not_transformed() { let bytes = instrument_wat(wat); validate(&bytes); let module = Module::from_buffer(&bytes).unwrap(); - let caller = func_by_name(&module, "caller"); - let f = local_func(&module, caller); - - // Carve-out functions have NO B1 transform applied. The try_table - // is preserved with its catches as-emitted by Phase 6 (which - // doesn't intercept plain Catch clauses; only catch_ref / catch_all_ref). - let try_tables = collect_try_tables(f); - assert_eq!( - try_tables.len(), - 1, - "carved-out function must still have exactly one try_table" - ); - let tt = &try_tables[0]; - let has_catch = tt - .catches - .iter() - .any(|c| matches!(c, ir::TryTableCatch::Catch { .. })); - assert!( - has_catch, - "carved-out function's plain Catch clause must be preserved \ - (B1 must NOT have transformed it)" - ); - - // The direct planner API must list the function in b2_carveout, - // not per_function. - let plan = fork_instrument::instrument::plan_plain_catches(&module, &[caller]); - assert!( - plan.b2_carveout.contains(&caller), - "carved-out function (ref-typed catch operand) must be in \ - b2_carveout" - ); - assert!( - !plan.per_function.contains_key(&caller), - "carved-out function must NOT have a per_function entry" - ); + assert_function_uses_exception_recipe(&module, "caller"); } #[test] fn b1_stage_2_byte_identity_for_module_without_plain_catch() { // A fork-using module with NO plain-catch should produce stable - // output that's byte-identical across repeated runs (instrument - // is deterministic) and produces ZERO try_tables — Stage 2's - // emission must not fire when there are no plain-catch arms. + // output that's byte-identical across repeated runs. The only + // try_table is the function-level private unwind boundary; Stage + // 2 must not introduce a user-catch capture table. let wat = r#" (module (import "kernel" "kernel_fork" (func $fork (result i32))) @@ -2365,12 +3775,19 @@ fn b1_stage_2_byte_identity_for_module_without_plain_catch() { let module = Module::from_buffer(&bytes_a).unwrap(); let caller = func_by_name(&module, "caller"); let f = local_func(&module, caller); - let try_tables = collect_try_tables(f); + let all_try_tables = collect_try_tables(f); + assert_eq!( + all_try_tables.len(), + 1, + "fork-only function should contain exactly one private transport boundary", + ); assert!( - try_tables.is_empty(), - "no try_tables expected for a fork-only function — Stage 2 \ - must not introduce any: got {} try_tables", - try_tables.len() + is_function_unwind_boundary(&all_try_tables[0], private_unwind_tag(&module)), + "the sole try_table must catch only the process-owned unwind tag", + ); + assert!( + collect_user_try_tables(&module, f).is_empty(), + "Stage 2 must not introduce a user catch table when the input has none", ); } @@ -2457,8 +3874,7 @@ fn fork_instrumentation_keeps_dylink_section_first() { #[test] fn b1_stage_2_rewind_stub_has_plain_catch_dispatch() { // The rewind-throw stub for a region with a plain-catch arm must - // include a `throw $tag` (in addition to Phase 6's existing - // `throw_ref`) so that on REWIND the original handler observes + // include a `throw $tag` so that on REWIND the original handler observes // the same exception class. The exact wat shape varies with // walrus's emitter, so we just check the key semantic markers. let wat = r#" @@ -2477,25 +3893,18 @@ fn b1_stage_2_rewind_stub_has_plain_catch_dispatch() { let printed = wasmprinter::print_bytes(&bytes).expect("wasmprinter"); let caller_section = extract_function_text(&printed, "caller"); - // Both stub paths must be present: - // - throw_ref (Phase 6 catch_ref re-throw) - // - throw $exn (B1 plain-catch arm dispatch) assert!( - caller_section.contains("throw_ref"), - "rewind stub must retain Phase 6's throw_ref path:\n{caller_section}" + !caller_section.contains("throw_ref"), + "ABI 43 replay must not depend on a saved exnref:\n{caller_section}" ); assert!( caller_section.contains("throw $exn") || caller_section.contains("throw 0"), "rewind stub must contain a `throw $exn` for the plain-catch \ arm dispatch:\n{caller_section}" ); - // The mode decision must be frame-owned. Stale contents in the - // static exnref table cannot decide whether this activation took - // a plain or ref catch. assert!( - !caller_section.contains("ref.is_null"), - "rewind stub must dispatch from frame-backed active_arm, not \ - exnref-table nullness:\n{caller_section}" + !caller_section.contains("_wpk_fork_exnref_stash"), + "rewind stub must be activation-owned:\n{caller_section}" ); } @@ -2534,14 +3943,16 @@ fn mixed_plain_and_catch_ref_uses_frame_backed_arm_kind() { let printed = wasmprinter::print_bytes(&bytes).expect("wasmprinter"); let caller_section = extract_function_text(&printed, "caller"); assert!( - caller_section.contains("i32.const -1"), - "catch_ref capture must mark the frame-backed active arm as \ - non-plain:\n{caller_section}", + caller_section.contains("throw $plain") || caller_section.contains("throw 0"), + "plain arm must have a tagged reconstruction path:\n{caller_section}", + ); + assert!( + caller_section.contains("throw $with_ref") || caller_section.contains("throw 1"), + "CatchRef arm must reconstruct by rethrowing its static tag:\n{caller_section}", ); assert!( - !caller_section.contains("ref.is_null"), - "mixed replay must not use static exnref-table contents as \ - capture-kind state:\n{caller_section}", + !caller_section.contains("throw_ref") && !caller_section.contains("_wpk_fork_exnref_stash"), + "mixed replay must not retain or reload an old-instance exnref:\n{caller_section}", ); } @@ -2586,10 +3997,7 @@ fn b1_stage_2_rewind_stub_dispatches_two_arms() { } #[test] -fn b1_stage_2_carved_out_function_no_b1_dispatch_emitted() { - // A function in `b2_carveout` (here: ref-typed catch payload) - // must NOT receive B1's plain-catch dispatch — its rewind stub - // should retain Phase 6's throw_ref-only form. +fn reference_payload_emits_complete_exception_dispatch() { let wat = r#" (module (import "kernel" "kernel_fork" (func $fork (result i32))) @@ -2605,18 +4013,6 @@ fn b1_stage_2_carved_out_function_no_b1_dispatch_emitted() { "#; let bytes = instrument_wat(wat); validate(&bytes); - - let printed = wasmprinter::print_bytes(&bytes).expect("wasmprinter"); - let caller_section = extract_function_text(&printed, "caller"); - // The carved-out function's rewind stub uses ONLY Phase 6's - // throw_ref path and no plain-catch tag throw. - assert!( - caller_section.contains("throw_ref"), - "carved-out function must retain Phase 6's throw_ref:\n{caller_section}" - ); - assert!( - !caller_section.contains("ref.is_null"), - "carved-out function must not consult exnref nullness as mode \ - state:\n{caller_section}" - ); + let module = Module::from_buffer(&bytes).unwrap(); + assert_function_uses_exception_recipe(&module, "caller"); } diff --git a/crates/fork-instrument/tests/large_dispatcher.rs b/crates/fork-instrument/tests/large_dispatcher.rs index c1ebe5a005..82ee5add41 100644 --- a/crates/fork-instrument/tests/large_dispatcher.rs +++ b/crates/fork-instrument/tests/large_dispatcher.rs @@ -182,14 +182,20 @@ fn dispatcher_call_count(bytes: &[u8]) -> usize { count } -/// `instrument_one_function_switch` places the preamble, unwind-save block, -/// and postamble inside one result-typed restart loop. +/// `instrument_one_function_switch` places the replay preamble before one +/// result-typed live-restart loop containing the unwind-save block and +/// postamble. fn dispatcher_unwind_save(local: &LocalFunction) -> InstrSeqId { let entry = local.block(local.entry_block()); - let restart = match entry.instrs.as_slice() { - [(Instr::Loop(ir::Loop { seq }), _)] => *seq, - other => panic!("expected one top-level restart Loop, got {other:?}"), - }; + let restart = entry + .instrs + .iter() + .filter_map(|(instruction, _)| match instruction { + Instr::Loop(ir::Loop { seq }) => Some(*seq), + _ => None, + }) + .next_back() + .unwrap_or_else(|| panic!("expected a top-level live-restart Loop: {:?}", entry.instrs)); let blocks: Vec = local .block(restart) .instrs @@ -382,13 +388,12 @@ fn bucketed_depth_indirect_dispatcher_passes_v8_limit() { } } -/// Every per-call UNWIND branch must target the function-level -/// `$unwind_save`. A regression re-pointing them at a leaf-local -/// `$child_K` / `$dispatch_normal` would still validate as wasm but -/// scramble the fork frame on the next REWIND. The dispatcher -/// fixtures emit only the successful-unwind branch and the allocation-failure -/// branch back to the restart loop, so their exact target counts pin -/// `(global.get state, const UNWINDING, i32.eq, if)` sequence. +/// Every per-call private-tag handler must target `$unwind_save` after a +/// successful reservation and the live-restart loop after synchronous +/// allocation failure. A regression re-pointing a site at a leaf-local +/// `$child_K` / `$dispatch_normal` would still validate as wasm but scramble +/// the fork frame on the next REWIND. Exact target counts pin one statically +/// indexed boundary per lexical call, with no function-wide selector handler. /// /// N=33 straddles `BUCKET_SIZE=32` to force one full leaf + one /// singleton leaf — exercises both first-leaf and last-leaf paths. @@ -411,10 +416,16 @@ fn leaf_unwind_br_targets_function_level_unwind_save() { }; let unwind_save = dispatcher_unwind_save(local); - let restart_loop = match local.block(local.entry_block()).instrs.as_slice() { - [(Instr::Loop(ir::Loop { seq }), _)] => *seq, - other => panic!("expected restart loop, got {other:?}"), - }; + let restart_loop = local + .block(local.entry_block()) + .instrs + .iter() + .filter_map(|(instruction, _)| match instruction { + Instr::Loop(ir::Loop { seq }) => Some(*seq), + _ => None, + }) + .next_back() + .expect("expected live-restart loop"); let targets = collect_br_targets(local); assert_eq!( @@ -423,7 +434,7 @@ fn leaf_unwind_br_targets_function_level_unwind_save() { .filter(|&&target| target == unwind_save) .count(), n, - "{label} N={n}: each call site must branch to unwind-save after commit", + "{label} N={n}: each static call boundary must branch to unwind-save after commit", ); assert_eq!( targets @@ -431,9 +442,15 @@ fn leaf_unwind_br_targets_function_level_unwind_save() { .filter(|&&target| target == restart_loop) .count(), n, - "{label} N={n}: each call site must branch to restart on allocation failure", + "{label} N={n}: each static call boundary must branch to restart on allocation failure", + ); + assert_eq!( + targets.len(), + 3 * n, + "{label} N={n}: expected one normal result-boundary branch, \ + one successful-unwind branch, and one abort-restart branch \ + per static call", ); - assert_eq!(targets.len(), 2 * n, "{label} N={n}: unexpected Br target"); } } } diff --git a/crates/fork-instrument/tests/legacy_dlopen.rs b/crates/fork-instrument/tests/legacy_dlopen.rs new file mode 100644 index 0000000000..e38009f36e --- /dev/null +++ b/crates/fork-instrument/tests/legacy_dlopen.rs @@ -0,0 +1,251 @@ +//! The ABI 43 transform must not leave the historical monolithic loader +//! callback beneath a forkable side-module initializer. + +use fork_instrument::{Options, instrument, legacy_dlopen}; +use walrus::{ + ElementItems, ExportItem, FunctionId, FunctionKind, ImportKind, LocalFunction, Module, + ir::{self, Instr, InstrSeqId}, +}; + +fn parse(wat: &str) -> Module { + let bytes = wat::parse_str(wat).expect("parse legacy dlopen fixture"); + Module::from_buffer(&bytes).expect("parse fixture with walrus") +} + +fn imported(module: &Module, name: &str) -> Vec { + module + .imports + .iter() + .filter_map(|import| { + if import.module != "env" || import.name != name { + return None; + } + match import.kind { + ImportKind::Function(function) => Some(function), + _ => None, + } + }) + .collect() +} + +fn exported_function(module: &Module, name: &str) -> FunctionId { + match module + .exports + .iter() + .find(|export| export.name == name) + .unwrap_or_else(|| panic!("missing export {name}")) + .item + { + ExportItem::Function(function) => function, + other => panic!("{name} is not a function: {other:?}"), + } +} + +fn local(module: &Module, function: FunctionId) -> &LocalFunction { + match &module.funcs.get(function).kind { + FunctionKind::Local(local) => local, + other => panic!("expected local function, got {other:?}"), + } +} + +fn children(instruction: &Instr) -> Vec { + match instruction { + Instr::Block(ir::Block { seq }) | Instr::Loop(ir::Loop { seq }) => vec![*seq], + Instr::IfElse(ir::IfElse { + consequent, + alternative, + }) => vec![*consequent, *alternative], + Instr::TryTable(ir::TryTable { seq, .. }) => vec![*seq], + Instr::Try(try_) => { + let mut result = vec![try_.seq]; + for catch in &try_.catches { + match catch { + ir::LegacyCatch::Catch { handler, .. } + | ir::LegacyCatch::CatchAll { handler } => result.push(*handler), + ir::LegacyCatch::Delegate { .. } => {} + } + } + result + } + _ => Vec::new(), + } +} + +fn walk(local: &LocalFunction, seq: InstrSeqId, visit: &mut impl FnMut(&Instr)) { + for (instruction, _) in &local.block(seq).instrs { + visit(instruction); + for child in children(instruction) { + walk(local, child, visit); + } + } +} + +#[test] +fn legacy_import_identity_becomes_a_staged_local_adapter() { + let mut module = parse( + r#" + (module + (type $legacy (func (param i32 i32 i32 i32) (result i32))) + (import "env" "__wasm_dlopen" (func $legacy (type $legacy))) + (memory 1) + (table (export "__indirect_function_table") 2 funcref) + (elem (i32.const 1) func $legacy) + (func $checkpoint (export "__wasm_posix_signal_checkpoint")) + (export "legacy_alias" (func $legacy))) + "#, + ); + let legacy = exported_function(&module, "legacy_alias"); + + assert_eq!(legacy_dlopen::lower(&mut module).expect("lower"), 1); + assert!(imported(&module, "__wasm_dlopen").is_empty()); + assert_eq!(exported_function(&module, "legacy_alias"), legacy); + assert!(matches!( + module.funcs.get(legacy).kind, + FunctionKind::Local(_) + )); + assert!(imported(&module, "__wasm_dlopen_main").len() == 1); + assert!(imported(&module, "__wasm_dlopen_prepare").len() == 1); + assert!(imported(&module, "__wasm_dlopen_next").len() == 1); + assert!(imported(&module, "__wasm_dlopen_commit").len() == 1); + + let element_kept_identity = module.elements.iter().any(|element| { + matches!( + &element.items, + ElementItems::Functions(functions) if functions.contains(&legacy) + ) + }); + assert!( + element_kept_identity, + "table aliases must name the local adapter without a partial rewrite", + ); + + let adapter = local(&module, legacy); + let mut tail_driver = None; + walk(adapter, adapter.entry_block(), &mut |instruction| { + if let Instr::ReturnCall(call) = instruction { + tail_driver = Some(call.func); + } + }); + let driver = tail_driver.expect("adapter tail-calls staged driver"); + assert_eq!( + local(&module, driver).args.len(), + 1, + "only the transaction token is a driver parameter", + ); + let checkpoint = exported_function(&module, "__wasm_posix_signal_checkpoint"); + let mut checkpoints = 0; + walk(local(&module, driver), local(&module, driver).entry_block(), &mut |instruction| { + if matches!(instruction, Instr::Call(call) if call.func == checkpoint) { + checkpoints += 1; + } + }); + assert_eq!( + checkpoints, 3, + "prepare, next, and commit each hand deferred signals back to libc", + ); + + let bytes = module.emit_wasm(); + wasmparser::Validator::new_with_features(wasmparser::WasmFeatures::all()) + .validate_all(&bytes) + .expect("lowered module validates"); +} + +#[test] +fn original_two_argument_loader_uses_the_staged_protocol() { + let input = wat::parse_str( + r#" + (module + (import "kernel" "kernel_fork" (func $fork (result i32))) + (import "env" "__wasm_dlopen" + (func $legacy (param i32 i32) (result i32))) + (memory 1) + (table (export "__indirect_function_table") 2 funcref) + (func $initializer + call $fork + drop) + (elem (i32.const 1) func $initializer) + (func (export "open") (param i32 i32) (result i32) + local.get 0 + local.get 1 + call $legacy)) + "#, + ) + .expect("parse original loader ABI fixture"); + + let output = instrument(&input, &Options::default()).expect("instrument"); + let module = Module::from_buffer(&output).expect("parse instrumented module"); + assert!(imported(&module, "__wasm_dlopen").is_empty()); + let prepare = imported(&module, "__wasm_dlopen_prepare"); + assert_eq!(prepare.len(), 1); + let prepare_ty = module.types.get(module.funcs.get(prepare[0]).ty()); + assert_eq!( + prepare_ty.params(), + [ + walrus::ValType::I32, + walrus::ValType::I32, + walrus::ValType::I32, + walrus::ValType::I32, + walrus::ValType::I32, + ], + "the two-argument form must supply an empty name range and default flags", + ); + wasmparser::Validator::new_with_features(wasmparser::WasmFeatures::all()) + .validate_all(&output) + .expect("instrumented original loader adapter validates"); +} + +#[test] +fn complete_transform_has_no_reentrant_loader_import() { + let input = wat::parse_str( + r#" + (module + (import "kernel" "kernel_fork" (func $fork (result i32))) + (import "env" "__wasm_dlopen" + (func $legacy (param i64 i32 i64 i32 i32) (result i32))) + (memory i64 1) + (table (export "__indirect_function_table") 2 funcref) + (func $initializer + call $fork + drop) + (elem (i32.const 1) func $initializer) + (func (export "open") (param i64 i32 i64 i32 i32) (result i32) + local.get 0 + local.get 1 + local.get 2 + local.get 3 + local.get 4 + call $legacy)) + "#, + ) + .expect("parse memory64 fixture"); + + let output = instrument(&input, &Options::default()).expect("instrument"); + let module = Module::from_buffer(&output).expect("parse instrumented module"); + assert!(imported(&module, "__wasm_dlopen").is_empty()); + assert!(!imported(&module, "__wasm_dlopen_prepare").is_empty()); + assert!(!imported(&module, "__wasm_dlopen_next").is_empty()); + assert!(!imported(&module, "__wasm_dlopen_commit").is_empty()); + wasmparser::Validator::new_with_features(wasmparser::WasmFeatures::all()) + .validate_all(&output) + .expect("instrumented staged adapter validates"); +} + +#[test] +fn malformed_reserved_signature_fails_before_runtime() { + let input = wat::parse_str( + r#" + (module + (import "env" "__wasm_dlopen" + (func (param externref) (result externref)))) + "#, + ) + .expect("parse malformed reserved import"); + let error = instrument(&input, &Options::default()) + .expect_err("reserved loader ABI mismatch must not retain reentrant import"); + assert!( + error + .to_string() + .contains("reserved env.__wasm_dlopen import has signature"), + "unexpected diagnostic: {error:#}", + ); +} diff --git a/crates/fork-instrument/tests/module_exception_codec.rs b/crates/fork-instrument/tests/module_exception_codec.rs new file mode 100644 index 0000000000..2e3793dd7d --- /dev/null +++ b/crates/fork-instrument/tests/module_exception_codec.rs @@ -0,0 +1,104 @@ +use fork_instrument::module_exception_codec::{ + FORMAT_HEADER_SIZE, FORMAT_SECTION, FORMAT_TAG_RECORD_SIZE, FORMAT_VERSION, + IMPORT_SCRATCH_RELEASE, IMPORT_SCRATCH_RESERVE, inject, +}; +use walrus::Module; + +const RETIRED_STAGING_MEMORY_EXPORT: &str = "__wpk_fork_ref_exn_staging"; + +fn codec_fixture() -> Vec { + wat::parse_str( + r#" + (module + (import "env" "memory" (memory 1)) + (tag $empty) + (tag $scalars (param i32 i64 f32 f64 v128)) + (tag $references + (param (ref null extern) (ref null func) (ref null exn) (ref null any)))) + "#, + ) + .expect("codec fixture WAT") +} + +#[test] +fn exact_tag_codec_uses_only_the_existing_process_memory() { + let mut module = Module::from_buffer(&codec_fixture()).expect("parse codec fixture"); + let memory = module.memories.iter().next().expect("fixture memory").id(); + let before_memories = module.memories.iter().count(); + let codec = inject(&mut module, memory).expect("inject codec"); + + assert_eq!(codec.memory, memory); + assert_eq!(module.memories.iter().count(), before_memories); + assert!( + module + .exports + .iter() + .all(|export| export.name != RETIRED_STAGING_MEMORY_EXPORT), + "the codec must not export or depend on a private staging memory", + ); + for name in [IMPORT_SCRATCH_RESERVE, IMPORT_SCRATCH_RELEASE] { + assert!( + module + .imports + .iter() + .any(|import| import.module == "env" && import.name == name), + "missing transaction scratch import {name}", + ); + } + + let output = module.emit_wasm(); + wasmparser::Validator::new_with_features(wasmparser::WasmFeatures::all()) + .validate_all(&output) + .expect("generated codec validates"); +} + +#[test] +fn descriptor_records_exact_scalar_and_reference_layouts() { + let mut module = Module::from_buffer(&codec_fixture()).expect("parse codec fixture"); + let memory = module.memories.iter().next().expect("fixture memory").id(); + inject(&mut module, memory).expect("inject codec"); + let output = module.emit_wasm(); + let engine_module = WebAssemblyModule::new(&output); + let section = engine_module + .custom_section(FORMAT_SECTION) + .expect("codec descriptor"); + + assert_eq!(section[0], FORMAT_VERSION); + assert_eq!(u32::from_le_bytes(section[4..8].try_into().unwrap()), 3,); + assert_eq!( + section.len(), + FORMAT_HEADER_SIZE + 3 * FORMAT_TAG_RECORD_SIZE, + ); + let scalar = §ion[FORMAT_HEADER_SIZE + FORMAT_TAG_RECORD_SIZE + ..FORMAT_HEADER_SIZE + 2 * FORMAT_TAG_RECORD_SIZE]; + assert_eq!(u32::from_le_bytes(scalar[8..12].try_into().unwrap()), 40); + assert_eq!(u32::from_le_bytes(scalar[12..16].try_into().unwrap()), 0); + let references = §ion[FORMAT_HEADER_SIZE + 2 * FORMAT_TAG_RECORD_SIZE..]; + assert_eq!(u32::from_le_bytes(references[8..12].try_into().unwrap()), 0,); + assert_eq!( + u32::from_le_bytes(references[12..16].try_into().unwrap()), + 4, + ); +} + +struct WebAssemblyModule<'a> { + bytes: &'a [u8], +} + +impl<'a> WebAssemblyModule<'a> { + fn new(bytes: &'a [u8]) -> Self { + Self { bytes } + } + + fn custom_section(&self, name: &str) -> Option> { + for payload in wasmparser::Parser::new(0).parse_all(self.bytes) { + let payload = payload.ok()?; + if let wasmparser::Payload::CustomSection(section) = payload + && section.name() == name + { + return Some(section.data().to_vec()); + } + } + None + } +} diff --git a/crates/fork-instrument/tests/module_exception_codec_node.rs b/crates/fork-instrument/tests/module_exception_codec_node.rs new file mode 100644 index 0000000000..930637afef --- /dev/null +++ b/crates/fork-instrument/tests/module_exception_codec_node.rs @@ -0,0 +1,333 @@ +use std::{ + fs, + process::Command, + time::{SystemTime, UNIX_EPOCH}, +}; + +use fork_instrument::module_exception_codec; +use walrus::Module; + +fn fixture_module() -> Vec { + let input = wat::parse_str( + r#" + (module + (import "env" "memory" (memory 2)) + (tag $test (export "test_tag") (param i32 i64)) + (tag $inner (export "inner_tag") (param i32)) + (tag $outer (export "outer_tag") (param (ref null exn)))) + "#, + ) + .expect("provider fixture WAT"); + let mut module = Module::from_buffer(&input).expect("provider fixture module"); + let memory = module.memories.iter().next().expect("provider memory").id(); + module_exception_codec::inject(&mut module, memory).expect("inject exception codec"); + module.emit_wasm() +} + +fn helper_module() -> Vec { + wat::parse_str( + r#" + (module + (import "provider" "tag" (tag $test (param i32 i64))) + (import "provider" "inner_tag" (tag $inner (param i32))) + (import "provider" "outer_tag" + (tag $outer (param (ref null exn)))) + (import "provider" "encode" + (func $encode (param (ref null exn)) (result i32))) + (import "provider" "decode" + (func $decode (param i32) (result (ref null exn)))) + + (func (export "capture") (param i32 i64) (result i32) + (local $exception (ref null exn)) + (block $caught (result i32 i64 (ref exn)) + (try_table (catch_ref $test $caught) + (local.get 0) + (local.get 1) + (throw $test)) + unreachable) + (local.set $exception) + drop + drop + (local.get $exception) + (call $encode)) + + (func (export "payload_i32") (param i32) (result i32) + (block $caught (result i32 i64) + (try_table (catch $test $caught) + (local.get 0) + (call $decode) + (ref.as_non_null) + (throw_ref)) + unreachable) + drop) + + (func (export "payload_i64") (param i32) (result i64) + (local $value i64) + (block $caught (result i32 i64) + (try_table (catch $test $caught) + (local.get 0) + (call $decode) + (ref.as_non_null) + (throw_ref)) + unreachable) + (local.set $value) + drop + (local.get $value)) + + (func (export "throw_decoded") (param i32) + (local.get 0) + (call $decode) + (ref.as_non_null) + (throw_ref)) + + (func (export "capture_nested") (param i32) (result i32) + (local $inner_exception (ref null exn)) + (local $outer_exception (ref null exn)) + (block $caught_inner (result i32 (ref exn)) + (try_table (catch_ref $inner $caught_inner) + (local.get 0) + (throw $inner)) + unreachable) + (local.set $inner_exception) + drop + (block $caught_outer (result (ref null exn) (ref exn)) + (try_table (catch_ref $outer $caught_outer) + (local.get $inner_exception) + (throw $outer)) + unreachable) + (local.set $outer_exception) + drop + (local.get $outer_exception) + (call $encode)) + + (func (export "nested_payload") (param i32) (result i32) + (block $caught_inner (result i32) + (try_table (catch $inner $caught_inner) + (block $caught_outer (result (ref null exn)) + (try_table (catch $outer $caught_outer) + (local.get 0) + (call $decode) + (ref.as_non_null) + (throw_ref)) + unreachable) + (ref.as_non_null) + (throw_ref)) + unreachable))) + "#, + ) + .expect("consumer helper WAT") +} + +fn anyref_dependencies() -> Vec { + wat::parse_str( + r#" + (module + (func (export "encode") (param (ref null any)) (result i32) + unreachable) + (func (export "decode") (param i32) (result (ref null any)) + unreachable)) + "#, + ) + .expect("anyref dependency WAT") +} + +#[test] +fn fresh_node_instance_reconstructs_exact_tag_and_alias_identity() { + let nonce = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("clock") + .as_nanos(); + let directory = std::env::temp_dir().join(format!( + "kandelo-module-exception-codec-{}-{nonce}", + std::process::id(), + )); + fs::create_dir(&directory).expect("create fixture directory"); + fs::write(directory.join("provider.wasm"), fixture_module()).expect("write provider"); + fs::write(directory.join("helper.wasm"), helper_module()).expect("write helper"); + fs::write(directory.join("anyref.wasm"), anyref_dependencies()).expect("write anyref"); + fs::write( + directory.join("test.mjs"), + r#" +import { readFileSync } from "node:fs"; + +const providerModule = new WebAssembly.Module(readFileSync( + new URL("./provider.wasm", import.meta.url), +)); +const helperModule = new WebAssembly.Module(readFileSync( + new URL("./helper.wasm", import.meta.url), +)); +const anyrefModule = new WebAssembly.Module(readFileSync( + new URL("./anyref.wasm", import.meta.url), +)); +const anyrefs = new WebAssembly.Instance(anyrefModule).exports; +const recipes = new Map(); +let nextRecipe = 1; + +function instantiate(memory) { + let provider; + const ids = new WeakMap(); + const scratch = []; + let scratchTop = 0x10000; + const thrown = (slot) => { + try { + provider.exports.__wpk_fork_ref_exn_throw_slot(slot); + } catch (value) { + return value; + } + throw new Error("exception scratch slot returned"); + }; + const imports = { + env: { + memory, + __wpk_fork_module_activation: + new WebAssembly.Global({ value: "i32", mutable: false }, 0), + __wpk_fork_ref_exn_lookup(slot) { + return ids.get(thrown(slot)) ?? 0; + }, + __wpk_fork_ref_exn_claim(slot) { + const value = thrown(slot); + let id = ids.get(value); + if (id === undefined) { + id = nextRecipe++; + ids.set(value, id); + } + return id; + }, + __wpk_fork_ref_exn_define( + id, activation, tag, layout, + scalarPointer, scalarLength, refsPointer, refCount, + ) { + recipes.set(id, { + activation, tag, layout, + scalars: new Uint8Array( + memory.buffer, + scalarPointer, + scalarLength, + ).slice(), + refs: new Uint32Array( + memory.buffer, + refsPointer, + refCount, + ).slice(), + }); + }, + __wpk_fork_ref_exn_load( + id, activation, tag, layout, + scalarPointer, scalarLength, refsPointer, refCount, + ) { + const recipe = recipes.get(id); + if ( + !recipe + || recipe.activation !== activation + || recipe.tag !== tag + || recipe.layout !== layout + || recipe.scalars.length !== scalarLength + || recipe.refs.length !== refCount + ) return 0; + new Uint8Array(memory.buffer, scalarPointer, scalarLength) + .set(recipe.scalars); + new Uint32Array(memory.buffer, refsPointer, refCount) + .set(recipe.refs); + return 1; + }, + __wpk_fork_ref_exn_route(id, activation) { + const recipe = recipes.get(id); + return recipe?.activation === activation ? recipe.layout : -1; + }, + __wpk_fork_ref_exn_cache_index(id) { + return id; + }, + __wpk_fork_ref_exn_broker_encode() { + throw new Error("known local tag unexpectedly reached broker encode"); + }, + __wpk_fork_ref_exn_broker_throw_recipe() { + throw new Error("known local tag unexpectedly reached broker decode"); + }, + __wpk_fork_ref_exn_ingress_throw() { + throw new Error("known local tag unexpectedly used ingress"); + }, + __wpk_fork_ref_scratch_reserve(size) { + const aligned = (size + 15) & ~15; + const address = scratchTop; + scratchTop += aligned; + scratch.push({ address, size, aligned }); + new Uint8Array(memory.buffer, address, aligned).fill(0); + return address; + }, + __wpk_fork_ref_scratch_release(address, size) { + const reservation = scratch.pop(); + if ( + !reservation + || reservation.address !== address + || reservation.size !== size + ) throw new Error("non-LIFO scratch release"); + new Uint8Array(memory.buffer, address, reservation.aligned).fill(0); + scratchTop = address; + }, + __wpk_fork_ref_encode_funcref() { throw new Error("unused funcref"); }, + __wpk_fork_ref_decode_funcref() { throw new Error("unused funcref"); }, + __wpk_fork_ref_encode_externref() { throw new Error("unused externref"); }, + __wpk_fork_ref_decode_externref() { throw new Error("unused externref"); }, + __wpk_fork_ref_encode_anyref: anyrefs.encode, + __wpk_fork_ref_decode_anyref: anyrefs.decode, + }, + }; + provider = new WebAssembly.Instance(providerModule, imports); + const helper = new WebAssembly.Instance(helperModule, { + provider: { + tag: provider.exports.test_tag, + inner_tag: provider.exports.inner_tag, + outer_tag: provider.exports.outer_tag, + encode: provider.exports.__wpk_fork_ref_encode_exnref, + decode: provider.exports.__wpk_fork_ref_decode_exnref, + }, + }); + return { provider, helper }; +} + +const parent = instantiate(new WebAssembly.Memory({ initial: 2 })); +const recipe = parent.helper.exports.capture(0x78563412, 0x102030405060708n); +if (recipe !== 1) throw new Error(`unexpected recipe ${recipe}`); +const nestedRecipe = parent.helper.exports.capture_nested(0x1234abcd); + +// This is a genuinely fresh provider instance with a different local Tag. +const child = instantiate(new WebAssembly.Memory({ initial: 2 })); +if (child.provider.exports.test_tag === parent.provider.exports.test_tag) { + throw new Error("fresh module unexpectedly reused local tag identity"); +} +if (child.helper.exports.payload_i32(recipe) !== 0x78563412) { + throw new Error("child lost i32 exception payload bits"); +} +if (child.helper.exports.payload_i64(recipe) !== 0x102030405060708n) { + throw new Error("child lost i64 exception payload bits"); +} +const catchDecoded = () => { + try { + child.helper.exports.throw_decoded(recipe); + } catch (value) { + return value; + } + throw new Error("decoded exception returned without throwing"); +}; +if (catchDecoded() !== catchDecoded()) { + throw new Error("child did not cache reconstructed exnref identity"); +} +if (child.helper.exports.nested_payload(nestedRecipe) !== 0x1234abcd) { + throw new Error("child lost recursively encoded exnref payload"); +} +"#, + ) + .expect("write Node test"); + + let output = Command::new("node") + .arg(directory.join("test.mjs")) + .output() + .expect("run Node"); + let _ = fs::remove_dir_all(&directory); + assert!( + output.status.success(), + "Node fresh-instance codec test failed:\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + ); +} diff --git a/crates/fork-instrument/tests/module_gc_codec.rs b/crates/fork-instrument/tests/module_gc_codec.rs new file mode 100644 index 0000000000..9fdb277002 --- /dev/null +++ b/crates/fork-instrument/tests/module_gc_codec.rs @@ -0,0 +1,482 @@ +use fork_instrument::module_gc_codec::{ + CONSTRUCTOR_ARRAY_FIXED, CONSTRUCTOR_ARRAY_GENERIC, CONSTRUCTOR_STRUCT, FIELD_FLAG_MUTABLE, + FIELD_FLAG_NULLABLE, FIELD_FLAG_REFERENCE, FORMAT_FIELD_RECORD_SIZE, FORMAT_HEADER_SIZE, + FORMAT_LAYOUT_RECORD_SIZE, FORMAT_MAGIC, FORMAT_VERSION, GcConstructorKind, GcLayoutKind, + KIND_ARRAY, KIND_STRUCT, LAYOUT_FLAG_REQUIRES_PROVENANCE, plan, +}; +use fork_instrument::{module_exception_codec, module_gc_codec, runtime}; +use walrus::Module; + +fn parse(wat: &str) -> Module { + let bytes = wat::parse_str(wat).expect("valid test WAT"); + Module::from_buffer(&bytes).expect("walrus accepts test module") +} + +fn u16_at(bytes: &[u8], offset: usize) -> u16 { + u16::from_le_bytes(bytes[offset..offset + 2].try_into().unwrap()) +} + +fn u32_at(bytes: &[u8], offset: usize) -> u32 { + u32::from_le_bytes(bytes[offset..offset + 4].try_into().unwrap()) +} + +fn finish_codec(mut module: Module) -> Vec { + let memory = module.memories.iter().next().expect("test memory").id(); + let declared = module_gc_codec::declare(&mut module, memory).expect("declare GC codec"); + let exception = module_exception_codec::inject_with_reference_overrides( + &mut module, + memory, + Some((declared.encode_externref, declared.decode_externref)), + Some((declared.encode_anyref, declared.decode_anyref)), + ) + .expect("inject exception codec"); + let runtime = runtime::inject_linked_runtime_with_reference_overrides( + &mut module, + runtime::ReferenceCodecOverrides { + funcref: Some(( + exception.references.encode_funcref, + exception.references.decode_funcref, + )), + externref: Some(( + exception.references.encode_externref, + exception.references.decode_externref, + )), + exnref: Some((exception.encode, exception.decode)), + anyref: Some((declared.encode_anyref, declared.decode_anyref)), + cleanup: Some(exception.clear), + }, + ); + module_gc_codec::finish_declaration(&mut module, declared, exception, &runtime) + .expect("emit GC codec"); + module.emit_wasm() +} + +#[test] +fn plans_exact_scalar_and_reference_layouts() { + let module = parse( + r#" + (module + (type $node + (sub (struct + (field (mut i8)) + (field i64) + (field (mut (ref null $node)))))) + (type $child + (sub $node (struct + (field (mut i8)) + (field i64) + (field (mut (ref null $node))) + (field f32)))) + (type $refs (array (mut (ref null $node)))) + (type $immutable (array i16)) + (func (export "make") (result (ref $child)) + i32.const 0 + i64.const 0 + ref.null $node + f32.const 0 + struct.new $child)) + "#, + ); + + let plan = plan(&module).expect("GC layouts plan"); + assert_eq!(plan.layouts().len(), 4); + + let node = &plan.layouts()[0]; + assert_eq!(node.kind, GcLayoutKind::Struct); + assert_eq!(node.constructor, GcConstructorKind::Struct); + assert_eq!(node.scalar_len_or_stride, 16); + assert!(!node.defaultable_shell); + assert_eq!(node.fields[0].scalar_offset, Some(0)); + assert_eq!(node.fields[1].scalar_offset, Some(8)); + assert_eq!(node.fields[2].reference_ordinal, Some(0)); + + let child = &plan.layouts()[1]; + assert_eq!(child.super_type_ordinal, Some(0)); + assert_eq!(child.subtype_depth, 1); + assert_eq!(child.scalar_len_or_stride, 20); + + let refs = &plan.layouts()[2]; + assert_eq!(refs.kind, GcLayoutKind::Array); + assert_eq!(refs.scalar_len_or_stride, 0); + assert!(refs.defaultable_shell); + assert_eq!(refs.fields[0].reference_ordinal, Some(0)); + + let immutable = &plan.layouts()[3]; + assert_eq!(immutable.kind, GcLayoutKind::Array); + assert_eq!(immutable.scalar_len_or_stride, 2); + assert!(!immutable.defaultable_shell); + + // Exact dynamic type dispatch must test the subtype before its parent. + assert_eq!(&plan.dispatch_layouts()[..2], &[child.id, node.id]); +} + +#[test] +fn assigns_constructor_layouts_only_when_shell_replay_is_not_safe() { + let module = parse( + r#" + (module + (type $mutable (array (mut i32))) + (type $immutable (array i32)) + (data $bytes "\01\00\00\00\02\00\00\00") + (func (export "mutable-fixed") (result (ref $mutable)) + i32.const 7 + i32.const 8 + array.new_fixed $mutable 2) + (func (export "immutable-fixed") (result (ref $immutable)) + i32.const 9 + i32.const 10 + array.new_fixed $immutable 2) + (func (export "immutable-data") (result (ref $immutable)) + i32.const 0 + i32.const 2 + array.new_data $immutable $bytes)) + "#, + ); + + let plan = plan(&module).expect("GC layouts plan"); + assert_eq!(plan.layouts().len(), 4); + assert_eq!( + plan.layouts()[0].constructor, + GcConstructorKind::ArrayGeneric + ); + assert_eq!( + plan.layouts()[1].constructor, + GcConstructorKind::ArrayGeneric + ); + assert_eq!( + plan.layouts()[2].constructor, + GcConstructorKind::ArrayFixed { len: 2 } + ); + assert_eq!( + plan.layouts()[3].constructor, + GcConstructorKind::ArrayData { segment_ordinal: 0 } + ); +} + +#[test] +fn descriptor_is_canonical_and_binds_constructor_safety() { + let module = parse( + r#" + (module + (type $pair + (struct + (field i32) + (field (mut (ref null $pair))))) + (type $immutable (array i16)) + (func (export "new-array") (result (ref $immutable)) + i32.const 1 + i32.const 2 + array.new_fixed $immutable 2)) + "#, + ); + let plan = plan(&module).expect("GC layouts plan"); + let bytes = plan.descriptor(); + + assert_eq!(&bytes[0..4], &FORMAT_MAGIC); + assert_eq!(u16_at(&bytes, 4), FORMAT_VERSION); + assert_eq!(u16_at(&bytes, 6), FORMAT_HEADER_SIZE); + assert_eq!(u32_at(&bytes, 8), 3); + assert_eq!(u32_at(&bytes, 12), 4); + assert_eq!( + bytes.len(), + usize::from(FORMAT_HEADER_SIZE) + + 3 * usize::from(FORMAT_LAYOUT_RECORD_SIZE) + + 4 * usize::from(FORMAT_FIELD_RECORD_SIZE) + ); + + let first = usize::from(FORMAT_HEADER_SIZE); + assert_eq!(bytes[first + 8], KIND_STRUCT); + assert_eq!(bytes[first + 9], CONSTRUCTOR_STRUCT); + assert_eq!(u16_at(&bytes, first + 10), 0); + + let second = first + usize::from(FORMAT_LAYOUT_RECORD_SIZE); + assert_eq!(bytes[second + 8], KIND_ARRAY); + assert_eq!(bytes[second + 9], CONSTRUCTOR_ARRAY_GENERIC); + assert_eq!(u16_at(&bytes, second + 10), LAYOUT_FLAG_REQUIRES_PROVENANCE); + + let third = second + usize::from(FORMAT_LAYOUT_RECORD_SIZE); + assert_eq!(bytes[third + 8], KIND_ARRAY); + assert_eq!(bytes[third + 9], CONSTRUCTOR_ARRAY_FIXED); + assert_eq!(u16_at(&bytes, third + 10), LAYOUT_FLAG_REQUIRES_PROVENANCE); + assert_eq!(u32_at(&bytes, third + 28), plan.layouts()[1].id); + assert_eq!(u32_at(&bytes, third + 32), 2); + + let fields = first + 3 * usize::from(FORMAT_LAYOUT_RECORD_SIZE); + assert_eq!(bytes[fields + 1], 0); + assert_eq!( + bytes[fields + usize::from(FORMAT_FIELD_RECORD_SIZE) + 1], + FIELD_FLAG_MUTABLE | FIELD_FLAG_NULLABLE | FIELD_FLAG_REFERENCE + ); +} + +#[test] +fn generated_probe_and_local_anyref_codec_validate_without_typed_anyref_imports() { + let mut module = parse( + r#" + (module + (memory 1) + (type $base (sub (struct (field (mut i32))))) + (type $child + (sub $base (struct (field (mut i32)) (field (mut i64))))) + (func (export "child") (result (ref $child)) + i32.const 1 + i64.const 2 + struct.new $child)) + "#, + ); + let memory = module.memories.iter().next().unwrap().id(); + let declared = module_gc_codec::declare(&mut module, memory).expect("declare GC codec"); + let exception = module_exception_codec::inject_with_reference_overrides( + &mut module, + memory, + Some((declared.encode_externref, declared.decode_externref)), + Some((declared.encode_anyref, declared.decode_anyref)), + ) + .expect("inject exception codec"); + let runtime = runtime::inject_linked_runtime_with_reference_overrides( + &mut module, + runtime::ReferenceCodecOverrides { + funcref: Some(( + exception.references.encode_funcref, + exception.references.decode_funcref, + )), + externref: Some(( + exception.references.encode_externref, + exception.references.decode_externref, + )), + exnref: Some((exception.encode, exception.decode)), + anyref: Some((declared.encode_anyref, declared.decode_anyref)), + cleanup: Some(exception.clear), + }, + ); + module_gc_codec::finish_declaration(&mut module, declared, exception, &runtime) + .expect("emit GC codec"); + let wasm = module.emit_wasm(); + + let mut validator = wasmparser::Validator::new_with_features(wasmparser::WasmFeatures::all()); + validator + .validate_all(&wasm) + .expect("generated module validates"); + let printed = wasmprinter::print_bytes(&wasm).expect("print generated module"); + assert!(printed.contains("(export \"__wpk_fork_ref_gc_probe\"")); + assert!(printed.contains("(export \"__wpk_fork_ref_gc_encode_slot\"")); + assert!(printed.contains("(export \"__wpk_fork_ref_gc_publish_externref\"")); + assert!(printed.contains("(table (;")); + assert!(printed.contains("anyref"), "{printed}"); + assert!(printed.contains("any.convert_extern"), "{printed}"); + assert!(printed.contains("extern.convert_any"), "{printed}"); + assert!(!printed.contains("(import \"env\" \"__wpk_fork_ref_encode_anyref\"",)); + assert!(!printed.contains("(import \"env\" \"__wpk_fork_ref_decode_anyref\"",)); + assert!(!printed.contains("(import \"env\" \"__wpk_fork_ref_encode_externref\"",)); + assert!(!printed.contains("(import \"env\" \"__wpk_fork_ref_decode_externref\"",)); +} + +#[test] +fn generated_allocate_and_fill_cover_non_shell_structs_and_array_constructors() { + let mut module = parse( + r#" + (module + (memory 1) + (type $leaf (struct (field (mut i32)))) + (type $holder + (struct + (field i64) + (field (mut (ref $leaf))))) + (type $bytes (array i8)) + (type $refs (array (ref null $leaf))) + (type $mutable-refs (array (mut (ref null $leaf)))) + (data $data "\01\02\03") + (elem $elements (ref null $leaf) + (item (ref.null $leaf)) + (item (ref.null $leaf))) + + (func (export "holder") (result (ref $holder)) + i64.const 9 + i32.const 1 + struct.new $leaf + struct.new $holder) + (func (export "fixed") (result (ref $bytes)) + i32.const 1 + i32.const 2 + array.new_fixed $bytes 2) + (func (export "data") (result (ref $bytes)) + i32.const 0 + i32.const 3 + array.new_data $bytes $data) + (func (export "elements") (result (ref $refs)) + i32.const 0 + i32.const 2 + array.new_elem $refs $elements) + (func (export "mutable") (result (ref $mutable-refs)) + ref.null $leaf + i32.const 2 + array.new $mutable-refs)) + "#, + ); + let memory = module.memories.iter().next().unwrap().id(); + let declared = module_gc_codec::declare(&mut module, memory).expect("declare GC codec"); + let exception = module_exception_codec::inject_with_reference_overrides( + &mut module, + memory, + Some((declared.encode_externref, declared.decode_externref)), + Some((declared.encode_anyref, declared.decode_anyref)), + ) + .expect("inject exception codec"); + let runtime = runtime::inject_linked_runtime_with_reference_overrides( + &mut module, + runtime::ReferenceCodecOverrides { + funcref: Some(( + exception.references.encode_funcref, + exception.references.decode_funcref, + )), + externref: Some(( + exception.references.encode_externref, + exception.references.decode_externref, + )), + exnref: Some((exception.encode, exception.decode)), + anyref: Some((declared.encode_anyref, declared.decode_anyref)), + cleanup: Some(exception.clear), + }, + ); + module_gc_codec::finish_declaration(&mut module, declared, exception, &runtime) + .expect("emit GC codec"); + let wasm = module.emit_wasm(); + wasmparser::Validator::new_with_features(wasmparser::WasmFeatures::all()) + .validate_all(&wasm) + .expect("all generated concrete helpers validate"); +} + +#[test] +fn generated_seeds_cover_mutable_nonnullable_function_external_and_exception_refs() { + let wasm = finish_codec(parse( + r#" + (module + (memory 1) + (type $holder + (struct + (field (mut (ref func))) + (field (mut (ref extern))) + (field (mut (ref exn))))) + (type $functions (array (mut (ref func)))) + (type $externals (array (mut (ref extern)))) + (type $exceptions (array (mut (ref exn))))) + "#, + )); + + wasmparser::Validator::new_with_features(wasmparser::WasmFeatures::all()) + .validate_all(&wasm) + .expect("generated non-null reference seeds validate"); + let printed = wasmprinter::print_bytes(&wasm).expect("print generated seeds"); + assert!( + printed.contains("__wpk_fork_ref_seed_func"), + "funcref allocation needs a locally typed temporary seed", + ); + assert!( + printed.contains("extern.convert_any"), + "externref allocation needs a non-null locally constructed seed", + ); + assert!( + printed.contains("__wpk_fork_ref_seed_exn"), + "exnref allocation needs a fresh instance-local exception seed", + ); +} + +#[test] +fn mutable_nonnullable_array_fixed_records_every_constructor_reference() { + let module = parse( + r#" + (module + (type $leaf (struct (field (mut i32)))) + (type $refs (array (mut (ref $leaf)))) + (func (export "fixed") (result (ref $refs)) + i32.const 1 + struct.new $leaf + i32.const 2 + struct.new $leaf + array.new_fixed $refs 2)) + "#, + ); + let plan = plan(&module).expect("GC layouts plan"); + let fixed = plan + .layouts() + .iter() + .find(|layout| layout.constructor == (GcConstructorKind::ArrayFixed { len: 2 })) + .expect("specialized array.fixed layout"); + assert!(fixed.requires_provenance); + assert_eq!( + fixed.provenance_reference_count, 2, + "each element is an allocation dependency: using only one static seed \ + would lose distinct constructor identities", + ); +} + +#[test] +fn parent_i31_capture_publishes_the_identity_to_the_transit_table() { + let wasm = finish_codec(parse( + r#" + (module + (memory 1) + (type $box (struct (field (mut i32))))) + "#, + )); + let module = Module::from_buffer(&wasm).expect("parse generated GC codec"); + let transit = module + .imports + .iter() + .find_map(|import| { + (import.name == module_gc_codec::IMPORT_TRANSIT_TABLE).then(|| match import.kind { + walrus::ImportKind::Table(table) => Some(table), + _ => None, + })? + }) + .expect("GC transit table import"); + let encode = module + .funcs + .iter() + .find(|function| function.name.as_deref() == Some(module_gc_codec::LOCAL_ENCODE_ANYREF)) + .expect("local anyref encoder"); + let encode = match &encode.kind { + walrus::FunctionKind::Local(local) => local, + _ => panic!("anyref encoder must be local"), + }; + + fn has_i31_publication( + function: &walrus::LocalFunction, + sequence: walrus::ir::InstrSeqId, + transit: walrus::TableId, + ) -> bool { + let instructions = &function.block(sequence).instrs; + let reads_i31 = instructions + .iter() + .any(|(instruction, _)| matches!(instruction, walrus::ir::Instr::I31GetS(_))); + let publishes = instructions.iter().any( + |(instruction, _)| { + matches!(instruction, walrus::ir::Instr::TableSet(set) if set.table == transit) + }, + ); + if reads_i31 && publishes { + return true; + } + instructions.iter().any(|(instruction, _)| { + let children: &[walrus::ir::InstrSeqId] = match instruction { + walrus::ir::Instr::Block(block) => std::slice::from_ref(&block.seq), + walrus::ir::Instr::Loop(block) => std::slice::from_ref(&block.seq), + walrus::ir::Instr::TryTable(table) => std::slice::from_ref(&table.seq), + walrus::ir::Instr::IfElse(branches) => { + return has_i31_publication(function, branches.consequent, transit) + || has_i31_publication(function, branches.alternative, transit); + } + _ => &[], + }; + children + .iter() + .any(|child| has_i31_publication(function, *child, transit)) + }) + } + + assert!( + has_i31_publication(encode, encode.entry_block(), transit), + "the parent encoder must publish recipe+1 -> i31ref because JavaScript \ + receives only the scalar i31 payload and cannot manufacture the value", + ); +} diff --git a/crates/fork-instrument/tests/module_gc_codec_node.rs b/crates/fork-instrument/tests/module_gc_codec_node.rs new file mode 100644 index 0000000000..97dd145ffc --- /dev/null +++ b/crates/fork-instrument/tests/module_gc_codec_node.rs @@ -0,0 +1,741 @@ +use std::{ + fs, + process::Command, + time::{SystemTime, UNIX_EPOCH}, +}; + +use fork_instrument::{module_exception_codec, module_gc_codec, runtime}; +use walrus::Module; + +fn fixture_module() -> Vec { + let input = wat::parse_str( + r#" + (module + (import "env" "memory" (memory 2)) + (type $node + (struct + (field (mut i32)) + (field (mut (ref null $node))) + (field (mut (ref null any))))) + (type $fixed (array i16)) + (type $data-bytes (array i8)) + (type $nullable-array (array (ref null $node))) + (table $objects (export "objects") 5 (ref null any)) + (data $bytes "\0b\16\21") + + (func (export "create_cycle") + (local $node (ref null $node)) + i32.const 77 + ref.null $node + ref.null any + struct.new $node + local.set $node + local.get $node + local.get $node + struct.set $node 1 + i32.const 0 + local.get $node + table.set $objects) + + (func (export "verify_cycle") (result i32) + (local $node (ref null $node)) + i32.const 0 + table.get $objects + ref.cast (ref $node) + local.set $node + local.get $node + ref.as_non_null + struct.get $node 0 + i32.const 77 + i32.eq + local.get $node + ref.as_non_null + local.get $node + ref.as_non_null + struct.get $node 1 + ref.as_non_null + ref.eq + i32.and) + + (func (export "create_externalized_cycle") + (param $token externref) + (result externref) + (local $node (ref null $node)) + i32.const 88 + ref.null $node + local.get $token + any.convert_extern + struct.new $node + local.set $node + local.get $node + local.get $node + struct.set $node 1 + i32.const 1 + local.get $node + table.set $objects + local.get $node + extern.convert_any) + + (func (export "verify_externalized_cycle") + (param $root externref) + (result i32) + (local $node (ref null $node)) + local.get $root + any.convert_extern + ref.cast (ref $node) + local.set $node + local.get $node + struct.get $node 0 + i32.const 88 + i32.eq + local.get $node + local.get $node + struct.get $node 1 + ref.as_non_null + ref.eq + i32.and) + + (func (export "externalized_cycle_token") + (param $root externref) + (result externref) + local.get $root + any.convert_extern + ref.cast (ref $node) + struct.get $node 2 + extern.convert_any) + + (func (export "create_fixed") + (local $array (ref null $fixed)) + i32.const 11 + i32.const 22 + array.new_fixed $fixed 2 + local.set $array + i32.const 2 + local.get $array + table.set $objects) + + (func (export "verify_fixed") (result i32) + (local $array (ref null $fixed)) + i32.const 2 + table.get $objects + ref.cast (ref $fixed) + local.set $array + local.get $array + i32.const 0 + array.get_u $fixed + i32.const 11 + i32.eq + local.get $array + i32.const 1 + array.get_u $fixed + i32.const 22 + i32.eq + i32.and) + + (func (export "create_data") + (local $array (ref null $data-bytes)) + i32.const 0 + i32.const 3 + array.new_data $data-bytes $bytes + local.set $array + i32.const 3 + local.get $array + table.set $objects) + + (func (export "verify_data") (result i32) + (local $array (ref null $data-bytes)) + i32.const 3 + table.get $objects + ref.cast (ref $data-bytes) + local.set $array + local.get $array + i32.const 0 + array.get_u $data-bytes + i32.const 11 + i32.eq + local.get $array + i32.const 1 + array.get_u $data-bytes + i32.const 22 + i32.eq + i32.and + local.get $array + i32.const 2 + array.get_u $data-bytes + i32.const 33 + i32.eq + i32.and) + + (func (export "create_nullable_empty") + (local $array (ref null $nullable-array)) + ref.null $node + i32.const 0 + array.new $nullable-array + local.set $array + i32.const 4 + local.get $array + table.set $objects) + + (func (export "verify_nullable_empty") (result i32) + i32.const 4 + table.get $objects + ref.cast (ref $nullable-array) + array.len + i32.eqz)) + "#, + ) + .expect("GC provider fixture WAT"); + let mut module = Module::from_buffer(&input).expect("GC provider fixture module"); + let memory = module.memories.iter().next().expect("provider memory").id(); + let declared = module_gc_codec::declare(&mut module, memory).expect("declare GC codec"); + module + .exports + .add("__test_encode_externref", declared.encode_externref); + module + .exports + .add("__test_decode_externref", declared.decode_externref); + let exception = module_exception_codec::inject_with_reference_overrides( + &mut module, + memory, + Some((declared.encode_externref, declared.decode_externref)), + Some((declared.encode_anyref, declared.decode_anyref)), + ) + .expect("inject exception codec"); + let runtime = runtime::inject_linked_runtime_with_reference_overrides( + &mut module, + runtime::ReferenceCodecOverrides { + funcref: Some(( + exception.references.encode_funcref, + exception.references.decode_funcref, + )), + externref: Some(( + exception.references.encode_externref, + exception.references.decode_externref, + )), + exnref: Some((exception.encode, exception.decode)), + anyref: Some((declared.encode_anyref, declared.decode_anyref)), + cleanup: Some(exception.clear), + }, + ); + module_gc_codec::finish_declaration(&mut module, declared, exception, &runtime) + .expect("finish GC codec"); + module.emit_wasm() +} + +fn transit_provider_module() -> Vec { + wat::parse_str( + r#" + (module + (table (export "transit") 64 (ref null any))) + "#, + ) + .expect("transit provider WAT") +} + +#[test] +fn fresh_node_instance_reconstructs_gc_cycle_and_identity() { + let nonce = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("clock") + .as_nanos(); + let directory = std::env::temp_dir().join(format!( + "kandelo-module-gc-codec-{}-{nonce}", + std::process::id(), + )); + fs::create_dir(&directory).expect("create fixture directory"); + fs::write(directory.join("provider.wasm"), fixture_module()).expect("write provider"); + fs::write(directory.join("transit.wasm"), transit_provider_module()) + .expect("write transit provider"); + fs::write( + directory.join("test.mjs"), + r#" +import { readFileSync } from "node:fs"; + +const providerModule = new WebAssembly.Module(readFileSync( + new URL("./provider.wasm", import.meta.url), +)); +const transitModule = new WebAssembly.Module(readFileSync( + new URL("./transit.wasm", import.meta.url), +)); +const transit = new WebAssembly.Instance(transitModule).exports.transit; +const recipes = new Map(); +const identities = new WeakMap(); +const capturedValues = new Map(); +const provenance = new WeakMap(); +const brokerValues = new Map(); +const vectors = [{ expected: 0, values: [] }]; +let nextRecipe = 1; +let nextProvenance = 1; +let pendingProvenance = null; + +function concatenate(left, right) { + const result = new Uint8Array(left.length + right.length); + result.set(left); + result.set(right, left.length); + return result; +} + +function instantiate() { + const memory = new WebAssembly.Memory({ initial: 2 }); + const scratch = []; + let scratchTop = 0x10000; + let instance; + + const implemented = { + __wpk_fork_ref_gc_lookup(slot) { + const value = transit.get(slot); + return value === null ? 0 : (identities.get(value) ?? 0); + }, + __wpk_fork_ref_gc_claim(slot) { + const value = transit.get(slot); + if (value === null) throw new Error("claim of null GC transit slot"); + let recipe = identities.get(value); + if (recipe === undefined) { + recipe = nextRecipe++; + identities.set(value, recipe); + capturedValues.set(recipe, value); + } + return recipe; + }, + __wpk_fork_ref_gc_broker_encode(slot) { + const value = transit.get(slot); + if (value === null) throw new Error("broker encode received null"); + let recipe = identities.get(value); + if (recipe === undefined) { + recipe = nextRecipe++; + identities.set(value, recipe); + brokerValues.set(recipe, value); + } + while (transit.length <= recipe + 1) transit.grow(1); + return recipe; + }, + __wpk_fork_ref_gc_i31(value) { + const recipe = nextRecipe++; + recipes.set(recipe, { + activation: 7, + type: 0xffffffff, + layout: 0, + kind: 0, + scalars: Uint8Array.of( + value & 0xff, + (value >>> 8) & 0xff, + (value >>> 16) & 0xff, + (value >>> 24) & 0xff, + ), + vector: 0, + }); + return recipe; + }, + __wpk_fork_ref_gc_capture_layout(slot, activation, baseLayout) { + if (activation !== 7) throw new Error("wrong capture activation"); + const record = provenance.get(transit.get(slot)); + if (!record) { + // Layout 1 is the default-constructible mutable struct used by the + // cycle fixture. Its field snapshot is a complete reconstruction + // recipe, so only the immutable-array layouts require constructor + // provenance. + if (baseLayout === 1) return baseLayout; + throw new Error("GC constructor provenance was not registered"); + } + if (record.activation !== activation) { + throw new Error("GC constructor provenance has the wrong activation"); + } + if (record.baseLayout !== baseLayout) { + throw new Error("GC constructor provenance has the wrong base layout"); + } + return record.specializedLayout; + }, + __wpk_fork_ref_gc_provenance_begin( + slot, activation, baseLayout, specializedLayout, + scalarLo, scalarHi, referenceCount, + ) { + if (pendingProvenance !== null) { + throw new Error("nested GC provenance registration"); + } + if (activation !== 7) throw new Error("wrong provenance activation"); + const object = transit.get(slot); + if (object === null) throw new Error("null GC provenance object"); + let scalars; + if (scalarLo === 0n && scalarHi === 0n) { + scalars = new Uint8Array(); + } else { + // array.new_data(0, 3) must occupy one packed eight-byte record. + if (scalarLo !== 0x0000000300000000n || scalarHi !== 0n) { + throw new Error( + `GC data constructor operands were not packed: ${scalarLo}:${scalarHi}`, + ); + } + scalars = new Uint8Array(8); + new DataView(scalars.buffer).setBigUint64(0, scalarLo, true); + } + const token = nextProvenance++; + pendingProvenance = { + token, + object, + activation, + baseLayout, + specializedLayout, + scalars, + referenceCount, + references: [], + }; + return token; + }, + __wpk_fork_ref_gc_provenance_ref(token, index, slot) { + if ( + pendingProvenance === null + || pendingProvenance.token !== token + || index !== pendingProvenance.references.length + || index >= pendingProvenance.referenceCount + ) { + throw new Error("invalid GC provenance reference"); + } + pendingProvenance.references.push(transit.get(slot)); + }, + __wpk_fork_ref_gc_provenance_end(token) { + if ( + pendingProvenance === null + || pendingProvenance.token !== token + || pendingProvenance.references.length + !== pendingProvenance.referenceCount + ) { + throw new Error("incomplete GC provenance registration"); + } + provenance.set(pendingProvenance.object, { + activation: pendingProvenance.activation, + baseLayout: pendingProvenance.baseLayout, + specializedLayout: pendingProvenance.specializedLayout, + scalars: pendingProvenance.scalars, + references: pendingProvenance.references, + }); + pendingProvenance = null; + }, + __wpk_fork_ref_gc_define( + recipe, activation, type, layout, kind, + scalarPointer, scalarLength, vector, + ) { + const refs = vectors[vector]; + if (!refs || refs.values.length !== refs.expected) { + throw new Error("incomplete capture vector"); + } + const source = capturedValues.get(recipe); + const constructor = source === undefined ? undefined : provenance.get(source); + const snapshot = new Uint8Array( + memory.buffer, + Number(scalarPointer), + scalarLength, + ).slice(); + const constructorRecipes = constructor?.references.map((reference) => { + if (reference === null) return 0; + transit.set(0, reference); + return instance.exports.__wpk_fork_ref_gc_encode_slot(0); + }) ?? []; + const combinedVector = constructorRecipes.length === 0 + ? vector + : vectors.push({ + expected: constructorRecipes.length + refs.values.length, + values: [...constructorRecipes, ...refs.values], + }) - 1; + recipes.set(recipe, { + activation, + type, + layout, + kind, + scalars: concatenate( + constructor?.scalars ?? new Uint8Array(), + snapshot, + ), + vector: combinedVector, + }); + }, + __wpk_fork_ref_gc_route(recipe, activation) { + const value = recipes.get(recipe); + return value?.activation === activation ? value.layout : -1; + }, + __wpk_fork_ref_gc_payload_len(recipe, activation, layout) { + const value = recipes.get(recipe); + if (!value || value.activation !== activation || value.layout !== layout) { + throw new Error("GC payload route mismatch"); + } + return value.scalars.length; + }, + __wpk_fork_ref_gc_load( + recipe, activation, type, layout, kind, destination, length, + ) { + const value = recipes.get(recipe); + if ( + !value + || value.activation !== activation + || value.type !== (type >>> 0) + || value.layout !== layout + || value.kind !== kind + || value.scalars.length !== length + ) throw new Error("GC load coordinate mismatch"); + new Uint8Array(memory.buffer, Number(destination), length) + .set(value.scalars); + return value.vector; + }, + __wpk_fork_ref_vector_begin(expected) { + const ordinal = vectors.length; + vectors.push({ expected, values: [] }); + return ordinal; + }, + __wpk_fork_ref_vector_append(ordinal, recipe) { + const vector = vectors[ordinal]; + if (!vector || vector.values.length >= vector.expected) { + throw new Error("invalid vector append"); + } + vector.values.push(recipe); + }, + __wpk_fork_ref_vector_finish(ordinal) { + const vector = vectors[ordinal]; + if (!vector || vector.values.length !== vector.expected) { + throw new Error("incomplete vector finish"); + } + return ordinal; + }, + __wpk_fork_ref_vector_get(ordinal, index) { + const value = vectors[ordinal]?.values[index]; + if (value === undefined) throw new Error("invalid vector lookup"); + return value; + }, + __wpk_fork_ref_scratch_reserve(size) { + const aligned = (Number(size) + 15) & ~15; + const address = scratchTop; + scratchTop += aligned; + scratch.push({ address, size: Number(size), aligned }); + new Uint8Array(memory.buffer, address, aligned).fill(0); + return address; + }, + __wpk_fork_ref_scratch_release(address, size) { + const reservation = scratch.pop(); + if ( + !reservation + || reservation.address !== Number(address) + || reservation.size !== Number(size) + ) throw new Error("non-LIFO scratch release"); + new Uint8Array( + memory.buffer, + reservation.address, + reservation.aligned, + ).fill(0); + scratchTop = reservation.address; + }, + }; + + const imports = {}; + for (const descriptor of WebAssembly.Module.imports(providerModule)) { + const namespace = imports[descriptor.module] ??= {}; + if (descriptor.kind === "memory") { + namespace[descriptor.name] = memory; + } else if (descriptor.kind === "global") { + namespace[descriptor.name] = descriptor.name === "__wpk_fork_module_activation" + ? new WebAssembly.Global({ value: "i32", mutable: false }, 7) + : new WebAssembly.Global({ value: "i32", mutable: true }, 0); + } else if (descriptor.kind === "table") { + namespace[descriptor.name] = + descriptor.name === "__wpk_fork_ref_gc_transit" + ? transit + : new WebAssembly.Table({ + element: "anyfunc", + initial: 1, + }); + } else if (descriptor.kind === "tag") { + namespace[descriptor.name] = new WebAssembly.Tag({ parameters: [] }); + } else if (descriptor.kind === "function") { + namespace[descriptor.name] = implemented[descriptor.name] ?? (() => { + throw new Error(`unexpected import call ${descriptor.name}`); + }); + } + } + instance = new WebAssembly.Instance(providerModule, imports); + return { instance, memory }; +} + +const parent = instantiate(); +parent.instance.exports.create_cycle(); +if (parent.instance.exports.verify_cycle() !== 1) { + throw new Error("parent fixture did not create its self-cycle"); +} +const parentObject = parent.instance.exports.objects.get(0); +transit.set(0, parentObject); +const recipe = parent.instance.exports.__wpk_fork_ref_gc_encode_slot(0); +if (recipe !== 1) throw new Error(`unexpected root recipe ${recipe}`); +if (vectors[recipes.get(recipe).vector].values[0] !== recipe) { + throw new Error("parent recipe did not preserve the self-edge"); +} +parent.instance.exports.create_fixed(); +parent.instance.exports.create_data(); +parent.instance.exports.create_nullable_empty(); +if ( + parent.instance.exports.verify_fixed() !== 1 + || parent.instance.exports.verify_data() !== 1 + || parent.instance.exports.verify_nullable_empty() !== 1 +) { + throw new Error("parent immutable-array fixture is invalid"); +} +transit.set(0, parent.instance.exports.objects.get(2)); +const fixedRecipe = + parent.instance.exports.__wpk_fork_ref_gc_encode_slot(0); +transit.set(0, parent.instance.exports.objects.get(3)); +const dataRecipe = + parent.instance.exports.__wpk_fork_ref_gc_encode_slot(0); +transit.set(0, parent.instance.exports.objects.get(4)); +const nullableEmptyRecipe = + parent.instance.exports.__wpk_fork_ref_gc_encode_slot(0); + +// Remove every parent-owned transit identity before creating the child. Replay +// must allocate a new object in the child's recursive type universe. +for (let index = 0; index < transit.length; index++) transit.set(index, null); +const child = instantiate(); +child.instance.exports.__wpk_fork_ref_gc_allocate(recipe); +child.instance.exports.__wpk_fork_ref_gc_fill(recipe); +const childObject = transit.get(recipe + 1); +if (childObject === null || childObject === parentObject) { + throw new Error("fresh child reused or lost the parent GC identity"); +} +child.instance.exports.objects.set(0, childObject); +if (child.instance.exports.verify_cycle() !== 1) { + throw new Error("fresh child lost scalar data, alias identity, or the cycle"); +} +for (const [arrayRecipe, tableIndex] of [ + [fixedRecipe, 2], + [dataRecipe, 3], + [nullableEmptyRecipe, 4], +]) { + child.instance.exports.__wpk_fork_ref_gc_allocate(arrayRecipe); + child.instance.exports.__wpk_fork_ref_gc_fill(arrayRecipe); + child.instance.exports.objects.set( + tableIndex, + transit.get(arrayRecipe + 1), + ); +} +if ( + child.instance.exports.verify_fixed() !== 1 + || child.instance.exports.verify_data() !== 1 + || child.instance.exports.verify_nullable_empty() !== 1 +) { + throw new Error("fresh child lost immutable-array constructor state"); +} + +// A replayed child is itself a valid future parent. Encoding its reconstructed +// arrays must use constructor provenance registered by the generated allocate +// helper, not a parent-Worker object or a stale transaction slot. +transit.set(0, child.instance.exports.objects.get(2)); +const nestedFixedRecipe = + child.instance.exports.__wpk_fork_ref_gc_encode_slot(0); +transit.set(0, child.instance.exports.objects.get(3)); +const nestedDataRecipe = + child.instance.exports.__wpk_fork_ref_gc_encode_slot(0); +transit.set(0, child.instance.exports.objects.get(4)); +const nestedNullableEmptyRecipe = + child.instance.exports.__wpk_fork_ref_gc_encode_slot(0); +for (let index = 0; index < transit.length; index++) transit.set(index, null); +const grandchild = instantiate(); +for (const [arrayRecipe, tableIndex] of [ + [nestedFixedRecipe, 2], + [nestedDataRecipe, 3], + [nestedNullableEmptyRecipe, 4], +]) { + grandchild.instance.exports.__wpk_fork_ref_gc_allocate(arrayRecipe); + grandchild.instance.exports.__wpk_fork_ref_gc_fill(arrayRecipe); + grandchild.instance.exports.objects.set( + tableIndex, + transit.get(arrayRecipe + 1), + ); +} +if ( + grandchild.instance.exports.verify_fixed() !== 1 + || grandchild.instance.exports.verify_data() !== 1 + || grandchild.instance.exports.verify_nullable_empty() !== 1 +) { + throw new Error("grandchild lost replay-registered GC constructor state"); +} + +// `extern.convert_any` is only a view of the same GC identity. Encoding that +// view must recover the typed object rather than assigning it an opaque host +// handle, while the token stored inside the object must become one broker leaf. +const parentToken = Object.freeze({ owner: "parent-token" }); +const externalizedRoot = + parent.instance.exports.create_externalized_cycle(parentToken); +if ( + parent.instance.exports.verify_externalized_cycle( + externalizedRoot, + ) !== 1 + || parent.instance.exports.externalized_cycle_token(externalizedRoot) + !== parentToken +) { + throw new Error("parent externalized fixture lost its GC/token identity"); +} +const externalizedRecipe = + parent.instance.exports.__test_encode_externref(externalizedRoot); +const externalizedRecord = recipes.get(externalizedRecipe); +if ( + !externalizedRecord + || vectors[externalizedRecord.vector].values[0] !== externalizedRecipe +) { + throw new Error("externalized GC recipe did not preserve its self-cycle"); +} +const tokenRecipe = vectors[externalizedRecord.vector].values[1]; +if (brokerValues.get(tokenRecipe) !== parentToken) { + throw new Error("opaque token was not captured as the graph's broker leaf"); +} +const directTokenRecipe = + parent.instance.exports.__test_encode_externref(parentToken); +if (directTokenRecipe !== tokenRecipe) { + throw new Error("externref/anyref token aliases received different recipes"); +} +transit.set(0, parent.instance.exports.objects.get(1)); +const directAnyRecipe = + parent.instance.exports.__wpk_fork_ref_gc_encode_slot(0); +if (directAnyRecipe !== externalizedRecipe) { + throw new Error("externalized/direct anyref aliases received different recipes"); +} + +for (let index = 0; index < transit.length; index++) transit.set(index, null); +const externalizedChild = instantiate(); +const childToken = Object.freeze({ owner: "child-token" }); +externalizedChild.instance.exports.__wpk_fork_ref_gc_publish_externref( + tokenRecipe, + childToken, +); +externalizedChild.instance.exports.__wpk_fork_ref_gc_allocate( + externalizedRecipe, +); +externalizedChild.instance.exports.__wpk_fork_ref_gc_fill( + externalizedRecipe, +); +const childExternalizedRoot = + externalizedChild.instance.exports.__test_decode_externref( + externalizedRecipe, + ); +if ( + childExternalizedRoot === externalizedRoot + || childToken === parentToken + || externalizedChild.instance.exports.verify_externalized_cycle( + childExternalizedRoot, + ) !== 1 + || externalizedChild.instance.exports.externalized_cycle_token( + childExternalizedRoot, + ) !== childToken +) { + throw new Error( + "fresh child lost externalized GC identity, cycle, or broker token", + ); +} +"#, + ) + .expect("write Node test"); + + let output = Command::new("node") + .arg(directory.join("test.mjs")) + .output() + .expect("run Node"); + let _ = fs::remove_dir_all(&directory); + assert!( + output.status.success(), + "Node fresh-instance GC codec test failed:\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + ); +} diff --git a/crates/fork-instrument/tests/module_state.rs b/crates/fork-instrument/tests/module_state.rs new file mode 100644 index 0000000000..952530c270 --- /dev/null +++ b/crates/fork-instrument/tests/module_state.rs @@ -0,0 +1,1790 @@ +//! Guest-owned module-state reconstruction for fresh fork children. + +use fork_instrument::runtime::names; +use fork_instrument::{FUNCTION_CATALOG_EXPORT, Options, instrument}; +use std::{ + fs, + process::Command, + time::{SystemTime, UNIX_EPOCH}, +}; +use walrus::{ + ExportItem, FunctionId, FunctionKind, ImportKind, LocalFunction, Module, ValType, + ir::{self, Instr, InstrSeqId}, +}; +use wasm_posix_shared::abi::{ + WPK_FORK_EXPORT_MODULE_BOOTSTRAP, WPK_FORK_EXPORT_MODULE_STATE_FINISH_RESTORE, + WPK_FORK_EXPORT_MODULE_STATE_RESTORE, WPK_FORK_EXPORT_MODULE_STATE_SAVE, + WPK_FORK_EXPORT_MODULE_TABLE_STATE_SAVE, + WPK_FORK_EXPORT_MODULE_THREAD_BOOTSTRAP, WPK_FORK_GLOBAL_CATALOG_EXPORT_PREFIX, + WPK_FORK_IMPORTED_GLOBALS_HEADER_SIZE, WPK_FORK_IMPORTED_GLOBALS_MAGIC, + WPK_FORK_IMPORTED_GLOBALS_RECORD_HEADER_SIZE, WPK_FORK_IMPORTED_GLOBALS_SECTION, + WPK_FORK_IMPORTED_TABLES_HEADER_SIZE, WPK_FORK_IMPORTED_TABLES_MAGIC, + WPK_FORK_IMPORTED_TABLES_RECORD_HEADER_SIZE, WPK_FORK_IMPORTED_TABLES_SECTION, + WPK_FORK_MODULE_STATE_IMPORT_RECORD_COMMIT, WPK_FORK_MODULE_STATE_IMPORT_RECORD_FIND, + WPK_FORK_MODULE_STATE_IMPORT_RECORD_RESERVE, WPK_FORK_MODULE_STATE_IMPORT_TABLE_DIRTY_COUNT, + WPK_FORK_MODULE_STATE_IMPORT_TABLE_DIRTY_MARK, WPK_FORK_MODULE_STATE_IMPORT_TABLE_DIRTY_PAGE, + WPK_FORK_MODULE_STATE_IMPORT_TABLE_MUTATION_ABORT, + WPK_FORK_MODULE_STATE_IMPORT_TABLE_MUTATION_BEGIN, + WPK_FORK_MODULE_STATE_IMPORT_TABLE_MUTATION_COMMIT, + WPK_FORK_MODULE_STATE_IMPORT_TABLE_RECONCILE, + WPK_FORK_MODULE_STATE_IMPORT_TABLE_STATE_OWNED, WPK_FORK_TABLE_CATALOG_EXPORT_PREFIX, +}; + +fn instrument_wat(wat: &str) -> Vec { + let input = wat::parse_str(wat).expect("parse WAT fixture"); + instrument(&input, &Options::default()).expect("instrument fixture") +} + +fn validate(bytes: &[u8]) { + wasmparser::Validator::new_with_features(wasmparser::WasmFeatures::default()) + .validate_all(bytes) + .expect("instrumented module validates"); +} + +fn export_function(module: &Module, name: &str) -> FunctionId { + match module + .exports + .iter() + .find(|export| export.name == name) + .unwrap_or_else(|| panic!("missing export {name}")) + .item + { + ExportItem::Function(function) => function, + other => panic!("{name} is not a function: {other:?}"), + } +} + +fn imported_function(module: &Module, name: &str) -> FunctionId { + module + .imports + .iter() + .find_map(|import| { + if import.module != "env" || import.name != name { + return None; + } + match &import.kind { + ImportKind::Function(function) => Some(*function), + _ => None, + } + }) + .unwrap_or_else(|| panic!("missing import env.{name}")) +} + +fn signature(module: &Module, function: FunctionId) -> (Vec, Vec) { + let ty = module.types.get(module.funcs.get(function).ty()); + (ty.params().to_vec(), ty.results().to_vec()) +} + +fn local(module: &Module, function: FunctionId) -> &LocalFunction { + match &module.funcs.get(function).kind { + FunctionKind::Local(local) => local, + other => panic!("expected local function, got {other:?}"), + } +} + +fn children(instr: &Instr) -> Vec { + match instr { + Instr::Block(ir::Block { seq }) | Instr::Loop(ir::Loop { seq }) => vec![*seq], + Instr::IfElse(ir::IfElse { + consequent, + alternative, + }) => vec![*consequent, *alternative], + Instr::TryTable(ir::TryTable { seq, .. }) => vec![*seq], + Instr::Try(try_) => { + let mut result = vec![try_.seq]; + for catch in &try_.catches { + match catch { + ir::LegacyCatch::Catch { handler, .. } + | ir::LegacyCatch::CatchAll { handler } => result.push(*handler), + ir::LegacyCatch::Delegate { .. } => {} + } + } + result + } + _ => Vec::new(), + } +} + +fn walk(local: &LocalFunction, seq: InstrSeqId, visit: &mut impl FnMut(&Instr)) { + for (instr, _) in &local.block(seq).instrs { + visit(instr); + for child in children(instr) { + walk(local, child, visit); + } + } +} + +fn assert_helper_signature(module: &Module, name: &str) { + assert_eq!( + signature(module, export_function(module, name)), + (vec![ValType::I32], vec![]), + "{name} must use the ABI activation-id signature", + ); +} + +fn custom_section<'a>(bytes: &'a [u8], name: &str) -> &'a [u8] { + wasmparser::Parser::new(0) + .parse_all(bytes) + .find_map(|payload| match payload.expect("parse custom section") { + wasmparser::Payload::CustomSection(section) if section.name() == name => { + Some(section.data()) + } + _ => None, + }) + .unwrap_or_else(|| panic!("missing custom section {name}")) +} + +fn emitted_local_count(bytes: &[u8], export_name: &str) -> u32 { + let mut imported_functions = 0u32; + let mut exported_function = None; + let mut defined_function = 0u32; + for payload in wasmparser::Parser::new(0).parse_all(bytes) { + match payload.expect("parse emitted module") { + wasmparser::Payload::ImportSection(imports) => { + for import in imports.into_imports() { + if matches!( + import.expect("parse emitted import").ty, + wasmparser::TypeRef::Func(_) | wasmparser::TypeRef::FuncExact(_) + ) { + imported_functions += 1; + } + } + } + wasmparser::Payload::ExportSection(exports) => { + for export in exports { + let export = export.expect("parse emitted export"); + if export.name == export_name + && matches!( + export.kind, + wasmparser::ExternalKind::Func + | wasmparser::ExternalKind::FuncExact + ) + { + exported_function = Some(export.index); + } + } + } + wasmparser::Payload::CodeSectionEntry(body) => { + let function_index = imported_functions + defined_function; + defined_function += 1; + if Some(function_index) != exported_function { + continue; + } + return body + .get_locals_reader() + .expect("read emitted locals") + .into_iter() + .map(|local| local.expect("parse emitted local").0) + .sum(); + } + _ => {} + } + } + panic!("missing emitted function export {export_name}"); +} + +fn codec_function(module: &Module, name: &str) -> FunctionId { + module + .exports + .iter() + .find_map(|export| { + (export.name == name) + .then_some(export.item) + .and_then(|item| match item { + ExportItem::Function(function) => Some(function), + _ => None, + }) + }) + .or_else(|| { + module.imports.iter().find_map(|import| { + (import.name == name) + .then_some(&import.kind) + .and_then(|kind| match kind { + ImportKind::Function(function) => Some(*function), + _ => None, + }) + }) + }) + .or_else(|| { + module + .funcs + .iter() + .find(|function| function.name.as_deref() == Some(name)) + .map(|function| function.id()) + }) + .unwrap_or_else(|| panic!("missing codec function {name}")) +} + +#[test] +fn table_synchronization_helpers_do_not_add_source_function_locals() { + let bytes = instrument_wat( + r#" + (module + (type $unary (func (param i32) (result i32))) + (memory 1) + (table $callbacks 8 32 funcref) + (func $identity (type $unary) (param i32) (result i32) + local.get 0) + (elem $passive func $identity) + (func (export "table_get") (param i32) (result funcref) + local.get 0 table.get $callbacks) + (func (export "table_set") (param i32 funcref) + local.get 0 local.get 1 table.set $callbacks) + (func (export "table_fill") (param i32 funcref i32) + local.get 0 local.get 1 local.get 2 table.fill $callbacks) + (func (export "table_copy") (param i32 i32 i32) + local.get 0 local.get 1 local.get 2 + table.copy $callbacks $callbacks) + (func (export "table_init") (param i32 i32 i32) + local.get 0 local.get 1 local.get 2 + table.init $callbacks $passive) + (func (export "table_grow") (param funcref i32) (result i32) + local.get 0 local.get 1 table.grow $callbacks) + (func (export "table_size") (result i32) + table.size $callbacks) + (func (export "call_indirect") (param i32 i32) (result i32) + local.get 0 local.get 1 + call_indirect $callbacks (type $unary)) + (func (export "return_call_indirect") (param i32 i32) (result i32) + local.get 0 local.get 1 + return_call_indirect $callbacks (type $unary))) + "#, + ); + validate(&bytes); + for name in [ + "table_get", + "table_set", + "table_fill", + "table_copy", + "table_init", + "table_grow", + "table_size", + "call_indirect", + "return_call_indirect", + ] { + assert_eq!( + emitted_local_count(&bytes, name), + 0, + "{name} gained an emitted source local; synchronization operands \ + must live only in generated non-suspendable helpers", + ); + } +} + +#[test] +fn deterministic_static_tables_do_not_pay_the_process_generation_fence() { + let static_bytes = instrument_wat( + r#" + (module + (memory 1) + (table $callbacks (export "__indirect_function_table") 1 funcref) + (func $callback) + (elem (i32.const 0) $callback) + (func (export "read") (param i32) (result funcref) + local.get 0 + table.get $callbacks)) + "#, + ); + validate(&static_bytes); + let static_module = + Module::from_buffer(&static_bytes).expect("parse static-table module"); + let read = local(&static_module, export_function(&static_module, "read")); + let mut direct_get = false; + let mut calls_generation_helper = false; + walk(read, read.entry_block(), &mut |instr| match instr { + Instr::TableGet(_) => direct_get = true, + Instr::Call(call) => { + calls_generation_helper |= static_module + .funcs + .get(call.func) + .name + .as_deref() + .is_some_and(|name| name.starts_with("__wpk_fork_table_get_")); + } + _ => {} + }); + assert!( + direct_get && !calls_generation_helper, + "a deterministic local table read must remain a direct Wasm operation", + ); + let table_save = local( + &static_module, + export_function( + &static_module, + WPK_FORK_EXPORT_MODULE_TABLE_STATE_SAVE, + ), + ); + let mut snapshots_static_table = false; + walk(table_save, table_save.entry_block(), &mut |instr| { + snapshots_static_table |= matches!(instr, Instr::TableSize(_) | Instr::TableGet(_)); + }); + assert!( + !snapshots_static_table, + "static element initialization is the reconstruction owner; a peer snapshot is redundant", + ); + + let process_bytes = instrument_wat( + r#" + (module + (import "env" "__wasm_dlopen" + (func (param i32 i32 i32 i32) (result i32))) + (memory 1) + (table $callbacks (export "__indirect_function_table") 1 funcref) + (func (export "read") (param i32) (result funcref) + local.get 0 + table.get $callbacks)) + "#, + ); + validate(&process_bytes); + let process_module = + Module::from_buffer(&process_bytes).expect("parse process-table module"); + let read = local(&process_module, export_function(&process_module, "read")); + direct_get = false; + calls_generation_helper = false; + walk(read, read.entry_block(), &mut |instr| match instr { + Instr::TableGet(_) => direct_get = true, + Instr::Call(call) => { + calls_generation_helper |= process_module + .funcs + .get(call.func) + .name + .as_deref() + .is_some_and(|name| name.starts_with("__wpk_fork_table_get_")); + } + _ => {} + }); + assert!( + !direct_get && calls_generation_helper, + "the dynamic linker's process table must reconcile before it is consumed", + ); +} + +#[test] +fn module_state_imports_and_exports_use_exact_wasm32_signatures() { + let bytes = instrument_wat( + r#" + (module + (import "kernel" "kernel_fork" (func $fork (result i32))) + (memory 1) + (global $root (mut funcref) (ref.null func)) + (table $callbacks 2 8 funcref) + (func $caller (result i32) call $fork)) + "#, + ); + validate(&bytes); + let module = Module::from_buffer(&bytes).expect("parse instrumented module"); + + assert_helper_signature(&module, WPK_FORK_EXPORT_MODULE_STATE_SAVE); + assert_helper_signature(&module, WPK_FORK_EXPORT_MODULE_STATE_RESTORE); + assert_helper_signature(&module, WPK_FORK_EXPORT_MODULE_STATE_FINISH_RESTORE); + assert_eq!( + signature( + &module, + export_function(&module, WPK_FORK_EXPORT_MODULE_BOOTSTRAP) + ), + (vec![], vec![]), + ); + assert_eq!( + signature( + &module, + export_function(&module, WPK_FORK_EXPORT_MODULE_THREAD_BOOTSTRAP), + ), + (vec![], vec![]), + ); + assert_eq!( + signature( + &module, + imported_function(&module, WPK_FORK_MODULE_STATE_IMPORT_RECORD_RESERVE), + ), + ( + vec![ValType::I32, ValType::I32, ValType::I32, ValType::I32], + vec![ValType::I32], + ), + ); + assert_eq!( + signature( + &module, + imported_function(&module, WPK_FORK_MODULE_STATE_IMPORT_TABLE_DIRTY_MARK), + ), + (vec![ValType::I32, ValType::I64, ValType::I64], vec![]), + ); + assert_eq!( + signature( + &module, + imported_function(&module, WPK_FORK_MODULE_STATE_IMPORT_TABLE_DIRTY_COUNT), + ), + (vec![ValType::I32], vec![ValType::I32]), + ); + assert_eq!( + signature( + &module, + imported_function(&module, WPK_FORK_MODULE_STATE_IMPORT_TABLE_DIRTY_PAGE), + ), + (vec![ValType::I32, ValType::I32], vec![ValType::I64]), + ); + assert_eq!( + signature( + &module, + imported_function(&module, WPK_FORK_MODULE_STATE_IMPORT_RECORD_COMMIT), + ), + (vec![ValType::I32], vec![]), + ); + assert_eq!( + signature( + &module, + imported_function(&module, WPK_FORK_MODULE_STATE_IMPORT_RECORD_FIND), + ), + ( + vec![ValType::I32, ValType::I32, ValType::I32, ValType::I32], + vec![ValType::I32], + ), + ); + assert_eq!( + signature( + &module, + imported_function( + &module, + WPK_FORK_MODULE_STATE_IMPORT_TABLE_MUTATION_BEGIN, + ), + ), + (vec![], vec![ValType::I64]), + ); + assert_eq!( + signature( + &module, + imported_function( + &module, + WPK_FORK_MODULE_STATE_IMPORT_TABLE_MUTATION_COMMIT, + ), + ), + ( + vec![ValType::I32, ValType::I64, ValType::I64], + vec![], + ), + ); + assert_eq!( + signature( + &module, + imported_function( + &module, + WPK_FORK_MODULE_STATE_IMPORT_TABLE_MUTATION_ABORT, + ), + ), + (vec![], vec![]), + ); + assert_eq!( + signature( + &module, + imported_function(&module, WPK_FORK_MODULE_STATE_IMPORT_TABLE_RECONCILE), + ), + (vec![], vec![ValType::I64]), + ); +} + +#[test] +fn module_state_record_pointers_follow_memory64() { + let bytes = instrument_wat( + r#" + (module + (import "kernel" "kernel_fork" (func $fork (result i32))) + (memory i64 1) + (global $root (mut externref) (ref.null extern)) + (func $caller (result i32) call $fork)) + "#, + ); + validate(&bytes); + let module = Module::from_buffer(&bytes).expect("parse instrumented memory64 module"); + assert_eq!( + signature( + &module, + imported_function(&module, WPK_FORK_MODULE_STATE_IMPORT_RECORD_RESERVE), + ), + ( + vec![ValType::I32, ValType::I32, ValType::I32, ValType::I64], + vec![ValType::I64], + ), + ); + assert_eq!( + signature( + &module, + imported_function(&module, WPK_FORK_MODULE_STATE_IMPORT_RECORD_COMMIT), + ), + (vec![ValType::I64], vec![]), + ); + assert_eq!( + signature( + &module, + imported_function(&module, WPK_FORK_MODULE_STATE_IMPORT_RECORD_FIND), + ), + ( + vec![ValType::I32, ValType::I32, ValType::I32, ValType::I32], + vec![ValType::I64], + ), + ); +} + +#[test] +fn save_and_restore_own_reference_globals_and_dirty_table_state() { + let bytes = instrument_wat( + r#" + (module + (import "kernel" "kernel_fork" (func $fork (result i32))) + (import "env" "shared_root" (global $shared_root (mut externref))) + (memory 1) + (global $callback (mut funcref) (ref.null func)) + (global $exception (mut exnref) (ref.null exn)) + (global $object (mut anyref) (ref.null any)) + (table $callbacks 2 8 funcref) + (table $exceptions 1 8 exnref) + (table $objects 1 8 anyref) + (func $mutate_tables + (param $callback funcref) + (param $exception exnref) + (param $object anyref) + i32.const 0 + local.get $callback + table.set $callbacks + i32.const 0 + local.get $exception + table.set $exceptions + i32.const 0 + local.get $object + table.set $objects) + (func $caller (result i32) call $fork)) + "#, + ); + validate(&bytes); + let module = Module::from_buffer(&bytes).expect("parse instrumented module"); + let save = local( + &module, + export_function(&module, WPK_FORK_EXPORT_MODULE_STATE_SAVE), + ); + let restore = local( + &module, + export_function(&module, WPK_FORK_EXPORT_MODULE_STATE_RESTORE), + ); + let finish_restore = local( + &module, + export_function(&module, WPK_FORK_EXPORT_MODULE_STATE_FINISH_RESTORE), + ); + let encode_funcref = codec_function(&module, names::IMPORT_REF_ENCODE_FUNCREF); + let encode_externref = codec_function(&module, names::IMPORT_REF_ENCODE_EXTERNREF); + let encode_exnref = codec_function(&module, names::IMPORT_REF_ENCODE_EXNREF); + let encode_anyref = codec_function(&module, names::IMPORT_REF_ENCODE_ANYREF); + let decode_funcref = codec_function(&module, names::IMPORT_REF_DECODE_FUNCREF); + let decode_externref = codec_function(&module, names::IMPORT_REF_DECODE_EXTERNREF); + let decode_exnref = codec_function(&module, names::IMPORT_REF_DECODE_EXNREF); + let decode_anyref = codec_function(&module, names::IMPORT_REF_DECODE_ANYREF); + + let mut save_has_table_size = false; + let mut save_has_table_get = false; + let mut save_codecs = Vec::new(); + walk(save, save.entry_block(), &mut |instr| match instr { + Instr::TableSize(_) => save_has_table_size = true, + Instr::TableGet(_) => save_has_table_get = true, + Instr::Call(call) => save_codecs.push(call.func), + _ => {} + }); + assert!(save_has_table_size && save_has_table_get); + assert!(save_codecs.contains(&encode_funcref)); + assert!(save_codecs.contains(&encode_externref)); + assert!(save_codecs.contains(&encode_exnref)); + assert!(save_codecs.contains(&encode_anyref)); + + let mut restore_has_global_set = false; + let mut restore_has_table_grow = false; + let mut restore_has_table_set = false; + let mut restore_codecs = Vec::new(); + walk( + finish_restore, + finish_restore.entry_block(), + &mut |instr| match instr { + Instr::GlobalSet(_) => restore_has_global_set = true, + Instr::TableGrow(_) => restore_has_table_grow = true, + Instr::TableSet(_) => restore_has_table_set = true, + Instr::Call(call) => restore_codecs.push(call.func), + _ => {} + }, + ); + walk(restore, restore.entry_block(), &mut |instr| match instr { + Instr::GlobalSet(_) => restore_has_global_set = true, + Instr::Call(call) => restore_codecs.push(call.func), + _ => {} + }); + assert!(restore_has_global_set); + assert!(restore_has_table_grow && restore_has_table_set); + assert!(restore_codecs.contains(&decode_funcref)); + assert!(restore_codecs.contains(&decode_externref)); + assert!(restore_codecs.contains(&decode_exnref)); + assert!(restore_codecs.contains(&decode_anyref)); +} + +#[test] +fn save_and_restore_own_every_scalar_global_type() { + let bytes = instrument_wat( + r#" + (module + (import "kernel" "kernel_fork" (func $fork (result i32))) + (memory 1) + (global $i32 (mut i32) (i32.const 1)) + (global $i64 (mut i64) (i64.const 2)) + (global $f32 (mut f32) (f32.const 3)) + (global $f64 (mut f64) (f64.const 4)) + (global $v128 (mut v128) (v128.const i32x4 5 6 7 8)) + (func $caller (result i32) call $fork)) + "#, + ); + validate(&bytes); + let module = Module::from_buffer(&bytes).expect("parse instrumented module"); + let save = local( + &module, + export_function(&module, WPK_FORK_EXPORT_MODULE_STATE_SAVE), + ); + let restore = local( + &module, + export_function(&module, WPK_FORK_EXPORT_MODULE_STATE_RESTORE), + ); + + let mut stores = [false; 5]; + walk(save, save.entry_block(), &mut |instr| { + let Instr::Store(store) = instr else { return }; + match store.kind { + ir::StoreKind::I32 { .. } => stores[0] = true, + ir::StoreKind::I64 { .. } => stores[1] = true, + ir::StoreKind::F32 => stores[2] = true, + ir::StoreKind::F64 => stores[3] = true, + ir::StoreKind::V128 => stores[4] = true, + _ => {} + } + }); + assert!(stores.into_iter().all(|seen| seen)); + + let mut loads = [false; 5]; + walk(restore, restore.entry_block(), &mut |instr| { + let Instr::Load(load) = instr else { return }; + match load.kind { + ir::LoadKind::I32 { .. } => loads[0] = true, + ir::LoadKind::I64 { .. } => loads[1] = true, + ir::LoadKind::F32 => loads[2] = true, + ir::LoadKind::F64 => loads[3] = true, + ir::LoadKind::V128 => loads[4] = true, + _ => {} + } + }); + assert!(loads.into_iter().all(|seen| seen)); +} + +#[test] +fn immutable_imports_keep_their_original_binding_and_preinstantiation_recipe() { + let bytes = instrument_wat( + r#" + (module + (import "kernel" "kernel_fork" (func $fork (result i32))) + (import "env" "immutable_callback" (global $callback funcref)) + (memory 1) + (export "immutable_callback_global" (global $callback)) + (global $callback_alias funcref (global.get $callback)) + (export "immutable_callback_alias" (global $callback_alias)) + (func $read (export "read") (result funcref) + global.get $callback) + (func $caller (result i32) call $fork)) + "#, + ); + validate(&bytes); + let module = Module::from_buffer(&bytes).expect("parse instrumented module"); + let imported = module + .imports + .iter() + .find_map(|import| { + if import.module != "env" || import.name != "immutable_callback" { + return None; + } + match import.kind { + ImportKind::Global(global) => Some(global), + _ => None, + } + }) + .expect("immutable imported global"); + let read = local(&module, export_function(&module, "read")); + let mut observed = Vec::new(); + walk(read, read.entry_block(), &mut |instr| { + if let Instr::GlobalGet(get) = instr { + observed.push(get.global); + } + }); + assert_eq!(observed.len(), 1); + assert_eq!(observed[0], imported); + assert!(matches!( + module + .exports + .iter() + .find(|export| export.name == "immutable_callback_global") + .map(|export| export.item), + Some(ExportItem::Global(global)) if global == imported + )); + let alias = module + .exports + .iter() + .find_map(|export| (export.name == "immutable_callback_alias").then_some(export.item)) + .expect("immutable alias export"); + let ExportItem::Global(alias) = alias else { + panic!("immutable alias export is not a global") + }; + assert!(matches!( + module.globals.get(alias).kind, + walrus::GlobalKind::Local(walrus::ConstExpr::Global(source)) if source == imported + )); + let catalog_globals: Vec<_> = module + .exports + .iter() + .filter_map(|export| { + if !export + .name + .starts_with(WPK_FORK_GLOBAL_CATALOG_EXPORT_PREFIX) + { + return None; + } + match export.item { + ExportItem::Global(global) => Some(global), + _ => panic!("private global catalog entry is not a global"), + } + }) + .collect(); + assert_eq!(catalog_globals.len(), 2); + assert!( + catalog_globals.contains(&imported), + "the imported provider cell needs an exact private Global wrapper" + ); + assert!( + catalog_globals.contains(&alias), + "local provider cells need the same deterministic catalog" + ); + + let save = local( + &module, + export_function(&module, WPK_FORK_EXPORT_MODULE_STATE_SAVE), + ); + let encode_funcref = imported_function(&module, names::IMPORT_REF_ENCODE_FUNCREF); + let mut saves_import = false; + walk(save, save.entry_block(), &mut |instr| { + if matches!(instr, Instr::Call(call) if call.func == encode_funcref) { + saves_import = true; + } + }); + assert!(saves_import); + + let restore = local( + &module, + export_function(&module, WPK_FORK_EXPORT_MODULE_STATE_RESTORE), + ); + let mut assigns_import = false; + walk(restore, restore.entry_block(), &mut |instr| { + if matches!(instr, Instr::GlobalSet(set) if set.global == imported) { + assigns_import = true; + } + }); + assert!(!assigns_import); + + let descriptor = custom_section(&bytes, WPK_FORK_IMPORTED_GLOBALS_SECTION); + assert!(descriptor.len() >= usize::from(WPK_FORK_IMPORTED_GLOBALS_HEADER_SIZE)); + assert_eq!(&descriptor[..4], &WPK_FORK_IMPORTED_GLOBALS_MAGIC); + let record = usize::from(WPK_FORK_IMPORTED_GLOBALS_HEADER_SIZE); + assert_eq!(u32::from_le_bytes(descriptor[8..12].try_into().unwrap()), 1); + assert_eq!( + u32::from_le_bytes(descriptor[record..record + 4].try_into().unwrap()) as usize, + usize::from(WPK_FORK_IMPORTED_GLOBALS_RECORD_HEADER_SIZE) + + "env".len() + + "immutable_callback".len(), + ); + assert_eq!( + u32::from_le_bytes(descriptor[record + 20..record + 24].try_into().unwrap()), + 1, + "KFIG must name the full import-section ordinal, including the preceding function", + ); +} + +#[test] +fn imported_global_recipe_preserves_full_wasm_name_lengths() { + let field = "x".repeat(70_000); + let bytes = instrument_wat(&format!( + r#" + (module + (import "env" "{field}" (global $value i32)) + (memory 1) + (func (export "read") (result i32) global.get $value)) + "#, + )); + validate(&bytes); + let descriptor = custom_section(&bytes, WPK_FORK_IMPORTED_GLOBALS_SECTION); + let record = usize::from(WPK_FORK_IMPORTED_GLOBALS_HEADER_SIZE); + assert_eq!( + u32::from_le_bytes(descriptor[record + 12..record + 16].try_into().unwrap()), + 3, + ); + assert_eq!( + u32::from_le_bytes(descriptor[record + 16..record + 20].try_into().unwrap()), + 70_000, + ); +} + +#[test] +fn imported_table_identity_has_exact_preinstantiation_recipe_and_catalog() { + let bytes = instrument_wat( + r#" + (module + (import "kernel" "kernel_fork" (func $fork (result i32))) + (import "env" "callbacks" (table $callbacks 2 8 funcref)) + (memory 1) + (func $caller (result i32) call $fork)) + "#, + ); + validate(&bytes); + let module = Module::from_buffer(&bytes).expect("parse instrumented module"); + let imported = module + .imports + .iter() + .find_map(|import| { + if import.module != "env" || import.name != "callbacks" { + return None; + } + match import.kind { + ImportKind::Table(table) => Some(table), + _ => None, + } + }) + .expect("imported table"); + assert!(module.exports.iter().any(|export| { + export.name == format!("{WPK_FORK_TABLE_CATALOG_EXPORT_PREFIX}1") + && matches!(export.item, ExportItem::Table(table) if table == imported) + })); + + let descriptor = custom_section(&bytes, WPK_FORK_IMPORTED_TABLES_SECTION); + assert_eq!(&descriptor[..4], &WPK_FORK_IMPORTED_TABLES_MAGIC); + assert_eq!(u32::from_le_bytes(descriptor[8..12].try_into().unwrap()), 1); + let record = usize::from(WPK_FORK_IMPORTED_TABLES_HEADER_SIZE); + assert_eq!( + u32::from_le_bytes(descriptor[record..record + 4].try_into().unwrap()) as usize, + usize::from(WPK_FORK_IMPORTED_TABLES_RECORD_HEADER_SIZE) + "env".len() + "callbacks".len(), + ); + assert_eq!( + u32::from_le_bytes(descriptor[record + 20..record + 24].try_into().unwrap()), + 1, + "KFIT must name the full import-section ordinal", + ); +} + +#[test] +fn active_segment_offsets_preserve_extended_const_semantics_without_a_shape_gate() { + let bytes = instrument_wat( + r#" + (module + (import "env" "base" (global $base i32)) + (memory 1) + (table 8 funcref) + (func $target) + (elem (i32.add (global.get $base) (i32.const 1)) $target) + (data (i32.mul (global.get $base) (i32.const 2)) "x")) + "#, + ); + validate(&bytes); + let module = Module::from_buffer(&bytes).expect("parse extended-const module"); + let preserved_offsets = module + .globals + .iter() + .filter(|global| { + matches!( + global.kind, + walrus::GlobalKind::Local(walrus::ConstExpr::Extended(_)) + ) + }) + .count(); + assert_eq!( + preserved_offsets, 2, + "each converted active segment must retain its original const expression", + ); + + for helper in [ + WPK_FORK_EXPORT_MODULE_BOOTSTRAP, + WPK_FORK_EXPORT_MODULE_THREAD_BOOTSTRAP, + WPK_FORK_EXPORT_MODULE_STATE_RESTORE, + ] { + let helper = local(&module, export_function(&module, helper)); + let mut reads_preserved_offset = false; + walk(helper, helper.entry_block(), &mut |instr| { + if let Instr::GlobalGet(get) = instr + && matches!( + module.globals.get(get.global).kind, + walrus::GlobalKind::Local(walrus::ConstExpr::Extended(_)) + ) + { + reads_preserved_offset = true; + } + }); + assert!( + reads_preserved_offset, + "segment helper must consume the naturally evaluated offset global", + ); + } +} + +#[test] +fn segment_lifetime_is_activation_owned_and_reapplied() { + let bytes = instrument_wat( + r#" + (module + (import "kernel" "kernel_fork" (func $fork (result i32))) + (memory 1) + (table 1 funcref) + (func $target) + (elem $functions funcref (ref.func $target)) + (data $bytes "payload") + (func $caller (result i32) + elem.drop $functions + data.drop $bytes + call $fork)) + "#, + ); + validate(&bytes); + let module = Module::from_buffer(&bytes).expect("parse instrumented module"); + let restore = local( + &module, + export_function(&module, WPK_FORK_EXPORT_MODULE_STATE_RESTORE), + ); + let finish_restore = local( + &module, + export_function(&module, WPK_FORK_EXPORT_MODULE_STATE_FINISH_RESTORE), + ); + let mut elem_drop = false; + let mut data_drop = false; + walk(restore, restore.entry_block(), &mut |instr| match instr { + Instr::ElemDrop(_) => elem_drop = true, + Instr::DataDrop(_) => data_drop = true, + _ => {} + }); + assert!( + !elem_drop && !data_drop, + "value/table restore must leave constructor segments live" + ); + walk( + finish_restore, + finish_restore.entry_block(), + &mut |instr| match instr { + Instr::ElemDrop(_) => elem_drop = true, + Instr::DataDrop(_) => data_drop = true, + _ => {} + }, + ); + assert!( + elem_drop, + "finish restore must reapply element-segment lifetime" + ); + assert!( + data_drop, + "finish restore must reapply data-segment lifetime" + ); + + let caller = module + .funcs + .iter() + .find(|func| func.name.as_deref() == Some("caller")) + .expect("named caller"); + let caller = local(&module, caller.id()); + let mut caller_drops = 0; + let mut tracker_updates = 0; + walk(caller, caller.entry_block(), &mut |instr| match instr { + Instr::ElemDrop(_) | Instr::DataDrop(_) => caller_drops += 1, + Instr::GlobalSet(_) => tracker_updates += 1, + _ => {} + }); + assert_eq!(caller_drops, 2); + assert!( + tracker_updates >= 2, + "each original segment drop must update its activation-owned bitmap", + ); +} + +#[test] +fn modules_outside_the_active_fork_stack_still_expose_reconstructible_state() { + let bytes = instrument_wat( + r#" + (module + (global $counter (mut i64) (i64.const 7)) + (global $root (mut funcref) (ref.null func)) + (table 1 funcref)) + "#, + ); + validate(&bytes); + let module = Module::from_buffer(&bytes).expect("parse no-seed instrumented module"); + assert_helper_signature(&module, WPK_FORK_EXPORT_MODULE_STATE_SAVE); + assert_helper_signature(&module, WPK_FORK_EXPORT_MODULE_STATE_RESTORE); + assert_helper_signature(&module, WPK_FORK_EXPORT_MODULE_STATE_FINISH_RESTORE); + assert!( + module + .exports + .iter() + .any(|export| export.name == FUNCTION_CATALOG_EXPORT), + "every module activation needs a deterministic funcref catalog", + ); + for reserved in [ + WPK_FORK_MODULE_STATE_IMPORT_RECORD_RESERVE, + WPK_FORK_MODULE_STATE_IMPORT_RECORD_COMMIT, + WPK_FORK_MODULE_STATE_IMPORT_RECORD_FIND, + WPK_FORK_MODULE_STATE_IMPORT_TABLE_DIRTY_COUNT, + WPK_FORK_MODULE_STATE_IMPORT_TABLE_DIRTY_MARK, + WPK_FORK_MODULE_STATE_IMPORT_TABLE_DIRTY_PAGE, + WPK_FORK_MODULE_STATE_IMPORT_TABLE_STATE_OWNED, + names::IMPORT_REF_ENCODE_FUNCREF, + names::IMPORT_REF_DECODE_FUNCREF, + ] { + assert!( + module.imports.iter().any(|import| import.name == reserved), + "no-seed state helper is missing {reserved}", + ); + } + assert!( + module.imports.iter().any(|import| { + import.module == "env" + && import.name == "memory" + && matches!(import.kind, ImportKind::Memory(_)) + }), + "a no-memory module must stage KFMS records through env.memory", + ); +} + +#[test] +fn node_fresh_instance_restores_no_seed_module_state_and_segment_lifetime() { + let module = instrument_wat( + r#" + (module + (import "env" "memory" (memory 0 65536 shared)) + (import "env" "shared_counter" (global $shared_counter (mut i64))) + (import "env" "shared_callbacks" (table $shared_callbacks 2 8 funcref)) + (import "env" "imported_callback" (func $imported_callback (result i32))) + (import "env" "immutable_callback" (global $immutable_callback funcref)) + (import "env" "immutable_token" (global $immutable_token externref)) + (export "immutable_callback_global" (global $immutable_callback)) + (export "immutable_token_global" (global $immutable_token)) + (global $immutable_callback_alias funcref (global.get $immutable_callback)) + (global $immutable_token_alias externref (global.get $immutable_token)) + (export "immutable_callback_alias" (global $immutable_callback_alias)) + (export "immutable_token_alias" (global $immutable_token_alias)) + + (global $counter (mut i32) (i32.const 0)) + (global $f32_bits (mut f32) (f32.const 0)) + (global $f64_bits (mut f64) (f64.const 0)) + (global $vector (mut v128) (v128.const i32x4 0 0 0 0)) + (global $callback (mut funcref) (ref.null func)) + (global $token (mut externref) (ref.null extern)) + (global $start_count (mut i32) (i32.const 0)) + + (table $callbacks (export "callbacks") 3 10 funcref) + (table $tokens (export "tokens") 2 10 externref) + (func $a (result i32) i32.const 11) + (func $b (result i32) i32.const 22) + (elem $baseline (table $callbacks) (i32.const 0) func $a $b $a) + (elem $late func $b $a) + (data $active_data (i32.const 16) "\31\32\33") + (data $late_data "xyz") + + (func $module_start + global.get $start_count + i32.const 1 + i32.add + global.set $start_count + i32.const 16 + i32.const 0x44 + i32.store8) + (start $module_start) + + (func (export "mutate") (param $owned externref) + i32.const 0x11223344 + global.set $counter + i64.const 0x1122334455667788 + global.set $shared_counter + i32.const 0x7fc12345 + f32.reinterpret_i32 + global.set $f32_bits + i64.const 0x7ff8123456789abc + f64.reinterpret_i64 + global.set $f64_bits + v128.const i32x4 101 202 303 404 + global.set $vector + ref.func $b + global.set $callback + local.get $owned + global.set $token + i32.const 16 + i32.const 0x7a + i32.store8 + + ref.func $b + i32.const 2 + table.grow $callbacks + drop + i32.const 0 + ref.func $b + table.set $callbacks + i32.const 1 + ref.func $a + i32.const 2 + table.fill $callbacks + i32.const 3 + i32.const 0 + i32.const 2 + table.copy $callbacks $callbacks + i32.const 1 + i32.const 0 + i32.const 2 + table.init $callbacks $late + elem.drop $late + + local.get $owned + i32.const 2 + table.grow $tokens + drop + i32.const 0 + local.get $owned + i32.const 4 + table.fill $tokens + i32.const 1 + i32.const 0 + i32.const 3 + table.copy $tokens $tokens + + ref.func $a + i32.const 1 + table.grow $shared_callbacks + drop + i32.const 0 + ref.func $b + i32.const 3 + table.fill $shared_callbacks + data.drop $late_data) + + (func (export "counter") (result i32) global.get $counter) + (func (export "shared_counter") (result i64) global.get $shared_counter) + (func (export "f32_bits") (result i32) + global.get $f32_bits + i32.reinterpret_f32) + (func (export "f64_bits") (result i64) + global.get $f64_bits + i64.reinterpret_f64) + (func (export "vector_lane_2") (result i32) + global.get $vector + i32x4.extract_lane 2) + (func (export "callback") (result funcref) global.get $callback) + (func (export "token") (result externref) global.get $token) + (func (export "start_count") (result i32) global.get $start_count) + (func (export "active_data_byte") (result i32) + i32.const 16 + i32.load8_u) + (func (export "immutable_callback") (result funcref) + global.get $immutable_callback) + (func (export "immutable_token") (result externref) + global.get $immutable_token) + (func (export "shared_callback") (param $index i32) (result funcref) + local.get $index + table.get $shared_callbacks) + (func (export "try_late_elem") + i32.const 0 + i32.const 0 + i32.const 1 + table.init $callbacks $late) + (func (export "try_late_data") + i32.const 0 + i32.const 0 + i32.const 1 + memory.init $late_data) + (func (export "try_active_elem") + i32.const 0 + i32.const 0 + i32.const 1 + table.init $callbacks $baseline) + (func (export "try_active_data") + i32.const 0 + i32.const 0 + i32.const 1 + memory.init $active_data)) + "#, + ); + let typed_codecs = wat::parse_str( + r#" + (module + (func (export "callback") (result i32) i32.const 31) + (func (export "__wpk_fork_ref_encode_exnref") + (param (ref null exn)) (result i32) i32.const 0) + (func (export "__wpk_fork_ref_decode_exnref") + (param i32) (result (ref null exn)) ref.null exn) + (func (export "__wpk_fork_ref_encode_anyref") + (param (ref null any)) (result i32) i32.const 0) + (func (export "__wpk_fork_ref_decode_anyref") + (param i32) (result (ref null any)) ref.null any)) + "#, + ) + .expect("compile typed reference codec fixture"); + + let nonce = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("clock after epoch") + .as_nanos(); + let directory = std::env::temp_dir().join(format!( + "kandelo-module-state-{}-{nonce}", + std::process::id(), + )); + fs::create_dir(&directory).expect("create module-state engine-test directory"); + let module_path = directory.join("module.wasm"); + let codecs_path = directory.join("typed-codecs.wasm"); + fs::write(&module_path, module).expect("write instrumented module fixture"); + fs::write(&codecs_path, typed_codecs).expect("write typed codec fixture"); + + let script = r#" + const fs = require("node:fs"); + const [modulePath, codecsPath] = process.argv.slice(1); + const module = new WebAssembly.Module(fs.readFileSync(modulePath)); + const codecModule = new WebAssembly.Module(fs.readFileSync(codecsPath)); + const typed = new WebAssembly.Instance(codecModule).exports; + const nodes = [{ kind: "null" }]; + const objectIds = new WeakMap(); + const records = []; + let parent; + let child; + let thread; + + function intern(value, node) { + if (value === null) return 0; + const known = objectIds.get(value); + if (known !== undefined) return known; + const id = nodes.length; + nodes.push(node()); + objectIds.set(value, id); + return id; + } + function functionOrdinal(instance, value) { + const catalog = instance.exports.__wpk_fork_function_catalog; + for (let ordinal = 0; ordinal < catalog.length; ordinal++) { + if (catalog.get(ordinal) === value) return ordinal; + } + throw new Error("funcref absent from function catalog"); + } + function encodeFuncref(value) { + return intern(value, () => ({ + kind: "funcref", + ordinal: functionOrdinal(parent, value), + })); + } + function encodeExternref(value) { + return intern(value, () => { + if (typeof value !== "object" || value === null || !Number.isInteger(value.handle)) { + throw new Error("externref bypassed the test process owner"); + } + return { kind: "externref", handle: value.handle }; + }); + } + const childExternrefs = new Map(); + function decodeFuncref(id) { + if (id === 0) return null; + const node = nodes[id]; + if (node?.kind !== "funcref") throw new Error(`recipe ${id} is not funcref`); + return child.exports.__wpk_fork_function_catalog.get(node.ordinal); + } + function decodeExternref(id) { + if (id === 0) return null; + const node = nodes[id]; + if (node?.kind !== "externref") throw new Error(`recipe ${id} is not externref`); + let token = childExternrefs.get(node.handle); + if (!token) { + token = Object.freeze({ handle: node.handle, child: true }); + childExternrefs.set(node.handle, token); + } + return token; + } + + function instantiate(mode, memory, sharedCounter, sharedCallbacks, imports) { + let cursor = 65536; + let pending = null; + const dirtyPages = new Map(); + let nextReferenceVector = 1; + const referenceVectors = new Map(); + // The real process owner sizes this typed transit table for the + // reference recipe transaction. Keep the engine fixture large enough + // for every scalar/global/table recipe it intentionally captures. + const gcTransit = new WebAssembly.Table({ + element: "anyref", initial: 1024, + }); + const beginReferenceVector = (capacity) => { + const id = nextReferenceVector++; + referenceVectors.set(id, { capacity, values: [] }); + return id; + }; + const appendReferenceVector = (id, value) => { + const vector = referenceVectors.get(id); + if (!vector || vector.values.length >= vector.capacity) { + throw new Error(`invalid reference vector append ${id}`); + } + vector.values.push(value); + }; + const finishReferenceVector = (id) => { + const vector = referenceVectors.get(id); + if (!vector || vector.values.length !== vector.capacity) { + throw new Error(`invalid reference vector finish ${id}`); + } + return id; + }; + const getReferenceVector = (id, index) => { + const vector = referenceVectors.get(id); + if (!vector || index < 0 || index >= vector.values.length) { + throw new Error(`invalid reference vector lookup ${id}:${index}`); + } + return vector.values[index]; + }; + const dirtyMark = (owner, firstValue, countValue) => { + const first = BigInt.asUintN(64, firstValue); + const count = BigInt.asUintN(64, countValue); + let pages = dirtyPages.get(owner); + if (!pages) { + pages = new Set(); + dirtyPages.set(owner, pages); + } + for (let offset = 0n; offset < count; offset++) { + pages.add(first + offset); + } + }; + const sortedDirtyPages = (owner) => + [...(dirtyPages.get(owner) ?? [])].sort((a, b) => a < b ? -1 : a > b ? 1 : 0); + const allocate = (size) => { + const pointer = (cursor + 7) & ~7; + cursor = pointer + size; + if (cursor > memory.buffer.byteLength) { + memory.grow(Math.ceil((cursor - memory.buffer.byteLength) / 65536)); + } + return pointer; + }; + const reserve = (kind, activation, owner, size) => { + if (pending) throw new Error("nested record reservation"); + const pointer = allocate(size); + pending = { kind, activation, owner, size, pointer }; + return pointer; + }; + const commit = (pointer) => { + if (!pending || pending.pointer !== pointer) throw new Error("bad record commit"); + records.push({ + kind: pending.kind, + activation: pending.activation, + owner: pending.owner, + payload: new Uint8Array(memory.buffer, pointer, pending.size).slice(), + }); + pending = null; + }; + const find = (kind, activation, owner, ordinal) => { + const matches = records.filter((record) => + record.kind === kind + && record.activation === activation + && record.owner === owner + ); + const record = matches[ordinal]; + if (!record) throw new Error(`missing record ${kind}:${activation}:${owner}:${ordinal}`); + const pointer = allocate(record.payload.length); + new Uint8Array(memory.buffer, pointer, record.payload.length).set(record.payload); + return pointer; + }; + const unreachableFrame = () => { + throw new Error("no-seed module unexpectedly used a continuation-frame hook"); + }; + const env = { + memory, + shared_counter: sharedCounter, + shared_callbacks: sharedCallbacks, + imported_callback: imports.importedCallback, + immutable_callback: imports.importedCallback, + immutable_token: imports.immutableToken, + __wpk_fork_module_activation: new WebAssembly.Global( + { value: "i32", mutable: false }, + 0, + ), + __wpk_fork_unwind: new WebAssembly.Tag({ parameters: [] }), + __wpk_fork_frame_reserve: unreachableFrame, + __wpk_fork_frame_commit: unreachableFrame, + __wpk_fork_frame_next: unreachableFrame, + __wpk_fork_frame_peek: unreachableFrame, + __wpk_fork_resume_peek: () => 0, + __wpk_fork_resume_table: new WebAssembly.Table({ + element: "anyfunc", initial: 1, + }), + __wpk_fork_ref_gc_transit: gcTransit, + __wpk_fork_module_state_record_reserve: + mode === "capture" ? reserve : unreachableFrame, + __wpk_fork_module_state_record_commit: + mode === "capture" ? commit : unreachableFrame, + __wpk_fork_module_state_record_find: + mode === "restore" ? find : unreachableFrame, + __wpk_fork_module_state_table_dirty_mark: dirtyMark, + __wpk_fork_module_state_table_dirty_count: + (owner) => sortedDirtyPages(owner).length, + __wpk_fork_module_state_table_dirty_page: + (owner, ordinal) => BigInt.asIntN(64, sortedDirtyPages(owner)[ordinal]), + __wpk_fork_module_state_table_state_owned: () => 1, + __wpk_fork_module_state_table_generation_addr: new WebAssembly.Global( + { value: "i64", mutable: false }, + 0n, + ), + __wpk_fork_module_state_table_reconcile: () => 0n, + __wpk_fork_module_state_table_mutation_begin: () => 0n, + __wpk_fork_module_state_table_mutation_commit: () => {}, + __wpk_fork_module_state_table_mutation_abort: () => {}, + __wpk_fork_ref_encode_funcref: encodeFuncref, + __wpk_fork_ref_decode_funcref: decodeFuncref, + __wpk_fork_ref_encode_externref: encodeExternref, + __wpk_fork_ref_decode_externref: decodeExternref, + __wpk_fork_ref_encode_exnref: typed.__wpk_fork_ref_encode_exnref, + __wpk_fork_ref_decode_exnref: typed.__wpk_fork_ref_decode_exnref, + __wpk_fork_ref_encode_anyref: typed.__wpk_fork_ref_encode_anyref, + __wpk_fork_ref_decode_anyref: typed.__wpk_fork_ref_decode_anyref, + __wpk_fork_ref_vector_begin: beginReferenceVector, + __wpk_fork_ref_vector_append: appendReferenceVector, + __wpk_fork_ref_vector_finish: finishReferenceVector, + __wpk_fork_ref_vector_get: getReferenceVector, + __wpk_fork_ref_exn_lookup: () => 0, + __wpk_fork_ref_exn_claim: () => 0, + __wpk_fork_ref_exn_define: () => {}, + __wpk_fork_ref_exn_load: () => 0, + __wpk_fork_ref_exn_route: () => 0, + __wpk_fork_ref_exn_cache_index: () => 1, + __wpk_fork_ref_exn_broker_encode: () => 0, + __wpk_fork_ref_exn_broker_throw_recipe: () => { + throw new Error("unused exception recipe route"); + }, + __wpk_fork_ref_exn_ingress_throw: () => { + throw new Error("unused exception ingress route"); + }, + __wpk_fork_ref_gc_lookup: () => 0, + __wpk_fork_ref_gc_claim: () => 0, + __wpk_fork_ref_gc_i31: () => 0, + __wpk_fork_ref_gc_define: () => {}, + __wpk_fork_ref_gc_route: () => 0, + __wpk_fork_ref_gc_payload_len: () => 0, + __wpk_fork_ref_gc_load: () => 0, + __wpk_fork_ref_gc_broker_encode: + (slot) => encodeExternref(gcTransit.get(slot)), + __wpk_fork_ref_gc_capture_layout: () => 0, + __wpk_fork_ref_gc_provenance_begin: () => 0, + __wpk_fork_ref_gc_provenance_ref: () => {}, + __wpk_fork_ref_gc_provenance_end: () => {}, + __wpk_fork_ref_scratch_reserve: (size) => allocate(Number(size)), + __wpk_fork_ref_scratch_release: () => {}, + }; + return new WebAssembly.Instance(module, { env }); + } + + const parentMemory = new WebAssembly.Memory({ + initial: 4, maximum: 65536, shared: true, + }); + const parentSharedCounter = new WebAssembly.Global( + { value: "i64", mutable: true }, 0n, + ); + const parentSharedCallbacks = new WebAssembly.Table({ + element: "anyfunc", initial: 2, maximum: 8, + }); + const parentImportedCallback = + new WebAssembly.Instance(codecModule).exports.callback; + const parentImmutableToken = Object.freeze({ handle: 88, parent: true }); + parent = instantiate( + "capture", + parentMemory, + parentSharedCounter, + parentSharedCallbacks, + { + importedCallback: parentImportedCallback, + immutableToken: parentImmutableToken, + }, + ); + if (parent.exports.start_count() !== 0) { + throw new Error("original start ran during raw instantiation"); + } + parent.exports.wpk_fork_module_bootstrap(); + if (parent.exports.start_count() !== 1 || parent.exports.active_data_byte() !== 0x44) { + throw new Error("parent bootstrap did not preserve segment/start ordering"); + } + const owned = Object.freeze({ handle: 77, parent: true }); + parent.exports.mutate(owned); + if (parent.exports.token() !== owned) { + throw new Error("parent mutable externref global was not assigned"); + } + parent.exports.wpk_fork_module_state_save(7); + if (records.length === 0) throw new Error("module state emitted no records"); + + thread = instantiate( + "thread", + parentMemory, + parentSharedCounter, + parentSharedCallbacks, + { + importedCallback: parentImportedCallback, + immutableToken: parentImmutableToken, + }, + ); + thread.exports.wpk_fork_module_thread_bootstrap(); + if (thread.exports.start_count() !== 0) { + throw new Error("pthread bootstrap reran original start"); + } + if (thread.exports.active_data_byte() !== 0x7a) { + throw new Error("pthread bootstrap overwrote shared linear memory"); + } + [11, 22, 11].forEach((value, index) => { + if (thread.exports.callbacks.get(index)() !== value) { + throw new Error(`pthread table baseline entry ${index} is missing`); + } + }); + for (const name of ["try_active_elem", "try_active_data"]) { + let trapped = false; + try { + thread.exports[name](); + } catch (error) { + trapped = error instanceof WebAssembly.RuntimeError; + } + if (!trapped) throw new Error(`${name} stayed live after pthread bootstrap`); + } + + const childMemory = new WebAssembly.Memory({ + initial: parentMemory.buffer.byteLength / 65536, + maximum: 65536, + shared: true, + }); + new Uint8Array(childMemory.buffer).set(new Uint8Array(parentMemory.buffer)); + const childSharedCounter = new WebAssembly.Global( + { value: "i64", mutable: true }, 0n, + ); + const childSharedCallbacks = new WebAssembly.Table({ + element: "anyfunc", initial: 2, maximum: 8, + }); + const childImportedCallback = + new WebAssembly.Instance(codecModule).exports.callback; + const childImmutableToken = Object.freeze({ handle: 88, child: true }); + childExternrefs.set(88, childImmutableToken); + child = instantiate( + "restore", + childMemory, + childSharedCounter, + childSharedCallbacks, + { + importedCallback: childImportedCallback, + immutableToken: childImmutableToken, + }, + ); + nodes.forEach((node, recipeId) => { + if (node.kind === "externref") { + child.exports.__wpk_fork_ref_gc_publish_externref( + recipeId, + decodeExternref(recipeId), + ); + } + }); + child.exports.wpk_fork_module_state_restore(7); + for (const name of [ + "try_late_elem", + "try_late_data", + "try_active_elem", + "try_active_data", + ]) { + try { + child.exports[name](); + } catch (error) { + throw new Error(`${name} was dropped before reference reconstruction`, { + cause: error, + }); + } + } + // Reapply the exact table/memory-owned state after probes, then cross + // the global segment-lifetime boundary. Both phases are idempotent. + child.exports.wpk_fork_module_state_restore(7); + child.exports.wpk_fork_module_state_finish_restore(7); + child.exports.wpk_fork_module_state_finish_restore(7); + + if (child.exports.start_count() !== 1) throw new Error("child reran original start"); + if (child.exports.active_data_byte() !== 0x7a) { + throw new Error("child reran active data initialization over copied memory"); + } + if (child.exports.counter() !== 0x11223344) throw new Error("i32 global reset"); + if (child.exports.shared_counter() !== 0x1122334455667788n) { + throw new Error("imported mutable scalar global reset"); + } + if ((child.exports.f32_bits() >>> 0) !== 0x7fc12345) { + throw new Error("f32 payload bits changed"); + } + if (child.exports.f64_bits() !== 0x7ff8123456789abcn) { + throw new Error("f64 payload bits changed"); + } + if (child.exports.vector_lane_2() !== 303) throw new Error("v128 global reset"); + if (child.exports.callback()() !== 22) throw new Error("funcref global reset"); + if (child.exports.immutable_callback() !== childImportedCallback) { + throw new Error("immutable imported funcref retained parent identity"); + } + if (child.exports.immutable_callback_global.value !== childImportedCallback) { + throw new Error("exported immutable funcref was not materialized before instantiation"); + } + if (child.exports.immutable_callback_alias.value !== childImportedCallback) { + throw new Error("immutable funcref const initializer saw the wrong child binding"); + } + const token = child.exports.token(); + if (token === owned || token.handle !== 77 || !token.child) { + throw new Error("externref global was not reconstructed"); + } + if (child.exports.immutable_token().handle !== 88) { + throw new Error("immutable imported externref recipe changed"); + } + if ( + child.exports.immutable_token_global.value !== childImmutableToken + || child.exports.immutable_token_alias.value !== childImmutableToken + || child.exports.immutable_token() !== childImmutableToken + ) { + throw new Error("immutable externref was not materialized before instantiation"); + } + + const callbackValues = [22, 22, 11, 22, 11]; + if (child.exports.callbacks.length !== callbackValues.length) { + throw new Error("funcref table length reset"); + } + callbackValues.forEach((value, index) => { + if (child.exports.callbacks.get(index)() !== value) { + throw new Error(`funcref table entry ${index} reset`); + } + }); + if (child.exports.tokens.length !== 4) throw new Error("externref table length reset"); + for (let index = 0; index < 4; index++) { + if (child.exports.tokens.get(index) !== token) { + throw new Error(`externref alias lost at table entry ${index}`); + } + } + if (childSharedCallbacks.length !== 3) { + throw new Error("imported table length reset"); + } + for (let index = 0; index < 3; index++) { + if (child.exports.shared_callback(index)() !== 22) { + throw new Error(`imported table entry ${index} reset`); + } + } + for (const name of ["try_late_elem", "try_late_data"]) { + let trapped = false; + try { + child.exports[name](); + } catch (error) { + trapped = error instanceof WebAssembly.RuntimeError; + } + if (!trapped) throw new Error(`${name} observed a live dropped segment`); + } + "#; + let output = Command::new("node") + .arg("-e") + .arg(script) + .arg(&module_path) + .arg(&codecs_path) + .output() + .expect("run Node module-state fresh-instance test"); + let _ = fs::remove_dir_all(&directory); + assert!( + output.status.success(), + "Node module-state engine test failed:\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + ); +} + +#[test] +fn node_fresh_instance_reimports_concrete_gc_global_before_const_initializers() { + let provider = wat::parse_str( + r#" + (module + (type $pair (struct (field i32))) + (global $root (ref $pair) + (struct.new $pair (i32.const 91))) + (export "__wpk_fork_global_1" (global $root)) + (export "root" (global $root))) + "#, + ) + .expect("compile concrete-GC provider"); + let consumer = wat::parse_str( + r#" + (module + (type $pair (struct (field i32))) + (import "provider" "root" (global $root (ref $pair))) + (global $alias (ref $pair) (global.get $root)) + (export "__wpk_fork_global_1" (global $root)) + (export "root" (global $root)) + (export "alias" (global $alias)) + (func (export "same") (result i32) + global.get $root + global.get $alias + ref.eq) + (func (export "value") (result i32) + global.get $alias + struct.get $pair 0)) + "#, + ) + .expect("compile concrete-GC consumer"); + + let nonce = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("clock after epoch") + .as_nanos(); + let directory = std::env::temp_dir().join(format!( + "kandelo-imported-gc-global-{}-{nonce}", + std::process::id(), + )); + fs::create_dir(&directory).expect("create concrete-GC test directory"); + let provider_path = directory.join("provider.wasm"); + let consumer_path = directory.join("consumer.wasm"); + fs::write(&provider_path, provider).expect("write concrete-GC provider"); + fs::write(&consumer_path, consumer).expect("write concrete-GC consumer"); + let script = r#" + const fs = require("node:fs"); + const [providerPath, consumerPath] = process.argv.slice(1); + const providerModule = + new WebAssembly.Module(fs.readFileSync(providerPath)); + const consumerModule = + new WebAssembly.Module(fs.readFileSync(consumerPath)); + + const parentProvider = new WebAssembly.Instance(providerModule); + const parentConsumer = new WebAssembly.Instance(consumerModule, { + provider: { root: parentProvider.exports.__wpk_fork_global_1 }, + }); + const childProvider = new WebAssembly.Instance(providerModule); + const childConsumer = new WebAssembly.Instance(consumerModule, { + provider: { root: childProvider.exports.__wpk_fork_global_1 }, + }); + + if ( + childConsumer.exports.root + !== childProvider.exports.__wpk_fork_global_1 + ) { + throw new Error("consumer did not bind the provider Global object"); + } + if (childConsumer.exports.root === parentConsumer.exports.root) { + throw new Error("fresh child retained the parent provider Global"); + } + if (childConsumer.exports.root.value === parentConsumer.exports.root.value) { + throw new Error("fresh child retained the parent concrete GC object"); + } + if ( + childConsumer.exports.alias.value + !== childConsumer.exports.root.value + ) { + throw new Error("concrete GC const initializer lost provider identity"); + } + if (childConsumer.exports.same() !== 1 || childConsumer.exports.value() !== 91) { + throw new Error("concrete GC provider recipe changed guest semantics"); + } + "#; + let output = Command::new("node") + .arg("-e") + .arg(script) + .arg(&provider_path) + .arg(&consumer_path) + .output() + .expect("run Node concrete-GC import test"); + let _ = fs::remove_dir_all(&directory); + assert!( + output.status.success(), + "Node concrete-GC import test failed:\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + ); +} diff --git a/crates/fork-instrument/tests/reference_analysis.rs b/crates/fork-instrument/tests/reference_analysis.rs new file mode 100644 index 0000000000..eedf67feeb --- /dev/null +++ b/crates/fork-instrument/tests/reference_analysis.rs @@ -0,0 +1,5 @@ +// Keep the reference planner independently compilable until its facts are +// wired into the transform. This avoids creating a temporary public API in +// `lib.rs` while still running the module's focused unit tests in CI. +#[path = "../src/reference_analysis.rs"] +mod reference_analysis; diff --git a/crates/fork-instrument/tests/roundtrip.rs b/crates/fork-instrument/tests/roundtrip.rs index c955979cae..eedf438f85 100644 --- a/crates/fork-instrument/tests/roundtrip.rs +++ b/crates/fork-instrument/tests/roundtrip.rs @@ -20,9 +20,8 @@ fn compile(wat_src: &str) -> Vec { fn validate(bytes: &[u8]) -> Result<(), wasmparser::BinaryReaderError> { // Independent validator (not walrus) — confirms the emitted bytes // are well-formed per the core spec. - let mut validator = wasmparser::Validator::new_with_features( - wasmparser::WasmFeatures::default(), - ); + let mut validator = + wasmparser::Validator::new_with_features(wasmparser::WasmFeatures::default()); validator.validate_all(bytes).map(|_| ()) } diff --git a/crates/fork-instrument/tests/runtime.rs b/crates/fork-instrument/tests/runtime.rs index cd5f3a0ce4..ef074ada85 100644 --- a/crates/fork-instrument/tests/runtime.rs +++ b/crates/fork-instrument/tests/runtime.rs @@ -15,11 +15,14 @@ use fork_instrument::linked_frames::{ FrameFormatDescriptor, LINKED_FRAME_FORMAT_SECTION, PointerWidth, }; use fork_instrument::runtime::names; -use fork_instrument::{ - FORK_CAP_DYLINK_MAIN, FORK_CAP_SIDE_ENTRY, FORK_CAPABILITIES_SECTION, - FORK_CAPABILITIES_VERSION, Options, instrument, +use fork_instrument::{Options, UNWIND_TRANSPORT_SECTION, UNWIND_TRANSPORT_VERSION, instrument}; +use walrus::{ExportItem, ImportKind, Module, ValType}; +use wasm_posix_shared::abi::{ + WPK_FORK_CAP_ACTIVATION_STATE_SAFE, WPK_FORK_CAP_DYLINK_MAIN, WPK_FORK_CAP_SIDE_ENTRY, + WPK_FORK_CAPABILITIES_SECTION, WPK_FORK_CAPABILITIES_VERSION, + WPK_FORK_IMPORTED_GLOBALS_SECTION, WPK_FORK_MODULE_STATE_FORMAT_SECTION, + WPK_FORK_REQUIRED_EXPORTS, WPK_FORK_REQUIRED_IMPORTS, WPK_FORK_REQUIRED_TABLE_IMPORTS, }; -use walrus::{ExportItem, Module, ValType}; use wasmparser::{Parser, Payload}; fn instrument_wat(wat_src: &str) -> Vec { @@ -37,7 +40,7 @@ fn fork_capabilities(bytes: &[u8]) -> Vec> { Parser::new(0) .parse_all(bytes) .filter_map(|payload| match payload.expect("parse payload") { - Payload::CustomSection(section) if section.name() == FORK_CAPABILITIES_SECTION => { + Payload::CustomSection(section) if section.name() == WPK_FORK_CAPABILITIES_SECTION => { Some(section.data().to_vec()) } _ => None, @@ -75,6 +78,60 @@ fn instrumented_module_validates() { validate(&bytes); } +#[test] +fn preinstrumented_artifact_cannot_be_restamped_as_activation_safe() { + let once = instrument_wat(EMPTY_MODULE_WITH_FORK); + let error = instrument(&once, &Options::default()) + .expect_err("an existing fork transform must not be restamped"); + let message = error.to_string(); + assert!( + message.contains("input already contains wasm-fork-instrument"), + "{message}" + ); + assert!(message.contains("raw linker output"), "{message}"); +} + +#[test] +fn source_module_cannot_spoof_private_global_catalog_exports() { + let bytes = wat::parse_str( + r#" + (module + (global $value (mut i32) (i32.const 0)) + (export "__wpk_fork_global_1" (global $value)) + (memory 1)) + "#, + ) + .expect("wat parse"); + let error = instrument(&bytes, &Options::default()) + .expect_err("a source export must not collide with the private global catalog"); + let message = error.to_string(); + assert!( + message.contains("input already contains wasm-fork-instrument"), + "{message}" + ); + assert!(message.contains("raw linker output"), "{message}"); +} + +#[test] +fn source_module_cannot_spoof_private_table_catalog_exports() { + let bytes = wat::parse_str( + r#" + (module + (table $value 1 funcref) + (export "__wpk_fork_table_1" (table $value)) + (memory 1)) + "#, + ) + .expect("wat parse"); + let error = instrument(&bytes, &Options::default()) + .expect_err("a source export must not collide with the private table catalog"); + assert!( + error + .to_string() + .contains("input already contains wasm-fork-instrument") + ); +} + #[test] fn linked_runtime_imports_transaction_hooks_and_emits_exact_prefix_metadata() { let bytes = instrument_wat(EMPTY_MODULE_WITH_FORK); @@ -108,6 +165,91 @@ fn linked_runtime_imports_transaction_hooks_and_emits_exact_prefix_metadata() { ); } +#[test] +fn linked_runtime_imports_exact_private_unwind_tag_and_metadata() { + let bytes = instrument_wat(EMPTY_MODULE_WITH_FORK); + let module = Module::from_buffer(&bytes).unwrap(); + let imports: Vec<_> = module + .imports + .iter() + .filter(|import| { + import.module == names::IMPORT_UNWIND_TAG_MODULE + && import.name == names::IMPORT_UNWIND_TAG + }) + .collect(); + assert_eq!(imports.len(), 1, "private transport must have one owner"); + let tag = match imports[0].kind { + ImportKind::Tag(tag) => tag, + ref other => panic!("private unwind transport must be a tag, got {other:?}"), + }; + let tag_ty = module.types.get(module.tags.get(tag).ty()); + assert!( + tag_ty.params().is_empty(), + "unwind tag payload must be empty" + ); + assert!(tag_ty.results().is_empty(), "tag type cannot return values"); + + let metadata: Vec<_> = Parser::new(0) + .parse_all(&bytes) + .filter_map(|payload| match payload.expect("parse payload") { + Payload::CustomSection(section) if section.name() == UNWIND_TRANSPORT_SECTION => { + Some(section.data().to_vec()) + } + _ => None, + }) + .collect(); + assert_eq!( + metadata, + vec![vec![UNWIND_TRANSPORT_VERSION, 0]], + "host must be able to reject a lookalike tag with a different contract", + ); +} + +#[test] +fn state_only_side_activation_carries_exact_private_unwind_metadata() { + let bytes = instrument_wat("(module (memory 1))"); + let metadata: Vec<_> = Parser::new(0) + .parse_all(&bytes) + .filter_map(|payload| match payload.expect("parse payload") { + Payload::CustomSection(section) if section.name() == UNWIND_TRANSPORT_SECTION => { + Some(section.data().to_vec()) + } + _ => None, + }) + .collect(); + assert_eq!( + metadata, + vec![vec![UNWIND_TRANSPORT_VERSION, 0]], + "uniform ABI 43 state helpers require the same exact-tag descriptor", + ); +} + +#[test] +fn raw_module_cannot_preclaim_reserved_unwind_transport() { + let bytes = wat::parse_str( + r#" + (module + (import "kernel" "kernel_fork" (func $fork (result i32))) + (import "env" "__wpk_fork_unwind" (tag $unwind)) + (memory 1) + (func (export "run") (result i32) + call $fork)) + "#, + ) + .expect("wat parse"); + let message = instrument(&bytes, &Options::default()) + .expect_err("reserved private tag collision must fail before rewrite") + .to_string(); + assert!( + message.contains("reserved private fork runtime hook"), + "{message}" + ); + assert!( + message.contains("instrumenter must own unwind transport"), + "{message}" + ); +} + #[test] fn plain_catches_do_not_expand_fixed_prefix_metadata() { let bytes = instrument_wat( @@ -179,24 +321,85 @@ fn memory64_plain_catches_do_not_expand_fixed_prefix_metadata() { } #[test] -fn module_without_fork_seed_does_not_import_linked_storage_hooks() { +fn module_without_fork_seed_has_the_uniform_linked_replay_contract() { let bytes = instrument_wat("(module (memory 1) (func (export \"run\")))"); let module = Module::from_buffer(&bytes).unwrap(); - for name in [ - names::IMPORT_FRAME_RESERVE, - names::IMPORT_FRAME_COMMIT, - names::IMPORT_FRAME_NEXT, - ] { + for requirement in WPK_FORK_REQUIRED_IMPORTS { assert!( - !module - .imports + module.imports.iter().any(|import| { + import.module == requirement.module + && import.name == requirement.name + && matches!(import.kind, ImportKind::Function(_)) + }), + "no-seed module is missing linked function import {}.{}", + requirement.module, + requirement.name, + ); + } + for requirement in WPK_FORK_REQUIRED_TABLE_IMPORTS { + assert!( + module.imports.iter().any(|import| { + import.module == requirement.module + && import.name == requirement.name + && matches!(import.kind, ImportKind::Table(_)) + }), + "no-seed module is missing linked table import {}.{}", + requirement.module, + requirement.name, + ); + } + for requirement in WPK_FORK_REQUIRED_EXPORTS { + assert!( + module + .exports .iter() - .any(|import| import.module == "env" && import.name == name), - "inert module unexpectedly imports linked continuation hook {name}", + .any(|export| export.name == requirement.name), + "no-seed module is missing linked export {}", + requirement.name, + ); + } + for section_name in [ + WPK_FORK_CAPABILITIES_SECTION, + LINKED_FRAME_FORMAT_SECTION, + WPK_FORK_MODULE_STATE_FORMAT_SECTION, + WPK_FORK_IMPORTED_GLOBALS_SECTION, + ] { + assert_eq!( + module + .customs + .iter() + .filter(|(_, section)| section.name() == section_name) + .count(), + 1, + "no-seed module must carry exactly one {section_name} descriptor", ); } } +#[test] +fn reference_vector_finish_import_returns_a_canonical_ordinal() { + let bytes = instrument_wat(EMPTY_MODULE_WITH_FORK); + let module = Module::from_buffer(&bytes).unwrap(); + let finish = module + .imports + .iter() + .find_map(|import| { + (import.module == wasm_posix_shared::abi::WPK_FORK_FRAME_IMPORT_MODULE + && import.name == names::IMPORT_REFERENCE_VECTOR_FINISH) + .then(|| match import.kind { + ImportKind::Function(function) => Some(function), + _ => None, + }) + .flatten() + }) + .expect("reference-vector finish import"); + assert_eq!( + func_signature(&module, finish), + (vec![ValType::I32], vec![ValType::I32]), + "finish consumes a transient builder handle and returns its canonical wire ordinal", + ); +} + #[test] fn marks_dlopen_main_indirect_boundary_separately() { let wat = r#" @@ -213,7 +416,10 @@ fn marks_dlopen_main_indirect_boundary_separately() { let output = instrument_wat(wat); assert_eq!( fork_capabilities(&output), - vec![vec![FORK_CAPABILITIES_VERSION, FORK_CAP_DYLINK_MAIN]], + vec![vec![ + WPK_FORK_CAPABILITIES_VERSION, + WPK_FORK_CAP_DYLINK_MAIN | WPK_FORK_CAP_ACTIVATION_STATE_SAFE, + ]], ); } @@ -237,7 +443,35 @@ fn marks_env_fork_side_entry_separately() { .expect("instrument side"); assert_eq!( fork_capabilities(&output), - vec![vec![FORK_CAPABILITIES_VERSION, FORK_CAP_SIDE_ENTRY]], + vec![vec![ + WPK_FORK_CAPABILITIES_VERSION, + WPK_FORK_CAP_SIDE_ENTRY | WPK_FORK_CAP_ACTIVATION_STATE_SAFE, + ]], + ); +} + +#[test] +fn dylink_module_without_env_fork_claims_complete_side_boundaries() { + let input = wat::parse_str( + r#" + (module + (@custom "dylink.0" (before first) "side") + (import "env" "side_b" (func $side_b (result i32))) + (memory 1) + (func (export "side_a") (result i32) call $side_b)) + "#, + ) + .expect("wat parse"); + let output = instrument(&input, &Options::default()).expect("instrument side boundaries"); + validate(&output); + assert_eq!( + fork_capabilities(&output), + vec![vec![ + WPK_FORK_CAPABILITIES_VERSION, + WPK_FORK_CAP_SIDE_ENTRY | WPK_FORK_CAP_ACTIVATION_STATE_SAFE, + ]], + "SIDE_ENTRY means every cross-module activation boundary is covered, \ + even when fork itself is downstream in another module", ); } @@ -246,7 +480,10 @@ fn generic_runtime_exports_do_not_claim_side_or_dylink_coverage() { let output = instrument_wat(EMPTY_MODULE_WITH_FORK); assert_eq!( fork_capabilities(&output), - vec![vec![FORK_CAPABILITIES_VERSION, 0]], + vec![vec![ + WPK_FORK_CAPABILITIES_VERSION, + WPK_FORK_CAP_ACTIVATION_STATE_SAFE, + ]], ); } @@ -496,8 +733,10 @@ fn wasm64_saved_globals_use_16_byte_header() { } #[test] -fn ref_typed_mutable_globals_are_skipped_in_4e() { - // Phase 4e handles scalar globals only; ref-typed ones await 4f. +fn linked_runtime_prefix_defers_reference_globals_to_kfms() { + // The fixed continuation prefix owns scalar control globals. KFMS owns + // reference-global recipes because it can reconstruct them in a fresh + // module instance without putting references in linear memory. let wat = r#" (module (import "kernel" "kernel_fork" (func $fork (result i32))) @@ -509,7 +748,7 @@ fn ref_typed_mutable_globals_are_skipped_in_4e() { let mut module = Module::from_buffer(&bytes).unwrap(); let runtime = inject_runtime(&mut module); - // Only the i32 scalar should have been picked up. + // Only the scalar is part of the fixed runtime prefix. assert_eq!(runtime.saved_globals.len(), 1); assert_eq!(runtime.saved_globals[0].ty, walrus::ValType::I32); } diff --git a/crates/fork-instrument/tests/static_reference_catalog.rs b/crates/fork-instrument/tests/static_reference_catalog.rs new file mode 100644 index 0000000000..9d96c12a52 --- /dev/null +++ b/crates/fork-instrument/tests/static_reference_catalog.rs @@ -0,0 +1,223 @@ +use std::{ + fs, + process::Command, + time::{SystemTime, UNIX_EPOCH}, +}; + +use fork_instrument::static_reference_catalog; +use walrus::Module; + +fn fixture() -> Vec { + wat::parse_str( + r#" + (module + (type $pair (struct (field i32))) + (global $root (ref $pair) + (struct.new $pair (i32.const 41))) + (global $alias (ref $pair) + (global.get $root)) + (table $values (export "values") 3 3 (ref null $pair)) + (elem $roots (ref $pair) + (global.get $root) + (global.get $alias) + (struct.new $pair (i32.const 99))) + + (func (export "initialize_values") + i32.const 0 + i32.const 0 + i32.const 3 + table.init $values $roots) + + (func (export "matches_root") + (param (ref null $pair)) (result i32) + (local.get 0) + (global.get $root) + ref.eq) + + (func (export "matches_table") + (param i32) (param (ref null $pair)) (result i32) + (local.get 0) + (table.get $values) + (local.get 1) + ref.eq)) + "#, + ) + .expect("static-root fixture WAT") +} + +fn catalogued_fixture() -> (Vec, usize) { + let mut module = Module::from_buffer(&fixture()).expect("parse static-root fixture"); + let plan = static_reference_catalog::plan(&mut module); + let root_count = plan.root_count(); + static_reference_catalog::inject(&mut module, plan); + (module.emit_wasm(), root_count) +} + +#[test] +fn aliases_keep_one_stable_ordinal_without_hoisting_allocating_elements() { + let (bytes, root_count) = catalogued_fixture(); + assert_eq!( + root_count, 2, + "the immutable global and its global.get aliases share ordinal zero; \ + the independently allocating element owns ordinal one", + ); + wasmparser::Validator::new_with_features(wasmparser::WasmFeatures::all()) + .validate_all(&bytes) + .expect("static-root catalog output validates"); + + let module = Module::from_buffer(&bytes).expect("reparse catalogued fixture"); + let catalog = module + .exports + .iter() + .find(|export| export.name == static_reference_catalog::EXPORT) + .expect("static-root table export"); + let walrus::ExportItem::Table(table) = catalog.item else { + panic!("static-root catalog export is not a table"); + }; + let table = module.tables.get(table); + assert_eq!(table.initial, 2); + assert_eq!(table.maximum, Some(2)); + assert_eq!(table.element_ty, walrus::RefType::ANYREF); + assert!( + module + .exports + .iter() + .any(|export| export.name == static_reference_catalog::HARVEST_EXPORT), + "static-root harvest helper must be exported", + ); + + let allocating_expression = module + .elements + .iter() + .find_map(|element| match &element.items { + walrus::ElementItems::Expressions(_, expressions) => expressions.get(2), + _ => None, + }) + .expect("allocating element expression"); + assert!( + !matches!(allocating_expression, walrus::ConstExpr::Global(_)), + "allocating element roots must remain segment-owned rather than being \ + hoisted into a permanent immutable global", + ); +} + +#[test] +fn allocating_local_table_initializer_is_harvested_without_hoisting() { + let input = wat::parse_str( + r#" + (module + (type $pair (struct (field i32))) + (table $values 2 2 (ref $pair) + (struct.new $pair (i32.const 73)))) + "#, + ) + .expect("table-initializer fixture WAT"); + let mut module = Module::from_buffer(&input).expect("parse table-initializer fixture"); + let plan = static_reference_catalog::plan(&mut module); + assert_eq!(plan.root_count(), 1); + + let table = module.tables.iter().next().expect("source table"); + assert!( + !matches!(table.init, Some(walrus::ConstExpr::Global(_))), + "allocating table initializer must not be hoisted into a permanent root", + ); +} + +#[test] +fn fresh_instance_catalog_decodes_to_the_identity_observed_by_ref_eq() { + let (bytes, _) = catalogued_fixture(); + let nonce = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("system clock") + .as_nanos(); + let dir = std::env::temp_dir().join(format!( + "kandelo-static-reference-catalog-{}-{nonce}", + std::process::id(), + )); + fs::create_dir_all(&dir).expect("create static-root test directory"); + let wasm = dir.join("fixture.wasm"); + let script = dir.join("verify.mjs"); + fs::write(&wasm, bytes).expect("write static-root fixture"); + fs::write( + &script, + r#" +import fs from "node:fs"; + +const module = new WebAssembly.Module(fs.readFileSync(process.argv[2])); +const parent = new WebAssembly.Instance(module); +const child = new WebAssembly.Instance(module); +const parentCatalog = parent.exports.__wpk_fork_static_root_catalog; +const childCatalog = child.exports.__wpk_fork_static_root_catalog; + +if (parentCatalog.length !== 2 || childCatalog.length !== 2) { + throw new Error("unexpected static-root catalog length"); +} +for (let index = 0; index < 2; index++) { + if (parentCatalog.get(index) !== null || childCatalog.get(index) !== null) { + throw new Error("static-root harvest tables did not instantiate empty"); + } +} +parent.exports.__wpk_fork_static_root_harvest(); +child.exports.__wpk_fork_static_root_harvest(); +const parentRoot = parentCatalog.get(0); +const childRoot = childCatalog.get(0); +const childElementRoot = childCatalog.get(1); +if (parentRoot === childRoot) { + throw new Error("fresh instances unexpectedly share a GC object"); +} +if (childCatalog.get(0) !== childRoot) { + throw new Error("repeated anyref table reads did not preserve JS wrapper identity"); +} +const transit = new WebAssembly.Table({ + element: "anyref", + initial: 1, + maximum: 1, +}); +transit.set(0, childRoot); +if (transit.get(0) !== childRoot) { + throw new Error("anyref table transit did not preserve JS wrapper identity"); +} +if (child.exports.matches_root(childRoot) !== 1) { + throw new Error("child catalog root does not ref.eq its immutable global"); +} +if (child.exports.matches_root(transit.get(0)) !== 1) { + throw new Error("anyref table transit did not preserve Wasm ref.eq identity"); +} +parent.exports.initialize_values(); +child.exports.initialize_values(); +if (child.exports.matches_root(parentRoot) !== 0) { + throw new Error("parent GC root incorrectly aliases the child's root"); +} +if (child.exports.matches_table(0, childRoot) !== 1 + || child.exports.matches_table(1, childRoot) !== 1) { + throw new Error("global.get element aliases lost their canonical root"); +} +if (child.exports.matches_table(2, childElementRoot) !== 1) { + throw new Error("harvested allocating element does not ref.eq its segment root"); +} +for (let index = 0; index < 2; index++) { + parentCatalog.set(index, null); + childCatalog.set(index, null); +} +for (let index = 0; index < 2; index++) { + if (parentCatalog.get(index) !== null || childCatalog.get(index) !== null) { + throw new Error("static-root harvest tables retained stale GC roots"); + } +} +"#, + ) + .expect("write static-root verifier"); + + let output = Command::new("node") + .arg(&script) + .arg(&wasm) + .output() + .expect("run Node static-root verifier"); + let _ = fs::remove_dir_all(&dir); + assert!( + output.status.success(), + "Node static-root verifier failed:\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + ); +} diff --git a/crates/fork-instrument/tests/switch_dispatch.rs b/crates/fork-instrument/tests/switch_dispatch.rs index 870438911d..3903c10ead 100644 --- a/crates/fork-instrument/tests/switch_dispatch.rs +++ b/crates/fork-instrument/tests/switch_dispatch.rs @@ -287,15 +287,22 @@ fn no_catch_switch_dispatch_omits_frame_header_state_locals() { let caller = extract_function_text(&printed, "caller"); let locals = declared_scalar_local_count(&caller); assert_eq!( - locals, 2, - "no-catch top-level fork path should declare only the original local and \ - abort_live_frame; \ - call_idx and frame_ptr are loaded from the frame header, and \ + locals, 1, + "no-catch top-level fork path should declare only the original local; \ + the static call boundary must not need an abort-frame/selector local, \ + saved call_idx and frame_ptr are loaded from the frame header, and \ unconditional catch metadata locals would raise this count:\n{caller}" ); assert!( - caller.contains("i32.store offset=4"), - "unwind call site must still write frame.call_index before the shared postamble:\n{caller}" + caller.contains("call $__wpk_fork_select_unwind_frame"), + "unwind call site must pass its static call index to the shared \ + frame selector before the postamble:\n{caller}" + ); + let selector = extract_function_text(&printed, "__wpk_fork_select_unwind_frame"); + assert!( + selector.contains("i32.store offset=4"), + "the shared frame selector must publish frame.call_index before \ + returning success or synchronous-abort routing:\n{selector}" ); } @@ -322,9 +329,9 @@ fn top_level_indirect_switch_dispatch_omits_frame_header_state_locals() { let caller = extract_function_text(&printed, "caller"); let locals = declared_scalar_local_count(&caller); assert_eq!( - locals, 1, - "top-level indirect call with a pure table index should need only \ - abort_live_frame, with no arg, frame_ptr, or call_idx locals:\n{caller}" + locals, 0, + "top-level indirect call with a pure table index should need no \ + arg, abort-frame, selector, frame_ptr, or saved-call-index locals:\n{caller}" ); } @@ -339,10 +346,10 @@ fn nested_direct_switch_dispatch_omits_frame_header_state_locals() { let main = extract_function_text(&printed, "main"); let locals = declared_scalar_local_count(&main); assert_eq!( - locals, 3, - "nested block dispatch should retain only the two source locals and \ - abort_live_frame; \ - frame_ptr and call_idx must not be declared locals:\n{main}" + locals, 2, + "nested block dispatch should retain only the two source locals; \ + static call boundaries do not require an activation-local selector, \ + frame_ptr and saved call_idx must not be declared locals:\n{main}" ); } @@ -368,10 +375,10 @@ fn nested_if_else_dispatch_omits_frame_header_state_locals() { let main = extract_function_text(&printed, "main"); let locals = declared_scalar_local_count(&main); assert_eq!( - locals, 1, + locals, 0, "nested if/else dispatch should replay a pure condition without cond_swap; \ - abort_live_frame is the only declared local, params are not declared locals, \ - and frame_ptr/call_idx must be loaded from the frame:\n{main}" + no abort-frame or call-selector local is declared, \ + params are not declared locals, and frame_ptr/saved call_idx come from the frame:\n{main}" ); } @@ -403,22 +410,594 @@ fn pr701_shape_replays_pure_condition_and_recursive_arg() { let walk = extract_function_text(&printed, "walk"); let locals = declared_scalar_local_count(&walk); assert_eq!( - locals, 1, + locals, 0, "PR701-shaped pure condition and recursive arg should not allocate \ - arg-spill or condition/carryover locals beyond abort_live_frame:\n{walk}" + arg-spill, condition/carryover, abort-frame, or active-call selector \ + locals:\n{walk}" ); + let normalized = walk.lines().map(str::trim).collect::>().join("\n"); assert!( - walk.contains("local.get 0\n i32.eqz\n global.get $_wpk_fork_state"), + normalized.contains("local.get 0\ni32.eqz\nglobal.get $_wpk_fork_state"), "rewritten IfElse landing should replay the pure eqz(depth) condition \ before selecting NORMAL vs REWIND:\n{walk}" ); assert!( - walk.contains("local.get 0\n i32.const 1\n i32.sub\n call $walk"), + !normalized.contains("local.set 1"), + "recursive call landing must use its statically known call index rather \ + than adding an activation-local selector:\n{walk}" + ); + assert!( + normalized.contains("local.get 0\ni32.const 1\ni32.sub\ncall $walk"), "recursive call landing should replay pure depth - 1 argument tail \ - before the call:\n{walk}" + on the lexical branch without allocating an argument local:\n{walk}" + ); +} + +#[test] +fn reference_recipe_vector_adds_no_ordinary_activation_local() { + let wat = r#" + (module + (import "kernel" "kernel_fork" (func $kernel_fork (result i32))) + (memory (export "memory") 1) + (func $walk (export "reference_walk") + (param $depth i32) + (param $value externref) + local.get $depth + i32.eqz + if + call $kernel_fork + drop + local.get $value + drop + else + local.get $depth + i32.const 1 + i32.sub + local.get $value + call $walk + end)) + "#; + let input = wat::parse_str(wat).expect("wat parse"); + let output = instrument(&input, &Options::default()).expect("instrument"); + validate(&output); + + let printed = wasmprinter::print_bytes(&output).expect("wasmprinter"); + let walk = extract_function_text(&printed, "walk"); + let locals = declared_scalar_local_count(&walk); + assert_eq!( + locals, 0, + "activation-owned reference recipes must use the reserved frame word \ + and process vector directly; adding a recipe/vector scratch local \ + would repeat the V8 recursion regression fixed by PR #713. Static \ + call boundaries must not add an abort-frame/selector local either:\n{walk}" + ); +} + +#[test] +fn catch_ref_arm_count_does_not_scale_native_local_tuple() { + fn fixture(arm_count: usize) -> String { + assert!(arm_count > 0); + let tags = (0..arm_count) + .map(|index| format!("(tag $tag{index} (param i32 i64))")) + .collect::>() + .join("\n"); + let catches = (0..arm_count) + .map(|index| format!("(catch_ref $tag{index} $handler)")) + .collect::>() + .join("\n"); + format!( + r#" + (module + (import "kernel" "kernel_fork" (func $kernel_fork (result i32))) + {tags} + (memory (export "memory") 1) + (func $caller (export "catch_ref_scaling") + (block $handler (result i32 i64 exnref) + (try_table (result i32 i64 exnref) + {catches} + call $kernel_fork + drop + i32.const 17 + i64.const 23 + throw $tag0)) + drop + drop + drop)) + "#, + ) + } + + fn counts(arm_count: usize) -> (GeneratedLocalCounts, String) { + let input = wat::parse_str(fixture(arm_count)).expect("wat parse"); + let output = instrument(&input, &Options::default()).expect("instrument"); + validate(&output); + let printed = wasmprinter::print_bytes(&output).expect("wasmprinter"); + ( + generated_local_counts(&output, "catch_ref_scaling"), + extract_function_text(&printed, "caller"), + ) + } + + let (one_arm, one_arm_wat) = counts(1); + let (many_arms, many_arms_wat) = counts(32); + assert_eq!( + one_arm, + GeneratedLocalCounts { + i32: 2, + i64: 1, + f32: 0, + f64: 0, + v128: 0, + nullable_exnref: 1, + other_reference: 0, + total: 4, + }, + "one scalar CatchRef arm should need one selector i32, one typed \ + i32/i64 payload union, and one forwarding exnref; the call boundary \ + adds no local:\n{one_arm_wat}", + ); + assert_eq!( + many_arms, one_arm, + "adding scalar CatchRef arms to one mutually-exclusive try_table must \ + not add native activation locals by type:\n{many_arms_wat}", + ); +} + +#[test] +fn catch_region_count_does_not_add_control_locals_or_frame_bytes() { + fn fixture(region_count: usize) -> String { + assert!(region_count > 0); + let regions = (0..region_count) + .map(|index| { + format!( + r#" + (block $handler{index} + (try_table (catch $tag $handler{index}) + nop)) + "#, + ) + }) + .collect::>() + .join("\n"); + format!( + r#" + (module + (import "kernel" "kernel_fork" (func $kernel_fork (result i32))) + (tag $tag) + (memory (export "memory") 1) + (func $caller (export "catch_region_scaling") + {regions} + call $kernel_fork + drop)) + "#, + ) + } + + fn measure(region_count: usize) -> (GeneratedLocalCounts, Vec, String) { + let input = wat::parse_str(fixture(region_count)).expect("wat parse"); + let output = instrument(&input, &Options::default()).expect("instrument"); + validate(&output); + let printed = wasmprinter::print_bytes(&output).expect("wasmprinter"); + ( + generated_local_counts(&output, "catch_region_scaling"), + frame_reserve_sizes(&output, "catch_region_scaling"), + extract_function_text(&printed, "caller"), + ) + } + + let (one_region, one_frame_sizes, one_region_wat) = measure(1); + let (many_regions, many_frame_sizes, many_regions_wat) = measure(32); + assert_eq!( + one_region, + GeneratedLocalCounts { + i32: 1, + i64: 0, + f32: 0, + f64: 0, + v128: 0, + nullable_exnref: 0, + other_reference: 0, + total: 1, + }, + "one empty-payload catch region needs only the activation's exact-arm \ + selector; the call boundary adds no local:\n{one_region_wat}", + ); + assert_eq!( + many_regions, one_region, + "static catch-region count must not recreate the old one-i32-per-region \ + marker cost in every native activation:\n{many_regions_wat}", + ); + assert!( + one_frame_sizes.iter().all(|size| *size == 16) + && many_frame_sizes.iter().all(|size| *size == 16), + "empty-payload catch regions reuse header selector word +8 and must not \ + enlarge a linked activation frame: one={one_frame_sizes:?}, \ + many={many_frame_sizes:?}", + ); +} + +#[test] +fn catch_region_count_uses_one_function_wide_operand_union() { + fn fixture(region_count: usize) -> String { + assert!(region_count > 0); + let regions = (0..region_count) + .map(|index| { + format!( + r#" + (block $handler{index} (result i32 i64 exnref) + (try_table (result i32 i64 exnref) + (catch_ref $tag $handler{index}) + i32.const {index} + i64.const {index} + throw $tag)) + drop + drop + drop + "#, + ) + }) + .collect::>() + .join("\n"); + format!( + r#" + (module + (import "kernel" "kernel_fork" (func $kernel_fork (result i32))) + (tag $tag (param i32 i64)) + (memory (export "memory") 1) + (func $caller (export "catch_region_operand_scaling") + {regions} + call $kernel_fork + drop)) + "#, + ) + } + + fn measure(region_count: usize) -> (GeneratedLocalCounts, Vec, String) { + let input = wat::parse_str(fixture(region_count)).expect("wat parse"); + let output = instrument(&input, &Options::default()).expect("instrument"); + validate(&output); + let printed = wasmprinter::print_bytes(&output).expect("wasmprinter"); + ( + generated_local_counts(&output, "catch_region_operand_scaling"), + frame_reserve_sizes(&output, "catch_region_operand_scaling"), + extract_function_text(&printed, "caller"), + ) + } + + let (one_region, one_frame_sizes, one_region_wat) = measure(1); + let (many_regions, many_frame_sizes, many_regions_wat) = measure(32); + assert_eq!( + one_region, + GeneratedLocalCounts { + i32: 2, + i64: 1, + f32: 0, + f64: 0, + v128: 0, + nullable_exnref: 1, + other_reference: 0, + total: 4, + }, + "one scalar CatchRef region needs one selector i32, one typed i32/i64 \ + operand union, and one forwarding exnref; the call boundary adds no \ + local:\n{one_region_wat}", + ); + assert_eq!( + many_regions, one_region, + "capture scratch belongs to the dynamically selected catch, so static \ + region count must not add native operand tuples:\n{many_regions_wat}", + ); + assert!( + one_frame_sizes.iter().all(|size| *size == 28) + && many_frame_sizes.iter().all(|size| *size == 28), + "all regions overlay the same 12-byte scalar catch payload range: \ + one={one_frame_sizes:?}, many={many_frame_sizes:?}", ); } +#[test] +fn recipe_backed_catch_arm_count_uses_one_region_local_and_header_only_frame() { + fn fixture(arm_count: usize) -> String { + assert!(arm_count > 0); + let tags = (0..arm_count) + .map(|index| format!("(tag $tag{index} (param externref))")) + .collect::>() + .join("\n"); + let catches = (0..arm_count) + .map(|index| format!("(catch_ref $tag{index} $handler)")) + .collect::>() + .join("\n"); + format!( + r#" + (module + (import "kernel" "kernel_fork" (func $kernel_fork (result i32))) + {tags} + (memory (export "memory") 1) + (func $caller (export "catch_ref_recipe_scaling") + (block $handler (result externref exnref) + (try_table (result externref exnref) + {catches} + call $kernel_fork + drop + ref.null extern + throw $tag0)) + drop + drop)) + "#, + ) + } + + fn counts(arm_count: usize) -> (GeneratedLocalCounts, Vec, String) { + let input = wat::parse_str(fixture(arm_count)).expect("wat parse"); + let output = instrument(&input, &Options::default()).expect("instrument"); + validate(&output); + let printed = wasmprinter::print_bytes(&output).expect("wasmprinter"); + ( + generated_local_counts(&output, "catch_ref_recipe_scaling"), + frame_reserve_sizes(&output, "catch_ref_recipe_scaling"), + extract_function_text(&printed, "caller"), + ) + } + + let (one_arm, one_frame_sizes, one_arm_wat) = counts(1); + let (many_arms, many_frame_sizes, many_arms_wat) = counts(32); + assert_eq!( + one_arm, + GeneratedLocalCounts { + i32: 1, + i64: 0, + f32: 0, + f64: 0, + v128: 0, + nullable_exnref: 1, + other_reference: 1, + total: 3, + }, + "one reference-payload CatchRef arm should need one selector i32, \ + one operand-forwarding externref, and one retained region exnref; \ + the call boundary adds no local:\n{one_arm_wat}", + ); + assert_eq!( + many_arms, one_arm, + "mutually exclusive recipe-backed arms in one try_table must share \ + both their typed operand union and retained exception local:\n\ + {many_arms_wat}", + ); + assert!( + one_frame_sizes.iter().all(|size| *size == 16) + && many_frame_sizes.iter().all(|size| *size == 16), + "reference-bearing catch payloads belong to the recipe vector; adding \ + static arms must not grow the 16-byte linked-frame payload header: \ + one={one_frame_sizes:?}, many={many_frame_sizes:?}", + ); +} + +#[test] +fn recipe_backed_catch_region_count_uses_one_function_local_and_header_only_frame() { + fn fixture(region_count: usize) -> String { + assert!(region_count > 0); + let regions = (0..region_count) + .map(|index| { + format!( + r#" + (block $handler{index} (result externref exnref) + (try_table (result externref exnref) + (catch_ref $tag $handler{index}) + ref.null extern + throw $tag)) + drop + drop + "#, + ) + }) + .collect::>() + .join("\n"); + format!( + r#" + (module + (import "kernel" "kernel_fork" (func $kernel_fork (result i32))) + (tag $tag (param externref)) + (memory (export "memory") 1) + (func $caller (export "catch_ref_recipe_region_scaling") + {regions} + call $kernel_fork + drop)) + "#, + ) + } + + fn measure(region_count: usize) -> (GeneratedLocalCounts, Vec, String) { + let input = wat::parse_str(fixture(region_count)).expect("wat parse"); + let output = instrument(&input, &Options::default()).expect("instrument"); + validate(&output); + let printed = wasmprinter::print_bytes(&output).expect("wasmprinter"); + ( + generated_local_counts(&output, "catch_ref_recipe_region_scaling"), + frame_reserve_sizes(&output, "catch_ref_recipe_region_scaling"), + extract_function_text(&printed, "caller"), + ) + } + + let (one_region, one_frame_sizes, one_region_wat) = measure(1); + let (many_regions, many_frame_sizes, many_regions_wat) = measure(32); + assert_eq!( + one_region, + GeneratedLocalCounts { + i32: 1, + i64: 0, + f32: 0, + f64: 0, + v128: 0, + nullable_exnref: 1, + other_reference: 1, + total: 3, + }, + "one recipe-backed region should need one selector i32, one shared \ + operand-forwarding externref, and one retained exception; the call \ + boundary adds no local:\n{one_region_wat}", + ); + assert_eq!( + many_regions, one_region, + "the one live catch selector can name only one complete-exception \ + recipe, so 32 static regions must not add 32 native exnref locals:\n\ + {many_regions_wat}", + ); + assert!( + one_frame_sizes.iter().all(|size| *size == 16) + && many_frame_sizes.iter().all(|size| *size == 16), + "recipe-backed regions must share the process reference vector's one \ + selected exception and must not grow the 16-byte linked frame: \ + one={one_frame_sizes:?}, many={many_frame_sizes:?}", + ); +} + +#[test] +fn v128_catch_arm_count_uses_one_region_local_and_header_only_frame() { + fn fixture(arm_count: usize) -> String { + assert!(arm_count > 0); + let tags = (0..arm_count) + .map(|index| format!("(tag $tag{index} (param v128))")) + .collect::>() + .join("\n"); + let catches = (0..arm_count) + .map(|index| format!("(catch_ref $tag{index} $handler)")) + .collect::>() + .join("\n"); + format!( + r#" + (module + (import "kernel" "kernel_fork" (func $kernel_fork (result i32))) + {tags} + (memory (export "memory") 1) + (func $caller (export "catch_ref_v128_scaling") + (block $handler (result v128 exnref) + (try_table (result v128 exnref) + {catches} + call $kernel_fork + drop + v128.const i32x4 1 2 3 4 + throw $tag0)) + drop + drop)) + "#, + ) + } + + fn counts(arm_count: usize) -> (GeneratedLocalCounts, Vec, String) { + let input = wat::parse_str(fixture(arm_count)).expect("wat parse"); + let output = instrument(&input, &Options::default()).expect("instrument"); + validate(&output); + let printed = wasmprinter::print_bytes(&output).expect("wasmprinter"); + ( + generated_local_counts(&output, "catch_ref_v128_scaling"), + frame_reserve_sizes(&output, "catch_ref_v128_scaling"), + extract_function_text(&printed, "caller"), + ) + } + + let (one_arm, one_frame_sizes, one_arm_wat) = counts(1); + let (many_arms, many_frame_sizes, many_arms_wat) = counts(32); + assert_eq!( + one_arm, + GeneratedLocalCounts { + i32: 1, + i64: 0, + f32: 0, + f64: 0, + v128: 1, + nullable_exnref: 1, + other_reference: 0, + total: 3, + }, + "one v128-payload CatchRef arm should need one selector i32, \ + one operand-forwarding v128, and one retained region exnref; the \ + call boundary adds no local:\n{one_arm_wat}", + ); + assert_eq!( + many_arms, one_arm, + "mutually exclusive v128 recipe-backed arms in one try_table must \ + share both their typed operand union and retained exception local:\n\ + {many_arms_wat}", + ); + assert!( + one_frame_sizes.iter().all(|size| *size == 16) + && many_frame_sizes.iter().all(|size| *size == 16), + "v128 catch payloads belong to the complete-exception recipe; adding \ + static arms must not grow the 16-byte linked-frame payload header: \ + one={one_frame_sizes:?}, many={many_frame_sizes:?}", + ); +} + +#[test] +fn catch_all_forms_use_one_retained_exception_local_and_no_frame_payload() { + let fixtures = [ + ( + "catch_all", + r#" + (module + (import "kernel" "kernel_fork" (func $kernel_fork (result i32))) + (tag $failure) + (memory (export "memory") 1) + (func $caller (export "catch_all_recipe_footprint") + (block $handler + (try_table (catch_all $handler) + call $kernel_fork + drop + throw $failure)))) + "#, + "catch_all_recipe_footprint", + ), + ( + "catch_all_ref", + r#" + (module + (import "kernel" "kernel_fork" (func $kernel_fork (result i32))) + (tag $failure) + (memory (export "memory") 1) + (func $caller (export "catch_all_ref_recipe_footprint") + (block $handler (result exnref) + (try_table (result exnref) (catch_all_ref $handler) + call $kernel_fork + drop + throw $failure)) + drop)) + "#, + "catch_all_ref_recipe_footprint", + ), + ]; + + for (label, wat, export_name) in fixtures { + let input = wat::parse_str(wat).unwrap_or_else(|error| panic!("{label}: {error}")); + let output = instrument(&input, &Options::default()) + .unwrap_or_else(|error| panic!("{label}: {error:#}")); + validate(&output); + let printed = wasmprinter::print_bytes(&output).expect("wasmprinter"); + let caller = extract_function_text(&printed, "caller"); + assert_eq!( + generated_local_counts(&output, export_name), + GeneratedLocalCounts { + i32: 1, + i64: 0, + f32: 0, + f64: 0, + v128: 0, + nullable_exnref: 1, + other_reference: 0, + total: 2, + }, + "{label}: an untagged catch needs one selector i32 and exactly \ + one retained region exception local; the call boundary adds no \ + local:\n{caller}", + ); + let frame_sizes = frame_reserve_sizes(&output, export_name); + assert!( + frame_sizes.iter().all(|size| *size == 16), + "{label}: the complete exception belongs to the recipe vector, \ + not additional linked-frame bytes: {frame_sizes:?}", + ); + } +} + // -- Helper predicates ---------------------------------------------- fn find_func(module: &Module, name: &str) -> FunctionId { @@ -480,6 +1059,147 @@ fn declared_scalar_local_count(func_text: &str) -> usize { .sum() } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct GeneratedLocalCounts { + i32: usize, + i64: usize, + f32: usize, + f64: usize, + v128: usize, + nullable_exnref: usize, + other_reference: usize, + total: usize, +} + +fn generated_local_counts(bytes: &[u8], export_name: &str) -> GeneratedLocalCounts { + let mut imported_functions = 0u32; + let mut exported_function = None; + let mut defined_function = 0u32; + for payload in wasmparser::Parser::new(0).parse_all(bytes) { + match payload.expect("parse generated module") { + wasmparser::Payload::ImportSection(imports) => { + for import in imports.into_imports() { + if matches!( + import.expect("parse generated import").ty, + wasmparser::TypeRef::Func(_) | wasmparser::TypeRef::FuncExact(_) + ) { + imported_functions += 1; + } + } + } + wasmparser::Payload::ExportSection(exports) => { + for export in exports { + let export = export.expect("parse generated export"); + if export.name == export_name && export.kind == wasmparser::ExternalKind::Func { + exported_function = Some(export.index); + } + } + } + wasmparser::Payload::CodeSectionEntry(body) => { + let function_index = imported_functions + defined_function; + defined_function += 1; + if Some(function_index) != exported_function { + continue; + } + + let mut counts = GeneratedLocalCounts { + i32: 0, + i64: 0, + f32: 0, + f64: 0, + v128: 0, + nullable_exnref: 0, + other_reference: 0, + total: 0, + }; + for local in body + .get_locals_reader() + .expect("read generated locals") + .into_iter() + { + let (count, ty) = local.expect("parse generated local"); + let count = count as usize; + counts.total += count; + match ty { + wasmparser::ValType::I32 => counts.i32 += count, + wasmparser::ValType::I64 => counts.i64 += count, + wasmparser::ValType::F32 => counts.f32 += count, + wasmparser::ValType::F64 => counts.f64 += count, + wasmparser::ValType::V128 => counts.v128 += count, + wasmparser::ValType::Ref(wasmparser::RefType::EXNREF) => { + counts.nullable_exnref += count; + } + wasmparser::ValType::Ref(_) => counts.other_reference += count, + } + } + return counts; + } + _ => {} + } + } + panic!("generated function export `{export_name}` has no code body"); +} + +fn frame_reserve_sizes(bytes: &[u8], export_name: &str) -> Vec { + let module = Module::from_buffer(bytes).expect("parse generated module"); + let frame_select = module + .funcs + .iter() + .find(|function| function.name.as_deref() == Some("__wpk_fork_select_unwind_frame")) + .expect("generated unwind-frame selector") + .id(); + let function_id = module + .exports + .iter() + .find_map(|export| { + (export.name == export_name).then(|| match export.item { + walrus::ExportItem::Function(function) => Some(function), + _ => None, + })? + }) + .unwrap_or_else(|| panic!("generated function export `{export_name}` not found")); + let function = local_func(&module, function_id); + let mut sizes = Vec::new(); + + fn collect( + function: &LocalFunction, + sequence: InstrSeqId, + frame_select: FunctionId, + sizes: &mut Vec, + ) { + let instructions = &function.block(sequence).instrs; + for (index, (instruction, _)) in instructions.iter().enumerate() { + if matches!(instruction, Instr::Call(call) if call.func == frame_select) { + let Some(( + Instr::Const(Const { + value: Value::I32(size), + }), + _, + )) = index + .checked_sub(2) + .and_then(|previous| instructions.get(previous)) + else { + panic!( + "unwind-frame selector must be preceded by its exact \ + static size and call index" + ); + }; + sizes.push(*size); + } + for child in nested_of(instruction) { + collect(function, child, frame_select, sizes); + } + } + } + + collect(function, function.entry_block(), frame_select, &mut sizes); + assert!( + !sizes.is_empty(), + "generated function export `{export_name}` has no unwind-frame selection" + ); + sizes +} + fn find_import_func(module: &Module, qualified: &str) -> FunctionId { let (mod_name, field) = qualified.split_once('.').expect("qualified name"); for imp in module.imports.iter() { diff --git a/crates/fork-instrument/tests/trampoline.rs b/crates/fork-instrument/tests/trampoline.rs index 35f2ca02bc..f552564ca8 100644 --- a/crates/fork-instrument/tests/trampoline.rs +++ b/crates/fork-instrument/tests/trampoline.rs @@ -37,8 +37,8 @@ //! `call_graph::reaching_closure` covers it. //! - **Legacy `try` body** — 2026-05-17 CI showed that shipping C //! ports can still contain legacy `try` in fork-path functions even -//! with explicit modern-EH flags. Forks in the try body are absorbed -//! by nested switch-dispatch; legacy catch-handler forks still panic. +//! with explicit modern-EH flags. Fork-reachable handlers are normalized to +//! activation-owned modern EH before nested switch-dispatch runs. //! //! Net result: the trampoline scaffolding is preserved in //! `crates/fork-instrument/src/instrument.rs` but currently has no @@ -56,6 +56,11 @@ //! | `nested_call_indirect.wat` | nested switch (2.1) | already handled empirically | use fork_instrument::{Options, instrument}; +use std::{ + fs, + process::Command, + sync::atomic::{AtomicU64, Ordering}, +}; use walrus::{ LocalFunction, Module, ir::{Block, IfElse, Instr, InstrSeqId, Loop, Try, TryTable}, @@ -76,6 +81,38 @@ fn try_parse(wat_src: &str) -> Option> { wat::parse_str(wat_src).ok() } +fn parse_legacy_wat(wat_src: &str) -> Vec { + if let Some(bytes) = try_parse(wat_src) { + return bytes; + } + + static NEXT_ID: AtomicU64 = AtomicU64::new(0); + let id = NEXT_ID.fetch_add(1, Ordering::Relaxed); + let base = std::env::temp_dir().join(format!( + "kandelo-fork-instrument-legacy-{}-{id}", + std::process::id(), + )); + let wat_path = base.with_extension("wat"); + let wasm_path = base.with_extension("wasm"); + fs::write(&wat_path, wat_src).expect("write legacy WAT fixture"); + let output = Command::new("wat2wasm") + .args(["--enable-exceptions"]) + .arg(&wat_path) + .arg("-o") + .arg(&wasm_path) + .output() + .expect("run wat2wasm for legacy EH fixture"); + assert!( + output.status.success(), + "wat2wasm failed for legacy EH fixture:\n{}", + String::from_utf8_lossy(&output.stderr), + ); + let bytes = fs::read(&wasm_path).expect("read compiled legacy EH fixture"); + let _ = fs::remove_file(wat_path); + let _ = fs::remove_file(wasm_path); + bytes +} + /// Walk every instruction sequence reachable from `seq` (including /// nested ones), invoking `visit(seq, depth, instr)` for each instr. /// Mirrored from tests/switch_dispatch.rs's `walk_all` so the two @@ -98,7 +135,10 @@ fn nested_of(instr: &Instr) -> Vec { match instr { Instr::Block(Block { seq }) => vec![*seq], Instr::Loop(Loop { seq }) => vec![*seq], - Instr::IfElse(IfElse { consequent, alternative }) => vec![*consequent, *alternative], + Instr::IfElse(IfElse { + consequent, + alternative, + }) => vec![*consequent, *alternative], Instr::TryTable(TryTable { seq, .. }) => vec![*seq], Instr::Try(Try { seq, .. }) => vec![*seq], _ => vec![], @@ -136,10 +176,12 @@ fn has_br_table_in(module: &Module, export_name: &str) -> bool { /// `_post_table` per fork-path function. #[allow(dead_code)] // used by the ignored trampoline_* tests in 2.3 fn has_table_with_prefix(module: &Module, prefix: &str) -> bool { - module - .tables - .iter() - .any(|t| t.name.as_deref().map(|n| n.starts_with(prefix)).unwrap_or(false)) + module.tables.iter().any(|t| { + t.name + .as_deref() + .map(|n| n.starts_with(prefix)) + .unwrap_or(false) + }) } // --------------------------------------------------------------------- @@ -250,8 +292,8 @@ fn nested_multivalue_params_uses_nested_switch_dispatch() { switch-dispatch (br_table emitted), not guard-dispatch" ); // Trampoline post-table is NOT emitted — switch-dispatch absorbs - // this case. The trampoline scaffolding stays reserved for - // unimplemented cases such as fork-from-legacy-catch. + // this case. Fork-reachable legacy catches are normalized to modern + // activation-owned EH before dispatch selection. assert!( !has_table_with_prefix(&module, "_start_post_table"), "post-2.6c: nested switch-dispatch absorbs multi-value-params; \ @@ -265,9 +307,8 @@ fn nested_multivalue_params_uses_nested_switch_dispatch() { // // 2026-05-17 CI disproved the "modern flags remove every legacy Try" // invariant for C ports such as bash, spidermonkey, and vim. A fork in the -// legacy try body can use the same per-region nested-switch route as -// Block/Loop/TryTable bodies. Legacy catch handlers still need their -// exception path reconstructed and remain unsupported. +// legacy try body and handlers use the same per-region nested-switch route as +// Block/Loop/TryTable after normalization to modern activation-owned EH. // // Note: the wat crate may not parse legacy try/catch on the host's // version (it's gated behind the legacy-EH feature). Skip cleanly @@ -276,10 +317,7 @@ fn nested_multivalue_params_uses_nested_switch_dispatch() { #[test] fn legacy_try_body_fork_uses_nested_switch_dispatch() { let wat = include_str!("fixtures/trampoline/legacy_try_fork.wat"); - let Some(input) = try_parse(wat) else { - eprintln!("skip: wat crate did not parse legacy try/catch fixture"); - return; - }; + let input = parse_legacy_wat(wat); let output = instrument(&input, &Options::default()).expect("instrument"); validate(&output); let module = Module::from_buffer(&output).expect("walrus parse"); @@ -290,16 +328,200 @@ fn legacy_try_body_fork_uses_nested_switch_dispatch() { ); } +#[test] +fn legacy_catch_handler_fork_is_lowered_to_activation_owned_modern_catch() { + let wat = include_str!("fixtures/trampoline/legacy_catch_fork.wat"); + let input = parse_legacy_wat(wat); + let output = instrument(&input, &Options::default()).expect("instrument legacy catch handler"); + validate(&output); + let module = Module::from_buffer(&output).expect("walrus parse"); + + assert!( + has_br_table_in(&module, "_start"), + "legacy catch-handler fork must route through nested switch replay", + ); + let start = module + .exports + .iter() + .find_map(|export| match export.item { + walrus::ExportItem::Function(id) if export.name == "_start" => Some(id), + _ => None, + }) + .expect("_start export"); + let start = match &module.funcs.get(start).kind { + walrus::FunctionKind::Local(local) => local, + _ => panic!("_start is not local"), + }; + let mut has_catch_ref = false; + let mut has_throw_ref = false; + let mut legacy_handlers = 0usize; + walk_all( + start, + start.entry_block(), + 0, + &mut |_, _, instr| match instr { + Instr::TryTable(table) => { + has_catch_ref |= table.catches.iter().any(|catch| { + matches!( + catch, + walrus::ir::TryTableCatch::CatchRef { .. } + | walrus::ir::TryTableCatch::CatchAllRef { .. } + ) + }); + } + Instr::ThrowRef(_) => has_throw_ref = true, + Instr::Try(legacy) => { + legacy_handlers += legacy + .catches + .iter() + .filter(|catch| !matches!(catch, walrus::ir::LegacyCatch::Delegate { .. })) + .count(); + } + _ => {} + }, + ); + assert_eq!( + legacy_handlers, 0, + "normalization must eliminate implicit legacy handler contexts before \ + continuation instrumentation", + ); + assert!(has_catch_ref, "legacy catch must lower through catch_ref"); + assert!( + has_throw_ref, + "rewind must reconstruct the caught exception with throw_ref", + ); +} + +#[test] +fn legacy_catch_all_handler_fork_uses_complete_exception_recipe() { + let input = parse_legacy_wat( + r#" + (module + (import "kernel" "kernel_fork" (func $fork (result i32))) + (tag $failure) + (memory 1) + (func (export "_start") (result i32) + (try (result i32) + (do + throw $failure) + (catch_all + call $fork)))) + "#, + ); + let output = instrument(&input, &Options::default()).expect("instrument legacy catch_all"); + validate(&output); + let printed = wasmprinter::print_bytes(&output).expect("print instrumented legacy catch_all"); + assert!( + printed.contains("catch_all_ref"), + "legacy catch_all must capture the complete exception, including unknown tags", + ); + assert!( + printed.contains("throw_ref"), + "legacy catch_all rewind must replay the complete exception recipe", + ); +} + +#[test] +fn legacy_rethrow_after_fork_uses_owned_exception_and_clears_legacy_opcode() { + let input = parse_legacy_wat( + r#" + (module + (import "kernel" "kernel_fork" (func $fork (result i32))) + (tag $number (param i32)) + (memory 1) + (func (export "_start") (result i32) + (try (result i32) + (do + (try (result i32) + (do + i32.const 73 + throw $number) + (catch $number + call $fork + drop + rethrow 0))) + (catch $number)))) + "#, + ); + let output = instrument(&input, &Options::default()).expect("instrument legacy rethrow"); + validate(&output); + let module = Module::from_buffer(&output).expect("parse normalized rethrow module"); + let start = module + .exports + .iter() + .find_map(|export| match export.item { + walrus::ExportItem::Function(id) if export.name == "_start" => Some(id), + _ => None, + }) + .expect("_start export"); + let start = match &module.funcs.get(start).kind { + walrus::FunctionKind::Local(local) => local, + _ => panic!("_start is not local"), + }; + let mut rethrows = 0usize; + let mut throw_refs = 0usize; + walk_all( + start, + start.entry_block(), + 0, + &mut |_, _, instr| match instr { + Instr::Rethrow(_) => rethrows += 1, + Instr::ThrowRef(_) => throw_refs += 1, + _ => {} + }, + ); + assert_eq!( + rethrows, 0, + "legacy implicit catch contexts must not survive normalization", + ); + assert!( + throw_refs >= 2, + "normalization and rewind must both use owned exnref throws", + ); +} + +#[test] +fn legacy_handler_br_table_exit_clears_owned_exception_through_typed_shim() { + let input = parse_legacy_wat( + r#" + (module + (import "kernel" "kernel_fork" (func $fork (result i32))) + (tag $number (param i32)) + (memory 1) + (func (export "_start") (result i32) + (block $done (result i32) + (try (result i32) + (do + i32.const 91 + throw $number) + (catch $number + call $fork + drop + i32.const 0 + br_table $done $done))))) + "#, + ); + let output = instrument(&input, &Options::default()).expect("instrument legacy br_table exit"); + validate(&output); + let printed = wasmprinter::print_bytes(&output).expect("print legacy br_table output"); + assert!( + printed.contains("br_table"), + "typed branch cleanup must preserve br_table rather than scalarizing dispatch", + ); + assert!( + printed.contains("ref.null exn"), + "the branch shim must clear its activation-local exception root", + ); +} + // --------------------------------------------------------------------- // (c) Nested call_indirect — empirically NOT a trampoline case // --------------------------------------------------------------------- // // Empirical finding (sub-commit 2.1): the simple nested call_indirect // case is already handled by nested switch-dispatch today. See the -// fixture's header comment for the explanation. The real class (c) -// trampoline gap is `call_indirect + another unsupported pattern` -// (e.g. carryover); a fixture for that lands in 2.5 once we audit -// which LLVM emission shapes actually trigger it. +// fixture's header comment for the explanation. Nested switch-dispatch now +// also owns call_indirect with typed carryovers. // // This test is a regression gate that nested call_indirect stays on // the switch-dispatch path. diff --git a/crates/shared/src/host_abi.rs b/crates/shared/src/host_abi.rs index 9801751bb9..d96092b04a 100644 --- a/crates/shared/src/host_abi.rs +++ b/crates/shared/src/host_abi.rs @@ -9,8 +9,7 @@ use core::mem::size_of; use crate::abi::extended_syscalls as extra_syscalls; use crate::{ - SCHED_AFFINITY_MASK_SIZE, Syscall, WASM_RUSAGE_WIRE_SIZE, WasmStat, WasmStatfs, - WasmTimespec, + SCHED_AFFINITY_MASK_SIZE, Syscall, WASM_RUSAGE_WIRE_SIZE, WasmStat, WasmStatfs, WasmTimespec, }; /// Direction of a marshalled pointer argument. @@ -330,10 +329,7 @@ pub const SYSCALL_ARG_DESCRIPTORS: &[SyscallArgDescriptor] = &[ entry!(Syscall::Sigsuspend as u32, [desc!(0, In, fixed!(8))]), entry!( Syscall::Pathconf as u32, - [ - desc!(0, In, cstring!()), - desc!(2, Out, fixed!(8), required), - ] + [desc!(0, In, cstring!()), desc!(2, Out, fixed!(8), required),] ), entry!( Syscall::Fpathconf as u32, @@ -625,8 +621,7 @@ mod tests { ); assert!(waitid[1].nullable); - let sched_getaffinity = - find(extra_syscalls::SYS_SCHED_GETAFFINITY).args[0]; + let sched_getaffinity = find(extra_syscalls::SYS_SCHED_GETAFFINITY).args[0]; assert_eq!(sched_getaffinity.arg_index, 2); assert_eq!(sched_getaffinity.direction, SyscallArgDirection::Out); assert_eq!( diff --git a/crates/shared/src/lib.rs b/crates/shared/src/lib.rs index 190b63fc46..5324ff0db8 100644 --- a/crates/shared/src/lib.rs +++ b/crates/shared/src/lib.rs @@ -84,7 +84,11 @@ pub mod host_abi; /// and fork exports return kernel-allocated identities; instrumented /// modules declare the continuation format and import reserve, commit, and /// replay hooks. -pub const ABI_VERSION: u32 = 42; +/// 43: fork artifacts prove activation-state safety explicitly. Activation +/// references, complete exceptions, mutable reference globals, and mutable +/// tables are serialized as versioned process-owned recipes and rebuilt +/// with fresh instance-local identities before continuation replay. +pub const ABI_VERSION: u32 = 43; /// Byte width of Kandelo's Linux-compatible kernel CPU-affinity mask. /// @@ -847,7 +851,7 @@ pub mod mode { /// 8 48B arguments (6 × i64) /// 56 8B return value (i64) /// 64 4B errno (i32) -/// 68 4B reserved/pad +/// 68 4B request flags /// 72 64KB data transfer buffer pub mod channel { /// Byte offset of the status field (i32, atomic). @@ -864,6 +868,13 @@ pub mod channel { pub const RETURN_OFFSET: usize = 56; /// Byte offset of the errno field (i32). pub const ERRNO_OFFSET: usize = 64; + /// Byte offset of host/process request flags (u32). + pub const REQUEST_FLAGS_OFFSET: usize = 68; + /// The request completion is consumed by process-worker JavaScript, not + /// the libc channel trampoline. Caught signals must remain kernel-pending + /// until an explicit guest checkpoint can invoke the handler after the + /// owning host transition returns. + pub const REQUEST_FLAG_DEFER_SIGNAL_DELIVERY: u32 = 1 << 0; /// Byte offset of the data buffer region. pub const DATA_OFFSET: usize = 72; /// Size of the data buffer. @@ -1232,8 +1243,7 @@ pub mod process_memory { /// Size of one fork save buffer in bytes. The control prefix and buffer /// together occupy exactly one dedicated 64 KiB scratch page. - pub const FORK_SAVE_BUFFER_SIZE: u32 = - WASM_PAGE_SIZE - FORK_SAVE_CONTROL_PREFIX_SIZE; + pub const FORK_SAVE_BUFFER_SIZE: u32 = WASM_PAGE_SIZE - FORK_SAVE_CONTROL_PREFIX_SIZE; /// Main-thread fork-save/scratch page, relative to `controlBasePage`. pub const MAIN_FORK_SAVE_PAGE: u32 = 0; @@ -1286,6 +1296,11 @@ pub mod abi { pub enum ProgramArtifactValueType { Pointer, I32, + I64, + FuncRef, + ExternRef, + ExnRef, + AnyRef, } /// One required function import in an instrumented program artifact. @@ -1297,6 +1312,17 @@ pub mod abi { pub results: &'static [ProgramArtifactValueType], } + /// One required private table import in an instrumented program artifact. + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + pub struct ProgramArtifactTableImport { + pub module: &'static str, + pub name: &'static str, + pub table64: bool, + pub element: ProgramArtifactValueType, + pub minimum: u64, + pub maximum: Option, + } + /// One required function export in an instrumented program artifact. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct ProgramArtifactExport { @@ -1305,7 +1331,7 @@ pub mod abi { pub results: &'static [ProgramArtifactValueType], } - /// ABI 42 linked-continuation metadata and function surface. + /// ABI 42+ linked-continuation metadata and function surface. /// /// WHY this lives in `shared::abi`: these names and descriptor fields are /// consumed before a program starts, by the instrumenter, host, package @@ -1324,20 +1350,391 @@ pub mod abi { WPK_FORK_LINKED_FRAME_FLAG_TRANSACTIONAL_NODES | WPK_FORK_LINKED_FRAME_FLAG_ABORT_UNWINDING; pub const WPK_FORK_LINKED_FRAME_POINTER_WIDTHS: &[u8] = &[4, 8]; + /// ABI 43+ activation-owned module-state recipe format. + /// + /// WHY this is shared ABI rather than host-private metadata: an + /// instrumented activation writes the arena before the host copies linear + /// memory, and a fresh child instance validates and consumes it. Every + /// literal below therefore crosses the instrumenter/guest/host boundary. + pub const WPK_FORK_MODULE_STATE_FORMAT_SECTION: &str = "kandelo.wpk_fork.module_state"; + pub const WPK_FORK_MODULE_STATE_FORMAT_MAGIC: [u8; 4] = *b"KFMD"; + pub const WPK_FORK_MODULE_STATE_FORMAT_VERSION: u16 = 1; + pub const WPK_FORK_MODULE_STATE_DESCRIPTOR_SIZE: u16 = 24; + pub const WPK_FORK_MODULE_STATE_RECORD_ALIGNMENT: u8 = 8; + pub const WPK_FORK_MODULE_STATE_FLAG_ROOT_PREFIX_POINTER: u16 = 1 << 0; + pub const WPK_FORK_MODULE_STATE_FLAG_EXPLICIT_OWNERS: u16 = 1 << 1; + pub const WPK_FORK_MODULE_STATE_FLAG_SPARSE_TABLES: u16 = 1 << 2; + pub const WPK_FORK_MODULE_STATE_REQUIRED_FLAGS: u16 = + WPK_FORK_MODULE_STATE_FLAG_ROOT_PREFIX_POINTER + | WPK_FORK_MODULE_STATE_FLAG_EXPLICIT_OWNERS + | WPK_FORK_MODULE_STATE_FLAG_SPARSE_TABLES; + pub const WPK_FORK_MODULE_STATE_KNOWN_FLAGS: u16 = WPK_FORK_MODULE_STATE_REQUIRED_FLAGS; + pub const WPK_FORK_MODULE_STATE_ARENA_VERSION: u16 = 1; + pub const WPK_FORK_MODULE_STATE_RECORD_VERSION: u16 = 1; + pub const WPK_FORK_MODULE_STATE_ROOT_POINTER_WORD_OFFSET: u32 = 1; + pub const WPK_FORK_MODULE_STATE_POINTER_WIDTHS: &[u8] = &[4, 8]; + + pub const WPK_FORK_MODULE_STATE_CHUNK_MAGIC: [u8; 4] = *b"KFMC"; + pub const WPK_FORK_MODULE_STATE_CHUNK_FLAG_ROOT: u16 = 1 << 0; + pub const WPK_FORK_MODULE_STATE_CHUNK_FLAG_SEALED: u16 = 1 << 1; + pub const WPK_FORK_MODULE_STATE_CHUNK_KNOWN_FLAGS: u16 = + WPK_FORK_MODULE_STATE_CHUNK_FLAG_ROOT | WPK_FORK_MODULE_STATE_CHUNK_FLAG_SEALED; + + pub const WPK_FORK_MODULE_STATE_RECORD_MAGIC: [u8; 4] = *b"KFMR"; + pub const WPK_FORK_MODULE_STATE_RECORD_HEADER_SIZE: u16 = 24; + pub const WPK_FORK_MODULE_STATE_RECORD_KIND_MODULE: u16 = 1; + pub const WPK_FORK_MODULE_STATE_RECORD_KIND_REFERENCE_RECIPE: u16 = 2; + pub const WPK_FORK_MODULE_STATE_RECORD_KIND_MUTABLE_GLOBAL: u16 = 3; + pub const WPK_FORK_MODULE_STATE_RECORD_KIND_TABLE: u16 = 4; + pub const WPK_FORK_MODULE_STATE_RECORD_KIND_TABLE_PAGE: u16 = 5; + pub const WPK_FORK_MODULE_STATE_RECORD_KIND_ELEMENT_SEGMENTS: u16 = 6; + pub const WPK_FORK_MODULE_STATE_RECORD_KIND_DATA_SEGMENTS: u16 = 7; + pub const WPK_FORK_MODULE_STATE_RECORD_KIND_REPLAY_EVENTS: u16 = 8; + pub const WPK_FORK_MODULE_STATE_RECORD_KIND_IMPORTED_GLOBAL_BINDINGS: u16 = 9; + pub const WPK_FORK_MODULE_STATE_RECORD_KIND_ACTIVATION_CONTINUATIONS: u16 = 10; + pub const WPK_FORK_MODULE_STATE_RECORD_KIND_IMPORTED_TABLE_BINDINGS: u16 = 11; + pub const WPK_FORK_MODULE_STATE_RECORD_KIND_REFERENCE_RECIPE_SEGMENT: u16 = 12; + pub const WPK_FORK_MODULE_STATE_RECORD_KIND_REPLAY_EVENT_SEGMENT: u16 = 13; + + /// One recognized module-state arena record kind. + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + pub struct ForkModuleStateRecordKind { + pub number: u16, + pub name: &'static str, + } + + pub const WPK_FORK_MODULE_STATE_RECORD_KINDS: &[ForkModuleStateRecordKind] = &[ + ForkModuleStateRecordKind { + number: WPK_FORK_MODULE_STATE_RECORD_KIND_MODULE, + name: "module", + }, + ForkModuleStateRecordKind { + number: WPK_FORK_MODULE_STATE_RECORD_KIND_REFERENCE_RECIPE, + name: "reference_recipe", + }, + ForkModuleStateRecordKind { + number: WPK_FORK_MODULE_STATE_RECORD_KIND_MUTABLE_GLOBAL, + name: "mutable_global", + }, + ForkModuleStateRecordKind { + number: WPK_FORK_MODULE_STATE_RECORD_KIND_TABLE, + name: "table", + }, + ForkModuleStateRecordKind { + number: WPK_FORK_MODULE_STATE_RECORD_KIND_TABLE_PAGE, + name: "table_page", + }, + ForkModuleStateRecordKind { + number: WPK_FORK_MODULE_STATE_RECORD_KIND_ELEMENT_SEGMENTS, + name: "element_segments", + }, + ForkModuleStateRecordKind { + number: WPK_FORK_MODULE_STATE_RECORD_KIND_DATA_SEGMENTS, + name: "data_segments", + }, + ForkModuleStateRecordKind { + number: WPK_FORK_MODULE_STATE_RECORD_KIND_REPLAY_EVENTS, + name: "replay_events", + }, + ForkModuleStateRecordKind { + number: WPK_FORK_MODULE_STATE_RECORD_KIND_IMPORTED_GLOBAL_BINDINGS, + name: "imported_global_bindings", + }, + ForkModuleStateRecordKind { + number: WPK_FORK_MODULE_STATE_RECORD_KIND_ACTIVATION_CONTINUATIONS, + name: "activation_continuations", + }, + ForkModuleStateRecordKind { + number: WPK_FORK_MODULE_STATE_RECORD_KIND_IMPORTED_TABLE_BINDINGS, + name: "imported_table_bindings", + }, + ForkModuleStateRecordKind { + number: WPK_FORK_MODULE_STATE_RECORD_KIND_REFERENCE_RECIPE_SEGMENT, + name: "reference_recipe_segment", + }, + ForkModuleStateRecordKind { + number: WPK_FORK_MODULE_STATE_RECORD_KIND_REPLAY_EVENT_SEGMENT, + name: "replay_event_segment", + }, + ]; + + pub const WPK_FORK_MODULE_STATE_MODULE_TEMPLATE_ID_SIZE: u16 = 32; + pub const WPK_FORK_MODULE_STATE_MODULE_RECORD_PAYLOAD_SIZE: u16 = + WPK_FORK_MODULE_STATE_MODULE_TEMPLATE_ID_SIZE + 8; + pub const WPK_FORK_MODULE_STATE_MODULE_RECORD_KNOWN_FLAGS: u32 = 0; + pub const WPK_FORK_MODULE_STATE_GLOBAL_HEADER_SIZE: u16 = 8; + pub const WPK_FORK_MODULE_STATE_GLOBAL_TYPE_I32: u8 = 1; + pub const WPK_FORK_MODULE_STATE_GLOBAL_TYPE_I64: u8 = 2; + pub const WPK_FORK_MODULE_STATE_GLOBAL_TYPE_F32: u8 = 3; + pub const WPK_FORK_MODULE_STATE_GLOBAL_TYPE_F64: u8 = 4; + pub const WPK_FORK_MODULE_STATE_GLOBAL_TYPE_V128: u8 = 5; + pub const WPK_FORK_MODULE_STATE_GLOBAL_TYPE_FUNCREF: u8 = 6; + pub const WPK_FORK_MODULE_STATE_GLOBAL_TYPE_EXTERNREF: u8 = 7; + pub const WPK_FORK_MODULE_STATE_GLOBAL_TYPE_EXNREF: u8 = 8; + pub const WPK_FORK_MODULE_STATE_GLOBAL_TYPE_ANYREF: u8 = 9; + pub const WPK_FORK_MODULE_STATE_TABLE_BASELINE_FINGERPRINT_SIZE: u16 = 32; + pub const WPK_FORK_MODULE_STATE_TABLE_DESCRIPTOR_PAYLOAD_SIZE: u16 = + WPK_FORK_MODULE_STATE_TABLE_BASELINE_FINGERPRINT_SIZE + 24; + pub const WPK_FORK_MODULE_STATE_TABLE_FLAG_SPARSE_OVERRIDES: u32 = 1 << 0; + pub const WPK_FORK_MODULE_STATE_TABLE_KNOWN_FLAGS: u32 = + WPK_FORK_MODULE_STATE_TABLE_FLAG_SPARSE_OVERRIDES; + pub const WPK_FORK_MODULE_STATE_TABLE_PAGE_HEADER_SIZE: u16 = 16; + pub const WPK_FORK_MODULE_STATE_TABLE_RUN_HEADER_SIZE: u16 = 8; + pub const WPK_FORK_MODULE_STATE_ELEMENT_SEGMENT_HEADER_SIZE: u16 = 8; + pub const WPK_FORK_MODULE_STATE_DATA_SEGMENT_HEADER_SIZE: u16 = 8; + pub const WPK_FORK_MODULE_STATE_REPLAY_EVENTS_OWNER: u32 = 1; + pub const WPK_FORK_MODULE_STATE_REPLAY_EVENTS_MAGIC: [u8; 4] = *b"KFRE"; + pub const WPK_FORK_MODULE_STATE_REPLAY_EVENTS_VERSION: u16 = 2; + pub const WPK_FORK_MODULE_STATE_REPLAY_EVENTS_HEADER_SIZE: u16 = 40; + pub const WPK_FORK_MODULE_STATE_REPLAY_EVENT_SIZE: u16 = 8; + pub const WPK_FORK_MODULE_STATE_REPLAY_EVENTS_KNOWN_FLAGS: u16 = 0; + pub const WPK_FORK_MODULE_STATE_REPLAY_EVENT_SEGMENT_VERSION: u16 = 1; + pub const WPK_FORK_MODULE_STATE_REPLAY_EVENT_SEGMENT_HEADER_SIZE: u16 = 24; + pub const WPK_FORK_MODULE_STATE_REPLAY_EVENT_SEGMENT_CAPACITY: u32 = 4080; + pub const WPK_FORK_MODULE_STATE_REPLAY_EVENT_SEGMENT_KNOWN_FLAGS: u16 = 0; + pub const WPK_FORK_REFERENCE_TRANSACTION_OWNER: u32 = 1; + pub const WPK_FORK_REFERENCE_TRANSACTION_MAGIC: [u8; 4] = *b"KFRV"; + pub const WPK_FORK_REFERENCE_TRANSACTION_VERSION: u16 = 2; + pub const WPK_FORK_REFERENCE_TRANSACTION_MANIFEST_SIZE: u16 = 96; + pub const WPK_FORK_REFERENCE_TRANSACTION_FLAG_SEALED: u32 = 1 << 0; + pub const WPK_FORK_REFERENCE_TRANSACTION_KNOWN_FLAGS: u32 = + WPK_FORK_REFERENCE_TRANSACTION_FLAG_SEALED; + pub const WPK_FORK_REFERENCE_SEGMENT_MAGIC: [u8; 4] = *b"KFRS"; + pub const WPK_FORK_REFERENCE_SEGMENT_HEADER_SIZE: u16 = 40; + pub const WPK_FORK_REFERENCE_SEGMENT_KNOWN_FLAGS: u16 = 0; + pub const WPK_FORK_REFERENCE_NODE_RECORD_SIZE: u16 = 48; + pub const WPK_FORK_REFERENCE_VECTOR_INDEX_SIZE: u16 = 16; + pub const WPK_FORK_REFERENCE_SECTION_NODES: u16 = 1; + pub const WPK_FORK_REFERENCE_SECTION_EDGES: u16 = 2; + pub const WPK_FORK_REFERENCE_SECTION_SCALARS: u16 = 3; + pub const WPK_FORK_REFERENCE_SECTION_VECTOR_INDEX: u16 = 4; + pub const WPK_FORK_REFERENCE_SECTION_VECTOR_ENTRIES: u16 = 5; + pub const WPK_FORK_IMPORTED_GLOBAL_BINDINGS_OWNER: u32 = 2; + pub const WPK_FORK_IMPORTED_GLOBAL_BINDINGS_MAGIC: [u8; 4] = *b"KFBG"; + pub const WPK_FORK_IMPORTED_GLOBAL_BINDINGS_VERSION: u16 = 1; + pub const WPK_FORK_IMPORTED_GLOBAL_BINDINGS_HEADER_SIZE: u16 = 24; + pub const WPK_FORK_IMPORTED_GLOBAL_BINDINGS_ENTRY_SIZE: u16 = 40; + pub const WPK_FORK_IMPORTED_GLOBAL_BINDINGS_KNOWN_FLAGS: u16 = 0; + pub const WPK_FORK_IMPORTED_GLOBAL_BINDING_RAW_NUMBER: u8 = 1; + pub const WPK_FORK_IMPORTED_GLOBAL_BINDING_RAW_BIGINT: u8 = 2; + pub const WPK_FORK_IMPORTED_GLOBAL_BINDING_RAW_REFERENCE: u8 = 3; + pub const WPK_FORK_IMPORTED_GLOBAL_BINDING_ACTIVATION_GLOBAL: u8 = 4; + pub const WPK_FORK_IMPORTED_GLOBAL_BINDING_BASE_IMPORT: u8 = 5; + pub const WPK_FORK_GLOBAL_CATALOG_EXPORT_PREFIX: &str = "__wpk_fork_global_"; + pub const WPK_FORK_ACTIVATION_CONTINUATIONS_OWNER: u32 = 3; + pub const WPK_FORK_ACTIVATION_CONTINUATIONS_MAGIC: [u8; 4] = *b"KFAC"; + pub const WPK_FORK_ACTIVATION_CONTINUATIONS_VERSION: u16 = 1; + pub const WPK_FORK_ACTIVATION_CONTINUATIONS_HEADER_SIZE: u16 = 24; + pub const WPK_FORK_ACTIVATION_CONTINUATION_ENTRY_SIZE: u16 = 16; + pub const WPK_FORK_ACTIVATION_CONTINUATIONS_KNOWN_FLAGS: u16 = 0; + pub const WPK_FORK_ACTIVATION_CONTINUATION_ENTRY_KNOWN_FLAGS: u32 = 0; + pub const WPK_FORK_IMPORTED_TABLE_BINDINGS_OWNER: u32 = 4; + pub const WPK_FORK_IMPORTED_TABLE_BINDINGS_MAGIC: [u8; 4] = *b"KFBT"; + pub const WPK_FORK_IMPORTED_TABLE_BINDINGS_VERSION: u16 = 1; + pub const WPK_FORK_IMPORTED_TABLE_BINDINGS_HEADER_SIZE: u16 = 24; + pub const WPK_FORK_IMPORTED_TABLE_BINDINGS_ENTRY_SIZE: u16 = 24; + pub const WPK_FORK_IMPORTED_TABLE_BINDINGS_KNOWN_FLAGS: u16 = 0; + pub const WPK_FORK_IMPORTED_TABLE_BINDING_ACTIVATION_TABLE: u8 = 1; + pub const WPK_FORK_IMPORTED_TABLE_BINDING_BASE_IMPORT: u8 = 2; + pub const WPK_FORK_TABLE_CATALOG_EXPORT_PREFIX: &str = "__wpk_fork_table_"; + pub const WPK_FORK_MODULE_STATE_MIN_TABLE_PAGE_SHIFT: u8 = 4; + pub const WPK_FORK_MODULE_STATE_MAX_TABLE_PAGE_SHIFT: u8 = 20; + /// Exact sparse-page geometry emitted by the ABI 43 instrumenter. + pub const WPK_FORK_MODULE_STATE_TABLE_PAGE_SHIFT: u8 = 10; + + /// ABI 43 imported-global ownership metadata. A fresh child consumes this + /// before module instantiation, which is earlier than KFMS restore and is + /// therefore the only phase that can preserve immutable exports and + /// constant initializers that observe imported globals. + pub const WPK_FORK_IMPORTED_GLOBALS_SECTION: &str = "kandelo.wpk_fork.imported_globals"; + pub const WPK_FORK_IMPORTED_GLOBALS_MAGIC: [u8; 4] = *b"KFIG"; + pub const WPK_FORK_IMPORTED_GLOBALS_VERSION: u16 = 1; + pub const WPK_FORK_IMPORTED_GLOBALS_HEADER_SIZE: u16 = 16; + pub const WPK_FORK_IMPORTED_GLOBALS_RECORD_HEADER_SIZE: u16 = 24; + pub const WPK_FORK_IMPORTED_GLOBAL_FLAG_MUTABLE: u8 = 1 << 0; + pub const WPK_FORK_IMPORTED_GLOBAL_FLAG_SHARED: u8 = 1 << 1; + pub const WPK_FORK_IMPORTED_GLOBAL_KNOWN_FLAGS: u8 = + WPK_FORK_IMPORTED_GLOBAL_FLAG_MUTABLE | WPK_FORK_IMPORTED_GLOBAL_FLAG_SHARED; + pub const WPK_FORK_IMPORTED_TABLES_SECTION: &str = "kandelo.wpk_fork.imported_tables"; + pub const WPK_FORK_IMPORTED_TABLES_MAGIC: [u8; 4] = *b"KFIT"; + pub const WPK_FORK_IMPORTED_TABLES_VERSION: u16 = 1; + pub const WPK_FORK_IMPORTED_TABLES_HEADER_SIZE: u16 = 16; + pub const WPK_FORK_IMPORTED_TABLES_RECORD_HEADER_SIZE: u16 = 24; + pub const WPK_FORK_IMPORTED_TABLE_FLAG_TABLE64: u8 = 1 << 0; + pub const WPK_FORK_IMPORTED_TABLE_KNOWN_FLAGS: u8 = WPK_FORK_IMPORTED_TABLE_FLAG_TABLE64; + + /// ABI 43 structural Wasm GC reconstruction catalog. + /// + /// GC object identities cannot cross Store or worker boundaries. The + /// catalog lets the fresh child allocate the same typed object graph and + /// then fill its scalar and reference fields without retaining parent + /// instance references. + pub const WPK_FORK_GC_CODEC_SECTION: &str = "kandelo.wpk_fork.gc_codec"; + pub const WPK_FORK_GC_CODEC_MAGIC: [u8; 4] = *b"KFGC"; + pub const WPK_FORK_GC_CODEC_VERSION: u16 = 1; + pub const WPK_FORK_GC_CODEC_HEADER_SIZE: u16 = 16; + pub const WPK_FORK_GC_CODEC_LAYOUT_RECORD_SIZE: u16 = 44; + pub const WPK_FORK_GC_CODEC_FIELD_RECORD_SIZE: u16 = 12; + + /// ABI 43 exact-tag exception reconstruction catalog. + /// + /// A tag is instance-local, so copied `exnref` values cannot be replayed + /// by importing a JavaScript-side reference. The instrumented activation + /// publishes deterministic tag ordinals and payload layouts instead. The + /// fresh child reconstructs each exception using its own corresponding tag. + pub const WPK_FORK_EXCEPTION_CODEC_SECTION: &str = "kandelo.wpk_fork.exception_codec"; + pub const WPK_FORK_EXCEPTION_CODEC_VERSION: u8 = 1; + pub const WPK_FORK_EXCEPTION_CODEC_HEADER_SIZE: u16 = 8; + pub const WPK_FORK_EXCEPTION_CODEC_TAG_RECORD_SIZE: u16 = 16; + + /// Private zero-payload tag used to unwind instrumented Wasm without + /// manufacturing values for arbitrary function result types. + pub const WPK_FORK_UNWIND_TAG_IMPORT_MODULE: &str = "env"; + pub const WPK_FORK_UNWIND_TAG_IMPORT_NAME: &str = "__wpk_fork_unwind"; + pub const WPK_FORK_UNWIND_TRANSPORT_SECTION: &str = "kandelo.wpk_fork.unwind_transport"; + pub const WPK_FORK_UNWIND_TRANSPORT_VERSION: u8 = 1; + pub const WPK_FORK_UNWIND_TRANSPORT_PAYLOAD_ARITY: u8 = 0; + + /// Fixed instance-local identities recreated by static initializers. + pub const WPK_FORK_STATIC_ROOT_CATALOG_EXPORT: &str = "__wpk_fork_static_root_catalog"; + pub const WPK_FORK_STATIC_ROOT_CATALOG_SECTION: &str = "kandelo.wpk_fork.static_root_catalog"; + pub const WPK_FORK_STATIC_ROOT_CATALOG_MAGIC: [u8; 4] = *b"KFSR"; + pub const WPK_FORK_STATIC_ROOT_CATALOG_VERSION: u16 = 1; + pub const WPK_FORK_STATIC_ROOT_CATALOG_HEADER_SIZE: u16 = 12; + pub const WPK_FORK_STATIC_ROOT_HARVEST_EXPORT: &str = "__wpk_fork_static_root_harvest"; + + /// Versioned instrumentation claims required by ABI 43. + /// + /// Role flags say which call graph was transformed. In ABI 43, + /// `SIDE_ENTRY` means complete side-module boundary coverage: callers of + /// every function import and unresolved function-reference dispatch are + /// resumable even if fork occurs in a different module. The + /// activation-state bit proves all replay state has a fresh-instance + /// reconstruction owner. + pub const WPK_FORK_CAPABILITIES_SECTION: &str = "kandelo.wpk_fork.capabilities"; + pub const WPK_FORK_CAPABILITIES_VERSION: u8 = 1; + pub const WPK_FORK_CAP_SIDE_ENTRY: u8 = 1 << 0; + pub const WPK_FORK_CAP_DYLINK_MAIN: u8 = 1 << 1; + pub const WPK_FORK_CAP_ACTIVATION_STATE_SAFE: u8 = 1 << 2; + pub const WPK_FORK_CAP_KNOWN_MASK: u8 = + WPK_FORK_CAP_SIDE_ENTRY | WPK_FORK_CAP_DYLINK_MAIN | WPK_FORK_CAP_ACTIVATION_STATE_SAFE; + pub const WPK_FORK_CAP_REQUIRED_FLAGS: u8 = WPK_FORK_CAP_ACTIVATION_STATE_SAFE; + pub const WPK_FORK_FRAME_IMPORT_MODULE: &str = "env"; pub const WPK_FORK_FRAME_IMPORT_RESERVE: &str = "__wpk_fork_frame_reserve"; pub const WPK_FORK_FRAME_IMPORT_COMMIT: &str = "__wpk_fork_frame_commit"; pub const WPK_FORK_FRAME_IMPORT_NEXT: &str = "__wpk_fork_frame_next"; + pub const WPK_FORK_FRAME_IMPORT_PEEK: &str = "__wpk_fork_frame_peek"; + pub const WPK_FORK_RESUME_IMPORT_PEEK: &str = "__wpk_fork_resume_peek"; + pub const WPK_FORK_RESUME_IMPORT_TABLE: &str = "__wpk_fork_resume_table"; + + pub const WPK_FORK_MODULE_STATE_IMPORT_MODULE: &str = "env"; + pub const WPK_FORK_MODULE_STATE_IMPORT_RECORD_COMMIT: &str = + "__wpk_fork_module_state_record_commit"; + pub const WPK_FORK_MODULE_STATE_IMPORT_RECORD_FIND: &str = + "__wpk_fork_module_state_record_find"; + pub const WPK_FORK_MODULE_STATE_IMPORT_RECORD_RESERVE: &str = + "__wpk_fork_module_state_record_reserve"; + pub const WPK_FORK_MODULE_STATE_IMPORT_TABLE_DIRTY_COUNT: &str = + "__wpk_fork_module_state_table_dirty_count"; + pub const WPK_FORK_MODULE_STATE_IMPORT_TABLE_DIRTY_MARK: &str = + "__wpk_fork_module_state_table_dirty_mark"; + pub const WPK_FORK_MODULE_STATE_IMPORT_TABLE_DIRTY_PAGE: &str = + "__wpk_fork_module_state_table_dirty_page"; + pub const WPK_FORK_MODULE_STATE_IMPORT_TABLE_STATE_OWNED: &str = + "__wpk_fork_module_state_table_state_owned"; + pub const WPK_FORK_MODULE_STATE_IMPORT_TABLE_MUTATION_BEGIN: &str = + "__wpk_fork_module_state_table_mutation_begin"; + pub const WPK_FORK_MODULE_STATE_IMPORT_TABLE_MUTATION_COMMIT: &str = + "__wpk_fork_module_state_table_mutation_commit"; + pub const WPK_FORK_MODULE_STATE_IMPORT_TABLE_MUTATION_ABORT: &str = + "__wpk_fork_module_state_table_mutation_abort"; + pub const WPK_FORK_MODULE_STATE_IMPORT_TABLE_RECONCILE: &str = + "__wpk_fork_module_state_table_reconcile"; + pub const WPK_FORK_MODULE_STATE_IMPORT_TABLE_GENERATION_ADDR: &str = + "__wpk_fork_module_state_table_generation_addr"; + + pub const WPK_FORK_EXCEPTION_CODEC_IMPORT_MODULE: &str = "env"; + pub const WPK_FORK_EXCEPTION_IMPORT_ACTIVATION: &str = "__wpk_fork_module_activation"; + pub const WPK_FORK_EXCEPTION_IMPORT_BROKER_ENCODE: &str = "__wpk_fork_ref_exn_broker_encode"; + pub const WPK_FORK_EXCEPTION_IMPORT_BROKER_THROW_RECIPE: &str = + "__wpk_fork_ref_exn_broker_throw_recipe"; + pub const WPK_FORK_EXCEPTION_IMPORT_CACHE_INDEX: &str = "__wpk_fork_ref_exn_cache_index"; + pub const WPK_FORK_EXCEPTION_IMPORT_CLAIM: &str = "__wpk_fork_ref_exn_claim"; + pub const WPK_FORK_EXCEPTION_IMPORT_DEFINE: &str = "__wpk_fork_ref_exn_define"; + pub const WPK_FORK_EXCEPTION_IMPORT_INGRESS_THROW: &str = "__wpk_fork_ref_exn_ingress_throw"; + pub const WPK_FORK_EXCEPTION_IMPORT_LOAD: &str = "__wpk_fork_ref_exn_load"; + pub const WPK_FORK_EXCEPTION_IMPORT_LOOKUP: &str = "__wpk_fork_ref_exn_lookup"; + pub const WPK_FORK_EXCEPTION_IMPORT_ROUTE: &str = "__wpk_fork_ref_exn_route"; + + pub const WPK_FORK_REFERENCE_CODEC_IMPORT_MODULE: &str = "env"; + pub const WPK_FORK_REFERENCE_IMPORT_DECODE_ANYREF: &str = "__wpk_fork_ref_decode_anyref"; + pub const WPK_FORK_REFERENCE_IMPORT_DECODE_EXNREF: &str = "__wpk_fork_ref_decode_exnref"; + pub const WPK_FORK_REFERENCE_IMPORT_DECODE_EXTERNREF: &str = "__wpk_fork_ref_decode_externref"; + pub const WPK_FORK_REFERENCE_IMPORT_DECODE_FUNCREF: &str = "__wpk_fork_ref_decode_funcref"; + pub const WPK_FORK_REFERENCE_IMPORT_ENCODE_ANYREF: &str = "__wpk_fork_ref_encode_anyref"; + pub const WPK_FORK_REFERENCE_IMPORT_ENCODE_EXNREF: &str = "__wpk_fork_ref_encode_exnref"; + pub const WPK_FORK_REFERENCE_IMPORT_ENCODE_EXTERNREF: &str = "__wpk_fork_ref_encode_externref"; + pub const WPK_FORK_REFERENCE_IMPORT_ENCODE_FUNCREF: &str = "__wpk_fork_ref_encode_funcref"; + pub const WPK_FORK_REFERENCE_IMPORT_GC_BROKER_ENCODE: &str = "__wpk_fork_ref_gc_broker_encode"; + pub const WPK_FORK_REFERENCE_IMPORT_GC_CAPTURE_LAYOUT: &str = + "__wpk_fork_ref_gc_capture_layout"; + pub const WPK_FORK_REFERENCE_IMPORT_GC_CLAIM: &str = "__wpk_fork_ref_gc_claim"; + pub const WPK_FORK_REFERENCE_IMPORT_GC_DEFINE: &str = "__wpk_fork_ref_gc_define"; + pub const WPK_FORK_REFERENCE_IMPORT_GC_I31: &str = "__wpk_fork_ref_gc_i31"; + pub const WPK_FORK_REFERENCE_IMPORT_GC_LOAD: &str = "__wpk_fork_ref_gc_load"; + pub const WPK_FORK_REFERENCE_IMPORT_GC_LOOKUP: &str = "__wpk_fork_ref_gc_lookup"; + pub const WPK_FORK_REFERENCE_IMPORT_GC_PAYLOAD_LEN: &str = "__wpk_fork_ref_gc_payload_len"; + pub const WPK_FORK_REFERENCE_IMPORT_GC_PROVENANCE_BEGIN: &str = + "__wpk_fork_ref_gc_provenance_begin"; + pub const WPK_FORK_REFERENCE_IMPORT_GC_PROVENANCE_END: &str = + "__wpk_fork_ref_gc_provenance_end"; + pub const WPK_FORK_REFERENCE_IMPORT_GC_PROVENANCE_REF: &str = + "__wpk_fork_ref_gc_provenance_ref"; + pub const WPK_FORK_REFERENCE_IMPORT_GC_ROUTE: &str = "__wpk_fork_ref_gc_route"; + pub const WPK_FORK_REFERENCE_IMPORT_GC_TRANSIT: &str = "__wpk_fork_ref_gc_transit"; + pub const WPK_FORK_REFERENCE_IMPORT_SCRATCH_RELEASE: &str = "__wpk_fork_ref_scratch_release"; + pub const WPK_FORK_REFERENCE_IMPORT_SCRATCH_RESERVE: &str = "__wpk_fork_ref_scratch_reserve"; + pub const WPK_FORK_REFERENCE_IMPORT_VECTOR_APPEND: &str = "__wpk_fork_ref_vector_append"; + pub const WPK_FORK_REFERENCE_IMPORT_VECTOR_BEGIN: &str = "__wpk_fork_ref_vector_begin"; + pub const WPK_FORK_REFERENCE_IMPORT_VECTOR_FINISH: &str = "__wpk_fork_ref_vector_finish"; + pub const WPK_FORK_REFERENCE_IMPORT_VECTOR_GET: &str = "__wpk_fork_ref_vector_get"; + + pub const WPK_FORK_EXCEPTION_EXPORT_DECODE: &str = "__wpk_fork_ref_decode_exnref"; + pub const WPK_FORK_EXCEPTION_EXPORT_ENCODE: &str = "__wpk_fork_ref_encode_exnref"; + pub const WPK_FORK_EXCEPTION_EXPORT_ABORT: &str = "__wpk_fork_ref_exn_abort"; + pub const WPK_FORK_EXCEPTION_EXPORT_CLEAR: &str = "__wpk_fork_ref_exn_clear"; + pub const WPK_FORK_EXCEPTION_EXPORT_ENCODE_INGRESS: &str = "__wpk_fork_ref_exn_encode_ingress"; + pub const WPK_FORK_EXCEPTION_EXPORT_MATERIALIZE: &str = "__wpk_fork_exception_materialize"; + pub const WPK_FORK_EXCEPTION_EXPORT_THROW_RECIPE: &str = "__wpk_fork_ref_exn_throw_recipe"; + pub const WPK_FORK_EXCEPTION_EXPORT_THROW_SLOT: &str = "__wpk_fork_ref_exn_throw_slot"; + pub const WPK_FORK_REFERENCE_EXPORT_GC_ALLOCATE: &str = "__wpk_fork_ref_gc_allocate"; + pub const WPK_FORK_REFERENCE_EXPORT_GC_ENCODE_SLOT: &str = "__wpk_fork_ref_gc_encode_slot"; + pub const WPK_FORK_REFERENCE_EXPORT_GC_FILL: &str = "__wpk_fork_ref_gc_fill"; + pub const WPK_FORK_REFERENCE_EXPORT_GC_PUBLISH_EXTERNREF: &str = + "__wpk_fork_ref_gc_publish_externref"; + pub const WPK_FORK_REFERENCE_EXPORT_GC_PROBE: &str = "__wpk_fork_ref_gc_probe"; pub const WPK_FORK_EXPORT_ABORT_BEGIN: &str = "wpk_fork_abort_begin"; pub const WPK_FORK_EXPORT_ABORT_END: &str = "wpk_fork_abort_end"; + pub const WPK_FORK_EXPORT_MODULE_BOOTSTRAP: &str = "wpk_fork_module_bootstrap"; + pub const WPK_FORK_EXPORT_MODULE_THREAD_BOOTSTRAP: &str = "wpk_fork_module_thread_bootstrap"; + pub const WPK_FORK_EXPORT_MODULE_STATE_FINISH_RESTORE: &str = + "wpk_fork_module_state_finish_restore"; + pub const WPK_FORK_EXPORT_MODULE_STATE_RESTORE: &str = "wpk_fork_module_state_restore"; + pub const WPK_FORK_EXPORT_MODULE_STATE_SAVE: &str = "wpk_fork_module_state_save"; + pub const WPK_FORK_EXPORT_MODULE_TABLE_STATE_SAVE: &str = "wpk_fork_module_table_state_save"; + pub const WPK_FORK_EXPORT_MODULE_TABLE_STATE_RESTORE: &str = + "wpk_fork_module_table_state_restore"; + pub const WPK_FORK_EXPORT_RESUME_START: &str = "wpk_fork_resume_start"; + pub const WPK_FORK_EXPORT_RESUME_THREAD: &str = "wpk_fork_resume_thread"; pub const WPK_FORK_EXPORT_REWIND_BEGIN: &str = "wpk_fork_rewind_begin"; pub const WPK_FORK_EXPORT_REWIND_END: &str = "wpk_fork_rewind_end"; pub const WPK_FORK_EXPORT_STATE: &str = "wpk_fork_state"; pub const WPK_FORK_EXPORT_UNWIND_BEGIN: &str = "wpk_fork_unwind_begin"; pub const WPK_FORK_EXPORT_UNWIND_END: &str = "wpk_fork_unwind_end"; - use ProgramArtifactValueType::{I32, Pointer}; + use ProgramArtifactValueType::{AnyRef, ExnRef, ExternRef, FuncRef, I32, I64, Pointer}; pub const WPK_FORK_REQUIRED_IMPORTS: &[ProgramArtifactImport] = &[ ProgramArtifactImport { @@ -1352,15 +1749,356 @@ pub mod abi { params: &[Pointer], results: &[Pointer], }, + ProgramArtifactImport { + module: WPK_FORK_FRAME_IMPORT_MODULE, + name: WPK_FORK_FRAME_IMPORT_PEEK, + params: &[Pointer], + results: &[Pointer], + }, ProgramArtifactImport { module: WPK_FORK_FRAME_IMPORT_MODULE, name: WPK_FORK_FRAME_IMPORT_RESERVE, params: &[Pointer], results: &[Pointer], }, + ProgramArtifactImport { + module: WPK_FORK_MODULE_STATE_IMPORT_MODULE, + name: WPK_FORK_MODULE_STATE_IMPORT_RECORD_COMMIT, + params: &[Pointer], + results: &[], + }, + ProgramArtifactImport { + module: WPK_FORK_MODULE_STATE_IMPORT_MODULE, + name: WPK_FORK_MODULE_STATE_IMPORT_RECORD_FIND, + params: &[I32, I32, I32, I32], + results: &[Pointer], + }, + ProgramArtifactImport { + module: WPK_FORK_MODULE_STATE_IMPORT_MODULE, + name: WPK_FORK_MODULE_STATE_IMPORT_RECORD_RESERVE, + params: &[I32, I32, I32, Pointer], + results: &[Pointer], + }, + ProgramArtifactImport { + module: WPK_FORK_MODULE_STATE_IMPORT_MODULE, + name: WPK_FORK_MODULE_STATE_IMPORT_TABLE_DIRTY_COUNT, + params: &[I32], + results: &[I32], + }, + ProgramArtifactImport { + module: WPK_FORK_MODULE_STATE_IMPORT_MODULE, + name: WPK_FORK_MODULE_STATE_IMPORT_TABLE_DIRTY_MARK, + params: &[I32, I64, I64], + results: &[], + }, + ProgramArtifactImport { + module: WPK_FORK_MODULE_STATE_IMPORT_MODULE, + name: WPK_FORK_MODULE_STATE_IMPORT_TABLE_DIRTY_PAGE, + params: &[I32, I32], + results: &[I64], + }, + ProgramArtifactImport { + module: WPK_FORK_MODULE_STATE_IMPORT_MODULE, + name: WPK_FORK_MODULE_STATE_IMPORT_TABLE_MUTATION_ABORT, + params: &[], + results: &[], + }, + ProgramArtifactImport { + module: WPK_FORK_MODULE_STATE_IMPORT_MODULE, + name: WPK_FORK_MODULE_STATE_IMPORT_TABLE_MUTATION_BEGIN, + params: &[], + results: &[I64], + }, + ProgramArtifactImport { + module: WPK_FORK_MODULE_STATE_IMPORT_MODULE, + name: WPK_FORK_MODULE_STATE_IMPORT_TABLE_MUTATION_COMMIT, + params: &[I32, I64, I64], + results: &[], + }, + ProgramArtifactImport { + module: WPK_FORK_MODULE_STATE_IMPORT_MODULE, + name: WPK_FORK_MODULE_STATE_IMPORT_TABLE_RECONCILE, + params: &[], + results: &[I64], + }, + ProgramArtifactImport { + module: WPK_FORK_MODULE_STATE_IMPORT_MODULE, + name: WPK_FORK_MODULE_STATE_IMPORT_TABLE_STATE_OWNED, + params: &[I32], + results: &[I32], + }, + ProgramArtifactImport { + module: WPK_FORK_REFERENCE_CODEC_IMPORT_MODULE, + name: WPK_FORK_REFERENCE_IMPORT_DECODE_FUNCREF, + params: &[I32], + results: &[FuncRef], + }, + ProgramArtifactImport { + module: WPK_FORK_REFERENCE_CODEC_IMPORT_MODULE, + name: WPK_FORK_REFERENCE_IMPORT_ENCODE_FUNCREF, + params: &[FuncRef], + results: &[I32], + }, + ProgramArtifactImport { + module: WPK_FORK_EXCEPTION_CODEC_IMPORT_MODULE, + name: WPK_FORK_EXCEPTION_IMPORT_BROKER_ENCODE, + params: &[I32], + results: &[I32], + }, + ProgramArtifactImport { + module: WPK_FORK_EXCEPTION_CODEC_IMPORT_MODULE, + name: WPK_FORK_EXCEPTION_IMPORT_BROKER_THROW_RECIPE, + params: &[I32], + results: &[], + }, + ProgramArtifactImport { + module: WPK_FORK_EXCEPTION_CODEC_IMPORT_MODULE, + name: WPK_FORK_EXCEPTION_IMPORT_CACHE_INDEX, + params: &[I32], + results: &[I32], + }, + ProgramArtifactImport { + module: WPK_FORK_EXCEPTION_CODEC_IMPORT_MODULE, + name: WPK_FORK_EXCEPTION_IMPORT_CLAIM, + params: &[I32], + results: &[I32], + }, + ProgramArtifactImport { + module: WPK_FORK_EXCEPTION_CODEC_IMPORT_MODULE, + name: WPK_FORK_EXCEPTION_IMPORT_DEFINE, + params: &[I32, I32, I32, I32, Pointer, I32, Pointer, I32], + results: &[], + }, + ProgramArtifactImport { + module: WPK_FORK_EXCEPTION_CODEC_IMPORT_MODULE, + name: WPK_FORK_EXCEPTION_IMPORT_INGRESS_THROW, + params: &[I32], + results: &[], + }, + ProgramArtifactImport { + module: WPK_FORK_EXCEPTION_CODEC_IMPORT_MODULE, + name: WPK_FORK_EXCEPTION_IMPORT_LOAD, + params: &[I32, I32, I32, I32, Pointer, I32, Pointer, I32], + results: &[I32], + }, + ProgramArtifactImport { + module: WPK_FORK_EXCEPTION_CODEC_IMPORT_MODULE, + name: WPK_FORK_EXCEPTION_IMPORT_LOOKUP, + params: &[I32], + results: &[I32], + }, + ProgramArtifactImport { + module: WPK_FORK_EXCEPTION_CODEC_IMPORT_MODULE, + name: WPK_FORK_EXCEPTION_IMPORT_ROUTE, + params: &[I32, I32], + results: &[I32], + }, + ProgramArtifactImport { + module: WPK_FORK_REFERENCE_CODEC_IMPORT_MODULE, + name: WPK_FORK_REFERENCE_IMPORT_GC_BROKER_ENCODE, + params: &[I32], + results: &[I32], + }, + ProgramArtifactImport { + module: WPK_FORK_REFERENCE_CODEC_IMPORT_MODULE, + name: WPK_FORK_REFERENCE_IMPORT_GC_CAPTURE_LAYOUT, + params: &[I32, I32, I32], + results: &[I32], + }, + ProgramArtifactImport { + module: WPK_FORK_REFERENCE_CODEC_IMPORT_MODULE, + name: WPK_FORK_REFERENCE_IMPORT_GC_CLAIM, + params: &[I32], + results: &[I32], + }, + ProgramArtifactImport { + module: WPK_FORK_REFERENCE_CODEC_IMPORT_MODULE, + name: WPK_FORK_REFERENCE_IMPORT_GC_DEFINE, + params: &[I32, I32, I32, I32, I32, Pointer, I32, I32], + results: &[], + }, + ProgramArtifactImport { + module: WPK_FORK_REFERENCE_CODEC_IMPORT_MODULE, + name: WPK_FORK_REFERENCE_IMPORT_GC_I31, + params: &[I32], + results: &[I32], + }, + ProgramArtifactImport { + module: WPK_FORK_REFERENCE_CODEC_IMPORT_MODULE, + name: WPK_FORK_REFERENCE_IMPORT_GC_LOAD, + params: &[I32, I32, I32, I32, I32, Pointer, I32], + results: &[I32], + }, + ProgramArtifactImport { + module: WPK_FORK_REFERENCE_CODEC_IMPORT_MODULE, + name: WPK_FORK_REFERENCE_IMPORT_GC_LOOKUP, + params: &[I32], + results: &[I32], + }, + ProgramArtifactImport { + module: WPK_FORK_REFERENCE_CODEC_IMPORT_MODULE, + name: WPK_FORK_REFERENCE_IMPORT_GC_PAYLOAD_LEN, + params: &[I32, I32, I32], + results: &[I32], + }, + ProgramArtifactImport { + module: WPK_FORK_REFERENCE_CODEC_IMPORT_MODULE, + name: WPK_FORK_REFERENCE_IMPORT_GC_PROVENANCE_BEGIN, + params: &[I32, I32, I32, I32, I64, I64, I32], + results: &[I32], + }, + ProgramArtifactImport { + module: WPK_FORK_REFERENCE_CODEC_IMPORT_MODULE, + name: WPK_FORK_REFERENCE_IMPORT_GC_PROVENANCE_END, + params: &[I32], + results: &[], + }, + ProgramArtifactImport { + module: WPK_FORK_REFERENCE_CODEC_IMPORT_MODULE, + name: WPK_FORK_REFERENCE_IMPORT_GC_PROVENANCE_REF, + params: &[I32, I32, I32], + results: &[], + }, + ProgramArtifactImport { + module: WPK_FORK_REFERENCE_CODEC_IMPORT_MODULE, + name: WPK_FORK_REFERENCE_IMPORT_GC_ROUTE, + params: &[I32, I32], + results: &[I32], + }, + ProgramArtifactImport { + module: WPK_FORK_REFERENCE_CODEC_IMPORT_MODULE, + name: WPK_FORK_REFERENCE_IMPORT_SCRATCH_RELEASE, + params: &[Pointer, Pointer], + results: &[], + }, + ProgramArtifactImport { + module: WPK_FORK_REFERENCE_CODEC_IMPORT_MODULE, + name: WPK_FORK_REFERENCE_IMPORT_SCRATCH_RESERVE, + params: &[Pointer], + results: &[Pointer], + }, + ProgramArtifactImport { + module: WPK_FORK_REFERENCE_CODEC_IMPORT_MODULE, + name: WPK_FORK_REFERENCE_IMPORT_VECTOR_APPEND, + params: &[I32, I32], + results: &[], + }, + ProgramArtifactImport { + module: WPK_FORK_REFERENCE_CODEC_IMPORT_MODULE, + name: WPK_FORK_REFERENCE_IMPORT_VECTOR_BEGIN, + params: &[I32], + results: &[I32], + }, + ProgramArtifactImport { + module: WPK_FORK_REFERENCE_CODEC_IMPORT_MODULE, + name: WPK_FORK_REFERENCE_IMPORT_VECTOR_FINISH, + params: &[I32], + results: &[I32], + }, + ProgramArtifactImport { + module: WPK_FORK_REFERENCE_CODEC_IMPORT_MODULE, + name: WPK_FORK_REFERENCE_IMPORT_VECTOR_GET, + params: &[I32, I32], + results: &[I32], + }, + ProgramArtifactImport { + module: WPK_FORK_FRAME_IMPORT_MODULE, + name: WPK_FORK_RESUME_IMPORT_PEEK, + params: &[I32], + results: &[I32], + }, + ]; + + pub const WPK_FORK_REQUIRED_TABLE_IMPORTS: &[ProgramArtifactTableImport] = &[ + ProgramArtifactTableImport { + module: WPK_FORK_REFERENCE_CODEC_IMPORT_MODULE, + name: WPK_FORK_REFERENCE_IMPORT_GC_TRANSIT, + table64: false, + element: AnyRef, + minimum: 1, + maximum: None, + }, + ProgramArtifactTableImport { + module: WPK_FORK_FRAME_IMPORT_MODULE, + name: WPK_FORK_RESUME_IMPORT_TABLE, + table64: false, + element: FuncRef, + minimum: 1, + maximum: None, + }, ]; pub const WPK_FORK_REQUIRED_EXPORTS: &[ProgramArtifactExport] = &[ + ProgramArtifactExport { + name: WPK_FORK_EXCEPTION_EXPORT_MATERIALIZE, + params: &[I32], + results: &[], + }, + ProgramArtifactExport { + name: WPK_FORK_EXCEPTION_EXPORT_DECODE, + params: &[I32], + results: &[ExnRef], + }, + ProgramArtifactExport { + name: WPK_FORK_EXCEPTION_EXPORT_ENCODE, + params: &[ExnRef], + results: &[I32], + }, + ProgramArtifactExport { + name: WPK_FORK_EXCEPTION_EXPORT_ABORT, + params: &[], + results: &[], + }, + ProgramArtifactExport { + name: WPK_FORK_EXCEPTION_EXPORT_CLEAR, + params: &[], + results: &[], + }, + ProgramArtifactExport { + name: WPK_FORK_EXCEPTION_EXPORT_ENCODE_INGRESS, + params: &[I32], + results: &[I32], + }, + ProgramArtifactExport { + name: WPK_FORK_EXCEPTION_EXPORT_THROW_RECIPE, + params: &[I32], + results: &[], + }, + ProgramArtifactExport { + name: WPK_FORK_EXCEPTION_EXPORT_THROW_SLOT, + params: &[I32], + results: &[], + }, + ProgramArtifactExport { + name: WPK_FORK_REFERENCE_EXPORT_GC_ALLOCATE, + params: &[I32], + results: &[], + }, + ProgramArtifactExport { + name: WPK_FORK_REFERENCE_EXPORT_GC_ENCODE_SLOT, + params: &[I32], + results: &[I32], + }, + ProgramArtifactExport { + name: WPK_FORK_REFERENCE_EXPORT_GC_FILL, + params: &[I32], + results: &[], + }, + ProgramArtifactExport { + name: WPK_FORK_REFERENCE_EXPORT_GC_PROBE, + params: &[I32], + results: &[I64], + }, + ProgramArtifactExport { + name: WPK_FORK_REFERENCE_EXPORT_GC_PUBLISH_EXTERNREF, + params: &[I32, ExternRef], + results: &[], + }, + ProgramArtifactExport { + name: WPK_FORK_STATIC_ROOT_HARVEST_EXPORT, + params: &[], + results: &[], + }, ProgramArtifactExport { name: WPK_FORK_EXPORT_ABORT_BEGIN, params: &[Pointer], @@ -1371,6 +2109,41 @@ pub mod abi { params: &[], results: &[], }, + ProgramArtifactExport { + name: WPK_FORK_EXPORT_MODULE_BOOTSTRAP, + params: &[], + results: &[], + }, + ProgramArtifactExport { + name: WPK_FORK_EXPORT_MODULE_STATE_FINISH_RESTORE, + params: &[I32], + results: &[], + }, + ProgramArtifactExport { + name: WPK_FORK_EXPORT_MODULE_STATE_RESTORE, + params: &[I32], + results: &[], + }, + ProgramArtifactExport { + name: WPK_FORK_EXPORT_MODULE_STATE_SAVE, + params: &[I32], + results: &[], + }, + ProgramArtifactExport { + name: WPK_FORK_EXPORT_MODULE_TABLE_STATE_RESTORE, + params: &[I32], + results: &[], + }, + ProgramArtifactExport { + name: WPK_FORK_EXPORT_MODULE_TABLE_STATE_SAVE, + params: &[I32], + results: &[], + }, + ProgramArtifactExport { + name: WPK_FORK_EXPORT_MODULE_THREAD_BOOTSTRAP, + params: &[], + results: &[], + }, ProgramArtifactExport { name: WPK_FORK_EXPORT_REWIND_BEGIN, params: &[Pointer], @@ -1417,6 +2190,16 @@ pub mod abi { } } + /// Return the version-1 module-state chunk header size for one pointer + /// width, including the required eight-byte alignment. + pub const fn wpk_fork_module_state_chunk_header_size(pointer_width: u8) -> Option { + match pointer_width { + 4 => Some(40), + 8 => Some(56), + _ => None, + } + } + /// Patterns (applied as prefix match) for kernel-wasm exports that /// are implementation details of the toolchain, not part of the /// host/kernel ABI. The snapshot excludes any export whose name @@ -2003,10 +2786,49 @@ pub mod abi { HOST_ADAPTER_MANIFEST_VERSION, HOST_ADAPTER_OPTIONAL_KERNEL_EXPORTS, HOST_ADAPTER_REQUIRED_KERNEL_EXPORTS, HOST_ADAPTER_REQUIRED_WORKER_FEATURES, HOST_ADAPTER_VERSION, HOST_ADAPTER_WORKER_FEATURES, + WPK_FORK_ACTIVATION_CONTINUATION_ENTRY_SIZE, + WPK_FORK_ACTIVATION_CONTINUATIONS_HEADER_SIZE, WPK_FORK_ACTIVATION_CONTINUATIONS_MAGIC, + WPK_FORK_ACTIVATION_CONTINUATIONS_OWNER, WPK_FORK_EXCEPTION_CODEC_HEADER_SIZE, + WPK_FORK_EXCEPTION_CODEC_SECTION, WPK_FORK_EXCEPTION_CODEC_TAG_RECORD_SIZE, + WPK_FORK_EXCEPTION_CODEC_VERSION, WPK_FORK_EXCEPTION_IMPORT_ACTIVATION, + WPK_FORK_GC_CODEC_FIELD_RECORD_SIZE, WPK_FORK_GC_CODEC_HEADER_SIZE, + WPK_FORK_GC_CODEC_LAYOUT_RECORD_SIZE, WPK_FORK_GC_CODEC_MAGIC, + WPK_FORK_IMPORTED_GLOBAL_BINDINGS_ENTRY_SIZE, + WPK_FORK_IMPORTED_GLOBAL_BINDINGS_HEADER_SIZE, WPK_FORK_IMPORTED_GLOBAL_BINDINGS_MAGIC, + WPK_FORK_IMPORTED_GLOBAL_BINDINGS_OWNER, WPK_FORK_IMPORTED_GLOBAL_FLAG_MUTABLE, + WPK_FORK_IMPORTED_GLOBAL_FLAG_SHARED, WPK_FORK_IMPORTED_GLOBAL_KNOWN_FLAGS, + WPK_FORK_IMPORTED_GLOBALS_HEADER_SIZE, WPK_FORK_IMPORTED_GLOBALS_MAGIC, + WPK_FORK_IMPORTED_GLOBALS_RECORD_HEADER_SIZE, + WPK_FORK_IMPORTED_TABLE_BINDINGS_ENTRY_SIZE, + WPK_FORK_IMPORTED_TABLE_BINDINGS_HEADER_SIZE, WPK_FORK_IMPORTED_TABLE_BINDINGS_MAGIC, + WPK_FORK_IMPORTED_TABLE_BINDINGS_OWNER, WPK_FORK_IMPORTED_TABLES_HEADER_SIZE, + WPK_FORK_IMPORTED_TABLES_MAGIC, WPK_FORK_IMPORTED_TABLES_RECORD_HEADER_SIZE, WPK_FORK_LINKED_FRAME_DESCRIPTOR_SIZE, WPK_FORK_LINKED_FRAME_FORMAT_MAGIC, WPK_FORK_LINKED_FRAME_POINTER_WIDTHS, WPK_FORK_LINKED_FRAME_REQUIRED_FLAGS, - WPK_FORK_REQUIRED_EXPORTS, WPK_FORK_REQUIRED_IMPORTS, extended_syscalls::SYSCALLS, - wpk_fork_linked_chunk_header_size, wpk_fork_linked_node_header_size, + WPK_FORK_MODULE_STATE_ARENA_VERSION, WPK_FORK_MODULE_STATE_CHUNK_KNOWN_FLAGS, + WPK_FORK_MODULE_STATE_CHUNK_MAGIC, WPK_FORK_MODULE_STATE_DATA_SEGMENT_HEADER_SIZE, + WPK_FORK_MODULE_STATE_DESCRIPTOR_SIZE, + WPK_FORK_MODULE_STATE_ELEMENT_SEGMENT_HEADER_SIZE, WPK_FORK_MODULE_STATE_FORMAT_MAGIC, + WPK_FORK_MODULE_STATE_KNOWN_FLAGS, WPK_FORK_MODULE_STATE_MAX_TABLE_PAGE_SHIFT, + WPK_FORK_MODULE_STATE_MIN_TABLE_PAGE_SHIFT, + WPK_FORK_MODULE_STATE_MODULE_RECORD_PAYLOAD_SIZE, + WPK_FORK_MODULE_STATE_MODULE_TEMPLATE_ID_SIZE, WPK_FORK_MODULE_STATE_POINTER_WIDTHS, + WPK_FORK_MODULE_STATE_RECORD_HEADER_SIZE, WPK_FORK_MODULE_STATE_RECORD_KINDS, + WPK_FORK_MODULE_STATE_RECORD_MAGIC, WPK_FORK_MODULE_STATE_RECORD_VERSION, + WPK_FORK_MODULE_STATE_REPLAY_EVENT_SEGMENT_CAPACITY, + WPK_FORK_MODULE_STATE_REPLAY_EVENT_SEGMENT_HEADER_SIZE, + WPK_FORK_MODULE_STATE_REPLAY_EVENT_SEGMENT_VERSION, + WPK_FORK_MODULE_STATE_REPLAY_EVENT_SIZE, + WPK_FORK_MODULE_STATE_REPLAY_EVENTS_HEADER_SIZE, + WPK_FORK_MODULE_STATE_REPLAY_EVENTS_MAGIC, WPK_FORK_MODULE_STATE_REPLAY_EVENTS_OWNER, + WPK_FORK_MODULE_STATE_REPLAY_EVENTS_VERSION, WPK_FORK_MODULE_STATE_REQUIRED_FLAGS, + WPK_FORK_MODULE_STATE_ROOT_POINTER_WORD_OFFSET, + WPK_FORK_MODULE_STATE_TABLE_BASELINE_FINGERPRINT_SIZE, + WPK_FORK_MODULE_STATE_TABLE_DESCRIPTOR_PAYLOAD_SIZE, + WPK_FORK_MODULE_STATE_TABLE_PAGE_SHIFT, WPK_FORK_REQUIRED_EXPORTS, + WPK_FORK_REQUIRED_IMPORTS, WPK_FORK_REQUIRED_TABLE_IMPORTS, + extended_syscalls::SYSCALLS, wpk_fork_linked_chunk_header_size, + wpk_fork_linked_node_header_size, wpk_fork_module_state_chunk_header_size, }; use crate::Syscall; @@ -2109,28 +2931,128 @@ pub mod abi { assert_eq!(wpk_fork_linked_chunk_header_size(16), None); assert_eq!(wpk_fork_linked_node_header_size(16), None); - assert_eq!(WPK_FORK_REQUIRED_IMPORTS.len(), 3); + assert_eq!(WPK_FORK_REQUIRED_IMPORTS.len(), 45); let mut previous_import = ("", ""); for requirement in WPK_FORK_REQUIRED_IMPORTS { let current = (requirement.module, requirement.name); assert!( previous_import < current, - "fork imports must be sorted and unique" + "fork imports must be sorted and unique: \ + previous={previous_import:?}, current={current:?}" ); previous_import = current; } - assert_eq!(WPK_FORK_REQUIRED_EXPORTS.len(), 7); + assert_eq!(WPK_FORK_REQUIRED_TABLE_IMPORTS.len(), 2); + let mut previous_table_import = ("", ""); + for requirement in WPK_FORK_REQUIRED_TABLE_IMPORTS { + let current = (requirement.module, requirement.name); + assert!( + previous_table_import < current, + "fork table imports must be sorted and unique" + ); + previous_table_import = current; + } + assert_eq!(WPK_FORK_REQUIRED_EXPORTS.len(), 28); let mut previous_export = ""; for requirement in WPK_FORK_REQUIRED_EXPORTS { assert!( previous_export < requirement.name, - "fork exports must be sorted and unique" + "fork exports must be sorted and unique: \ + previous={previous_export:?}, current={:?}", + requirement.name, ); previous_export = requirement.name; } } + #[test] + fn module_state_recipe_contract_is_complete_and_sorted() { + assert_eq!(WPK_FORK_MODULE_STATE_FORMAT_MAGIC, *b"KFMD"); + assert_eq!(WPK_FORK_MODULE_STATE_DESCRIPTOR_SIZE, 24); + assert_eq!(WPK_FORK_MODULE_STATE_REQUIRED_FLAGS, 0b111); + assert_eq!( + WPK_FORK_MODULE_STATE_KNOWN_FLAGS, + WPK_FORK_MODULE_STATE_REQUIRED_FLAGS + ); + assert_eq!(WPK_FORK_MODULE_STATE_ARENA_VERSION, 1); + assert_eq!(WPK_FORK_MODULE_STATE_RECORD_VERSION, 1); + assert_eq!(WPK_FORK_MODULE_STATE_ROOT_POINTER_WORD_OFFSET, 1); + assert_eq!(WPK_FORK_MODULE_STATE_CHUNK_MAGIC, *b"KFMC"); + assert_eq!(WPK_FORK_MODULE_STATE_CHUNK_KNOWN_FLAGS, 0b11); + assert_eq!(WPK_FORK_MODULE_STATE_RECORD_MAGIC, *b"KFMR"); + assert_eq!(WPK_FORK_MODULE_STATE_RECORD_HEADER_SIZE, 24); + assert_eq!(WPK_FORK_MODULE_STATE_POINTER_WIDTHS, &[4, 8]); + assert_eq!(wpk_fork_module_state_chunk_header_size(4), Some(40)); + assert_eq!(wpk_fork_module_state_chunk_header_size(8), Some(56)); + assert_eq!(wpk_fork_module_state_chunk_header_size(16), None); + + let mut previous_number = 0; + for kind in WPK_FORK_MODULE_STATE_RECORD_KINDS { + assert!( + previous_number < kind.number, + "module-state record kinds must be sorted and unique" + ); + assert!(!kind.name.is_empty()); + previous_number = kind.number; + } + assert_eq!(WPK_FORK_MODULE_STATE_RECORD_KINDS.len(), 13); + + assert_eq!(WPK_FORK_MODULE_STATE_MODULE_TEMPLATE_ID_SIZE, 32); + assert_eq!(WPK_FORK_MODULE_STATE_MODULE_RECORD_PAYLOAD_SIZE, 40); + assert_eq!(WPK_FORK_MODULE_STATE_TABLE_BASELINE_FINGERPRINT_SIZE, 32); + assert_eq!(WPK_FORK_MODULE_STATE_TABLE_DESCRIPTOR_PAYLOAD_SIZE, 56); + assert_eq!(WPK_FORK_MODULE_STATE_ELEMENT_SEGMENT_HEADER_SIZE, 8); + assert_eq!(WPK_FORK_MODULE_STATE_DATA_SEGMENT_HEADER_SIZE, 8); + assert_eq!(WPK_FORK_MODULE_STATE_REPLAY_EVENTS_OWNER, 1); + assert_eq!(WPK_FORK_MODULE_STATE_REPLAY_EVENTS_MAGIC, *b"KFRE"); + assert_eq!(WPK_FORK_MODULE_STATE_REPLAY_EVENTS_VERSION, 2); + assert_eq!(WPK_FORK_MODULE_STATE_REPLAY_EVENTS_HEADER_SIZE, 40); + assert_eq!(WPK_FORK_MODULE_STATE_REPLAY_EVENT_SIZE, 8); + assert_eq!(WPK_FORK_MODULE_STATE_REPLAY_EVENT_SEGMENT_VERSION, 1); + assert_eq!(WPK_FORK_MODULE_STATE_REPLAY_EVENT_SEGMENT_HEADER_SIZE, 24); + assert_eq!(WPK_FORK_MODULE_STATE_REPLAY_EVENT_SEGMENT_CAPACITY, 4080); + assert_eq!(WPK_FORK_IMPORTED_GLOBAL_BINDINGS_MAGIC, *b"KFBG"); + assert_eq!(WPK_FORK_IMPORTED_GLOBAL_BINDINGS_OWNER, 2); + assert_eq!(WPK_FORK_IMPORTED_GLOBAL_BINDINGS_HEADER_SIZE, 24); + assert_eq!(WPK_FORK_IMPORTED_GLOBAL_BINDINGS_ENTRY_SIZE, 40); + assert_eq!(WPK_FORK_ACTIVATION_CONTINUATIONS_MAGIC, *b"KFAC"); + assert_eq!(WPK_FORK_ACTIVATION_CONTINUATIONS_OWNER, 3); + assert_eq!(WPK_FORK_ACTIVATION_CONTINUATIONS_HEADER_SIZE, 24); + assert_eq!(WPK_FORK_ACTIVATION_CONTINUATION_ENTRY_SIZE, 16); + assert_eq!(WPK_FORK_IMPORTED_TABLE_BINDINGS_MAGIC, *b"KFBT"); + assert_eq!(WPK_FORK_IMPORTED_TABLE_BINDINGS_OWNER, 4); + assert_eq!(WPK_FORK_IMPORTED_TABLE_BINDINGS_HEADER_SIZE, 24); + assert_eq!(WPK_FORK_IMPORTED_TABLE_BINDINGS_ENTRY_SIZE, 24); + assert_eq!(WPK_FORK_IMPORTED_GLOBALS_MAGIC, *b"KFIG"); + assert_eq!(WPK_FORK_IMPORTED_GLOBALS_HEADER_SIZE, 16); + assert_eq!(WPK_FORK_IMPORTED_GLOBALS_RECORD_HEADER_SIZE, 24); + assert_eq!(WPK_FORK_IMPORTED_GLOBAL_KNOWN_FLAGS, 0b11); + assert_eq!(WPK_FORK_IMPORTED_GLOBAL_FLAG_MUTABLE, 0b01); + assert_eq!(WPK_FORK_IMPORTED_GLOBAL_FLAG_SHARED, 0b10); + assert_eq!(WPK_FORK_IMPORTED_TABLES_MAGIC, *b"KFIT"); + assert_eq!(WPK_FORK_IMPORTED_TABLES_HEADER_SIZE, 16); + assert_eq!(WPK_FORK_IMPORTED_TABLES_RECORD_HEADER_SIZE, 24); + assert_eq!(WPK_FORK_GC_CODEC_MAGIC, *b"KFGC"); + assert_eq!(WPK_FORK_GC_CODEC_HEADER_SIZE, 16); + assert_eq!(WPK_FORK_GC_CODEC_LAYOUT_RECORD_SIZE, 44); + assert_eq!(WPK_FORK_GC_CODEC_FIELD_RECORD_SIZE, 12); + assert_eq!( + WPK_FORK_EXCEPTION_CODEC_SECTION, + "kandelo.wpk_fork.exception_codec" + ); + assert_eq!(WPK_FORK_EXCEPTION_CODEC_VERSION, 1); + assert_eq!(WPK_FORK_EXCEPTION_CODEC_HEADER_SIZE, 8); + assert_eq!(WPK_FORK_EXCEPTION_CODEC_TAG_RECORD_SIZE, 16); + assert_eq!( + WPK_FORK_EXCEPTION_IMPORT_ACTIVATION, + "__wpk_fork_module_activation" + ); + assert_eq!(WPK_FORK_MODULE_STATE_MIN_TABLE_PAGE_SHIFT, 4); + assert_eq!(WPK_FORK_MODULE_STATE_MAX_TABLE_PAGE_SHIFT, 20); + assert_eq!(WPK_FORK_MODULE_STATE_TABLE_PAGE_SHIFT, 10); + } + fn assert_sorted_unique(items: &[&str]) { let mut prev = None; for item in items { @@ -2299,7 +3221,6 @@ pub mod oss { pub const AFMT_S16_LE: u32 = 0x10; } - /// GLES / EGL ABI: ioctl numbers, opcode tables, and marshalled argument /// structs for `/dev/dri/renderD128`. /// @@ -2331,16 +3252,16 @@ pub mod gl { // built against an older op-table can't talk to a newer kernel (and vice // versa) without the divergence being caught at first contact rather than // surfacing later as a silent decode error. See A6's GLIO_INIT handler. - pub const GLIO_INIT: u32 = 0x40; - pub const GLIO_TERMINATE: u32 = 0x41; - pub const GLIO_CREATE_CONTEXT: u32 = 0x42; + pub const GLIO_INIT: u32 = 0x40; + pub const GLIO_TERMINATE: u32 = 0x41; + pub const GLIO_CREATE_CONTEXT: u32 = 0x42; pub const GLIO_DESTROY_CONTEXT: u32 = 0x43; - pub const GLIO_CREATE_SURFACE: u32 = 0x44; + pub const GLIO_CREATE_SURFACE: u32 = 0x44; pub const GLIO_DESTROY_SURFACE: u32 = 0x45; - pub const GLIO_MAKE_CURRENT: u32 = 0x46; - pub const GLIO_SUBMIT: u32 = 0x47; - pub const GLIO_PRESENT: u32 = 0x48; - pub const GLIO_QUERY: u32 = 0x49; + pub const GLIO_MAKE_CURRENT: u32 = 0x46; + pub const GLIO_SUBMIT: u32 = 0x47; + pub const GLIO_PRESENT: u32 = 0x48; + pub const GLIO_QUERY: u32 = 0x49; // --- surface kind tags ------------------------------------------------- @@ -2368,88 +3289,88 @@ pub mod gl { // endian. Payload formats are documented inline next to the libGLESv2 // stub call sites in glue/libglesv2_stub.c (Phase C). - pub const OP_CLEAR: u16 = 0x0001; - pub const OP_CLEAR_COLOR: u16 = 0x0002; - pub const OP_VIEWPORT: u16 = 0x0003; - pub const OP_SCISSOR: u16 = 0x0004; - pub const OP_ENABLE: u16 = 0x0005; - pub const OP_DISABLE: u16 = 0x0006; - pub const OP_BLEND_FUNC: u16 = 0x0007; - pub const OP_DEPTH_FUNC: u16 = 0x0008; - pub const OP_CULL_FACE: u16 = 0x0009; - pub const OP_FRONT_FACE: u16 = 0x000A; - pub const OP_LINE_WIDTH: u16 = 0x000B; - pub const OP_PIXEL_STOREI: u16 = 0x000C; - - pub const OP_GEN_BUFFERS: u16 = 0x0100; - pub const OP_DELETE_BUFFERS: u16 = 0x0101; - pub const OP_BIND_BUFFER: u16 = 0x0102; - pub const OP_BUFFER_DATA: u16 = 0x0103; - pub const OP_BUFFER_SUB_DATA: u16 = 0x0104; - - pub const OP_GEN_TEXTURES: u16 = 0x0200; - pub const OP_DELETE_TEXTURES: u16 = 0x0201; - pub const OP_BIND_TEXTURE: u16 = 0x0202; - pub const OP_TEX_IMAGE_2D: u16 = 0x0203; - pub const OP_TEX_SUB_IMAGE_2D: u16 = 0x0204; - pub const OP_TEX_PARAMETERI: u16 = 0x0205; - pub const OP_ACTIVE_TEXTURE: u16 = 0x0206; - pub const OP_GENERATE_MIPMAP: u16 = 0x0207; - - pub const OP_CREATE_SHADER: u16 = 0x0300; - pub const OP_SHADER_SOURCE: u16 = 0x0301; - pub const OP_COMPILE_SHADER: u16 = 0x0302; - pub const OP_DELETE_SHADER: u16 = 0x0303; - pub const OP_CREATE_PROGRAM: u16 = 0x0304; - pub const OP_ATTACH_SHADER: u16 = 0x0305; - pub const OP_LINK_PROGRAM: u16 = 0x0306; - pub const OP_USE_PROGRAM: u16 = 0x0307; - pub const OP_BIND_ATTRIB_LOCATION: u16 = 0x0308; - pub const OP_DELETE_PROGRAM: u16 = 0x0309; - - pub const OP_UNIFORM1I: u16 = 0x0400; - pub const OP_UNIFORM1F: u16 = 0x0401; - pub const OP_UNIFORM2F: u16 = 0x0402; - pub const OP_UNIFORM3F: u16 = 0x0403; - pub const OP_UNIFORM4F: u16 = 0x0404; - pub const OP_UNIFORM_MATRIX4FV: u16 = 0x0405; + pub const OP_CLEAR: u16 = 0x0001; + pub const OP_CLEAR_COLOR: u16 = 0x0002; + pub const OP_VIEWPORT: u16 = 0x0003; + pub const OP_SCISSOR: u16 = 0x0004; + pub const OP_ENABLE: u16 = 0x0005; + pub const OP_DISABLE: u16 = 0x0006; + pub const OP_BLEND_FUNC: u16 = 0x0007; + pub const OP_DEPTH_FUNC: u16 = 0x0008; + pub const OP_CULL_FACE: u16 = 0x0009; + pub const OP_FRONT_FACE: u16 = 0x000A; + pub const OP_LINE_WIDTH: u16 = 0x000B; + pub const OP_PIXEL_STOREI: u16 = 0x000C; + + pub const OP_GEN_BUFFERS: u16 = 0x0100; + pub const OP_DELETE_BUFFERS: u16 = 0x0101; + pub const OP_BIND_BUFFER: u16 = 0x0102; + pub const OP_BUFFER_DATA: u16 = 0x0103; + pub const OP_BUFFER_SUB_DATA: u16 = 0x0104; + + pub const OP_GEN_TEXTURES: u16 = 0x0200; + pub const OP_DELETE_TEXTURES: u16 = 0x0201; + pub const OP_BIND_TEXTURE: u16 = 0x0202; + pub const OP_TEX_IMAGE_2D: u16 = 0x0203; + pub const OP_TEX_SUB_IMAGE_2D: u16 = 0x0204; + pub const OP_TEX_PARAMETERI: u16 = 0x0205; + pub const OP_ACTIVE_TEXTURE: u16 = 0x0206; + pub const OP_GENERATE_MIPMAP: u16 = 0x0207; + + pub const OP_CREATE_SHADER: u16 = 0x0300; + pub const OP_SHADER_SOURCE: u16 = 0x0301; + pub const OP_COMPILE_SHADER: u16 = 0x0302; + pub const OP_DELETE_SHADER: u16 = 0x0303; + pub const OP_CREATE_PROGRAM: u16 = 0x0304; + pub const OP_ATTACH_SHADER: u16 = 0x0305; + pub const OP_LINK_PROGRAM: u16 = 0x0306; + pub const OP_USE_PROGRAM: u16 = 0x0307; + pub const OP_BIND_ATTRIB_LOCATION: u16 = 0x0308; + pub const OP_DELETE_PROGRAM: u16 = 0x0309; + + pub const OP_UNIFORM1I: u16 = 0x0400; + pub const OP_UNIFORM1F: u16 = 0x0401; + pub const OP_UNIFORM2F: u16 = 0x0402; + pub const OP_UNIFORM3F: u16 = 0x0403; + pub const OP_UNIFORM4F: u16 = 0x0404; + pub const OP_UNIFORM_MATRIX4FV: u16 = 0x0405; /// `glUniform4fv(location, count, value)` — vector form. es2gears uses /// this for the directional light position. `OP_UNIFORM4F` (scalar) is a /// different signature; both are needed. - pub const OP_UNIFORM4FV: u16 = 0x0406; + pub const OP_UNIFORM4FV: u16 = 0x0406; - pub const OP_ENABLE_VERTEX_ATTRIB_ARRAY: u16 = 0x0500; + pub const OP_ENABLE_VERTEX_ATTRIB_ARRAY: u16 = 0x0500; pub const OP_DISABLE_VERTEX_ATTRIB_ARRAY: u16 = 0x0501; - pub const OP_VERTEX_ATTRIB_POINTER: u16 = 0x0502; - pub const OP_DRAW_ARRAYS: u16 = 0x0503; - pub const OP_DRAW_ELEMENTS: u16 = 0x0504; - - pub const OP_GEN_VERTEX_ARRAYS: u16 = 0x0600; - pub const OP_DELETE_VERTEX_ARRAYS: u16 = 0x0601; - pub const OP_BIND_VERTEX_ARRAY: u16 = 0x0602; - - pub const OP_GEN_FRAMEBUFFERS: u16 = 0x0700; - pub const OP_BIND_FRAMEBUFFER: u16 = 0x0701; - pub const OP_FRAMEBUFFER_TEXTURE_2D: u16 = 0x0702; - pub const OP_GEN_RENDERBUFFERS: u16 = 0x0703; - pub const OP_BIND_RENDERBUFFER: u16 = 0x0704; - pub const OP_RENDERBUFFER_STORAGE: u16 = 0x0705; - pub const OP_FRAMEBUFFER_RENDERBUFFER: u16 = 0x0706; + pub const OP_VERTEX_ATTRIB_POINTER: u16 = 0x0502; + pub const OP_DRAW_ARRAYS: u16 = 0x0503; + pub const OP_DRAW_ELEMENTS: u16 = 0x0504; + + pub const OP_GEN_VERTEX_ARRAYS: u16 = 0x0600; + pub const OP_DELETE_VERTEX_ARRAYS: u16 = 0x0601; + pub const OP_BIND_VERTEX_ARRAY: u16 = 0x0602; + + pub const OP_GEN_FRAMEBUFFERS: u16 = 0x0700; + pub const OP_BIND_FRAMEBUFFER: u16 = 0x0701; + pub const OP_FRAMEBUFFER_TEXTURE_2D: u16 = 0x0702; + pub const OP_GEN_RENDERBUFFERS: u16 = 0x0703; + pub const OP_BIND_RENDERBUFFER: u16 = 0x0704; + pub const OP_RENDERBUFFER_STORAGE: u16 = 0x0705; + pub const OP_FRAMEBUFFER_RENDERBUFFER: u16 = 0x0706; // --- sync query op tags (used in GlQueryInfo.op) ----------------------- - pub const QOP_GET_ERROR: u32 = 0x01; - pub const QOP_GET_STRING: u32 = 0x02; - pub const QOP_GET_INTEGERV: u32 = 0x03; - pub const QOP_GET_FLOATV: u32 = 0x04; - pub const QOP_GET_UNIFORM_LOC: u32 = 0x05; - pub const QOP_GET_ATTRIB_LOC: u32 = 0x06; - pub const QOP_GET_SHADERIV: u32 = 0x07; - pub const QOP_GET_SHADER_INFO_LOG: u32 = 0x08; - pub const QOP_GET_PROGRAMIV: u32 = 0x09; - pub const QOP_GET_PROGRAM_INFO_LOG: u32 = 0x0A; - pub const QOP_READ_PIXELS: u32 = 0x0B; - pub const QOP_CHECK_FB_STATUS: u32 = 0x0C; + pub const QOP_GET_ERROR: u32 = 0x01; + pub const QOP_GET_STRING: u32 = 0x02; + pub const QOP_GET_INTEGERV: u32 = 0x03; + pub const QOP_GET_FLOATV: u32 = 0x04; + pub const QOP_GET_UNIFORM_LOC: u32 = 0x05; + pub const QOP_GET_ATTRIB_LOC: u32 = 0x06; + pub const QOP_GET_SHADERIV: u32 = 0x07; + pub const QOP_GET_SHADER_INFO_LOG: u32 = 0x08; + pub const QOP_GET_PROGRAMIV: u32 = 0x09; + pub const QOP_GET_PROGRAM_INFO_LOG: u32 = 0x0A; + pub const QOP_READ_PIXELS: u32 = 0x0B; + pub const QOP_CHECK_FB_STATUS: u32 = 0x0C; // --- marshalled ioctl argument structs --------------------------------- @@ -2583,14 +3504,14 @@ pub mod dri { #[repr(C)] #[derive(Clone, Copy, Default)] pub struct WpkDrmModeCreateDumb { - pub height: u32, // 0 in - pub width: u32, // 4 in - pub bpp: u32, // 8 in bits-per-pixel (32 for ARGB8888) - pub flags: u32, // 12 in must be 0 - pub handle: u32, // 16 out process-local bo handle - pub pitch: u32, // 20 out stride in bytes - pub size: u64, // 24 out total bytes (pitch * height) - // total: 32 + pub height: u32, // 0 in + pub width: u32, // 4 in + pub bpp: u32, // 8 in bits-per-pixel (32 for ARGB8888) + pub flags: u32, // 12 in must be 0 + pub handle: u32, // 16 out process-local bo handle + pub pitch: u32, // 20 out stride in bytes + pub size: u64, // 24 out total bytes (pitch * height) + // total: 32 } /// Linux `struct drm_mode_map_dumb` (16 bytes). @@ -2652,16 +3573,16 @@ pub mod dri { #[repr(C)] #[derive(Clone, Copy, Default)] pub struct WpkDrmVersion { - pub version_major: i32, // 0 - pub version_minor: i32, // 4 - pub version_patchlevel: i32, // 8 - pub name_len: u32, // 12 in/out - pub name_ptr: u32, // 16 wasm32 user pointer - pub date_len: u32, // 20 in/out - pub date_ptr: u32, // 24 wasm32 user pointer - pub desc_len: u32, // 28 in/out - pub desc_ptr: u32, // 32 wasm32 user pointer - // total: 36 + pub version_major: i32, // 0 + pub version_minor: i32, // 4 + pub version_patchlevel: i32, // 8 + pub name_len: u32, // 12 in/out + pub name_ptr: u32, // 16 wasm32 user pointer + pub date_len: u32, // 20 in/out + pub date_ptr: u32, // 24 wasm32 user pointer + pub desc_len: u32, // 28 in/out + pub desc_ptr: u32, // 32 wasm32 user pointer + // total: 36 } // --- WPK extensions ('d' magic, nrs 0xE0+ — unused by Linux 6.x) ---- @@ -2695,11 +3616,11 @@ pub mod dri { #[repr(C)] #[derive(Clone, Copy, Default)] pub struct WpkDrmGpuBoCreate { - pub width: u32, // 0 in - pub height: u32, // 4 in - pub format: u32, // 8 in DRM_FORMAT_* (ARGB8888 etc.) - pub usage: u32, // 12 in GBM_BO_USE_* bitmask - // total: 16 + pub width: u32, // 0 in + pub height: u32, // 4 in + pub format: u32, // 8 in DRM_FORMAT_* (ARGB8888 etc.) + pub usage: u32, // 12 in GBM_BO_USE_* bitmask + // total: 16 } /// `BIND_FOREIGN_TEXTURE` argument. 16 bytes on wasm32 (4 × u32). After @@ -2713,11 +3634,11 @@ pub mod dri { #[repr(C)] #[derive(Clone, Copy, Default)] pub struct WpkDrmBindForeignTexture { - pub bo_handle: u32, // 0 in caller's local GEM handle - pub gl_target: u32, // 4 in GL_TEXTURE_2D etc. - pub ctx_id: u32, // 8 in caller's GL ctx_id + pub bo_handle: u32, // 0 in caller's local GEM handle + pub gl_target: u32, // 4 in GL_TEXTURE_2D etc. + pub ctx_id: u32, // 8 in caller's GL ctx_id pub gl_texture_id: u32, // 12 out the WebGLTexture id assigned - // (also writable as a sampler binding) + // (also writable as a sampler binding) } // --- KMS ioctls ('d' magic, Linux UAPI) ------------------------------- @@ -2771,92 +3692,92 @@ pub mod dri { #[repr(C)] #[derive(Clone, Copy, Default)] pub struct WpkDrmModeCardRes { - pub fb_id_ptr: u64, // 0 in - pub crtc_id_ptr: u64, // 8 in - pub connector_id_ptr: u64, // 16 in - pub encoder_id_ptr: u64, // 24 in - pub count_fbs: u32, // 32 in/out - pub count_crtcs: u32, // 36 in/out - pub count_connectors: u32, // 40 in/out - pub count_encoders: u32, // 44 in/out - pub min_width: u32, // 48 out - pub max_width: u32, // 52 out - pub min_height: u32, // 56 out - pub max_height: u32, // 60 out - // total: 64 + pub fb_id_ptr: u64, // 0 in + pub crtc_id_ptr: u64, // 8 in + pub connector_id_ptr: u64, // 16 in + pub encoder_id_ptr: u64, // 24 in + pub count_fbs: u32, // 32 in/out + pub count_crtcs: u32, // 36 in/out + pub count_connectors: u32, // 40 in/out + pub count_encoders: u32, // 44 in/out + pub min_width: u32, // 48 out + pub max_width: u32, // 52 out + pub min_height: u32, // 56 out + pub max_height: u32, // 60 out + // total: 64 } /// `struct drm_mode_modeinfo`. 68 bytes. #[repr(C)] #[derive(Clone, Copy, Default)] pub struct WpkDrmModeModeinfo { - pub clock: u32, // 0 - pub hdisplay: u16, // 4 - pub hsync_start: u16, // 6 - pub hsync_end: u16, // 8 - pub htotal: u16, // 10 - pub hskew: u16, // 12 - pub vdisplay: u16, // 14 - pub vsync_start: u16, // 16 - pub vsync_end: u16, // 18 - pub vtotal: u16, // 20 - pub vscan: u16, // 22 - pub vrefresh: u32, // 24 - pub flags: u32, // 28 - pub mode_type: u32, // 32 - pub name: [u8; 32], // 36..68 - // total: 68 + pub clock: u32, // 0 + pub hdisplay: u16, // 4 + pub hsync_start: u16, // 6 + pub hsync_end: u16, // 8 + pub htotal: u16, // 10 + pub hskew: u16, // 12 + pub vdisplay: u16, // 14 + pub vsync_start: u16, // 16 + pub vsync_end: u16, // 18 + pub vtotal: u16, // 20 + pub vscan: u16, // 22 + pub vrefresh: u32, // 24 + pub flags: u32, // 28 + pub mode_type: u32, // 32 + pub name: [u8; 32], // 36..68 + // total: 68 } /// `struct drm_mode_crtc`. 104 bytes (embedded modeinfo at offset 36). #[repr(C)] #[derive(Clone, Copy, Default)] pub struct WpkDrmModeGetCrtc { - pub set_connectors_ptr: u64, // 0 in (SETCRTC only) - pub count_connectors: u32, // 8 in (SETCRTC only) - pub crtc_id: u32, // 12 in/out - pub fb_id: u32, // 16 in/out - pub x: u32, // 20 in/out - pub y: u32, // 24 in/out - pub gamma_size: u32, // 28 out - pub mode_valid: u32, // 32 in/out - pub mode: WpkDrmModeModeinfo, // 36..104 - // total: 104 + pub set_connectors_ptr: u64, // 0 in (SETCRTC only) + pub count_connectors: u32, // 8 in (SETCRTC only) + pub crtc_id: u32, // 12 in/out + pub fb_id: u32, // 16 in/out + pub x: u32, // 20 in/out + pub y: u32, // 24 in/out + pub gamma_size: u32, // 28 out + pub mode_valid: u32, // 32 in/out + pub mode: WpkDrmModeModeinfo, // 36..104 + // total: 104 } /// `struct drm_mode_get_connector`. 80 bytes. #[repr(C)] #[derive(Clone, Copy, Default)] pub struct WpkDrmModeGetConnector { - pub encoders_ptr: u64, // 0 in - pub modes_ptr: u64, // 8 in - pub props_ptr: u64, // 16 in - pub prop_values_ptr: u64, // 24 in - pub count_modes: u32, // 32 in/out - pub count_props: u32, // 36 in/out - pub count_encoders: u32, // 40 in/out - pub encoder_id: u32, // 44 out - pub connector_id: u32, // 48 in/out - pub connector_type: u32, // 52 out - pub connector_type_id: u32, // 56 out - pub connection: u32, // 60 out - pub mm_width: u32, // 64 out - pub mm_height: u32, // 68 out - pub subpixel: u32, // 72 out - pub pad: u32, // 76 - // total: 80 + pub encoders_ptr: u64, // 0 in + pub modes_ptr: u64, // 8 in + pub props_ptr: u64, // 16 in + pub prop_values_ptr: u64, // 24 in + pub count_modes: u32, // 32 in/out + pub count_props: u32, // 36 in/out + pub count_encoders: u32, // 40 in/out + pub encoder_id: u32, // 44 out + pub connector_id: u32, // 48 in/out + pub connector_type: u32, // 52 out + pub connector_type_id: u32, // 56 out + pub connection: u32, // 60 out + pub mm_width: u32, // 64 out + pub mm_height: u32, // 68 out + pub subpixel: u32, // 72 out + pub pad: u32, // 76 + // total: 80 } /// `struct drm_mode_get_encoder`. 20 bytes. #[repr(C)] #[derive(Clone, Copy, Default)] pub struct WpkDrmModeGetEncoder { - pub encoder_id: u32, // 0 in/out - pub encoder_type: u32, // 4 out - pub crtc_id: u32, // 8 out - pub possible_crtcs: u32, // 12 out - pub possible_clones: u32, // 16 out - // total: 20 + pub encoder_id: u32, // 0 in/out + pub encoder_type: u32, // 4 out + pub crtc_id: u32, // 8 out + pub possible_crtcs: u32, // 12 out + pub possible_clones: u32, // 16 out + // total: 20 } /// `struct drm_mode_fb_cmd2`. 104 bytes — `[u64; 4] modifier` aligns @@ -2864,28 +3785,28 @@ pub mod dri { #[repr(C)] #[derive(Clone, Copy, Default)] pub struct WpkDrmModeFbCmd2 { - pub fb_id: u32, // 0 out - pub width: u32, // 4 in - pub height: u32, // 8 in - pub pixel_format: u32, // 12 in - pub flags: u32, // 16 in - pub handles: [u32; 4], // 20 in - pub pitches: [u32; 4], // 36 in - pub offsets: [u32; 4], // 52 in - pub modifier: [u64; 4], // 72..104 in - // total: 104 + pub fb_id: u32, // 0 out + pub width: u32, // 4 in + pub height: u32, // 8 in + pub pixel_format: u32, // 12 in + pub flags: u32, // 16 in + pub handles: [u32; 4], // 20 in + pub pitches: [u32; 4], // 36 in + pub offsets: [u32; 4], // 52 in + pub modifier: [u64; 4], // 72..104 in + // total: 104 } /// `struct drm_mode_crtc_page_flip`. 24 bytes. #[repr(C)] #[derive(Clone, Copy, Default)] pub struct WpkDrmModeCrtcPageFlip { - pub crtc_id: u32, // 0 in - pub fb_id: u32, // 4 in - pub flags: u32, // 8 in - pub reserved: u32, // 12 - pub user_data: u64, // 16 in - // total: 24 + pub crtc_id: u32, // 0 in + pub fb_id: u32, // 4 in + pub flags: u32, // 8 in + pub reserved: u32, // 12 + pub user_data: u64, // 16 in + // total: 24 } /// `struct drm_event_vblank`. 32 bytes — `drm_event` header (8) + @@ -2895,35 +3816,35 @@ pub mod dri { #[repr(C)] #[derive(Clone, Copy, Default)] pub struct WpkDrmEventVblank { - pub ev_type: u32, // 0 - pub length: u32, // 4 - pub user_data: u64, // 8 - pub tv_sec: u32, // 16 - pub tv_usec: u32, // 20 - pub sequence: u32, // 24 - pub crtc_id: u32, // 28 - // total: 32 + pub ev_type: u32, // 0 + pub length: u32, // 4 + pub user_data: u64, // 8 + pub tv_sec: u32, // 16 + pub tv_usec: u32, // 20 + pub sequence: u32, // 24 + pub crtc_id: u32, // 28 + // total: 32 } /// `struct drm_wait_vblank_request`. Union member (input). 16 bytes. #[repr(C)] #[derive(Clone, Copy, Default)] pub struct WpkDrmWaitVblankRequest { - pub req_type: u32, // 0 - pub sequence: u32, // 4 - pub signal: u64, // 8 - // total: 16 + pub req_type: u32, // 0 + pub sequence: u32, // 4 + pub signal: u64, // 8 + // total: 16 } /// `struct drm_wait_vblank_reply`. Union member (output). 16 bytes. #[repr(C)] #[derive(Clone, Copy, Default)] pub struct WpkDrmWaitVblankReply { - pub rep_type: u32, // 0 - pub sequence: u32, // 4 - pub tv_sec: u32, // 8 - pub tv_usec: u32, // 12 - // total: 16 + pub rep_type: u32, // 0 + pub sequence: u32, // 4 + pub tv_sec: u32, // 8 + pub tv_usec: u32, // 12 + // total: 16 } } @@ -2954,22 +3875,68 @@ mod dri_tests { #[test] fn ioctl_numbers_match_linux_uapi() { let iowr = IOC_READ | IOC_WRITE; - assert_eq!(DRM_IOCTL_VERSION, - ioc(iowr, 'd' as u32, 0x00, size_of::() as u32)); - assert_eq!(DRM_IOCTL_GET_CAP, - ioc(iowr, 'd' as u32, 0x0c, size_of::() as u32)); - assert_eq!(DRM_IOCTL_GEM_CLOSE, - ioc(IOC_WRITE, 'd' as u32, 0x09, size_of::() as u32)); - assert_eq!(DRM_IOCTL_PRIME_HANDLE_TO_FD, - ioc(iowr, 'd' as u32, 0x2d, size_of::() as u32)); - assert_eq!(DRM_IOCTL_PRIME_FD_TO_HANDLE, - ioc(iowr, 'd' as u32, 0x2e, size_of::() as u32)); - assert_eq!(DRM_IOCTL_MODE_CREATE_DUMB, - ioc(iowr, 'd' as u32, 0xb2, size_of::() as u32)); - assert_eq!(DRM_IOCTL_MODE_MAP_DUMB, - ioc(iowr, 'd' as u32, 0xb3, size_of::() as u32)); - assert_eq!(DRM_IOCTL_MODE_DESTROY_DUMB, - ioc(iowr, 'd' as u32, 0xb4, size_of::() as u32)); + assert_eq!( + DRM_IOCTL_VERSION, + ioc(iowr, 'd' as u32, 0x00, size_of::() as u32) + ); + assert_eq!( + DRM_IOCTL_GET_CAP, + ioc(iowr, 'd' as u32, 0x0c, size_of::() as u32) + ); + assert_eq!( + DRM_IOCTL_GEM_CLOSE, + ioc( + IOC_WRITE, + 'd' as u32, + 0x09, + size_of::() as u32 + ) + ); + assert_eq!( + DRM_IOCTL_PRIME_HANDLE_TO_FD, + ioc( + iowr, + 'd' as u32, + 0x2d, + size_of::() as u32 + ) + ); + assert_eq!( + DRM_IOCTL_PRIME_FD_TO_HANDLE, + ioc( + iowr, + 'd' as u32, + 0x2e, + size_of::() as u32 + ) + ); + assert_eq!( + DRM_IOCTL_MODE_CREATE_DUMB, + ioc( + iowr, + 'd' as u32, + 0xb2, + size_of::() as u32 + ) + ); + assert_eq!( + DRM_IOCTL_MODE_MAP_DUMB, + ioc( + iowr, + 'd' as u32, + 0xb3, + size_of::() as u32 + ) + ); + assert_eq!( + DRM_IOCTL_MODE_DESTROY_DUMB, + ioc( + iowr, + 'd' as u32, + 0xb4, + size_of::() as u32 + ) + ); } #[test] @@ -2981,10 +3948,24 @@ mod dri_tests { #[test] fn wpk_extension_ioctl_numbers() { let iowr = IOC_READ | IOC_WRITE; - assert_eq!(DRM_IOCTL_WPK_CREATE_GPU_BO, - ioc(iowr, 'd' as u32, 0xE0, size_of::() as u32)); - assert_eq!(DRM_IOCTL_WPK_BIND_FOREIGN_TEXTURE, - ioc(iowr, 'd' as u32, 0xE1, size_of::() as u32)); + assert_eq!( + DRM_IOCTL_WPK_CREATE_GPU_BO, + ioc( + iowr, + 'd' as u32, + 0xE0, + size_of::() as u32 + ) + ); + assert_eq!( + DRM_IOCTL_WPK_BIND_FOREIGN_TEXTURE, + ioc( + iowr, + 'd' as u32, + 0xE1, + size_of::() as u32 + ) + ); } #[test] @@ -3014,28 +3995,76 @@ mod dri_tests { #[test] fn kms_ioctl_numbers_match_linux_uapi() { let iowr = IOC_READ | IOC_WRITE; - assert_eq!(DRM_IOCTL_SET_MASTER, - ioc(0, 'd' as u32, 0x1e, 0)); - assert_eq!(DRM_IOCTL_DROP_MASTER, - ioc(0, 'd' as u32, 0x1f, 0)); - assert_eq!(DRM_IOCTL_WAIT_VBLANK, - ioc(iowr, 'd' as u32, 0x3a, size_of::() as u32)); - assert_eq!(DRM_IOCTL_MODE_GETRESOURCES, - ioc(iowr, 'd' as u32, 0xa0, size_of::() as u32)); - assert_eq!(DRM_IOCTL_MODE_GETCRTC, - ioc(iowr, 'd' as u32, 0xa1, size_of::() as u32)); - assert_eq!(DRM_IOCTL_MODE_SETCRTC, - ioc(iowr, 'd' as u32, 0xa2, size_of::() as u32)); - assert_eq!(DRM_IOCTL_MODE_GETENCODER, - ioc(iowr, 'd' as u32, 0xa6, size_of::() as u32)); - assert_eq!(DRM_IOCTL_MODE_GETCONNECTOR, - ioc(iowr, 'd' as u32, 0xa7, size_of::() as u32)); - assert_eq!(DRM_IOCTL_MODE_RMFB, - ioc(iowr, 'd' as u32, 0xaf, 4)); - assert_eq!(DRM_IOCTL_MODE_PAGE_FLIP, - ioc(iowr, 'd' as u32, 0xb0, size_of::() as u32)); - assert_eq!(DRM_IOCTL_MODE_ADDFB2, - ioc(iowr, 'd' as u32, 0xb8, size_of::() as u32)); + assert_eq!(DRM_IOCTL_SET_MASTER, ioc(0, 'd' as u32, 0x1e, 0)); + assert_eq!(DRM_IOCTL_DROP_MASTER, ioc(0, 'd' as u32, 0x1f, 0)); + assert_eq!( + DRM_IOCTL_WAIT_VBLANK, + ioc( + iowr, + 'd' as u32, + 0x3a, + size_of::() as u32 + ) + ); + assert_eq!( + DRM_IOCTL_MODE_GETRESOURCES, + ioc( + iowr, + 'd' as u32, + 0xa0, + size_of::() as u32 + ) + ); + assert_eq!( + DRM_IOCTL_MODE_GETCRTC, + ioc( + iowr, + 'd' as u32, + 0xa1, + size_of::() as u32 + ) + ); + assert_eq!( + DRM_IOCTL_MODE_SETCRTC, + ioc( + iowr, + 'd' as u32, + 0xa2, + size_of::() as u32 + ) + ); + assert_eq!( + DRM_IOCTL_MODE_GETENCODER, + ioc( + iowr, + 'd' as u32, + 0xa6, + size_of::() as u32 + ) + ); + assert_eq!( + DRM_IOCTL_MODE_GETCONNECTOR, + ioc( + iowr, + 'd' as u32, + 0xa7, + size_of::() as u32 + ) + ); + assert_eq!(DRM_IOCTL_MODE_RMFB, ioc(iowr, 'd' as u32, 0xaf, 4)); + assert_eq!( + DRM_IOCTL_MODE_PAGE_FLIP, + ioc( + iowr, + 'd' as u32, + 0xb0, + size_of::() as u32 + ) + ); + assert_eq!( + DRM_IOCTL_MODE_ADDFB2, + ioc(iowr, 'd' as u32, 0xb8, size_of::() as u32) + ); } #[test] @@ -3054,10 +4083,10 @@ mod gl_tests { #[test] fn struct_sizes_match_abi() { - assert_eq!(size_of::(), 8); + assert_eq!(size_of::(), 8); assert_eq!(size_of::(), 16); assert_eq!(size_of::(), 32); - assert_eq!(size_of::(), 24); + assert_eq!(size_of::(), 24); } #[test] @@ -3068,27 +4097,62 @@ mod gl_tests { #[test] fn opcodes_are_unique() { let ops: &[u16] = &[ - OP_CLEAR, OP_CLEAR_COLOR, OP_VIEWPORT, OP_SCISSOR, - OP_ENABLE, OP_DISABLE, OP_BLEND_FUNC, OP_DEPTH_FUNC, - OP_CULL_FACE, OP_FRONT_FACE, OP_LINE_WIDTH, OP_PIXEL_STOREI, - OP_GEN_BUFFERS, OP_DELETE_BUFFERS, OP_BIND_BUFFER, - OP_BUFFER_DATA, OP_BUFFER_SUB_DATA, - OP_GEN_TEXTURES, OP_DELETE_TEXTURES, OP_BIND_TEXTURE, - OP_TEX_IMAGE_2D, OP_TEX_SUB_IMAGE_2D, OP_TEX_PARAMETERI, - OP_ACTIVE_TEXTURE, OP_GENERATE_MIPMAP, - OP_CREATE_SHADER, OP_SHADER_SOURCE, OP_COMPILE_SHADER, - OP_DELETE_SHADER, OP_CREATE_PROGRAM, OP_ATTACH_SHADER, - OP_LINK_PROGRAM, OP_USE_PROGRAM, OP_BIND_ATTRIB_LOCATION, + OP_CLEAR, + OP_CLEAR_COLOR, + OP_VIEWPORT, + OP_SCISSOR, + OP_ENABLE, + OP_DISABLE, + OP_BLEND_FUNC, + OP_DEPTH_FUNC, + OP_CULL_FACE, + OP_FRONT_FACE, + OP_LINE_WIDTH, + OP_PIXEL_STOREI, + OP_GEN_BUFFERS, + OP_DELETE_BUFFERS, + OP_BIND_BUFFER, + OP_BUFFER_DATA, + OP_BUFFER_SUB_DATA, + OP_GEN_TEXTURES, + OP_DELETE_TEXTURES, + OP_BIND_TEXTURE, + OP_TEX_IMAGE_2D, + OP_TEX_SUB_IMAGE_2D, + OP_TEX_PARAMETERI, + OP_ACTIVE_TEXTURE, + OP_GENERATE_MIPMAP, + OP_CREATE_SHADER, + OP_SHADER_SOURCE, + OP_COMPILE_SHADER, + OP_DELETE_SHADER, + OP_CREATE_PROGRAM, + OP_ATTACH_SHADER, + OP_LINK_PROGRAM, + OP_USE_PROGRAM, + OP_BIND_ATTRIB_LOCATION, OP_DELETE_PROGRAM, - OP_UNIFORM1I, OP_UNIFORM1F, OP_UNIFORM2F, OP_UNIFORM3F, - OP_UNIFORM4F, OP_UNIFORM_MATRIX4FV, OP_UNIFORM4FV, - OP_ENABLE_VERTEX_ATTRIB_ARRAY, OP_DISABLE_VERTEX_ATTRIB_ARRAY, - OP_VERTEX_ATTRIB_POINTER, OP_DRAW_ARRAYS, OP_DRAW_ELEMENTS, - OP_GEN_VERTEX_ARRAYS, OP_DELETE_VERTEX_ARRAYS, + OP_UNIFORM1I, + OP_UNIFORM1F, + OP_UNIFORM2F, + OP_UNIFORM3F, + OP_UNIFORM4F, + OP_UNIFORM_MATRIX4FV, + OP_UNIFORM4FV, + OP_ENABLE_VERTEX_ATTRIB_ARRAY, + OP_DISABLE_VERTEX_ATTRIB_ARRAY, + OP_VERTEX_ATTRIB_POINTER, + OP_DRAW_ARRAYS, + OP_DRAW_ELEMENTS, + OP_GEN_VERTEX_ARRAYS, + OP_DELETE_VERTEX_ARRAYS, OP_BIND_VERTEX_ARRAY, - OP_GEN_FRAMEBUFFERS, OP_BIND_FRAMEBUFFER, - OP_FRAMEBUFFER_TEXTURE_2D, OP_GEN_RENDERBUFFERS, - OP_BIND_RENDERBUFFER, OP_RENDERBUFFER_STORAGE, + OP_GEN_FRAMEBUFFERS, + OP_BIND_FRAMEBUFFER, + OP_FRAMEBUFFER_TEXTURE_2D, + OP_GEN_RENDERBUFFERS, + OP_BIND_RENDERBUFFER, + OP_RENDERBUFFER_STORAGE, OP_FRAMEBUFFER_RENDERBUFFER, ]; for (i, &a) in ops.iter().enumerate() { @@ -3101,10 +4165,18 @@ mod gl_tests { #[test] fn query_opcodes_are_unique() { let qops: &[u32] = &[ - QOP_GET_ERROR, QOP_GET_STRING, QOP_GET_INTEGERV, - QOP_GET_FLOATV, QOP_GET_UNIFORM_LOC, QOP_GET_ATTRIB_LOC, - QOP_GET_SHADERIV, QOP_GET_SHADER_INFO_LOG, QOP_GET_PROGRAMIV, - QOP_GET_PROGRAM_INFO_LOG, QOP_READ_PIXELS, QOP_CHECK_FB_STATUS, + QOP_GET_ERROR, + QOP_GET_STRING, + QOP_GET_INTEGERV, + QOP_GET_FLOATV, + QOP_GET_UNIFORM_LOC, + QOP_GET_ATTRIB_LOC, + QOP_GET_SHADERIV, + QOP_GET_SHADER_INFO_LOG, + QOP_GET_PROGRAMIV, + QOP_GET_PROGRAM_INFO_LOG, + QOP_READ_PIXELS, + QOP_CHECK_FB_STATUS, ]; for (i, &a) in qops.iter().enumerate() { for &b in &qops[i + 1..] { diff --git a/docs/abi-versioning.md b/docs/abi-versioning.md index e7278d9e10..0b88d5ebc7 100644 --- a/docs/abi-versioning.md +++ b/docs/abi-versioning.md @@ -49,7 +49,8 @@ kernel. Specifically, any of the following requires an `ABI_VERSION` bump: explicitly and coordinate the host implementation in the same ABI epoch. - Changing the name, version, encoding, or role semantics of the `kandelo.wpk_fork.capabilities` custom section. The host uses these claims to - decide whether a main/side-module pair can safely coordinate fork replay. + decide whether a main/side-module pair can safely coordinate fork replay and, + in ABI 43, whether the artifact satisfies activation-state ownership. - Renaming the ABI custom section or the process-expected globals. - Changing the meaning of a syscall argument, errno, or blocking behavior without changing its signature. **This is not caught @@ -58,7 +59,11 @@ kernel. Specifically, any of the following requires an `ABI_VERSION` bump: The fork-capability section has an explicit ABI transition rule. ABI 16 accepts an absent section through the pre-existing five-export fallback, while treating a present marker as authoritative. ABI 17 was intentionally skipped; ABI 18 -was the first epoch above 16 and made the role marker mandatory. +was the first epoch above 16 and made the role marker mandatory. ABI 43 adds +`FORK_CAP_ACTIVATION_STATE_SAFE` and requires it on every fork-instrumented +main or side module. An ABI 42 artifact does not become ABI 43-compatible by +copying the new capability byte: the embedded ABI version and the capability +contract are validated together. ABI 26 also makes `kernel_get_process_exit_signal` a required host-adapter export. The host uses the query unconditionally to distinguish signal death @@ -254,6 +259,109 @@ later failure enters `ABORT_UNWINDING`, reconstructs the committed inner frames, releases the partial continuation, and returns the errno from the original `fork()` call without terminating the parent. +### ABI 43 activation-owned fork replay + +ABI 43 closes the remaining dependency on mutable state in the parent Wasm +instance. A fork child receives copied linear memory but a newly instantiated +module, globals, tables, exception tags, and host Store. Module-static +reference tables therefore cannot prove that a replay value survived fork. + +Every ABI 43 fork artifact carries the version-1 +`kandelo.wpk_fork.capabilities` section with +`FORK_CAP_ACTIVATION_STATE_SAFE`. Instrumentation, package guards, Node and +browser executable resolution, worker launch, pthread launch, and side-module +loading treat the capability as part of the artifact contract. Missing, +duplicate, malformed, unknown-version, unknown-bit, or safety-bit-free +capabilities fail before execution. + +The instrumenter also rejects any input that already carries fork control +exports, linked-frame imports, or fork metadata. This prevents a transformed +ABI 42 module from being run through the ABI 43 tool merely to acquire the new +safety claim; package builds must instrument raw linker output. + +The frame contract remains version 1 and keeps its existing 16-byte header. +Offset `+8` carries the exact dynamic catch selector and the formerly +reference-stash-related word at `+12` carries a process reference-vector +ordinal. The instrumenter no longer creates +`_wpk_fork_funcref_stash`, `_wpk_fork_externref_stash`, or +`_wpk_fork_exnref_stash`. + +Live reference locals, parameters, call operands/results, `call_ref` callees, +mutable reference globals, typed table entries, and complete exceptions use +one process-owned KFRV (Kandelo Fork Reference Vectors) recipe transaction +inside the KFMS (Kandelo Fork Module State) arena copied through linear memory. +Function/static-root catalogs reconstruct fresh instance-local identities; +typed GC recipes preserve concrete layout, cycles, aliases, and externalized +views. Materialization also re-registers weak constructor provenance for the +new object, including packed segment operands and nullable recipe-zero seeds, +so that the child can itself become the parent of a later fork. Durable +process-image handles represent opaque `externref` values. +Generated module-state helpers restore globals, table length/content, and +segment lifetime before frame replay. A generation-published sparse table +journal keeps pthread and late-dlopen replicas coherent without copying +WebAssembly functions or `exnref` values through JavaScript. + +ABI 43's POSIX dynamic-loader path is staged and non-reentrant. +`__wasm_dlopen_prepare` validates and owns a private transaction without +entering Wasm. Each `__wasm_dlopen_next` advances host-only +compilation/instantiation as needed and returns one initializer table entry. +Instrumentation removes the native start section and exposes its initialization +as an explicit bootstrap stage, so instance construction cannot run that guest +path; libc invokes each returned entry only after the import returns. +Instrumentation lowers the historical canonical two-, four-, and five-argument +`__wasm_dlopen` imports to the same protocol before computing fork +reachability. The two-argument form retains its historical +`dlopen::` identity. The original imported +function identity becomes a local tail adapter, preserving table and `ref.func` +aliases without leaving a host callback under initialization. +Artifact publication and host launch reject an ABI 43 safety claim if the +legacy import or a native start section remains. Input modules may use a start +section, but an accepted completed transform must expose it only through +`wpk_fork_module_bootstrap`. The lower-level `DynamicLinker.dlopenSync()` +driver is an embedder API, not an accepted process import. The process Worker +must perform final instantiation and Store-local function/tag registration +even when kernel policy coordinates the load, because those identities cannot +be cloned from the kernel Worker. + +ABI 43 also assigns channel-header offset 68 to `request_flags`. +`REQUEST_FLAG_DEFER_SIGNAL_DELIVERY` marks a request whose completion is +consumed by process-worker JavaScript rather than libc's ordinary post-syscall +signal trampoline. The kernel leaves a caught signal pending for such a +completion instead of dequeuing it into a channel record that JavaScript +cannot deliver. After `fork`, `clone`, or a staged-loader import returns, libc +issues a side-effect-free `getpid` checkpoint through the ordinary channel +path; that completion owns normal handler delivery and signal-mask restoration. +The flag changes neither the continuation encoding nor any activation's frame +size. + +Statically tagged scalar `Catch`/`CatchRef` arms serialize their exact selector +and maximum live scalar tag tuple. During rewind the tool executes `throw` with +that reconstructed payload; the original clause creates a fresh +child-instance exnref. Reference/vector payloads, `CatchAll`, `CatchAllRef`, +JSTag ingress, and normalized legacy-EH cleanup paths use the +complete-exception recipe and likewise throw inside Wasm. Transaction cleanup +clears temporary tables, roots, and owner leases after replay or abort. + +The capability therefore attests to present reconstruction machinery, not a +conservative source-shape rejection pass. Valid reference-bearing code outside +the fork closure remains unmodified; valid reference-bearing code inside the +closure receives the typed ownership path. Artifact validation still rejects +malformed/version-mismatched contracts and pre-instrumented ABI 42 input before +execution. + +This is an incompatible artifact epoch even though the linked-frame descriptor +version is unchanged. All fork-instrumented programs, side modules, package +bottles, binary indexes, shell closures, and VFS images must be rebuilt from +source. Existing C++ modern-EH outputs that retain exnref locals or use +`CatchAllRef`, and the Dash `expandstr` cleanup path, are supported rebuild +inputs through those recipes; they are not candidates for metadata relabeling +or package-specific bypasses. The ABI 43 development shell/rootfs closure can +be rebuilt from source. Broad bottle, index, shell, and image publication still +requires explicit release coordination. The exact archive-generation, +rootfs/image, and Homebrew sequencing and isolation boundary is recorded in +the [ABI 43 activation-state-safe artifact rebuild +plan](plans/2026-07-25-abi-43-activation-state-safe-rebuild-plan.md). + ## The snapshot `abi/snapshot.json` is generated by `cargo xtask dump-abi` from the @@ -290,15 +398,17 @@ captures: contract. - `custom_sections` — names of wasm custom sections that participate in the ABI: `wasm-posix-abi` for the per-binary version and - `kandelo.wpk_fork.linked_frames` for the linked-continuation layout. + `kandelo.wpk_fork.linked_frames` for the linked-continuation layout, and + `kandelo.wpk_fork.capabilities` for fork role and activation-safety claims. - `process_expected_globals` — globals every user process instance is expected to expose for the host to thread through fork/exec. - `program_artifact` — requirements checked on instrumented user programs before they can be published: the linked-frame descriptor schema, its wasm32/wasm64 header sizes, the three transactional frame imports, and - the seven `wpk_fork_*` control exports with pointer-width-aware signatures. - The descriptor width, function signatures, and the module's single memory - address width are validated as one contract. + the seven `wpk_fork_*` control exports with pointer-width-aware signatures, + plus the capability-section version, known bits, and required safety bit. + The descriptor width, function signatures, capability claims, and the + module's single memory address width are validated as one contract. WHY this is snapshot-owned: a program can otherwise pass kernel ABI checks yet fail only when its first `fork()` reaches a newer host. - `kernel_exports` — every non-toolchain export in the built kernel diff --git a/docs/architecture.md b/docs/architecture.md index 0d1d28a0b2..6566ac1b31 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -280,7 +280,7 @@ Offset Size Field 8 48 arguments (6 × i64) 56 8 return_value (i64) 64 4 errno_value (i32) -68 4 reserved/padding +68 4 request_flags (i32) 72 65536 data_buffer (for path strings, read/write buffers, etc.) ``` @@ -292,6 +292,15 @@ because that is the C calling convention its callers use. The non-variadic `__syscallN` and cancellation-point `__syscall_cp` paths widen values to 64 bits before calling the glue layer so offsets and lengths are not truncated. +Bit 0 of `request_flags`, +`REQUEST_FLAG_DEFER_SIGNAL_DELIVERY`, identifies a completion consumed by +process-worker JavaScript instead of libc's post-syscall signal trampoline. +The kernel leaves caught signals pending on those completions. Fork, clone, +continuation allocation/cleanup, and staged-loader VFS/memory requests set the +bit and clear it before returning control to guest code. Libc then uses an +ordinary side-effect-free `getpid` syscall as the signal-delivery checkpoint +after the owning import returns. Ordinary guest syscalls clear the flags word. + ### Status Values | Value | Name | Meaning | @@ -324,6 +333,13 @@ Process Worker Kernel Worker (host) 15. Return to caller ``` +Steps 10–15 normally include caught-signal publication and libc handler +dispatch. A request marked `REQUEST_FLAG_DEFER_SIGNAL_DELIVERY` deliberately +omits that publication because JavaScript owns its completion and has no +signal-handler trampoline. The next explicit guest checkpoint performs the +same delivery only after the host import has returned; this avoids both signal +loss and a reentrant host-to-Wasm callback. + ### Blocking Syscalls and Retry Some syscalls (read from empty pipe, accept on socket, poll with timeout) cannot complete immediately. The kernel returns `-EAGAIN` and the host enters a retry loop: @@ -416,18 +432,46 @@ channel, fork-context, and clear-TID metadata. ### fork() Fork uses the in-tree `wasm-fork-instrument` tool to snapshot the Wasm call stack (details in [fork-instrumentation.md](fork-instrumentation.md)): +Before compilation or worker launch, Node and browser hosts validate the +embedded ABI version, linked-frame contract, control exports, and ABI 43 +`FORK_CAP_ACTIVATION_STATE_SAFE` claim. Pthread and side-module entry points +apply the same policy. 1. User calls `fork()` → musl → `__syscall(SYS_clone, ...)` → glue -2. The host's `kernel_fork` override maps a root continuation chunk and calls `wpk_fork_unwind_begin(root + chunk_header_size)`. The tool-injected export sets state to UNWINDING and snapshots every mutable scalar global (including `__tls_base` and `__stack_pointer`) into the root's fixed prefix. -3. The return-to-caller chain unwinds. After each fork-path call returns in the unwinding state, the caller asks the host to reserve a complete node before its first frame write; its postamble commits the node only after all scalar and reference state has been saved. The host maps additional page-rounded chunks when necessary. +2. The host's `kernel_fork` override begins one process continuation + transaction. It captures activation catalogs and module state, maps each + participating activation's root continuation chunk, and calls + `wpk_fork_unwind_begin(root + chunk_header_size)`. The tool-injected export + sets state to UNWINDING and snapshots every mutable scalar global (including + `__tls_base` and `__stack_pointer`) into that activation's fixed prefix. +3. The return-to-caller chain unwinds. After each fork-path call returns in the + unwinding state, the caller asks the host to reserve a complete node before + its first frame write; its postamble commits the node only after all + activation-owned scalar state has been saved. Live references are interned + into one process recipe graph and the frame stores only its reference-vector + ordinal. The host maps additional page-rounded chunks when necessary. No + accepted frame names a module-instance reference-table slot. 4. Once `_start` returns (top-of-stack), the host sends SYS_FORK through the channel. 5. Kernel's `kernel_fork_process(parent_pid, caller_tid)` validates the caller, allocates the child PID from the global task-ID sequence, and copies process metadata and the fd/OFD tables. The child receives the calling task's blocked signal mask, while inherited stateful descriptors retain references to their existing kernel-global backings. -6. Host copies the parent's linear memory, including continuation mappings, to a new `WebAssembly.Memory` and spawns a child worker. Kernel mmap metadata is inherited with the process state. -7. Child worker attaches to the copied root and calls `wpk_fork_rewind_begin(buf)` — the tool's export restores all saved globals. The host then calls `setupChannelBase(...)` (which reads the now-correct `__tls_base`) and invokes `_start`. +6. Host copies the parent's linear memory, including continuation mappings, to + a new `WebAssembly.Memory` and spawns a child worker. Kernel mmap metadata is + inherited with the process state. The worker creates a fresh Wasm instance: + mutable globals, tables, exception references, and Store-owned references + are not copied and are not evidence that replay state survived. +7. The child validates the copied KFRV/KFMS arena, instantiates every required + main/side activation, materializes static roots, typed GC objects, opaque + owner tokens, and complete exceptions, then restores reference globals, + table contents/length, and segment lifetime. Only after those owners are + ready does it call `wpk_fork_rewind_begin(buf)` to restore scalar globals. + Typed object allocation also installs the child's own weak constructor + provenance, so a later nested fork encodes child-local objects rather than + depending on identities retained from the original parent. + The host then calls `setupChannelBase(...)` (which reads the now-correct + `__tls_base`) and invokes the selected main or pthread resume root. 8. Each instrumented function's preamble requests and validates the next committed frame, then re-enters the call site where the parent was interrupted. Eventually it reaches the `kernel_fork` call site in the leaf function, which returns 0. Libc then refreshes the copied pthread TID from the kernel through `set_tid_address` before returning to user code. 9. `wpk_fork_rewind_end` resets state; parent and child independently unmap their continuation chunks; fork returns 0 in child and the child PID in the parent. @@ -440,17 +484,34 @@ errno. A negative `SYS_FORK` result after step 4 instead uses the complete parent rewind. These resource failures create no child and leave the parent in `NORMAL`, able to continue or retry `fork()`. -The instrumentation handles LLVM's new-EH `try_table` output correctly, including fork from inside C++ catch handlers. See [fork-instrumentation.md](fork-instrumentation.md) for the current guarantees and documented unanticipated Wasm-level carve-outs. - -A fork reached directly inside an instrumented dlopened side module uses two -ordered state machines and two linked continuations: side then main during unwind, main -then side during rewind. Versioned fork-instrument capability metadata lets -marker-present artifacts prove their role. ABI 16 defines the historical +ABI 43 reconstructs reference locals/parameters/carryovers, concrete and +abstract GC objects, mutable reference globals, and complete exception state. +Statically tagged scalar `Catch`/`CatchRef` arms keep their exact selector and +maximum live operand tuple in activation-owned bytes; rewind rethrows the tag +so the original clause creates a fresh instance-local exnref. +Reference/vector payloads, `CatchAll`/`CatchAllRef`, JSTag ingress, and modern +C++ cleanup exnrefs use the complete-exception recipe and likewise re-enter the +original Wasm handler without parent-instance scratch. See +[fork-instrumentation.md](fork-instrumentation.md) for the ownership formats +and cleanup ordering. + +A fork reached inside instrumented dlopened side modules uses one process-wide +event journal plus one linked continuation per active module. Unwind records +the exact leaf-to-root activation/function order; fresh-child replay consumes +the reverse order. This supports nested main-to-side-to-side stacks without +assuming that the main activation is present at the leaf. Versioned +fork-instrument capability metadata lets +marker-present artifacts prove their role, and ABI 43 additionally requires +`FORK_CAP_ACTIVATION_STATE_SAFE` before launch. ABI 16 defines the historical five-export fallback, while ABI 18 and later require role claims and reject stale call-graph artifacts. The ABI 36 epoch combines that contract with side-module replay state and concurrent pthread-fork arbitration. Dlopen replay records both the parent's memory base and exact table base, including null gaps -left by failed loads. TLS-bearing side modules additionally record their live, +left by failed loads, then re-instantiates each side module's static element +initialization at that exact base in the child. The process table journal then +applies later loader and guest `table.set`/`fill`/`copy`/`init`/`grow` effects +through typed recipes after every referenced activation catalog is present. +TLS-bearing side modules additionally record their live, positive `__tls_base`. A child restores the pointer-width-correct mutable global without calling `__wasm_init_tls`, because copied memory already holds the parent's live TLS bytes and reinitialization would reset C++ unwinder state @@ -458,21 +519,66 @@ and application `thread_local` values. C++ exceptions and longjmp use one canonical pointer-width tag identity across the main image and all side modules; a main-exported tag wins over the host-created fallback. +ABI 43 libc drives dynamic initialization as a non-reentrant transaction. +`__wasm_dlopen_prepare` validates and owns a private transaction without +entering guest code. Each `__wasm_dlopen_next` advances host-only compilation +and instantiation of the `DT_NEEDED` closure as needed. Instrumentation has +removed the native start section and converted active segments plus the +original start function into an explicit bootstrap, so +`new WebAssembly.Instance(...)` cannot run that guest path inside `next`. +The call publishes one canonical initializer table entry and returns; libc +then calls that entry as ordinary Wasm before requesting the next stage. A +constructor that calls `fork()` thus has a normal instrumentable Wasm call +chain rather than a suspended host import beneath it. Before reachability +analysis, the instrumenter turns the +historical two-, four-, and five-argument private `__wasm_dlopen` function +imports into in-place local adapters that prepare the transaction and +tail-call an ordinary Wasm driver for their initializers. This retains all +aliases of the old function without retaining the reentrant host boundary. +The two-argument form retains its original +`dlopen::` identity. ABI 43 artifact and launch +validation require both the legacy import and the completed artifact's native +start section to be absent. Valid source modules may contain a start section; +instrumentation transfers it to `wpk_fork_module_bootstrap` before admission. +The lower-level `DynamicLinker.dlopenSync()` driver remains available to +standalone embedders, but it is not a process import. + +The kernel cannot replace the process-local half of this protocol. It can own +path authorization, loader scheduling, and replay policy, but the kernel +Worker cannot inject Store-local functions, exception tags, or GC identities +into the process Worker's tables. Core Wasm cannot instantiate arbitrary +runtime module bytes itself, and those JavaScript references are not +structured-clonable between Workers. A kernel syscall would therefore need a +request/yield/resume protocol that still delegates instantiation and catalog +registration to the process Worker; it would relocate, rather than remove, the +boundary. In particular, using the generic syscall import instead of a named +loader import would not itself change reentrancy. The contract is that every +process-local host operation returns before guest initialization begins. + +The process worker can issue VFS and mapping requests while a staged loader +import is active, but those JavaScript-owned channel completions cannot invoke +libc's signal trampoline. They set +`REQUEST_FLAG_DEFER_SIGNAL_DELIVERY`, leaving any caught signal in the kernel, +and libc performs an ordinary checkpoint after every `prepare`/`next` return +and after `dlclose`. Constructors still begin only after both the import and +checkpoint return, so signal delivery and dynamic initialization add no +host-to-Wasm reentrancy. + The dlopen replay list and its atomic pthread-fork lock live in a transient, host-private control record. The same host build writes and reads that record during one process lifetime; guest code and persisted artifacts never interpret it. Changing that record's size is therefore not a guest ABI change, while the public ABI snapshot/classifier remains authoritative. -Pthread workers have separate Wasm instances, tables, and exception tags, none -of which can be structured-cloned from the process worker. `dlopen()` from a -pthread therefore fails normally with `dlerror()`. If the process has loaded a -side module, `fork()` from a pthread returns `ENOTSUP`; an atomic process lock -excludes a racing main-worker dlopen across the pthread's archive check, -unwind, memory copy, and parent rewind. The supported -direct-main-to-side boundary and the remaining opaque cross-side callback -limitation are specified in -[fork-instrumentation.md](fork-instrumentation.md#fork-from-a-dlopened-side-module). +Pthread workers have separate Wasm instances, tables, and exception tags, so +no JavaScript reference is structured-cloned from the process worker. Each +pthread owns a local dynamic-linker replica. Under the process archive lock it +compares a shared generation, instantiates missing side modules at their exact +bases, registers fresh function/exception catalogs, and applies the typed table +journal. The same mechanism supports `dlopen`/`dlsym` from a pthread and fork +from a pthread after dynamic loading; the fork child reconstructs only the +calling thread but receives the process module/table recipe state. The +generation fast path avoids reparsing or reinstantiating unchanged state. Fork and non-forking spawn still copy each process's fd and OFD metadata. The objects whose mutable state must remain identical across those copies use @@ -1149,6 +1255,14 @@ Signals are delivered at syscall boundaries. When a process has a pending signal Features: RT signal queuing with `si_value`, cross-process `kill`/`killpg`, `sigaltstack` with shadow stack swap, `sigsuspend`, `sigtimedwait`, `setitimer`/`alarm` via host timers. +The exception is a channel request whose completion is owned by +process-worker JavaScript. Its ABI 43 request flag tells the kernel not to +dequeue a caught signal into a record that JavaScript cannot consume. The +signal remains pending until libc makes the explicit ordinary-channel +checkpoint after `fork`, `clone`, or a staged-loader import. Handler invocation +and `rt_sigreturn` therefore still occur on the guest side after the host +import has returned. + Exact-thread delivery never degrades into process-wide delivery. `tkill` and `tgkill` resolve their target against retained live task records in the calling process; TID 0 and unknown or exited TIDs return `ESRCH`. Cross-process diff --git a/docs/fork-instrumentation.md b/docs/fork-instrumentation.md index f0b17173d3..9ba2b943f6 100644 --- a/docs/fork-instrumentation.md +++ b/docs/fork-instrumentation.md @@ -20,7 +20,7 @@ For motivation, tradeoffs, and the rollout plan that led here, read for the post-rollout switch-dispatch redesign and non-fork-path-call gating that fix the kernel-side-effect re-fire bug, read [`plans/2026-04-22-fork-instrument-switch-dispatch-redesign.md`](plans/2026-04-22-fork-instrument-switch-dispatch-redesign.md). -ABI version: `42` (see +ABI version: `43` (see [`crates/shared/src/lib.rs`](../crates/shared/src/lib.rs) — see [abi-versioning.md](abi-versioning.md) for the policy). @@ -33,6 +33,16 @@ ABI version: `42` (see - Missing instrumentation is a build/runtime error, not an optional feature loss. A fork-using program without complete `wpk_fork_*` exports cannot resume the child at the fork call site. +- Every value needed after replay must be either activation-owned bytes in the + linked continuation or the output of a versioned deterministic + reconstruction recipe in the fresh child. Module globals and tables are + instance state, not evidence that a value survived `fork()`. +- ABI 43 fork artifacts must carry the activation-state-safe capability. The + capability means that activation references, exceptions, mutable reference + globals, and mutable table state have versioned reconstruction owners. An + incomplete, malformed, or old-ABI ownership contract fails during + instrumentation or pre-launch artifact validation; it must not become a + child-only trap. - Binaries exporting legacy `asyncify_*` symbols are stale and must be rebuilt. Do not add host support for them. - Do not keep compiler/linker flags solely for the retired legacy path. The @@ -125,17 +135,17 @@ wpk_fork_state() -> i32 Returns current state. Exported for host-side assertions. ``` -ABI 42 modules additionally import three exact `env` functions. A module that +ABI 42 and later modules additionally import three exact `env` functions. A module that imports any one of them must import all three and carry the linked-frame custom section described below. ``` __wpk_fork_frame_reserve(frame_size: ptr) -> ptr Reserves a complete node and returns its payload address before any frame - bytes or reference-table entries are written. + bytes are written. __wpk_fork_frame_commit(payload: ptr) -> () - Publishes the pending node after all payload and reference writes complete. + Publishes the pending node after the activation-owned payload is complete. __wpk_fork_frame_next(expected_frame_size: ptr) -> ptr Returns the next committed payload during rewind and rejects size/order @@ -151,27 +161,52 @@ section `kandelo.wpk_fork.capabilities`. Its two-byte payload is `env.fork`-importing side module has complete side-entry coverage; - bit 1 (`0x02`): a default-entry main module imported Kandelo's dynamic-linker functions and conservatively instrumented every `call_indirect` boundary - plus its direct callers. - -Capability enforcement follows the compiled kernel ABI. ABI 16 predates this -section, so an artifact with no section retains the legacy five-export fallback -and can still coordinate a main/side-module fork. If an ABI-16 artifact does -carry the section, its marker is authoritative and malformed, unknown, or -role-inconsistent claims fail loudly. Starting with ABI 18, the -role-appropriate bit is mandatory: generic five-export artifacts and binaries -produced by the older call-graph pass fail with a rebuild diagnostic. This -threshold ensures that mandatory enforcement and the incompatible artifact -contract activate in the same ABI-bump commit. Changing the meaning or encoding -of these capability claims changes fork replay assumptions and must follow the -ABI-versioning policy. - -ABI 17 was intentionally skipped. ABI 18 activated the mandatory role marker. -ABI 16 remains only a historical compatibility boundary. The reconstructed -line enforces role claims and this ABI 36 epoch adds side-module replay state -and pthread-fork arbitration. ABI-16 artifacts without a marker remain -historical inputs for the parser's explicit compatibility tests; they do not -satisfy an ABI-36 launch. Any future capability-contract change must still -advance `ABI_VERSION` and regenerate `abi/snapshot.json` atomically. + plus its direct callers; +- bit 2 (`0x04`, `WPK_FORK_CAP_ACTIVATION_STATE_SAFE`): the instrumenter + emitted and validated the complete ABI 43 activation, reference, exception, + module-state, and table-reconstruction contracts required by a fresh module + instance. + +ABI 43 requires exactly one two-byte capability section, version 1, with bit 2 +set. Unknown bits, missing/duplicate/malformed sections, or a missing safety bit +fail artifact validation before execution. Role bits retain their existing +meaning and are still required for side-entry and dynamic-linking replay where +applicable. The safety bit is not inferred from the seven control exports: +ABI 42 emitted those exports while still depending on module-instance +reference tables. A copied safety claim also cannot upgrade a normal ABI 42 +program because the program ABI marker must match 43. + +ABI 16/18 role-marker compatibility remains historical parser-test coverage; +it is not a launch fallback for an ABI 43 kernel. Changing the capability +encoding or meaning requires another ABI bump and regenerated snapshot. + +### ABI 43 deployment and rebuild boundary + +ABI 43 is an artifact epoch, not a host-only update. All fork-instrumented main +programs and side modules must be rebuilt with the ABI 43 instrumenter, then +the affected package archives, bottles, binary indexes, shell closure, and VFS +images must be regenerated against the new cache keys. Kernel, host, SDK/libc, +and generated ABI constants must ship as one coordinated set. + +Do not republish ABI 42 artifacts with edited metadata or a copied capability. +The ABI 43 instrumenter refuses inputs that already contain fork control +exports, linked-frame imports, or fork metadata; builds must start from raw +linker output so the new validation sees the original activation and table +state. +The source package projection may be regenerated while developing this epoch, +but broad bottle/index/VFS publication requires explicit release coordination. +Reference-bearing modern C++ exception output and Dash's fork-reachable +`exnref` cleanup state are reconstruction inputs, not package-specific +exceptions. The ABI 43 source-build path now accepts those shapes through the +typed reference and complete-exception recipes. A development rootfs containing +the configured shell closure can therefore be rebuilt, but it is not a +published release artifact. Existing ABI 42 packages, bottles, indexes, shell +closures, and VFS images must still be rebuilt through their normal source +paths and must never be relabeled. Broad publication remains a separately +coordinated release action. See the [ABI 43 activation-state-safe artifact +rebuild plan](plans/2026-07-25-abi-43-activation-state-safe-rebuild-plan.md) +for the exact registry generation count, derived-image order, and ABI 42 +Homebrew-proof isolation boundary. `ptr` is `i32` on wasm32 user programs and `i64` on wasm64 user programs. The tool picks the pointer width from the module's primary memory — a memory64 @@ -247,42 +282,96 @@ P-06 (`pthread_create` worker calls `fork`), and K-03 ## Fork from a dlopened side module -The supported dynamic-linking shape is a direct main-module `call_indirect` -into one side-module instance whose call stack reaches `env.fork`: - -1. Instrument the main program normally. If it imports Kandelo's dlopen host - functions, the tool marks and preserves all possible dynamic indirect-call - boundaries. -2. Instrument the fork-capable side module with `--entry env.fork`. It receives - its own linked continuation and versioned side-entry capability. -3. The process worker unwinds the side module, then the main module. Fork replay - restores dlopen instances at their exact memory and table bases, rewinds the - main module, then rewinds the active side module. +Dynamic linking participates in the same process-wide activation protocol as +the main module. The supported stack is not limited to one main-to-side call: +the event journal records arbitrary main-to-side-to-side nesting, including +calls through shared-table function pointers and side-originated +`dlopen`/`dlsym`. Each participating module owns a separate linked +continuation; the journal supplies their exact leaf-to-root +activation/function order. + +The ABI 43 POSIX `dlopen()` path deliberately separates host-owned +instantiation from guest initialization: + +1. Libc reads the side-module bytes through ordinary file operations and calls + `__wasm_dlopen_prepare`. +2. The process worker copies and validates the request, claims loader + ownership, creates a private transaction, and returns its token without + entering guest Wasm. +3. Each `__wasm_dlopen_next` advances host-only compilation/instantiation of + the complete `DT_NEEDED` closure as needed, recording provisional module + identities, memory/table bases, symbol visibility, dependency/provider + edges, and rollback ownership. ABI 43 instrumentation has removed every + native start section and converted active segments plus the original start + function into an explicit bootstrap, so instantiation cannot enter that + guest path. The call publishes at most one canonical `() -> ()` bootstrap, + relocation, or constructor entry in a transaction-owned table slot and + returns its index. +4. Libc calls that entry only after the import has returned. The initializer is + therefore an ordinary Wasm-to-Wasm activation and may call another side + module or `fork()`. The next host call acknowledges the completed stage and + either returns another entry or atomically commits the public handle. + +This staged path is non-reentrant: no host import calls back into Wasm before +returning. ABI 43 libc uses only the staged `prepare`/`next` path. Before call +graph discovery, the instrumenter also replaces either historical canonical +two-, four-, or five-argument `env.__wasm_dlopen` import in place with a local +adapter. Retaining the original function identity preserves direct calls, +exports, table elements, and `ref.func` aliases. The adapter prepares the load +and then tail-calls a generated driver; after the import has returned, that +driver invokes each initializer through the process function table and commits +the transaction. The original two-argument form had no pathname and retains +its deterministic historical `dlopen::` module +identity. ABI 43 host and publication guards require the legacy import count +and native start-section count to be zero, so stale or forged safety metadata +cannot expose either the monolithic callback or instantiation-time guest +reentry. Source modules may contain a start section; the zero-count rule +applies to the completed instrumented artifact after its start has become an +explicit bootstrap. `DynamicLinker.dlopenSync()` remains a lower-level embedder +API, not an import reachable from an accepted ABI 43 process artifact. + +The staged host can issue internal VFS and mapping channel requests while an +import is active. Those completions set the ABI 43 +`REQUEST_FLAG_DEFER_SIGNAL_DELIVERY` bit because process-worker JavaScript +cannot run libc's signal trampoline. The kernel leaves a caught signal pending, +and libc performs an ordinary `getpid` checkpoint after each staged import +returns and before it calls a guest initializer. Fork and clone use the same +ownership handoff. This preserves signal delivery without calling back into a +suspended Wasm import frame and adds no continuation or activation-frame +bytes. + +Moving preparation behind a kernel syscall would not eliminate the +process-worker portion of loading. Core Wasm cannot compile arbitrary module +bytes or manufacture the fresh Store-local function, exception-tag, and GC +identities needed by the process table. Those JavaScript objects also cannot +be structured-cloned from the kernel Worker. The kernel may own pathname +authorization, process policy, and serialization, but the process Worker must +still instantiate and register each side module. A syscall-based loader would +therefore require a loader request/yield/resume protocol around the same +process-local work. It could replace the named loader import with the generic +syscall import, but it would not remove the host transition. Safety comes from +returning to Wasm before any initializer is called, not from which import +performs the process-local work. ABI 43 uses the smaller staged-import +protocol. The main fork trampoline is captured before side exports enter the symbol table, so a later extension cannot interpose the coordinator's `fork` target. -Failed dlopen attempts may leave non-shrinkable null table gaps; each successful -archive entry records its exact parent table base, and child replay pads to and -validates that base. - -The loader preserves ordinary independent multi-extension loading. When a -fork-capable extension participates, it rejects statically visible -side-to-side function/GOT linkage and side-originated `dlopen`/`dlsym`, because -an intervening side-module frame would need a third ordered unwind. Opaque -function pointers passed through main-module memory or the shared table cannot -currently be attributed to their originating module; using such a pointer to -create a side A -> side B -> fork path is unsupported and is not yet guaranteed -to fail before control-flow corruption. A future module-activation protocol is -required to close that residual. - -Pthread workers do not own the process worker's side-module instances, table, -or exception-tag identities. `dlopen()` from a pthread consequently returns -NULL with a precise `dlerror()`. Once the process main worker has published a -dlopen archive entry, `fork()` from a pthread returns `ENOTSUP` without -creating a child. A host-private atomic lock prevents main-worker dlopen from -racing the pthread's archive check and is held through unwind, SYS_FORK/memory -copy, and parent rewind; the child clears its copied lock before replay. Fork -from a pthread remains supported while that process-wide archive is empty. +Failed loads may leave non-shrinkable null table gaps. Successful archive +events retain exact parent memory/table bases, handle values, +`RTLD_LOCAL`/`RTLD_GLOBAL` visibility, dependency/provider edges, and nested +transaction rollback state. A fresh child recreates modules in dependency +order, pads to and validates their exact bases, registers their function and +exception catalogs, and only then applies the process table journal and +activation replay. + +Pthread workers have distinct Wasm instances, tables, tags, and Stores; no +JavaScript reference is copied between them. Each pthread therefore owns a +local dynamic-linker replica driven by the process archive's generation +journal. A host-private loader-owner lease serializes staged initialization +across workers, while a shorter archive lock publishes complete records. +Unchanged generations take a fast path. This supports `dlopen`/`dlsym` from a +pthread and `fork()` after dynamic loading; the fork child recreates the +calling thread's local replica and process module/table state. For TLS-bearing side modules, each archive entry also preserves the live positive `__tls_base`. Replay restores only that mutable global using the @@ -325,7 +414,7 @@ The module prefix retains the runtime's active-frame pointer word, a reserved pointer word, saved scalar globals, and a 16-byte abort selector. `frames_start_offset = 2P + N` identifies the selector, while the host-visible fixed-prefix size is `frames_start_offset + 16`. Frame nodes and -plain-catch activation state are not stored in that prefix. +tagged-catch activation state are not stored in that prefix. This does not introduce a new linked-frame encoding. `fixed_prefix_size` has always been a module-specific value in the version-1 descriptor, and each node @@ -342,7 +431,7 @@ not fit the next complete node, the host maps another page-rounded chunk. A single node larger than a WebAssembly page receives a multi-page chunk. Allocation is transactional: a reserved node is not linked from the committed -tail until all scalar and reference writes finish. If a later chunk allocation +tail until all activation-owned bytes are written. If a later chunk allocation fails, the reserve import records the positive errno, enters `ABORT_UNWINDING`, and returns a zero pointer. The still-live activation stores only its call-site selector in the fixed-prefix scratch and restarts; already @@ -362,9 +451,13 @@ Parent and child independently walk and unmap their copies after rewind. The linked format makes chunk boundaries explicit, but version 1 does not rebase internal pointers or relocate the chain in the child. -Ref-typed mutable globals (`funcref` / `externref` / `exnref`) are not stored -in the linear-memory header — they would need aux-table spill slots, which is -a future extension. The tool currently ignores them when snapshotting globals. +Mutable reference globals (`funcref`, `externref`, `exnref`, and typed GC +references) are not stored in the scalar linear-memory header. Generated +module-state helpers encode them into the process reference graph during +capture and restore them in the fresh activation before any continuation frame +executes. Immutable imported references use the same activation/template +catalog during early instantiation. A global outside the fork closure remains +ordinary Wasm state and does not pay activation-frame overhead. ## Frame format @@ -376,32 +469,58 @@ format version, transactional state, previous-node pointer, payload size, and total aligned node size. That header costs 24 bytes on wasm32 and 32 bytes on wasm64 before alignment. -| Offset | Size | Field | Purpose | -|--------|------|-------------------|------------------------------------------| -| `+0` | 4 | `func_index` | Ordinal assigned at instrument time | -| `+4` | 4 | `call_index` | Which call site within the function | -| `+8` | 4 | `catch_region_id` | 0 in normal flow; non-zero for catches | -| `+12` | 4 | `exnref_slot` | Aux-table slot for `_ref` catch replay | -| `+16` | var | `saved_locals[]` | User and synthetic scalars, aligned | - -Ref-typed user locals (funcref, externref, exnref) do **not** appear in this -frame. They are spilled to auxiliary tables — see [Auxiliary -tables](#auxiliary-tables) below. The frame only records the ordinal identity -of the function and its call-site, which together with the ref-table slot -assignment is sufficient to restore the ref-typed locals during rewind. - -Synthetic frame locals include call-argument and operand-stack carryover -spills. For each supported plain-catch region they also include one -`active_arm` i32 and typed scalar operand locals for every static arm. Capture -therefore belongs to one function activation, and recursive activations -serialize distinct values in distinct linked frames. - -`catch_region_id` is zero in the common case (the frame was captured outside -any catch handler). When non-zero, it identifies the `try_table` whose catch -handler the frame lives in. For a `_ref` catch, `exnref_slot` identifies the -auxiliary-table entry. For a plain catch, the restored `active_arm` and operand -locals select and reconstruct the exact arm. See [Catch-handler -resume](#catch-handler-resume). +| Offset | Size | Field | Purpose | +|--------|------|----------------------------|---------| +| `+0` | 4 | `func_index` | Ordinal assigned at instrument time | +| `+4` | 4 | `call_index` | Which call site within the function | +| `+8` | 4 | exact catch selector | Zero outside reconstructed catch flow; otherwise the exact region/arm | +| `+12` | 4 | reference-vector ordinal | Process-transaction recipe vector for this landing; zero when none | +| `+16` | var | `saved_scalars[]` | User/synthetic scalars and scalar catch payload union, aligned | + +References are deliberately not copied into the frame and never name a +module-static stash slot. Existing live reference locals and parameters are +encoded into a call-specific process recipe vector; the frame owns only the +ordinal in its existing header word. Definitely-null values need no recipe. +The child decodes each recipe against its own activation, function/static-root +catalog, imported-global owner, GC layout, exception codec, or durable +externref owner. + +This constant-per-frame reference representation is a stack-depth requirement, +not only a space optimization. The standalone PR #701 V8 reproducer measured +an instrumented recursive function falling from 9,959 surviving calls to 6,639 +when its declaration grew from four to twelve locals. PR #713 reduced the +generated-local count to eight and recovered 8,536 calls in the same +measurement context; PR #714 replayed pure scalar inputs and restored the +fixture's original four-local declaration. ABI 43 therefore does not add a +generated local or linked-frame field per live reference, recipe, catch arm, or +catch region. Reference-vector entries live in the process transaction arena, +and catch scratch is pooled by simultaneously live type/width rather than +static source count. Absolute engine limits remain platform- and tier-specific, +so these historical measurements are constraints on generated shape, not a +current performance claim. + +Synthetic scalar locals include only call arguments and operand-stack +carryovers that cannot be replayed directly. Catch code uses one exact-arm +selector per function and one typed operand-scratch union sized to the maximum +simultaneously selected payload, not one tuple per static arm or region. +Reference-bearing and untagged exceptions are retained through the complete +exception recipe and do not add linked-frame payload bytes. Rewind either +rethrows a saved scalar tag payload or asks the exception codec to materialize +the complete exception before the original `Catch`, `CatchRef`, `CatchAll`, or +`CatchAllRef` control path resumes. + +Rewind also avoids inserting a no-argument resume thunk in front of every +materialized direct activation. When the event journal proves that the next +activation is the direct lexical callee, the caller executes the original call +with its reconstructed arguments; the callee preamble validates the expected +activation and function through `frame_next` before consuming it. A universal +thunk would add a second native engine frame for each recursive Wasm +activation and can exhaust the engine stack well before the continuation +chain is exhausted. Indirect/reference calls, cross-module or +tail-transparent boundaries, and targets whose lexical identity is not proven +still use the process resume catalog. The lexical fast path adds no +ordinary-activation local and no continuation bytes; its second +non-consuming event lookup runs only during replay. ## Dispatch schemes @@ -437,8 +556,8 @@ Both shapes share: - The state machine, exported ABI, and save-buffer header. - The per-function frame layout (header + scalar locals). -- Aux-table spill for ref-typed user locals (Phase 4f). -- Catch-handler resume via `throw_ref` (Phase 6). +- Activation-owned tagged-catch arm and scalar payload state. +- Catch-handler reconstruction by throwing the saved static tag. Switch-dispatch avoids the need for per-call gating: no chunk before the chosen `POST_K` runs on REWIND, so non-fork-path calls and side-effect ops @@ -546,7 +665,7 @@ Numbered callouts: test. Under `REWINDING`, the preamble calls `__wpk_fork_frame_next(frame_size)`, stores the returned payload in `*(buf + 0)`, and deserializes every frame scalar: user locals, - argument/carryover spills, and plain-catch activation state. Dispatch reads + argument/carryover spills, and tagged-catch activation state. Dispatch reads `call_index` directly from that active frame payload. 2. **Body wrapper (Phase 4b/4c).** The original body is wrapped in a `$unwind_save` block. On `REWINDING`, a `br_table` keyed by `frame.call_index` jumps to @@ -560,10 +679,10 @@ Numbered callouts: `frame.call_index` and exits `$unwind_save`. If the callee did not begin unwinding, execution continues normally. 5. **Postamble (Phase 4d).** Emits the remaining frame header fields - (func_index, catch_region_id, exnref_slot), writes every user and synthetic - frame scalar, commits the reserved node, and returns a default value of the - function's result type. Callers see the default on the unwind path but - discard it because their own postamble runs next. + (`func_index`, `catch_region_id`, and a zero reserved word), writes every + user and synthetic frame scalar, commits the reserved node, and returns a + default value of the function's result type. Callers see the default on the + unwind path but discard it because their own postamble runs next. ### (b) Fork from inside a catch handler @@ -572,65 +691,67 @@ Fixture: `FIXTURE_FORK_FROM_CATCH_HANDLER` (see ```wat (func $caller (result i32) - (block $handler (result (ref null exn)) - (try_table (result (ref null exn)) (catch_ref $exn $handler) - ref.null exn)) + (local $caught i32) + (block $handler (result i32 exnref) + (try_table (result i32 exnref) (catch_ref $exn $handler) + i32.const 7 + throw $exn)) drop + local.set $caught call $fork) ``` -After instrumentation the try_table clause gets wrapped in two injected -blocks, `$outer` and `$capture`, and the try_table body gets a rewind-throw -stub prepended: +After instrumentation the `CatchRef` clause targets an injected capture block +and the try_table body gets a rewind-throw stub. The exact emitted nesting is +omitted here; the important dataflow is: ```wat -(block $outer (result (ref null exn)) - (block $capture (result (ref null exn) exnref) - (try_table (result (ref null exn)) (catch_ref $exn $capture) - ;; [6c] Rewind-throw stub: executed lexically first on every entry. - (if (i32.and - (i32.eq (global.get $_wpk_fork_state) (i32.const 2)) - (i32.eq (local.get $catch_region_id_local) (i32.const 1))) - (then - ;; Resume into this try_table's catch handler by re-throwing - ;; the saved exnref. - (throw_ref - (ref.as_non_null - (table.get $_wpk_fork_exnref_stash - (local.get $exnref_slot_local)))))) - - ;; Original try_table body. - (ref.null exn))) - - ;; [6d] On catch_ref dispatch, stack = (ref null exn, exnref). - (local.tee $captured_exnref_1) - (local.set $in_catch_1 (i32.const 1)) - (table.set $_wpk_fork_exnref_stash - (i32.const 0 (; slot ;)) - (local.get $captured_exnref_1)) - (br $outer)) ;; fall through to the original handler continuation +;; Inside the original try_table body: +(if (i32.and + (i32.ge_u (global.get $_wpk_fork_state) (i32.const 2)) + (i32.eq (local.get $catch_region_id) (i32.const 1))) + (then + (if (i32.eq (local.get $active_arm) (i32.const 0)) + (then + ;; The frame restored 7 (or the actual scalar payload). + local.get $saved_payload + throw $exn) + (else unreachable)))) + +;; On CatchRef dispatch, stack = (i32 payload, non-null exnref): +local.set $temporary_exnref +local.set $saved_payload +i32.const 0 +local.set $active_arm +i32.const 1 +local.set $in_catch +i32.const 1 +local.set $catch_region_id + +;; Forward the original handler values, but retain no synthetic GC root. +local.get $saved_payload +local.get $temporary_exnref +ref.as_non_null +ref.null exn +local.set $temporary_exnref +br $handler ``` Numbered callouts: -- **6c — Rewind-throw stub.** Prepended to every fork-path try_table body. - On `REWINDING` with a matching `catch_region_id`, it re-throws the saved - exnref using `throw_ref`. The try_table's own catch clause catches it, - which dispatches into `$capture` exactly as if the original exception had - been thrown by the body. -- **6d — Capture block.** The tool rewrites every `catch_ref` / `catch_all_ref` - clause to target an injected `$capture` block rather than the user's - original handler. `$capture` stashes the exnref into - `_wpk_fork_exnref_stash`, sets the `$in_catch_K` flag, then unconditionally - branches to `$outer`, which is the block the user's original handler falls - through from. The net effect: the user's handler code runs with the exnref - already stashed and `in_catch_K == 1`, ready for a later fork call to - record it. -- **6e — Call-site region writes.** Any call site inside the handler - observes `$in_catch_K == 1` and writes the active region's id and exnref - slot into `$catch_region_id_local` / `$exnref_slot_local` before the - unwind-only call-index store and `$unwind_save` branch, so the frame - carries the handler identity into the save buffer. +- **Rewind-throw stub.** On replay with a matching `catch_region_id`, dispatch + validates the restored arm index, pushes that arm's restored scalar payload, + and executes `throw $tag`. The original `CatchRef` clause catches this new + exception and creates a fresh exnref in the child instance. +- **Capture block.** Every statically tagged `Catch` and `CatchRef` clause is + retargeted through a per-arm capture. It stores only the arm index and scalar + payload in frame-backed locals. A `CatchRef` exnref is temporarily forwarded + to the original handler; the synthetic local is nulled before the branch so + successful replay and abort paths do not retain a stale GC root. +- **Call-site region writes.** A call inside the handler observes the + activation-local `$in_catch_K` flag and records the lexical region in the + frame before unwinding. There is no reference slot or module-global + reference state. ### (c) Indirect fork through `call_indirect` @@ -813,10 +934,10 @@ The SubRegion spill list is computed by `analyze_subregion_spill_types` tracks the typed operand stack as `Vec>` and reports the full list of values to spill per landing — covering both the SubRegion's declared type-params AND any extra carryover above them on the parent -stack. `seq_has_unsupported_carryover` runs first as a gate; post-2.6c -it rejects only IfElse-with-carryover and SubRegions with unsupported -result types (multi-value RESULTs are still gated, though body PARAMS -are now supported). +stack. The current analyser covers scalar, vector, reference/GC, direct, +indirect, and `call_ref` producers as well as multi-value structured-control +parameters and results. Scalar/vector spill locals join the linked payload; +reference spills join the landing's process recipe vector. **Multi-value-params bodies (sub-commit 2.6c).** When a SubRegion is a multi-value `Block`/`Loop`/`TryTable` whose body uses its declared input @@ -841,67 +962,98 @@ replayed from an empty stack. The whitelist is deliberately small: - non-trapping i32/i64 binary arithmetic, bit operations, shifts, rotates, and integer comparisons. -The whitelist excludes calls, memory/table operations, globals, reference -operations, integer div/rem, floating-point operators, `local.set`/`local.tee`, -and any instruction that needs stack input from before the suffix. Unsupported -or type-mismatched suffixes fall back to the existing spill-local path. This -keeps REWIND behavior tied to the same post-call/post-landing sequence while -avoiding frame locals for common compiler shapes like recursive -`walk(depth - 1)` arguments and `eqz(depth)` branch conditions. - -**Function-level analyser gate.** When `walk_seq_for_carryovers` or -`compute_nested_carryover_types` encounters a producer whose pushed type -the analyser can't statically track (Unop, Cmpxchg, ref-typed -CallIndirect/CallRef, multi-value structured control), the unknown slot -is tracked as `None` and tolerated as long as it's consumed before any -fork-path call. Only if a `None` slot ends up IN a carryover does the -analyser fail the switch-dispatch classification for that shape. -The same `Option` policy applies to the top-level -`compute_carryover_types` for switch-dispatch (top-level) routing. If a -function still reaches an unsupported carryover shape, the tool rejects that -shape loudly; there is no guard-dispatch fallback after the mega-PR cleanup. - -## Auxiliary tables - -When the module has at least one fork-path ref-typed user local of a given -class, the tool emits a per-class stash table: - -``` -(table $_wpk_fork_funcref_stash funcref) -(table $_wpk_fork_externref_stash externref) -(table $_wpk_fork_exnref_stash (ref null exn)) -``` - -Modules with no ref-typed fork-path locals of a given class emit no table for -that class. A module with no fork-path try_tables and no fork-path ref-typed -locals emits zero aux tables. - -Slot assignment is per-class and contiguous: - -- The tool walks the fork-path functions in deterministic order. -- For each function, each ref-typed user local gets the next slot in its - class's table. -- For each fork-path `try_table`, the exnref class additionally reserves one - slot to hold the currently-caught exnref while a handler runs. - -Each table's `initial` size is set to exactly the assigned slot count so the -cost is bounded. Slot indices are baked into the postamble (as `table.set`) -and preamble (as `table.get`) of the owning function, and into the `$capture` -blocks emitted for fork-path `catch_ref` / `catch_all_ref` clauses. - -Scalar operand-stack values at call sites are spilled to synthetic scalar -locals, not tables — they are scoped to a single call-site window and do not -cross the unwind/rewind boundary. +The materialization whitelist excludes calls, memory/table operations, globals, +reference operations, integer div/rem, floating-point operators, +`local.set`/`local.tee`, and any instruction that needs stack input from before +the suffix. A suffix outside that replay-safe optimization is still supported: +the typed spill/recipe path preserves it instead. This keeps REWIND behavior +tied to the same post-call/post-landing sequence while avoiding frame locals +for common compiler shapes like recursive `walk(depth - 1)` arguments and +`eqz(depth)` branch conditions. + +**Function-level analyser invariant.** `walk_seq_for_carryovers`, +`compute_carryover_types`, and `compute_nested_carryover_types` must determine +the exact pushed types for every valid producer that reaches a fork landing. +An unknown slot consumed earlier is irrelevant; an unknown live carryover is an +instrumenter typing defect to fix, not an accepted source-program limitation. +There is no guard-dispatch fallback after the mega-PR cleanup. + +## Reference and table-state ownership + +ABI 43 retires `_wpk_fork_funcref_stash`, +`_wpk_fork_externref_stash`, and `_wpk_fork_exnref_stash`. The tool never +emits them. Static slots were unsafe twice over: recursive/reentrant +activations could alias, and every fork child starts from a fresh module +instance whose tables are empty. JavaScript cannot generically transfer +`funcref`/`externref` across workers or Stores, and the Table API cannot copy +`exnref`. + +Closure and liveness analysis runs before rewriting so functions wholly outside +the fork closure remain untouched. Within a live activation, the generated +representation is selected by value class: + +- scalar locals, parameters, arguments, and carryovers use the linked frame; +- `funcref` values use an activation-scoped immutable function catalog; +- static references use the fresh instance's static-root catalog; +- concrete and abstract GC references use versioned typed struct/array/i31 + recipes with graph identity established before recursive fields, preserving + cycles and aliases; +- externalized GC values pass through Wasm's `any.convert_extern` / + `extern.convert_any` bridge so their typed identity is not mistaken for an + opaque host object; +- opaque `externref` values use a process-image owner handle. Each Worker has a + generation-branded canonical token; imports resolve the token at the owner + boundary rather than transferring the JavaScript object; +- complete Wasm/JSTag exceptions use an exception recipe whose payload + references the same process graph. + +Reference-bearing function signatures, `call_ref`/`return_call_ref`, nullable +and non-null concrete types, reference arguments/results, and reference +operand-stack carryovers all use those same recipes. A fresh child materializes +providers first, restores module state second, then consumes continuation +frames. Capture, successful replay, abort replay, process-image replacement, +and worker teardown clear transaction-local tables and leases so +instrumentation does not retain stale GC roots. + +Constructor provenance is recreated as part of typed GC materialization. +Immutable arrays and mutable aggregates with non-defaultable reference seeds +cannot always be allocated from their final field snapshot alone, so the +recipe records the exact constructor layout, up to sixteen scalar operand +bytes, and typed seed edges. The generated allocate helper registers that same +weak provenance for the fresh child object before releasing its staging +record. Consequently a child can fork again and reconstruct an equivalent +grandchild; it never needs a weak-map entry keyed by the parent's Store-local +object. Nullable constructor seeds, including the unobservable seed of a +zero-length array, remain canonical recipe zero. + +Mutable reference globals and tables are module-state, not activation-frame +fields. Generated KFMS helpers save mutable globals, table length, sparse dirty +pages, element/data segment lifetime, and typed entries. Static initialization +is recreated by instantiation; runtime `table.set`, `table.fill`, `table.copy`, +`table.init`, and `table.grow` effects are restored from the process-owned +state. A generation-published table journal brings pthread replicas to the +same state before indirect/table-reference use. The dynamic-link archive first +recreates side modules at their exact memory/table bases and registers their +function catalogs; table-state replay then resolves entries against those +fresh functions. Publication writes records before the generation fence, so a +reader can never treat a partially initialized recipe as current. ## Catch-handler resume -Catch-handler resume is the subtlest piece of the tool. The overall idea: -at unwind time, save the caught exnref into the stash table and record the -try_table's `catch_region_id` in the frame. At rewind time, re-throw the -saved exnref *from inside the same try_table body*, so the normal wasm -exception-dispatch rules deliver it back to the original catch clause, which -sends control into the handler — whose own state-machine preamble then -continues to the fork call site. +Catch-handler resume saves a reconstruction recipe, never a parent-instance +exception reference. Normal handler entry records one function-wide exact +region/arm selector. A statically tagged scalar arm stores its tag payload in a +typed scratch union that overlays the maximum active tuple in the linked frame. +Rewind dispatches inside the same `try_table` body, restores the selected tuple, +and executes the selected arm's `throw $tag`. Normal Wasm exception dispatch +reaches the original clause; `CatchRef` receives a new exnref owned by the child +instance. + +Reference-bearing tag payloads, vector payloads, `CatchAll`, `CatchAllRef`, and +legacy-EH cleanup handlers use the complete-exception codec. The codec retains +the caught value only for the activation lifetime needed to encode its recipe, +then reconstructs and throws it inside Wasm during replay. It never asks +JavaScript to return an `exnref`. ``` ┌────────────────────────────────────────────────────────────────────┐ @@ -915,9 +1067,8 @@ continues to the fork call site. │ more_handler_code │ └────────────────────────────────────────────────────────────────────┘ │ - │ unwind: save exnref X to stash, - │ frame.catch_region_id = K, - │ frame.exnref_slot = S, + │ unwind: save region K, arm A, + │ and scalar tag payload in this frame, │ drain frames to top. ▼ ┌────────────────────────────────────────────────────────────────────┐ @@ -927,11 +1078,10 @@ continues to the fork call site. │ state == REWINDING, load our frame │ │ │ │ try_table body rewind-throw stub: │ -│ state == REWINDING && catch_region_id == K → │ -│ throw_ref (table.get $_wpk_fork_exnref_stash S) │ -│ ← caught by try_table's own catch clause, dispatches to │ -│ the $capture block; $capture branches to $outer, placing │ -│ control at the top of the user's handler code. │ +│ state == REWINDING && catch_selector == (K, A) → │ +│ validate arm A; push saved scalar payload; throw $tag_A │ +│ ← caught by the original Catch/CatchRef clause; CatchRef │ +│ creates a fresh child-instance exnref. │ │ │ │ handler-level preamble (state still REWINDING): │ │ resume at the fork() call site with return value = child pid 0 │ @@ -940,20 +1090,14 @@ continues to the fork call site. └────────────────────────────────────────────────────────────────────┘ ``` -`catch_ref` and `catch_all_ref` clauses use the exnref stash + `throw_ref` -flow above. Plain (non-`_ref`) `catch` clauses have no exnref to re-throw. -For those clauses, normal capture stores the exact nonnegative catch-list arm -index and scalar operand tuple in activation-local frame locals. Rewind -compares the restored arm index, pushes that arm's restored operands, and -throws the original tag back through the same `try_table`. - -In a region that mixes `_ref` and plain catches, `_ref` capture stores -`active_arm = -1`; exact nonnegative IDs select plain arms and every other -value falls through to `throw_ref`. The mode is deliberately frame-owned. -Auxiliary-table nullness is neither activation identity nor reliable handler -kind state because a static table entry can retain an older value. See -[Fork-from-plain-catch](#fork-from-plain-catch) under "Maintainer notes" for -the implementation. +Mixed `Catch`/`CatchRef` lists, multiple arms, distinct target labels, +`CatchAll`/`CatchAllRef`, reference-bearing payloads, recursion, loop re-entry, +and catches followed by an ordinary merged fork all use the same exact +activation selector and process recipe graph. An unknown selector, malformed +recipe, stale process generation, or catalog mismatch traps before child code +can consume partial state; replay cannot fall back to old instance state. See +[Fork from a tagged catch](#fork-from-a-tagged-catch) under "Maintainer notes" +for the implementation. ## Call-graph discovery @@ -1011,9 +1155,11 @@ K-04, and K-07 cover the current behavior. invoked `fork()`. - **Scalar user locals.** All i32, i64, f32, f64, and v128 locals on the fork-path are saved to linear memory at unwind and restored at rewind. -- **Ref-typed user locals.** funcref, externref, and exnref locals are - spilled to aux tables at unwind and restored at rewind. Slot assignments - are deterministic per module. +- **Fresh-instance ownership.** Every accepted replay value is either scalar + activation state in the linked continuation or state rebuilt by an explicit, + versioned reconstruction owner. Instrumented modules carry + `FORK_CAP_ACTIVATION_STATE_SAFE`; ABI 43 hosts and artifact guards reject a + fork-shaped artifact without that capability before execution. - **Byte-reproducible instrumentation.** Given the same input bytes, CLI options, and built tool, separate processes emit byte-identical Wasm. Synthetic locals and nested regions are assigned in canonical sequence-ID @@ -1022,10 +1168,30 @@ K-04, and K-07 cover the current behavior. `wpk_fork_unwind_begin` and restored in `wpk_fork_rewind_begin`. Includes `__stack_pointer`, `__tls_base`, and any program-declared mutable globals. -- **try_table context.** Frames captured inside a supported fork-path catch - handler carry the active `catch_region_id`. `_ref` catches replay from their - exnref stash slot; plain catches replay from activation-local arm and scalar - payload state serialized in the same frame. +- **Reference activation state.** Live reference locals, parameters, + reference call arguments/results, call-ref callees, and operand-stack + carryovers are represented by typed process recipes and decoded into the + fresh activation. Definitely-null references consume no recipe entry. +- **Mutable reference globals and tables.** KFMS module-state helpers restore + reference globals, sparse table contents and length, and passive-segment + lifetime before frame replay. Process generation fencing keeps pthread + replicas and late dynamic-link consumers coherent. +- **Exception context.** Frames captured inside a catch handler carry the + exact dynamic region/arm selector. Scalar tagged payloads occupy an overlaid + maximum-size tuple; reference/vector payloads and untagged catches use a + complete-exception recipe. Replay throws inside Wasm so `CatchRef` and + `CatchAllRef` receive fresh child-instance exnrefs. +- **No stale replay roots.** The instrumenter emits none of the historical + `_wpk_fork_*ref_stash` tables. Temporary codec slots, retained caught + exceptions, anyref transit entries, and externref handle leases are cleared + on normal completion, successful replay, abort, process-image replacement, + and worker teardown. +- **Frame-pressure bounds.** Ordinary reference recipes add no source-function + local and no bytes beyond the existing 16-byte linked-frame header: + catch selector and vector ordinal reuse `+8` and `+12`. Catch operand storage + is colored by maximum simultaneously live type tuple rather than static arm + count. Generated helper-function locals and the process recipe arena are + outside every ordinary native activation. - **Kernel-side-effect calls don't re-fire during REWIND.** Switch-dispatch (the only live scheme post-commit-4) skips the body chunks before the matching `POST_K` entirely on REWIND, so non-fork-path direct calls @@ -1033,42 +1199,18 @@ K-04, and K-07 cover the current behavior. side-effect ops in those chunks run exactly once, on the parent's NORMAL pass. No per-call or per-op gating is needed. -### Not guaranteed (unsupported patterns) +### Boundaries outside activation replay - **`makecontext` / `swapcontext` / `getcontext` / `setcontext`.** Userspace stack-switching primitives are unsupported and not on any roadmap. See [posix-status.md](posix-status.md) for rationale. -- **Functions whose plain-catch arms carry ref-typed operands.** Catch arms - whose operand tuple includes an `(ref ...)` value (typically a function or - GC ref) are excluded from plain-catch replay support at instrument time. - The function may still be instrumented for other fork sites, but a fork - reached from the affected handler remains unsupported. Spilling ref-typed - catch operands would require per-arm activation-aware auxiliary storage. - The current implementation keeps that explicit - `PlainCatchPlan::b2_carveout` boundary and covers it with WAT-level tests. - This is an unanticipated Wasm-level case, - not expected output from ordinary C++ EH lowering: C++ exception payloads - live in linear memory / libc++abi state rather than as `funcref` or - `externref` plain-catch tag operands. If a future language frontend or - hand-written Wasm module needs it, implement per-arm funcref/externref - aux-table stashing and promote C-08/C-09 from carve-out validation to full - replay tests. -- **Recursive/reentrant ref activation state.** Auxiliary-table slots for - ref-typed user locals and caught exnrefs are assigned statically per - function or `try_table`. Recursive or reentrant activations that keep - distinct ref values live across one fork can therefore alias those slots. - Scalar plain-catch arm/payload state does not have this limitation because - it is serialized per activation. Closing the ref case requires - activation-aware auxiliary storage and executable recursive regressions. -- **IfElse with operand-stack carryover.** A fork-bearing `if/else` - enclosing a stack value that survives across the branch is rejected by - `seq_has_unsupported_carryover` — the cond rewrite via `select` (see - §IfElse cond rewrite) doesn't currently compose with carryover spilling. - Rare in LLVM output; not tracked as a current blocker. -- **Wasm-GC refs.** Abstract `any` / `eq` / `struct` / `array` / `i31` refs - and concrete GC refs are rejected at the `classify_ref` step — the tool - panics rather than produce a silently-broken module. Add classes in - `crates/fork-instrument/src/instrument.rs` when a real program needs them. +- **Host engine proposal support.** The input must be a valid module for both + the transform's parser and the target Node/browser engine. The ABI does not + emulate a WebAssembly proposal that the selected engine itself cannot + instantiate. +- **Stale or incomplete artifacts.** ABI 42 fork artifacts, copied capability + bytes, malformed recipe metadata, and mixed-version host/module contracts + fail before execution. They are rebuild inputs, not compatibility modes. #### Closed since the mega-PR's 2.5/2.6 sub-commits @@ -1135,15 +1277,20 @@ and loop structure but does not allocate continuation memory. The module-format fixed cost is three imports, two abort exports, plus the 24-byte `kandelo.wpk_fork.linked_frames` descriptor and normal Wasm section/name encoding. The fixed 60 KiB host-reserved control-region geometry remains in -ABI 42, but it is no longer continuation capacity: only its anchor word is -used to find the dynamically allocated root chunk. +place from ABI 42, but it is no longer continuation capacity: only its anchor +word is used to find the dynamically allocated root chunk. -A function with supported plain catches adds one i32 `active_arm` local per -plain-capable region plus typed scalar operand locals for every supported arm -to each activation's frame payload. This can use more aggregate continuation -bytes than one module-global tuple, but distinct activation storage is required -for recursion and reentrancy correctness and removes normal-execution writes -to continuation-owned memory. +The ABI 43 deferred-signal request flag occupies the channel header, and the +post-import checkpoint is an ordinary syscall. Neither adds bytes to the +linked continuation, its 16-byte frame header, or an activation payload. + +A function with tagged catches uses one function-wide exact-arm selector and +one typed scalar operand union colored to the maximum simultaneously live +payload. The fixed frame header remains 16 bytes, references add no frame +bytes, and additional catch arms do not each allocate a tuple. A scalar catch +payload can still enlarge that function's frame by the maximum live tuple; +activation-owned storage is required for recursion and reentrancy correctness +and replaces the unsafe module-global tuple. As a narrow size check, instrumenting the P-10 deep-recursion fixture from the same 27,886-byte raw Wasm produced 50,873 bytes with the ABI 41 instrumenter @@ -1240,22 +1387,26 @@ interceptor recurses into ASAN init which holds a spin mutex). The fuzzer targets validator/semantic divergence rather than memory-safety, so ASAN is not load-bearing. -### Adding a new ref type +### Supporting additional reference state + +Do not add a module-static reference stash. A fresh fork child has a new Wasm +instance, table, Store, and exception-tag identity, so a slot number is not a +transferable value even if it happens to fix same-instance recursion. + +Support for a new reference shape extends one of two complete designs: -Ref types accepted for local / global spilling are gated by `classify_ref` -in `crates/fork-instrument/src/instrument.rs`. To add support for a new -class: +1. Encode every value needed by replay as versioned activation-owned bytes in + the linked continuation, then reconstruct the reference deterministically + in the child. +2. Name an explicit host reconstruction owner, version its recipe, and prove + Node, browser, pthread, and side-module parity. -1. Extend the `RefClass` enum with the new class. -2. Map the corresponding `HeapType` variant in `classify_ref` to the new - class. -3. If the new class cannot share an existing stash table (e.g. it is a - wasm-GC ref that requires `ref.cast` at reload time), add a new table to - `AuxTables`, size it the same way the existing classes do, and extend - the spill / reload emitters to target it. -4. Add a fixture test under `tests/instrument.rs` that exercises the new - type both as a local and as a function parameter, and confirms the - module validates after round-tripping through the tool. +Add a positive fresh-instance replay test that would fail if the parent +module's globals or tables were consulted, plus malformed/version-mismatch +tests for the ownership contract. A valid source shape is not converted into +an instrumentation rejection merely because its reconstruction provider is +new work. Update the capability contract and bump the ABI if the accepted +artifact surface or reconstruction format changes. ### Extending side-effect coverage @@ -1265,48 +1416,54 @@ the containing switch-dispatch shape skips that opcode on REWIND. Existing examples are the S-01..S-08 host fixtures plus the WAT-level table-operation tests in `crates/fork-instrument/tests/coverage_wat.rs`. -### Fork-from-plain-catch +### Fork from a tagged catch -Plain (non-`_ref`) `catch` arms unwrap the thrown exception's operand tuple -onto the operand stack at handler entry, but unlike `catch_ref` / -`catch_all_ref` they do not push an exnref. The Phase 6 rewind-throw stub -reaches the handler by `throw_ref`-ing a saved exnref into the original -try_table's catch clause; with a plain catch there is no exnref to save, so -some other resume path is needed. +`Catch` arms unwrap the thrown exception's operand tuple at handler entry. +`CatchRef` arms additionally push an instance-local exnref. Neither reference +identity nor module scratch is available in a fresh child, so both forms replay +from activation-owned selectors and typed recipes. The implementation adds that path without accessing continuation memory during ordinary catch execution: 1. **Static discovery (`plan_plain_catches`).** Walk each fork-path function - and collect every supported plain-catch arm's tag, target label, catch-list - index, and operand types. This plan contains no runtime addresses or - activation state. -2. **Activation allocation (`allocate_plain_catch_state`).** Allocate one - `PlainCatchRegionState` per static region: an i32 `active_arm` plus typed - scalar operand locals for every supported arm. -3. **Frame ownership (`append_plain_catch_frame_scalars`).** Add those locals - before frame offsets and size are assigned. Each recursive activation then - saves and restores its own catch values in its own linked frame. -4. **Capture and replay.** `apply_plain_catch_handlers` stores the incoming - operand tuple and exact arm index in those locals, then re-pushes the - operands for the user handler. `inject_rewind_throw_stubs` dispatches on - the restored exact arm index and rethrows its tag with the restored tuple. - Mixed `_ref` capture records `-1`, which selects the `throw_ref` fallback - without consulting stale table nullness. -5. **Carve-out (`PlainCatchPlan::b2_carveout`).** A function whose plain-catch - arms carry ref-typed operands or use an unverified multi-target shape is - excluded from supported plain-catch replay. Other fork sites in that - function may still be instrumented. Scalar frame serialization does not - solve reference ownership; that needs activation-aware auxiliary storage. - -The lifetime boundary is load-bearing: a plain catch can run before any fork -or after a prior continuation has been released. Its normal capture path must + and collect every `Catch`, `CatchRef`, `CatchAll`, and `CatchAllRef` arm's + tag when present, target label, exact catch-list index, kind, and operand + types. Legacy `try` handlers are normalized to the same modern-EH control + representation. This plan has no runtime addresses or activation state. +2. **Activation allocation.** Allocate one function-wide exact-arm selector + and a typed operand-scratch union sized by maximum simultaneous use. + Scalar `CatchRef` forwarding uses one short-lived nullable exnref scratch; + complete-exception arms retain only the liveness-colored recipe roots + required at a fork landing. +3. **Frame ownership.** Header word `+8` stores the selector. Scalar payload + types overlay one maximum-size frame range; reference/vector payloads are + edges in the process recipe graph. Each recursive or reentrant activation + therefore owns a distinct recipe without cost proportional to static arm + count. +4. **Capture.** A generated block records the incoming tuple or complete + exception and exact selector, then restores the original handler stack. + Short-lived forwarding scratch is cleared before user code; retained recipe + roots are cleared by transaction completion or abort. +5. **Replay.** `inject_rewind_throw_stubs` dispatches on the restored selector. + Scalar arms push their tuple and execute `throw` with the original tag. + Complete-exception arms materialize and throw inside Wasm. The original + clause then reconstructs its payload and, for reference clauses, a fresh + child-local exnref. An unknown selector traps instead of consulting old + instance state. + +The lifetime boundary is load-bearing: a catch can run before any fork or +after a prior continuation has been released. Its normal capture path must therefore never dereference `_wpk_fork_buf`. -C-08/C-09 in -`crates/fork-instrument/tests/coverage_wat.rs` verify that funcref and -externref catch operands retain this explicit unsupported boundary rather than -being serialized as scalars. +C-08/C-09 verify the transformed funcref/externref catch shapes. The Node +`catch-ref-fresh-worker` test and Chromium continuation gate additionally +execute non-null funcref and nullable externref payloads through `CatchRef` in +new process Workers; the child calls the reconstructed funcref and receives a +fresh child-local exnref. The module-exception, GC-codec, process-owner, and +mailbox suites separately cover vector payloads and non-null opaque externref +ownership. Together these gates prove that catch operands use typed recipes +rather than being misclassified as scalars or placed in a module-static table. ## See also diff --git a/docs/package-management.md b/docs/package-management.md index f2fd44c659..9c2199b1d1 100644 --- a/docs/package-management.md +++ b/docs/package-management.md @@ -658,9 +658,12 @@ parsed `DepsManifest` at load time) and defaults to 1 when Program packages that use fork instrumentation also hash the fork-instrument host tool inputs (`crates/fork-instrument`, the -workspace Cargo lockfile, and the wrapper/build scripts). Programs -that declare `fork_instrumentation = "disabled"` do not hash that -tooling. +target-unfiltered non-dev Cargo dependency closure selected from the +workspace lockfile, and the wrapper/build scripts). The dependency closure is +the union across build-host target predicates: package cache paths do not have +a build-host dimension, so filtering through the current macOS or Linux host +would give identical source trees different identities. Programs that declare +`fork_instrumentation = "disabled"` do not hash that tooling. The global toolchain/sysroot fingerprint covers the reproducible build environment and sysroot recipe: the Nix flake, Rust toolchain file, diff --git a/docs/plans/2026-07-25-abi-43-activation-state-safe-rebuild-plan.md b/docs/plans/2026-07-25-abi-43-activation-state-safe-rebuild-plan.md new file mode 100644 index 0000000000..ccd2161c72 --- /dev/null +++ b/docs/plans/2026-07-25-abi-43-activation-state-safe-rebuild-plan.md @@ -0,0 +1,156 @@ +# ABI 43 activation-state-safe artifact rebuild plan + +Status: development plan only. No canonical package, bottle, index, shell, or +VFS publication is authorized by this document. + +## Why + +ABI 43 changes the ownership contract for fork continuations. A fresh child +must reconstruct reference locals, exceptions, mutable reference globals, and +mutable tables from activation/process-owned state; an ABI 42 artifact cannot +be made safe by relabeling it. The ABI number and +`FORK_CAP_ACTIVATION_STATE_SAFE` capability must therefore move together +through every executable, archive, index, and derived image. + +This rebuild must not delay, mutate, reuse, or publish over the separate ABI 42 +Bash/Homebrew proof. In particular, do not modify the +`emdash/homebrew-complete-qk044` worktree, PR #1094, its branch, its commits, or +its publication namespaces. + +## Frozen-input gate + +Do not start a publishable rebuild until all of the following are true: + +1. The instrumenter, host imports, table journal, pthread path, side-module + replay, and cleanup contracts have stopped changing. +2. `ABI_VERSION` is 43, `abi/snapshot.json` and generated TypeScript constants + are regenerated, and both ABI checks pass. +3. The source-controlled program-package projection is regenerated after the + final instrumenter/tool digest. That digest must use the target-unfiltered + non-dev Cargo dependency union; a current-host-filtered graph gives macOS + and Linux different keys in a cache namespace that has no host dimension. + A stale projection must fail the limited rootfs-scope derivation instead of + selecting an incomplete rebuild. +4. Fresh-instance Node and browser tests, pthread fork, side-module/dlopen + replay, artifact guards, and the selected POSIX/package gates are green. +5. Brandon has explicitly approved the exact final head for kernel/fork + integration. This plan does not authorize merge or publication. + +## Kandelo package archive scope + +The guest ABI is an input to every library/program cache key. The registry +currently contains 77 ABI-bound packages: + +- 10 libraries, producing 14 architecture generations: + `icu`, `libcurl`, `libcxx`, `libiconv`, `libpng`, `libxml2`, `libzip`, + `openssl`, `sqlite`, and `zlib`; +- 67 programs, producing 69 architecture generations. The committed program + projection covers 65 of those packages/67 generations; the special `kernel` + and `userspace` packages add one wasm32 generation each. + +That is 83 ABI-bound `(package, architecture)` archive generations. The +source-only `pcre2-source` package is not itself a guest-ABI artifact; it is +rebuilt only if its own source-package identity changes. Do not bump package +`revision` merely for the ABI epoch: ABI 43 already changes the cache key. + +Of the 65 projected program packages, 57 packages/59 generations contain an +output whose fork-instrumentation policy is `auto`. These outputs must be +rebuilt from raw linker output with the ABI 43 instrumenter. The eight +all-disabled packages (`homebrew-bootstrap`, `nginx-php-vfs`, `nginx-vfs`, +`node`, `node-vfs`, `redis-vfs`, `spidermonkey`, and +`spidermonkey-node`) still need ABI 43 archive generations because the archive +ledger and any embedded ABI-bound dependencies are single-epoch; disabling +fork instrumentation is not permission to reuse an ABI 42 archive. + +Build libraries before their transitive program consumers, then publish only +to an isolated PR-staging or run-specific merge-candidate ledger. The complete +candidate index must have top-level ABI 43, contain only `-abi43-` archive +identities, and pass archive/artifact guards before it can be considered for +canonical activation. + +### Current isolated staging evidence + +Exact-head run `30193794024` is prepublication evidence, not a complete +candidate. All 14 library generations and 56 of 62 attempted program +generations built successfully. The six attempted failures are `shell` plus +its five direct consumers (`lamp`, `nginx-php-vfs`, `nginx-vfs`, `node-vfs`, +and `wordpress`); every one failed closed because the available Homebrew VFS +metadata declares ABI 41 while the candidate requires ABI 43. This proves one +coordinated missing-input boundary rather than six independent transform +failures. + +Seven further ABI-bound program generations were not attempted by the +inherited staging workflow: the expensive-package exclusion covers +`erlang-vfs`, `perl`, `perl-vfs`, `python-vfs`, `redis`, and `texlive`, while +`sqlite-cli` has no staging build block. The isolated `pr-1096-staging` index +therefore contains 70 successful entries and six failed entries, not all 83 +required generations. It must not be promoted, used as a complete test-gate +input, or described as a publication candidate. The successful independent +rootfs job built all 336 declared paths into a 16,787,687-byte ABI 43 image, +which is useful closure evidence but does not fill the missing generations. + +## Rootfs and derived image scope + +The exact current wasm32 rootfs closure is 15 package generations: + +`bash`, `bc`, `coreutils`, `dash`, `diffutils`, `file`, `findutils`, `gawk`, +`grep`, `m4`, `make`, `ncurses`, `posix-utils-lite`, `sed`, and `rootfs`. + +A local source build of this closure is useful early evidence, but the +`stage-rootfs-closure-only` path is deliberately incomplete and cannot be used +for prepare-merge. After the final tool digest, regenerate the package +projection, derive the scope mechanically, and require it to select these +generations with their new cache keys. + +After the full dependency archive set is available, rebuild every composite +runtime/image output whose cache key or embedded executable changes. The +current projection includes: + +- `rootfs.vfs`, `shell.vfs.zst`, and `kandelo-sdk.vfs.zst`; +- `erlang-vfs`, `lamp`, `mariadb-test`, `mariadb-vfs`, `nginx-php-vfs`, + `nginx-vfs`, `node-vfs`, `perl-vfs`, `python-vfs`, `redis-vfs`, and + `wordpress` VFS outputs; +- CPython/Ruby runtime archives, Nethack/Vim browser bundles, and the Texlive + bundle where their owning package generation changes. + +The browser-facing checked or published images must be created from the exact +ABI 43 candidate index and tested as immutable candidate bytes in Node and +Chromium/Firefox/WebKit. Do not copy an ABI 42 executable into a new image or +rewrite its metadata. + +## Homebrew scope and isolation + +Homebrew uses the separate `bottles-abi-v43` namespace and Formula-controlled +bottle identities. An ABI 43 acceptance run must rebuild the 36 direct roots +in `homebrew/main-shell.Brewfile` plus the exact transitive bottle closure, +regenerate sidecars/provenance, and build a new content-addressed Homebrew VFS +acceptance image. Formula revision/bottle-rebuild changes belong to that +coordinated run; do not make speculative bumps in the fork implementation PR. + +The existing ABI 42 Bash/Homebrew proof remains an independent input and +historical result. Do not edit its worktree, commits, sidecars, bottle +namespace, index, shell lock files, or VFS image to make ABI 43 validation pass. +ABI 43 must succeed from its own rebuilt bytes. + +## Ordered execution + +1. Freeze code and generated ABI/package metadata. +2. Run focused instrumenter/host/fresh-instance tests and the full required + dev-shell validation on the implementation head. +3. Rebuild the 15-generation rootfs closure locally as an early source-build + proof; do not publish it. +4. Build all 83 ABI-bound registry generations into isolated staging, + dependency order first, and seal one complete ABI 43 candidate index. +5. Build all derived VFS/runtime/bundle artifacts from that exact candidate. +6. Run Node, browser, pthread, side-module/dlopen, libc/POSIX/Sortix, Bash, + shell, package-guard, and lifecycle coverage against the candidate bytes. +7. In a separate coordinated Homebrew run, build the ABI 43 bottle closure, + sidecars, shell closure, and content-addressed VFS evidence without touching + the ABI 42 proof. +8. Report exact successful, failed, and unrun generations/tests. Canonical + index activation, bottle publication, VFS publication, and merge require + explicit coordination and Brandon's approval of the exact head. + +ABI 42 releases remain immutable historical state. If ABI 43 validation fails, +leave its candidate/staging evidence isolated and fix the platform or rebuild +input; do not fall back to a mixed-ABI index or relabeled artifact. diff --git a/docs/porting-guide.md b/docs/porting-guide.md index 7f08239583..5fa43eee31 100644 --- a/docs/porting-guide.md +++ b/docs/porting-guide.md @@ -584,7 +584,7 @@ library dep) for canonical references; the schema reference is in kind = "program" # or "library" or "source" name = "myprog" version = "1.2.3" -kernel_abi = 42 # current ABI_VERSION; required for packages with a [build] block +kernel_abi = 43 # current ABI_VERSION; required for packages with a [build] block depends_on = ["zlib@1.3.1"] # transitive deps the resolver will pull first [source] diff --git a/docs/posix-status.md b/docs/posix-status.md index bd4f5eb469..9afcb7ea4d 100644 --- a/docs/posix-status.md +++ b/docs/posix-status.md @@ -121,7 +121,7 @@ same final-OFD lifetime rules. | Function | Status | Notes | |----------|--------|-------| -| `fork()` | Partial | The kernel validates the calling task, allocates the child PID, and copies process state; the host starts a child Worker with copied Memory. The child inherits the calling task's blocked signal mask, and libc refreshes a copied pthread TID from the kernel before returning from `fork()`. Initial launch mirrors the environment into kernel-owned process state; fork copies that metadata while instrumented rewind preserves the live libc `environ` in copied Memory, and `execve()` replaces both from its supplied `envp`. `wasm-fork-instrument` resumes the child at the call site with preserved stack locals and mutable globals. Root or later continuation-allocation failure and a negative `SYS_FORK` result unwind transactionally, create no child, and return the failure to the still-running parent. Main-thread and pthread fork are supported, as is the documented direct main-to-one-side-module path; nested/opaque cross-side callbacks and fork from a pthread inside a side module remain unsupported. Pipes, sockets, PTYs, eventfd/timerfd/signalfd, memfd, procfs snapshots, and shared mappings retain their existing backings; signal and wait lifecycle state is copied/coordinated by the kernel. An inherited directory drops the parent's process-local host iterator and lazily reopens at the copied next-record cookie, so handles cannot alias, but later parent/child cursor movement is not shared. Ordinary regular-file OFD seek positions/status flags have the same copied rather than shared boundary. See [fork-instrumentation.md](fork-instrumentation.md) and the known OFD gap below. | +| `fork()` | Partial | The kernel validates the calling task, allocates the child PID, and copies process state; the host starts a child Worker with copied Memory. The child inherits the calling task's blocked signal mask, and libc refreshes a copied pthread TID from the kernel before returning from `fork()`. Host-owned continuation and fork channel requests leave caught signals kernel-pending; after the import returns, libc performs an ordinary syscall checkpoint so the guest signal trampoline owns handler invocation and mask restoration without host-to-Wasm reentrancy. Initial launch mirrors the environment into kernel-owned process state; fork copies that metadata while instrumented rewind preserves the live libc `environ` in copied Memory, and `execve()` replaces both from its supplied `envp`. `wasm-fork-instrument` resumes the child at the call site with scalar locals in linked frames and versioned reconstruction recipes for references, exceptions, globals, tables, and dynamic-link activations. Root or later continuation-allocation failure and a negative `SYS_FORK` result unwind transactionally, create no child, and return the failure to the still-running parent. Main-thread and pthread fork are supported, including nested main/side-module stacks and process-owned dynamic-link/table replay. Pipes, sockets, PTYs, eventfd/timerfd/signalfd, memfd, procfs snapshots, and shared mappings retain their existing backings; signal and wait lifecycle state is copied/coordinated by the kernel. An inherited directory drops the parent's process-local host iterator and lazily reopens at the copied next-record cookie, so handles cannot alias, but later parent/child cursor movement is not shared. Ordinary regular-file OFD seek positions/status flags have the same copied rather than shared boundary. See [fork-instrumentation.md](fork-instrumentation.md) and the known OFD gap below. | | `exec()` | Partial | Kernel-initiated via SYS_EXECVE (syscall 211). The host preflights the module, ABI, replacement memory, caller, deferred file actions, and a 4 MiB combined argv/environment representation (strings, terminators, and pointer entries) before replacing the image in place; individual strings are limited to 64 KiB and oversize returns `E2BIG` without truncation. Preserves PID, non-CLOEXEC fds and their exact kernel-backed object state, new argv/envp (including an explicitly empty environment), CWD, the calling pthread's signal mask and directed queue, terminal queues, and `alarm()`/`ITIMER_REAL`; closes directory streams, deletes `timer_create()` timers, publishes and detaches old mappings, terminates sibling threads, and resets the program break before installing the new `__heap_base`. File mappings retain a stable writeback handle even after their original fd closes. Remaining gaps: POSIX message-queue descriptors are not process-owned and therefore cannot yet be closed on exec; epoll registrations track numeric fds rather than OFD identity, so close/dup and same-number replacement cases are incomplete; and main-thread-directed signals share the process-pending queue and therefore cannot be distinguished from process-directed signals when a worker pthread execs. | | `wait()` / `waitpid()` / `wait4()` / `waitid()` | Partial | Rust-owned child status covers stop, continue, normal exit, and signal death. New status replaces older unconsumed status; `waitid(WNOWAIT)` preserves the current record. `WNOHANG`, `WUNTRACED`/`WSTOPPED`, `WEXITED`, and `WCONTINUED` are supported, as are specific-PID, any-child, same-process-group, and specific-process-group selection. Stop/continue reports do not reap; consuming exit status does. `wait4()` returns the zero-filled resource-usage wire record described under `getrusage()`. Remaining gap: a blocked `pid == 0` / `P_PGID,id == 0` wait currently re-evaluates the caller's process group on each host retry instead of freezing it at call entry. | | `exit()` / `_exit()` | Full | Closes all fds and dir streams, releases locks and mapping/backing ownership, and retains the low eight status bits. Normal codes 128–255 remain distinct from signal termination, which is stored separately. SIGCHLD is delivered to the parent and zombie state remains until `waitpid()` reaps it. | @@ -144,7 +144,7 @@ same final-OFD lifetime rules. | `futex()` | Partial | FUTEX_WAIT, FUTEX_WAKE, FUTEX_REQUEUE, FUTEX_CMP_REQUEUE, and FUTEX_WAKE_OP operate on one process's shared memory. Main-process WAIT uses host `Atomics.waitAsync`; pthread workers use direct `Atomics.wait`. Separate processes have separate `SharedArrayBuffer` objects, so these operations do not wake or synchronize a peer PID even when the futex word lies in a host-coordinated MAP_SHARED mapping. | | `execve()` | Partial | Delegates to the in-place `exec()` path and has the same remaining descriptor/signal/mapping limitations described above. | | `execveat()` | Partial | SYS_EXECVEAT (386). Resolves fd path via `kernel_get_fd_path`, supports AT_EMPTY_PATH for `fexecve()`, and resolves relative paths against process CWD; otherwise has the same remaining `exec()` limitations. | -| `fork()` (syscall) | Partial | Glue traps through channel IPC; the kernel copies process state, the host starts a child Worker, and `wasm-fork-instrument` replays the supported call stack so parent/child receive the POSIX return values. Negative results replay to the caller without terminating the parent, including continuation-allocation failure before or during unwind. The side-module and ordinary-OFD limitations in the main `fork()` row still apply. | +| `fork()` (syscall) | Partial | Glue traps through channel IPC; the kernel copies process state, the host starts a child Worker, and `wasm-fork-instrument` replays the call stack so parent/child receive the POSIX return values. ABI 43 requires the activation-state-safe artifact capability before launch and validates the linked-frame, reference/exception recipe, mutable module-state, table-journal, and activation-catalog contracts. Unsafe ABI 42, malformed, or mixed-version artifacts fail before execution. Negative results replay to the caller without terminating the parent, including continuation-allocation failure before or during unwind. The ordinary-OFD limitations in the main `fork()` row still apply. | | `vfork()` | Partial | Alias for `fork()` and therefore has the same continuation/OFD limitations; it does not provide distinct vfork address-space semantics. | | `posix_spawn()` | Partial | **Non-forking implementation** (this kernel's invention; no Linux equivalent). Glue issues `SYS_SPAWN` (500) with a marshalled blob (argv + envp + file actions + spawn attrs). The host passes the calling TID to `kernel_spawn_process`; the Rust `ProcessTable` validates that task, allocates the child PID from the global task-ID sequence, and builds the child Process descriptor before `onSpawn` launches a fresh Worker. The child inherits the calling task's signal mask unless `POSIX_SPAWN_SETSIGMASK` replaces it. No fork, no `wpk_fork_*` rewind, no exec replay. Supports POSIX_SPAWN_SETSID / SETPGROUP / SETSIGMASK / SETSIGDEF and FDOP_OPEN / CLOSE / DUP2 / CHDIR / FCHDIR. SIG_IGN dispositions persist across the implicit exec; custom handlers reset to SIG_DFL (POSIX exec semantics). An inherited directory never aliases the parent's live host iterator: spawn preserves its next-record cookie and lazily reopens a child-owned iterator there. Its later cursor movement and ordinary-file OFD metadata are still process-local rather than shared; see the known OFD gap below. Regression-guarded: `kernel_get_fork_count` exposes a per-process counter the test suite asserts is unchanged across SYS_SPAWN. See `docs/plans/2026-05-04-non-forking-posix-spawn-design.md`. | | `posix_spawnp()` | Partial | PATH search lives in libc (`libc/musl-overlay/src/process/wasm32posix/posix_spawnp.c`); resolves the absolute path then delegates to `posix_spawn()`. Empty PATH entries are treated as `.` and EACCES is deferred per `__execvpe` policy. It inherits `posix_spawn()`'s cross-process open-file-description limitation. | @@ -169,7 +169,7 @@ same final-OFD lifetime rules. | `ioperm()` / `iopl()` | Stub | Returns EPERM. No I/O port access. | | `remap_file_pages()` | Stub | Returns ENOSYS. | | `getcontext()` / `setcontext()` / `makecontext()` / `swapcontext()` | Unsupported | Userspace stack-switching primitives, deprecated in POSIX.1-2008, not planned. See the "ucontext API unsupported" row under [Wasm-Inherent gaps](#wasm-inherent--gaps-that-cannot-be-fully-resolved-in-wasm) for rationale. | -| `fork()` called from a C++/Ruby exception catch handler | Partial | Modern wasm-EH `try_table` catch replay is implemented. Plain-catch arm identity and scalar operands are activation-local and serialized in each linked frame, so distinct recursive activations do not share module scratch; multi-arm capture/rethrow restores the exact arm. `_ref` catches use `_wpk_fork_exnref_stash`. Existing C-01 through C-07, C-10, C-11, and S-08 coverage plus `host/test/plain-catch-payload-lifetime.test.ts` exercise the supported surface. Remaining gaps are explicit: ref-typed plain-catch operands are carved out, and static auxiliary-table slots can alias caught exnrefs or ref-typed locals across recursive/reentrant activations. See [docs/fork-instrumentation.md §Not guaranteed](fork-instrumentation.md#not-guaranteed-unsupported-patterns). | +| `fork()` called from an exception catch handler | Partial | ABI 43 supports mixed `Catch`, `CatchRef`, `CatchAll`, and `CatchAllRef` arms, including scalar, vector, reference, JSTag, and modern C++ cleanup payloads. Scalar tagged arms serialize one exact activation selector and maximum live operand tuple; complete exceptions use the process reference graph and are thrown inside the fresh Wasm instance so reference clauses receive child-local exnrefs. Multiple arms/targets, recursion, loop re-entry, nested catches, later merged-flow forks, reference locals/carryovers, mutable reference globals, and mutated tables use the same versioned ownership machinery without module-static stashes. Dash and the configured shell/rootfs closure rebuild through this path. This row remains Partial only because `fork()` retains the ordinary open-file-description gaps in the main row, not because catch/reference replay is intentionally excluded. See [fork-instrumentation.md](fork-instrumentation.md). | ## Signals @@ -652,9 +652,23 @@ These PHP needs are well-handled by the current kernel: RTLD_DEFAULT), dlclose, dlerror (Wasm dylink on the process worker) for both wasm32 and wasm64 processes. The wasm64 path uses memory64 pointer globals, GOT entries, and table64 indices without narrowing them to JavaScript - numbers at the Wasm boundary. RTLD_NEXT lookup is not currently supported. - Pthread workers cannot share the process's Wasm table/tag graph, so pthread - `dlopen` fails and pthread `fork` after a process dlopen returns `ENOTSUP`. + numbers at the Wasm boundary. `DT_NEEDED`, `RTLD_LOCAL`/`RTLD_GLOBAL`, + dependency/provider lifetimes, nested loader transactions, pthread + `dlopen`/`dlsym`, and fork after dynamic loading use a process archive plus a + fresh local linker replica in each Worker. ABI 43 libc stages + prepare/initialization so a host import never calls back into Wasm before + returning; constructors and relocation helpers run as ordinary + Wasm-to-Wasm calls. Instrumentation removes native start sections from + accepted ABI 43 modules and lowers the historical canonical two-, four-, and + five-argument `env.__wasm_dlopen` forms to that same staged path while + preserving direct, table, export, and `ref.func` aliases. The earliest + two-argument form retains its deterministic historical buffer-derived module + name. ABI 43 publication and launch guards reject a remaining monolithic + import or native start section in a completed instrumented artifact; source + start sections remain supported through the explicit module bootstrap. + Loader-owned VFS/mapping completions leave caught signals pending, and libc + performs an ordinary signal-delivery checkpoint after each staged import + returns and after `dlclose`. RTLD_NEXT lookup is not currently supported. - POSIX timers: `SIGEV_SIGNAL`, `SIGEV_NONE`, and `SIGEV_THREAD` timer creation, timer_settime, timer_gettime, overrun reporting, and deletion. Timer timing remains host-scheduled at millisecond granularity, and direct wasm64 diff --git a/examples/wait_lifecycle_test.c b/examples/wait_lifecycle_test.c index ba66130047..cf9fbdc204 100644 --- a/examples/wait_lifecycle_test.c +++ b/examples/wait_lifecycle_test.c @@ -493,6 +493,31 @@ static int test_getrusage_pointer_validation(void) return expect_zero_rusage(&usage); } +struct delayed_stop_ctx { + int fd; + atomic_int armed; + int error; +}; + +static void *release_delayed_stop(void *opaque) +{ + struct delayed_stop_ctx *ctx = opaque; + while (!atomic_load_explicit(&ctx->armed, memory_order_acquire)) + usleep(1000); + + /* + * The main thread sets armed immediately before entering waitpid. + * Leave enough time for it to publish the blocking wait to the kernel; + * otherwise a fast fresh child can stop and deliver SIGCHLD before the + * wait begins, in which case POSIX correctly permits that later wait to + * remain blocked. + */ + usleep(50000); + if (write(ctx->fd, "s", 1) != 1) + ctx->error = errno != 0 ? errno : EIO; + return NULL; +} + static int test_nonmatching_sigchld_interrupts_wait(void) { struct sigaction action; @@ -504,16 +529,76 @@ static int test_nonmatching_sigchld_interrupts_wait(void) sigchld_count = 0; int gate[2]; - pid_t pid = spawn_stopping_child(gate, 27); + if (pipe(gate) != 0) + return fail("interrupt test pipe"); + pid_t pid = fork(); if (pid < 0) - return -1; + return fail("interrupt test fork"); + if (pid == 0) { + close(gate[1]); + char byte = 0; + if (read(gate[0], &byte, 1) != 1) + _exit(121); + if (raise(SIGSTOP) != 0) + _exit(120); + if (read(gate[0], &byte, 1) != 1) + _exit(121); + close(gate[0]); + _exit(27); + } + close(gate[0]); + + /* + * SIGCHLD is process-directed. Block it while creating the helper so the + * helper inherits the blocked mask, then restore the main thread's mask. + * The child's stop notification therefore has exactly one eligible + * recipient: the thread blocked in waitpid below. + */ + sigset_t block; + sigset_t previous; + sigemptyset(&block); + sigaddset(&block, SIGCHLD); + int error = pthread_sigmask(SIG_BLOCK, &block, &previous); + if (error != 0) { + errno = error; + return fail("interrupt test block SIGCHLD"); + } + struct delayed_stop_ctx stop = { + .fd = gate[1], + .armed = ATOMIC_VAR_INIT(0), + .error = 0, + }; + pthread_t releaser; + error = pthread_create(&releaser, NULL, release_delayed_stop, &stop); + if (error != 0) { + pthread_sigmask(SIG_SETMASK, &previous, NULL); + errno = error; + return fail("interrupt test pthread_create"); + } + error = pthread_sigmask(SIG_SETMASK, &previous, NULL); + if (error != 0) { + errno = error; + return fail("interrupt test restore SIGCHLD mask"); + } int status = 0; errno = 0; - if (waitpid(pid, &status, 0) != -1 || errno != EINTR || sigchld_count != 1) { + atomic_store_explicit(&stop.armed, 1, memory_order_release); + pid_t got = waitpid(pid, &status, 0); + int wait_errno = errno; + error = pthread_join(releaser, NULL); + if (error != 0) { + errno = error; + return fail("interrupt test pthread_join"); + } + if (stop.error != 0) { + errno = stop.error; + return fail("interrupt test release stop"); + } + if (got != -1 || wait_errno != EINTR || sigchld_count != 1) { fprintf(stderr, "nonmatching stop SIGCHLD did not interrupt wait: errno=%d count=%d\n", - errno, (int)sigchld_count); + wait_errno, (int)sigchld_count); return -1; } diff --git a/flake.nix b/flake.nix index b39e295e3a..aad4442b44 100644 --- a/flake.nix +++ b/flake.nix @@ -200,6 +200,13 @@ export LLVM_BIN=${llvmTree}/bin export LLVM_PREFIX=${llvmTree} export LLVM_VERSION=${llvmVersion} + # mkShell's generic AR=ar/RANLIB=ranlib names fall through to + # /usr/bin on Darwin because LLVM exposes llvm-* names. Apple ar + # exits 255 when cc-rs sets ZERO_AR_DATE=1 for reproducible native + # Rust archives, so bind these variables to the declared LLVM + # tools instead of ambient host binaries. + export AR="$LLVM_BIN/llvm-ar" + export RANLIB="$LLVM_BIN/llvm-ranlib" export WASM_POSIX_LLVM_LIBCXX_SOURCE=${llvmPkg.libcxx.src} export WASM_POSIX_LLVM_LIBUNWIND_SOURCE=${llvmPkg.libunwind.src} # CA bundle for HTTPS — pure-shell strips the user's diff --git a/host/src/browser-kernel-worker-entry.ts b/host/src/browser-kernel-worker-entry.ts index ed03c73c10..e961334c14 100644 --- a/host/src/browser-kernel-worker-entry.ts +++ b/host/src/browser-kernel-worker-entry.ts @@ -41,7 +41,13 @@ import { import type { MountConfig } from "./vfs/types"; import { TlsNetworkBackend } from "./networking/tls-network-backend"; import { patchWasmForThread } from "./worker-main"; -import { detectPtrWidth, extractAbiVersion, extractHeapBase, isWasmModuleBytes } from "./constants"; +import { + describeWasmArtifactPolicyFailures, + detectPtrWidth, + extractAbiVersion, + extractHeapBase, + isWasmModuleBytes, +} from "./constants"; import { ThreadExitCoordinator } from "./thread-exit-coordinator"; import { readForkContinuationAnchor } from "./fork-continuation"; import { @@ -56,6 +62,16 @@ import { } from "./thread-worker-disposition"; import { VmInterruptTimerManager } from "./vm-interrupt-timer"; import { RootfsSnapshotGate } from "./rootfs-snapshot-gate"; +import { + ForkReplayGateCoordinator, + observeForkReplayWorker, +} from "./fork-replay-gate"; +import { ForkExternrefProcessOwner } from "./fork-externref-process-owner"; +import type { ForkExternrefGeneration } from "./fork-reference-broker"; +import { + ForkHostImportOwnerRuntime, + type ForkHostImportOwnerWorker, +} from "./fork-host-import-runtime"; import type { CentralizedWorkerInitMessage, CentralizedThreadInitMessage, @@ -116,10 +132,17 @@ interface ProcessInfo { ptrWidth: 4 | 8; layout: ProcessMemoryLayout; threadAllocator: ThreadPageAllocator; + /** Exact broker authority for this PID's current Wasm image. */ + externrefGeneration: ForkExternrefGeneration; /** Non-_start continuation root inherited from a pthread fork until exec. */ forkReplayContext?: ForkReplayContext; } const processes = new Map(); +const externrefProcessOwner = new ForkExternrefProcessOwner(); +const forkHostImportOwnerRuntime = + new ForkHostImportOwnerRuntime(externrefProcessOwner); +const forkHostImportsByWorker = + new WeakMap(); const processTeardowns = new Map>(); const vmInterruptTimers = new VmInterruptTimerManager( (pid) => processes.get(pid), @@ -172,6 +195,10 @@ async function resolveExecutableForLaunch( const shebang = parseShebang(bytes); if (!shebang) { if (!isWasmModuleBytes(bytes)) return { errno: ENOEXEC }; + const artifactFailures = describeWasmArtifactPolicyFailures(bytes, { + expectedAbi: kernelWorker.getKernelAbiVersion(), + }); + if (artifactFailures.length > 0) return { errno: ENOEXEC }; let programModule: WebAssembly.Module; try { programModule = await WebAssembly.compile(bytes); @@ -272,6 +299,7 @@ async function terminateTrackedWorker( settleMs = 0, ): Promise { intentionallyTerminated.add(worker as object); + forkHostImportsByWorker.get(worker as object)?.close(); const teardown = (async () => { await worker.terminate().catch(() => {}); if (settleMs > 0) await delay(settleMs); @@ -281,6 +309,29 @@ async function terminateTrackedWorker( await teardown; } +function bindForkHostImports( + worker: ReturnType, + owner: ForkHostImportOwnerWorker, +): void { + forkHostImportsByWorker.set(worker as object, owner); +} + +function dispatchForkHostImport( + worker: ReturnType, + message: Extract, +): void { + const owner = forkHostImportsByWorker.get(worker as object); + if (!owner || !owner.dispatch(message.wake)) { + reportHostDiagnostic({ + pid: message.wake.pid, + source: "fork host-import protocol", + message: + `[kernel-worker] ignored stale or unbound fork host-import wake ` + + `pid=${message.wake.pid} sender=${message.wake.senderId}`, + }, "warn"); + } +} + async function terminateThreadWorkers(pid: number): Promise { const threads = threadWorkers.get(pid); if (!threads) return; @@ -795,6 +846,8 @@ async function handleInit(msg: Extract) { async function handleSpawn(msg: Extract) { let releaseMutation: (() => void) | undefined; let createdPid: number | undefined; + let createdExternrefGeneration: ForkExternrefGeneration | undefined; + let createdForkHostImports: ForkHostImportOwnerWorker | undefined; try { releaseMutation = rootfsSnapshotGate.beginMutation("spawn a process"); await waitForProcessTeardowns(); @@ -870,12 +923,32 @@ async function handleSpawn(msg: Extract) } } + const externrefGeneration = externrefProcessOwner.startGeneration(pid); + createdExternrefGeneration = externrefGeneration; + let worker: ReturnType; + const forkHostImports = forkHostImportOwnerRuntime.createWorker({ + pid, + generationId: externrefGeneration.id, + authorizeSender: () => { + const current = processes.get(pid); + if ( + !current + || current.worker !== worker + || current.externrefGeneration !== externrefGeneration + ) { + throw new Error(`stale fork host-import sender for pid=${pid}`); + } + }, + }); + createdForkHostImports = forkHostImports; const initData: CentralizedWorkerInitMessage = { type: "centralized_init", pid, programBytes, memory, channelOffset, + externrefGenerationId: externrefGeneration.id, + forkHostImports: forkHostImports.init, env: launchEnv, argv: msg.argv, cwd: msg.cwd, @@ -883,7 +956,8 @@ async function handleSpawn(msg: Extract) kernelAbiVersion: kernelWorker.getKernelAbiVersion(), }; - const worker = workerAdapter.createWorker(initData); + worker = workerAdapter.createWorker(initData); + bindForkHostImports(worker, forkHostImports); processes.set(pid, { memory, programBytes, @@ -893,13 +967,20 @@ async function handleSpawn(msg: Extract) ptrWidth, layout, threadAllocator, + externrefGeneration, }); installProcessWorkerListeners(worker, pid); createdPid = undefined; + createdExternrefGeneration = undefined; + createdForkHostImports = undefined; respond(msg.requestId, pid); } catch (e) { + createdForkHostImports?.close(); + if (createdExternrefGeneration) { + externrefProcessOwner.releaseGeneration(createdExternrefGeneration); + } if (createdPid !== undefined) { kernelWorker.unregisterProcess(createdPid); kernelWorker.removeProcessFromKernelTable(createdPid); @@ -1021,6 +1102,8 @@ function installProcessWorkerListeners( finalize(m.status ?? 0); } else if (m.type === "vm_interrupt_timer") { handleVmInterruptTimer(m, pid, process); + } else if (m.type === "fork_host_import") { + dispatchForkHostImport(worker, m); } }); } @@ -1084,58 +1167,123 @@ async function handleFork( ? { ...parentInfo.forkReplayContext, forkBufAddr: activeForkBufAddr } : undefined; const forkBufAddr = activeForkBufAddr; - const childInitData: CentralizedWorkerInitMessage = { - type: "centralized_init", - pid: childPid, - programBytes: parentInfo.programBytes, - programModule: parentInfo.programModule, - memory: childMemory, - channelOffset: childChannelOffset, - isForkChild: true, - forkBufAddr, - forkChildThreadFnPtr: forkReplayContext?.fnPtr, - forkChildThreadArgPtr: forkReplayContext?.argPtr, + const forkReplay = new ForkReplayGateCoordinator( + `fork child pid=${childPid}`, + ); + const externrefGrant = externrefProcessOwner.forkGenerationFromContinuation( + parentInfo.externrefGeneration, + childPid, + parentMemory, ptrWidth, - kernelAbiVersion: kernelWorker.getKernelAbiVersion(), - }; - - const childWorker = new DeferredWorkerHandle( - () => workerAdapter.createWorker(childInitData), + forkBufAddr, ); + let childWorker: DeferredWorkerHandle | undefined; + let childForkHostImports: ForkHostImportOwnerWorker | undefined; + try { + let launchedWorker: DeferredWorkerHandle; + const forkHostImports = forkHostImportOwnerRuntime.createWorker({ + pid: childPid, + generationId: externrefGrant.generation.id, + authorizeSender: () => { + const current = processes.get(childPid); + if ( + !current + || current.worker !== launchedWorker + || current.externrefGeneration !== externrefGrant.generation + ) { + throw new Error( + `stale fork host-import sender for child pid=${childPid}`, + ); + } + }, + }); + childForkHostImports = forkHostImports; + const childInitData: CentralizedWorkerInitMessage = { + type: "centralized_init", + pid: childPid, + programBytes: parentInfo.programBytes, + programModule: parentInfo.programModule, + memory: childMemory, + channelOffset: childChannelOffset, + externrefGenerationId: externrefGrant.generation.id, + forkHostImports: forkHostImports.init, + isForkChild: true, + forkBufAddr, + forkReplayGate: forkReplay.gate, + forkChildThreadFnPtr: forkReplayContext?.fnPtr, + forkChildThreadArgPtr: forkReplayContext?.argPtr, + ptrWidth, + kernelAbiVersion: kernelWorker.getKernelAbiVersion(), + }; - processes.set(childPid, { - memory: childMemory, - programBytes: parentInfo.programBytes, - programModule: parentInfo.programModule, - worker: childWorker, - argv: parentInfo.argv, - channelOffset: childChannelOffset, - ptrWidth, - layout: childLayout, - threadAllocator: threadAllocatorForLayout(childLayout, ptrWidth, childPid), - forkReplayContext, - }); + childWorker = new DeferredWorkerHandle( + () => workerAdapter.createWorker(childInitData), + ); + launchedWorker = childWorker; + bindForkHostImports(launchedWorker, forkHostImports); + processes.set(childPid, { + memory: childMemory, + programBytes: parentInfo.programBytes, + programModule: parentInfo.programModule, + worker: launchedWorker, + argv: parentInfo.argv, + channelOffset: childChannelOffset, + ptrWidth, + layout: childLayout, + threadAllocator: threadAllocatorForLayout(childLayout, ptrWidth, childPid), + forkReplayContext, + externrefGeneration: externrefGrant.generation, + }); - installProcessWorkerListeners(childWorker, childPid); + observeForkReplayWorker( + forkReplay, + launchedWorker, + childPid, + () => processes.get(childPid)?.worker === launchedWorker, + ); + installProcessWorkerListeners(launchedWorker, childPid); - try { const startDisposition = kernelWorker.startProcessWorkerWhenRunnable( childPid, childMemory, - () => { childWorker.start(); }, - () => { void childWorker.terminate(); }, + () => { launchedWorker.start(); }, + () => { + forkReplay.cancel( + new Error( + `Fork child ${childPid} launch was cancelled before replay readiness`, + ), + ); + forkHostImports.close(); + void launchedWorker.terminate(); + }, ); if (startDisposition === "stale") { throw new Error(`Fork child ${childPid} changed generation before Worker launch`); } + await forkReplay.waitUntilReady(); + if (processes.get(childPid)?.worker !== launchedWorker) { + throw new Error( + `Fork child ${childPid} changed generation before replay commit`, + ); + } + if (!kernelWorker.shouldLaunchPendingChild(childPid)) { + throw new Error(`Fork child ${childPid} exited before replay commit`); + } + // WHY: keep the child blocked inside its inherited fork import until the + // exact fresh Worker generation proves reconstruction completed. onFork + // resolves only after the shared gate is committed. + forkReplay.commit(); } catch (error) { - if (processes.get(childPid)?.worker === childWorker) { + forkReplay.cancel(error); + childForkHostImports?.close(); + externrefProcessOwner.releaseGeneration(externrefGrant.generation); + if (childWorker && processes.get(childPid)?.worker === childWorker) { processes.delete(childPid); threadModuleCache.delete(childPid); ptyByPid.delete(childPid); vmInterruptTimers.clear(childPid); } - void childWorker.terminate(); + if (childWorker) void childWorker.terminate(); throw error; } @@ -1187,6 +1335,8 @@ async function handleExec( const addressSpaceResult = kernelWorker.prepareAddressSpaceForExec(pid); if (addressSpaceResult < 0) return addressSpaceResult; let replacementWorker: ReturnType | undefined; + let replacementExternrefGeneration: ForkExternrefGeneration | undefined; + let replacementForkHostImports: ForkHostImportOwnerWorker | undefined; try { const setupResult = kernelWorker.kernelExecSetup(pid, callerTid); if (setupResult < 0) return setupResult; @@ -1200,6 +1350,9 @@ async function handleExec( } threadedProcessPids.delete(pid); kernelWorker.prepareProcessForExec(pid); + replacementExternrefGeneration = externrefProcessOwner.replaceGeneration( + initiatingInfo.externrefGeneration, + ); const finalizeResult = kernelWorker.finalizeAddressSpaceForExec(pid); if (finalizeResult < 0) { @@ -1208,9 +1361,14 @@ async function handleExec( await terminateThreadWorkers(pid); if (initiatingInfo.worker) { + forkHostImportsByWorker.get(initiatingInfo.worker as object)?.close(); await initiatingInfo.worker.terminate().catch(() => {}); } - if (kernelWorker.finalizeExecHandoffTermination(pid) > 0) return 0; + if (kernelWorker.finalizeExecHandoffTermination(pid) > 0) { + externrefProcessOwner.releaseGeneration(replacementExternrefGeneration); + replacementExternrefGeneration = undefined; + return 0; + } // DIAGNOSTIC: track pid → exec path so the sysprof dump can name // each pid (otherwise the table is just opaque numbers). @@ -1225,6 +1383,21 @@ async function handleExec( threadAllocator: newThreadAllocator, } = prepared; const newChannelOffset = newLayout.channelOffset; + replacementForkHostImports = forkHostImportOwnerRuntime.createWorker({ + pid, + generationId: replacementExternrefGeneration.id, + authorizeSender: () => { + const current = processes.get(pid); + if ( + !replacementWorker + || !current + || current.worker !== replacementWorker + || current.externrefGeneration !== replacementExternrefGeneration + ) { + throw new Error(`stale fork host-import sender for exec pid=${pid}`); + } + }, + }); const execInitData: CentralizedWorkerInitMessage = { type: "centralized_init", @@ -1233,6 +1406,8 @@ async function handleExec( programModule, memory: newMemory, channelOffset: newChannelOffset, + externrefGenerationId: replacementExternrefGeneration.id, + forkHostImports: replacementForkHostImports.init, argv: launchArgv, env: envp, ptrWidth, @@ -1253,6 +1428,7 @@ async function handleExec( replacementWorker = new DeferredWorkerHandle( () => workerAdapter.createWorker(execInitData), ); + bindForkHostImports(replacementWorker, replacementForkHostImports); // Clear cached thread module — the new program binary is different threadModuleCache.delete(pid); @@ -1267,6 +1443,7 @@ async function handleExec( ptrWidth, layout: newLayout, threadAllocator: newThreadAllocator, + externrefGeneration: replacementExternrefGeneration, }); // Wire post-exec error/exit handling. The handleFork listener (on the @@ -1277,12 +1454,16 @@ async function handleExec( pid, newMemory, () => { (replacementWorker as DeferredWorkerHandle).start(); }, - () => { void replacementWorker?.terminate(); }, + () => { + replacementForkHostImports?.close(); + void replacementWorker?.terminate(); + }, ); if (startDisposition === "stale") { throw new Error(`Exec pid ${pid} changed generation before Worker launch`); } if (startDisposition === "dead") { + replacementForkHostImports.close(); kernelWorker.finishProcessExecHandoff(pid); kernelWorker.finalizeExecHandoffTermination(pid); return 0; @@ -1290,6 +1471,11 @@ async function handleExec( kernelWorker.finishProcessExecHandoff(pid); return 0; } catch (err) { + replacementForkHostImports?.close(); + if (replacementExternrefGeneration) { + externrefProcessOwner.releaseGeneration(replacementExternrefGeneration); + replacementExternrefGeneration = undefined; + } if (initiatingInfo.worker) { intentionallyTerminated.add(initiatingInfo.worker as object); } @@ -1394,6 +1580,24 @@ async function handlePosixSpawn( maxAddr: newLayout.maxAddr, }); + const externrefGeneration = externrefProcessOwner.startGeneration(childPid); + let newWorker: DeferredWorkerHandle; + const forkHostImports = forkHostImportOwnerRuntime.createWorker({ + pid: childPid, + generationId: externrefGeneration.id, + authorizeSender: () => { + const current = processes.get(childPid); + if ( + !current + || current.worker !== newWorker + || current.externrefGeneration !== externrefGeneration + ) { + throw new Error( + `stale fork host-import sender for spawn pid=${childPid}`, + ); + } + }, + }); const initData: CentralizedWorkerInitMessage = { type: "centralized_init", pid: childPid, @@ -1401,15 +1605,18 @@ async function handlePosixSpawn( programModule, memory: newMemory, channelOffset: newChannelOffset, + externrefGenerationId: externrefGeneration.id, + forkHostImports: forkHostImports.init, argv, env: envp, ptrWidth, kernelAbiVersion: kernelWorker.getKernelAbiVersion(), }; - const newWorker = new DeferredWorkerHandle( + newWorker = new DeferredWorkerHandle( () => workerAdapter.createWorker(initData), ); + bindForkHostImports(newWorker, forkHostImports); processes.set(childPid, { memory: newMemory, @@ -1421,6 +1628,7 @@ async function handlePosixSpawn( ptrWidth, layout: newLayout, threadAllocator, + externrefGeneration, }); installProcessWorkerListeners(newWorker, childPid); @@ -1430,12 +1638,17 @@ async function handlePosixSpawn( childPid, newMemory, () => { newWorker.start(); }, - () => { void newWorker.terminate(); }, + () => { + forkHostImports.close(); + void newWorker.terminate(); + }, ); if (startDisposition === "stale") { throw new Error(`Spawn child ${childPid} changed generation before Worker launch`); } } catch (error) { + forkHostImports.close(); + externrefProcessOwner.releaseGeneration(externrefGeneration); if (processes.get(childPid)?.worker === newWorker) { processes.delete(childPid); threadModuleCache.delete(childPid); @@ -1508,6 +1721,25 @@ async function handleClone( throw err; } + let threadWorker: DeferredWorkerHandle; + let threadEntry: ThreadWorkerInfo; + const forkHostImports = forkHostImportOwnerRuntime.createWorker({ + pid, + generationId: processInfo.externrefGeneration.id, + authorizeSender: () => { + const entries = threadWorkers.get(pid); + if ( + !belongsToCurrentProcessImage() + || !threadEntry + || threadEntry.worker !== threadWorker + || !entries?.includes(threadEntry) + ) { + throw new Error( + `stale fork host-import sender for pid=${pid} tid=${tid}`, + ); + } + }, + }); const threadInitData: CentralizedThreadInitMessage = { type: "centralized_thread_init", pid, @@ -1517,6 +1749,8 @@ async function handleClone( memory, processChannelOffset: processInfo.channelOffset, channelOffset: alloc.channelOffset, + externrefGenerationId: processInfo.externrefGeneration.id, + forkHostImports: forkHostImports.init, fnPtr, argPtr, stackPtr, @@ -1528,11 +1762,12 @@ async function handleClone( kernelAbiVersion: kernelWorker.getKernelAbiVersion(), }; - const threadWorker = new DeferredWorkerHandle( + threadWorker = new DeferredWorkerHandle( () => workerAdapter.createWorker(threadInitData), ); + bindForkHostImports(threadWorker, forkHostImports); if (!threadWorkers.has(pid)) threadWorkers.set(pid, []); - const threadEntry: ThreadWorkerInfo = { + threadEntry = { worker: threadWorker, channelOffset: alloc.channelOffset, tid, @@ -1608,6 +1843,8 @@ async function handleClone( } else if (m.type === "vm_interrupt_timer") { if (!isCurrentThreadGeneration() || m.pid !== pid) return; handleVmInterruptTimer(m, pid, processInfo); + } else if (m.type === "fork_host_import") { + dispatchForkHostImport(threadWorker, m); } }); threadWorker.on("error", (err: Error) => { @@ -1622,7 +1859,10 @@ async function handleClone( pid, memory, () => { threadWorker.start(); }, - () => { void threadWorker.terminate(); }, + () => { + forkHostImports.close(); + void threadWorker.terminate(); + }, () => { kernelWorker.finalizeThreadExit(pid, tid, alloc.channelOffset); const failedClone = kernelWorker.failDeferredCloneLaunch(pid, tid, 12); @@ -1710,6 +1950,7 @@ async function finishProcessExit( // channel once the worker is gone. kernelWorker.deactivateProcess(pid); + externrefProcessOwner.releaseGeneration(info.externrefGeneration); processes.delete(pid); threadModuleCache.delete(pid); ptyByPid.delete(pid); @@ -1875,6 +2116,9 @@ async function handleTerminateProcess(msg: Extract= buf.length) throw new Error(`${context} is truncated`); + if ((buf[off++]! & 0x80) === 0) return off; + } + throw new Error(`${context} has an overlong LEB128 encoding`); +} + +/** + * Read one complete value/reference type. + * + * Concrete and exact references are multi-byte (`ref[ null] heaptype`), and + * recursive GC modules can use them in function, table, and global types. + * Treating every value type as one byte desynchronizes the artifact guard and + * can make a valid ABI function appear to have an arbitrary signature. + */ +function readWasmValueType( + buf: Uint8Array, + off: number, + context: string, +): ParsedWasmValueType { + if (off >= buf.length) throw new Error(`${context} is truncated`); + const code = buf[off++]!; + switch (code) { + case 0x7f: // i32 + case 0x7e: // i64 + case 0x7d: // f32 + case 0x7c: // f64 + case 0x7b: // v128 + case 0x75: // nocontref + case 0x74: // noexnref + case 0x73: // nofuncref + case 0x72: // noexternref + case 0x71: // nullref + case 0x70: // funcref + case 0x6f: // externref + case 0x6e: // anyref + case 0x6d: // eqref + case 0x6c: // i31ref + case 0x6b: // structref + case 0x6a: // arrayref + case 0x69: // exnref + case 0x68: // contref + return { code, shared: false, next: off }; + case 0x62: // exact heaptype + case 0x63: // ref null heaptype + case 0x64: { // ref heaptype + // Shared abstract heap types add a prefix before the signed heap type. + const shared = buf[off] === 0x65; + if (shared) off++; + const next = skipSignedLeb128(buf, off, 5, `${context} heap type`); + const [heapType] = readSLEB128_i64(buf, off); + return { + code, + heapType: Number(heapType), + shared, + next, + }; + } + default: + throw new Error( + `${context} has unknown value type 0x${code.toString(16)}`, + ); + } +} + +function readWasmStorageType( + buf: Uint8Array, + off: number, + context: string, +): ParsedWasmValueType { + if (buf[off] === 0x78 || buf[off] === 0x77) { + return { code: buf[off]!, shared: false, next: off + 1 }; + } + return readWasmValueType(buf, off, context); +} + function skipWasmBlockType(buf: Uint8Array, off: number): number { const first = buf[off]; if ( @@ -116,6 +275,163 @@ function skipWasmBlockType(buf: Uint8Array, off: number): number { return off + bytes; } +function readWasmFunctionType( + src: Uint8Array, + pos: number, + context: string, +): { signature: WasmFunctionSignature; next: number } { + const [paramCount, paramCountBytes] = readULEB128(src, pos); + pos += paramCountBytes; + const params: number[] = []; + const paramTypes: WasmValueType[] = []; + for (let index = 0; index < paramCount; index++) { + const value = readWasmValueType( + src, + pos, + `${context} parameter ${index}`, + ); + params.push(value.code); + paramTypes.push({ + code: value.code, + heapType: value.heapType, + shared: value.shared, + }); + pos = value.next; + } + const [resultCount, resultCountBytes] = readULEB128(src, pos); + pos += resultCountBytes; + const results: number[] = []; + const resultTypes: WasmValueType[] = []; + for (let index = 0; index < resultCount; index++) { + const value = readWasmValueType( + src, + pos, + `${context} result ${index}`, + ); + results.push(value.code); + resultTypes.push({ + code: value.code, + heapType: value.heapType, + shared: value.shared, + }); + pos = value.next; + } + return { + signature: { params, results, paramTypes, resultTypes }, + next: pos, + }; +} + +function readWasmFieldType( + src: Uint8Array, + pos: number, + context: string, +): number { + const storage = readWasmStorageType(src, pos, context); + pos = storage.next; + if (pos >= src.length) throw new Error(`${context} mutability is truncated`); + const mutability = src[pos++]!; + if (mutability !== 0 && mutability !== 1) { + throw new Error(`${context} has invalid mutability ${mutability}`); + } + return pos; +} + +function readWasmCompositeType( + src: Uint8Array, + opcode: number, + pos: number, + context: string, +): { signature?: WasmFunctionSignature; next: number } { + if (opcode === 0x65) { + if (pos >= src.length) throw new Error(`${context} shared type is truncated`); + opcode = src[pos++]!; + } + // Descriptor types may prefix the actual composite type. + for (const prefix of [0x4c, 0x4d]) { + if (opcode !== prefix) continue; + const [, indexBytes] = readULEB128(src, pos); + pos += indexBytes; + if (pos >= src.length) throw new Error(`${context} descriptor is truncated`); + opcode = src[pos++]!; + } + + if (opcode === 0x60) { + return readWasmFunctionType(src, pos, context); + } + if (opcode === 0x5f) { + const [fieldCount, fieldCountBytes] = readULEB128(src, pos); + pos += fieldCountBytes; + for (let index = 0; index < fieldCount; index++) { + pos = readWasmFieldType(src, pos, `${context} field ${index}`); + } + return { next: pos }; + } + if (opcode === 0x5e) { + return { + next: readWasmFieldType(src, pos, `${context} array field`), + }; + } + if (opcode === 0x5d) { + return { + next: skipSignedLeb128(src, pos, 5, `${context} continuation type`), + }; + } + throw new Error( + `${context} has unknown composite type 0x${opcode.toString(16)}`, + ); +} + +function readWasmSubtype( + src: Uint8Array, + pos: number, + context: string, +): { signature?: WasmFunctionSignature; next: number } { + if (pos >= src.length) throw new Error(`${context} is truncated`); + let opcode = src[pos++]!; + if (opcode === 0x4f || opcode === 0x50) { + const [supertypeCount, countBytes] = readULEB128(src, pos); + pos += countBytes; + for (let index = 0; index < supertypeCount; index++) { + const [, indexBytes] = readULEB128(src, pos); + pos += indexBytes; + } + if (pos >= src.length) throw new Error(`${context} body is truncated`); + opcode = src[pos++]!; + } + return readWasmCompositeType(src, opcode, pos, context); +} + +function readWasmTypeSection( + src: Uint8Array, + pos: number, +): { types: Array; next: number } { + const [groupCount, groupCountBytes] = readULEB128(src, pos); + pos += groupCountBytes; + const types: Array = []; + for (let groupIndex = 0; groupIndex < groupCount; groupIndex++) { + if (src[pos] === 0x4e) { + pos++; + const [typeCount, typeCountBytes] = readULEB128(src, pos); + pos += typeCountBytes; + for (let typeIndex = 0; typeIndex < typeCount; typeIndex++) { + const parsed = readWasmSubtype( + src, + pos, + `recursive type ${groupIndex}:${typeIndex}`, + ); + types.push(parsed.signature); + pos = parsed.next; + } + } else { + const parsed = readWasmSubtype(src, pos, `type ${groupIndex}`); + types.push(parsed.signature); + pos = parsed.next; + } + } + return { types, next: pos }; +} + function skipVectorMemarg(buf: Uint8Array, off: number): number { const [, alignBytes] = readULEB128(buf, off); off += alignBytes; @@ -230,19 +546,20 @@ function skipImportEntry( const [, n] = readULEB128(src, pos); pos += n; } else if (kind === 1) { // table: reftype + limits - pos++; // reftype - const f = src[pos++]; - const [, n] = readULEB128(src, pos); pos += n; - if (f & 1) { const [, n2] = readULEB128(src, pos); pos += n2; } + pos = readWasmValueType(src, pos, "table import type").next; + pos = readLimits(src, pos).next; } else if (kind === 2) { // memory: limits - const f = src[pos++]; - const [, n] = readULEB128(src, pos); pos += n; - if (f & 1) { const [, n2] = readULEB128(src, pos); pos += n2; } + pos = readLimits(src, pos).next; } else if (kind === 3) { // global: valtype + mutability counts.globalImports++; - pos += 2; + pos = readWasmValueType(src, pos, "global import type").next; + pos++; + } else if (kind === 4) { + // exception tag: attribute byte + function type index + pos++; + const [, n] = readULEB128(src, pos); pos += n; } return pos; } @@ -283,19 +600,137 @@ function containsAscii(src: Uint8Array, needle: string): boolean { */ export const WPK_FORK_EXPORTS = WPK_FORK_REQUIRED_EXPORTS.map(({ name }) => name); -interface WasmFunctionSignature { - params: number[]; - results: number[]; +export interface WasmFunctionSignature { + readonly params: readonly number[]; + readonly results: readonly number[]; + /** Complete binary value types, including concrete heap type and nullability. */ + readonly paramTypes: readonly WasmValueType[]; + /** Complete binary value types, including concrete heap type and nullability. */ + readonly resultTypes: readonly WasmValueType[]; +} + +export interface WasmFunctionImportType { + readonly module: string; + readonly name: string; + /** Ordinal among every import-section entry, regardless of import kind. */ + readonly importOrdinal: number; + /** Function index assigned by the core Wasm index space. */ + readonly functionIndex: number; + readonly signature: WasmFunctionSignature; +} + +interface WasmGlobalImportType { + module: string; + name: string; + importOrdinal: number; + index: number; + valueType: number; + recipeTypeCode: number | null; + mutable: boolean; + shared: boolean; +} + +interface WasmTableType { + elementType: number; + table64: boolean; + minimum: number; + maximum: number | null; +} + +interface WasmTableImportType extends WasmTableType { + module: string; + name: string; + importOrdinal: number; + index: number; + recipeTypeCode: number | null; +} + +interface WasmExportEntry { + kind: number; + index: number; } interface WasmForkArtifactFacts { functionImports: Map; + functionImportEntries: WasmFunctionImportType[]; + globalImports: Map; + tableImports: Map; + tables: WasmTableType[]; + tagImports: Map; functionExports: Map; + globalExports: Map; + tableExports: Map; + exports: Map; memoryPointerWidths: number[]; + forkCapabilities: Uint8Array[]; linkedFrameDescriptors: Uint8Array[]; + exceptionCodecDescriptors: Uint8Array[]; + importedGlobalsDescriptors: Uint8Array[]; + importedTablesDescriptors: Uint8Array[]; + moduleStateDescriptors: Uint8Array[]; + staticRootDescriptors: Uint8Array[]; + unwindTransportDescriptors: Uint8Array[]; + nativeStartCount: number; importsKernelFork: boolean; } +function forkGlobalRecipeTypeCode( + valueType: ParsedWasmValueType, + functionTypes: readonly (WasmFunctionSignature | undefined)[], +): number | null { + switch (valueType.code) { + case 0x7f: + return WPK_FORK_MODULE_STATE_GLOBAL_TYPE_I32; + case 0x7e: + return WPK_FORK_MODULE_STATE_GLOBAL_TYPE_I64; + case 0x7d: + return WPK_FORK_MODULE_STATE_GLOBAL_TYPE_F32; + case 0x7c: + return WPK_FORK_MODULE_STATE_GLOBAL_TYPE_F64; + case 0x7b: + return WPK_FORK_MODULE_STATE_GLOBAL_TYPE_V128; + case 0x70: // funcref + case 0x73: // nofuncref + return WPK_FORK_MODULE_STATE_GLOBAL_TYPE_FUNCREF; + case 0x6f: // externref + case 0x72: // noexternref + return WPK_FORK_MODULE_STATE_GLOBAL_TYPE_EXTERNREF; + case 0x69: // exnref + case 0x74: // noexnref + return WPK_FORK_MODULE_STATE_GLOBAL_TYPE_EXNREF; + case 0x68: // contref + case 0x6a: // arrayref + case 0x6b: // structref + case 0x6c: // i31ref + case 0x6d: // eqref + case 0x6e: // anyref + case 0x71: // nullref + case 0x75: // nocontref + return WPK_FORK_MODULE_STATE_GLOBAL_TYPE_ANYREF; + case 0x62: // exact heaptype + case 0x63: // ref null heaptype + case 0x64: { // ref heaptype + const heapType = valueType.heapType; + if (heapType === undefined) return null; + if (heapType === -16 || heapType === -13) { + return WPK_FORK_MODULE_STATE_GLOBAL_TYPE_FUNCREF; + } + if (heapType === -17 || heapType === -14) { + return WPK_FORK_MODULE_STATE_GLOBAL_TYPE_EXTERNREF; + } + if (heapType === -23 || heapType === -12) { + return WPK_FORK_MODULE_STATE_GLOBAL_TYPE_EXNREF; + } + if (heapType >= 0 && functionTypes[heapType] !== undefined) { + return WPK_FORK_MODULE_STATE_GLOBAL_TYPE_FUNCREF; + } + return WPK_FORK_MODULE_STATE_GLOBAL_TYPE_ANYREF; + } + default: + return null; + } +} + function appendSignature( signatures: Map, identity: string, @@ -309,23 +744,40 @@ function appendSignature( signatures.set(identity, values); } +function appendImportType( + imports: Map, + identity: string, + value: T, +): void { + const values = imports.get(identity) ?? []; + values.push(value); + imports.set(identity, values); +} + function readLimits( src: Uint8Array, pos: number, -): { flags: number; next: number } { +): { + flags: number; + minimum: number; + maximum: number | null; + next: number; +} { const [flags, flagBytes] = readULEB128(src, pos); pos += flagBytes; - const [, minBytes] = readULEB128(src, pos); + const [minimum, minBytes] = readULEB128(src, pos); pos += minBytes; + let maximum: number | null = null; if ((flags & 1) !== 0) { - const [, maxBytes] = readULEB128(src, pos); + const [value, maxBytes] = readULEB128(src, pos); pos += maxBytes; + maximum = value; } - return { flags, next: pos }; + return { flags, minimum, maximum, next: pos }; } /** - * Parse the portions of a final Wasm module that jointly define the ABI 42 + * Parse the portions of a final Wasm module that jointly define the ABI 43 * fork-artifact contract. * * WHY: names alone can look complete while the host and guest disagree about @@ -336,17 +788,35 @@ function readWasmForkArtifactFacts(programBytes: ArrayBuffer): WasmForkArtifactF const src = new Uint8Array(programBytes); if (!hasWasmMagic(src)) throw new Error("not a wasm binary"); - const functionTypes: WasmFunctionSignature[] = []; + const functionTypes: Array = []; const functionTypeIndices: number[] = []; const pendingFunctionExports: Array<{ name: string; index: number }> = []; const facts: WasmForkArtifactFacts = { functionImports: new Map(), + functionImportEntries: [], + globalImports: new Map(), + tableImports: new Map(), + tables: [], + tagImports: new Map(), functionExports: new Map(), + globalExports: new Map(), + tableExports: new Map(), + exports: new Map(), memoryPointerWidths: [], + forkCapabilities: [], linkedFrameDescriptors: [], + exceptionCodecDescriptors: [], + importedGlobalsDescriptors: [], + importedTablesDescriptors: [], + moduleStateDescriptors: [], + staticRootDescriptors: [], + unwindTransportDescriptors: [], + nativeStartCount: 0, importsKernelFork: false, }; + let globalImportCount = 0; + let tableImportCount = 0; let offset = 8; while (offset < src.length) { const sectionId = src[offset]; @@ -361,25 +831,26 @@ function readWasmForkArtifactFacts(programBytes: ArrayBuffer): WasmForkArtifactF const [name, afterName] = readName(src, pos); if (name === WPK_FORK_LINKED_FRAME_FORMAT_SECTION) { facts.linkedFrameDescriptors.push(src.slice(afterName, sectionEnd)); + } else if (name === WPK_FORK_CAPABILITIES_SECTION) { + facts.forkCapabilities.push(src.slice(afterName, sectionEnd)); + } else if (name === WPK_FORK_EXCEPTION_CODEC_SECTION) { + facts.exceptionCodecDescriptors.push(src.slice(afterName, sectionEnd)); + } else if (name === WPK_FORK_IMPORTED_GLOBALS_SECTION) { + facts.importedGlobalsDescriptors.push(src.slice(afterName, sectionEnd)); + } else if (name === WPK_FORK_IMPORTED_TABLES_SECTION) { + facts.importedTablesDescriptors.push(src.slice(afterName, sectionEnd)); + } else if (name === WPK_FORK_MODULE_STATE_FORMAT_SECTION) { + facts.moduleStateDescriptors.push(src.slice(afterName, sectionEnd)); + } else if (name === FORK_STATIC_ROOT_CATALOG_SECTION) { + facts.staticRootDescriptors.push(src.slice(afterName, sectionEnd)); + } else if (name === FORK_UNWIND_TRANSPORT_SECTION) { + facts.unwindTransportDescriptors.push(src.slice(afterName, sectionEnd)); } } else if (sectionId === 1) { requireFullyConsumed = true; - const [count, countBytes] = readULEB128(src, pos); - pos += countBytes; - for (let i = 0; i < count; i++) { - if (src[pos++] !== 0x60) { - throw new Error("unsupported non-function type in fork artifact"); - } - const [paramCount, paramCountBytes] = readULEB128(src, pos); - pos += paramCountBytes; - const params = [...src.slice(pos, pos + paramCount)]; - pos += paramCount; - const [resultCount, resultCountBytes] = readULEB128(src, pos); - pos += resultCountBytes; - const results = [...src.slice(pos, pos + resultCount)]; - pos += resultCount; - functionTypes.push({ params, results }); - } + const parsed = readWasmTypeSection(src, pos); + functionTypes.push(...parsed.types); + pos = parsed.next; } else if (sectionId === 2) { requireFullyConsumed = true; const [count, countBytes] = readULEB128(src, pos); @@ -392,23 +863,101 @@ function readWasmForkArtifactFacts(programBytes: ArrayBuffer): WasmForkArtifactF if (kind === 0) { const [typeIndex, typeBytes] = readULEB128(src, pos); pos += typeBytes; + const functionIndex = functionTypeIndices.length; functionTypeIndices.push(typeIndex); const identity = `${moduleName}.${fieldName}`; - appendSignature(facts.functionImports, identity, functionTypes[typeIndex]); + const signature = functionTypes[typeIndex]; + appendSignature(facts.functionImports, identity, signature); + facts.functionImportEntries.push({ + module: moduleName, + name: fieldName, + importOrdinal: i, + functionIndex, + signature: signature!, + }); if (identity === "kernel.kernel_fork") facts.importsKernelFork = true; } else if (kind === 1) { - pos++; // reference type - pos = readLimits(src, pos).next; + const element = readWasmValueType( + src, + pos, + `table import ${moduleName}.${fieldName}`, + ); + pos = element.next; + const limits = readLimits(src, pos); + pos = limits.next; + appendImportType( + facts.tableImports, + `${moduleName}.${fieldName}`, + { + module: moduleName, + name: fieldName, + importOrdinal: i, + index: tableImportCount++, + elementType: element.code, + recipeTypeCode: forkGlobalRecipeTypeCode( + element, + functionTypes, + ), + table64: (limits.flags & 4) !== 0, + minimum: limits.minimum, + maximum: limits.maximum, + }, + ); + facts.tables.push({ + elementType: element.code, + table64: (limits.flags & 4) !== 0, + minimum: limits.minimum, + maximum: limits.maximum, + }); } else if (kind === 2) { const limits = readLimits(src, pos); pos = limits.next; facts.memoryPointerWidths.push((limits.flags & 4) !== 0 ? 8 : 4); } else if (kind === 3) { - pos += 2; // value type + mutability + const valueType = readWasmValueType( + src, + pos, + `global import ${moduleName}.${fieldName}`, + ); + pos = valueType.next; + if (pos >= src.length) { + throw new Error(`global import ${moduleName}.${fieldName} is truncated`); + } + const flags = src[pos++]!; + if ((flags & ~0b11) !== 0) { + throw new Error( + `global import ${moduleName}.${fieldName} has invalid flags ${flags}`, + ); + } + appendImportType( + facts.globalImports, + `${moduleName}.${fieldName}`, + { + module: moduleName, + name: fieldName, + importOrdinal: i, + index: globalImportCount++, + valueType: valueType.code, + recipeTypeCode: forkGlobalRecipeTypeCode( + valueType, + functionTypes, + ), + mutable: (flags & 0b01) !== 0, + shared: (flags & 0b10) !== 0, + }, + ); } else if (kind === 4) { - pos++; // tag attribute - const [, typeBytes] = readULEB128(src, pos); + const attribute = src[pos++]; + if (attribute !== 0) { + throw new Error(`unsupported wasm tag attribute ${attribute}`); + } + const [typeIndex, typeBytes] = readULEB128(src, pos); pos += typeBytes; + appendSignature( + facts.tagImports, + `${moduleName}.${fieldName}`, + functionTypes[typeIndex], + ); } else { throw new Error(`unsupported wasm import kind ${kind}`); } @@ -422,6 +971,26 @@ function readWasmForkArtifactFacts(programBytes: ArrayBuffer): WasmForkArtifactF pos += typeBytes; functionTypeIndices.push(typeIndex); } + } else if (sectionId === 4) { + requireFullyConsumed = true; + const [count, countBytes] = readULEB128(src, pos); + pos += countBytes; + for (let index = 0; index < count; index++) { + const element = readWasmValueType( + src, + pos, + `defined table ${index}`, + ); + pos = element.next; + const limits = readLimits(src, pos); + pos = limits.next; + facts.tables.push({ + elementType: element.code, + table64: (limits.flags & 4) !== 0, + minimum: limits.minimum, + maximum: limits.maximum, + }); + } } else if (sectionId === 5) { requireFullyConsumed = true; const [count, countBytes] = readULEB128(src, pos); @@ -441,8 +1010,20 @@ function readWasmForkArtifactFacts(programBytes: ArrayBuffer): WasmForkArtifactF const kind = src[pos++]; const [index, indexBytes] = readULEB128(src, pos); pos += indexBytes; - if (kind === 0) pendingFunctionExports.push({ name, index }); + appendImportType(facts.exports, name, { kind, index }); + if (kind === 0) { + pendingFunctionExports.push({ name, index }); + } else if (kind === 3) { + appendImportType(facts.globalExports, name, index); + } else if (kind === 1) { + appendImportType(facts.tableExports, name, index); + } } + } else if (sectionId === 8) { + requireFullyConsumed = true; + facts.nativeStartCount++; + const [, functionIndexBytes] = readULEB128(src, pos); + pos += functionIndexBytes; } if (requireFullyConsumed && pos !== sectionEnd) { @@ -511,18 +1092,713 @@ function validateLinkedFrameDescriptor(descriptor: Uint8Array): number { return pointerFormat.bytes; } +function validateForkCapabilities(sections: Uint8Array[]): string[] { + if (sections.length === 0) { + return [`missing required ${WPK_FORK_CAPABILITIES_SECTION} capability`]; + } + if (sections.length !== 1) { + return [ + `has ${sections.length} ${WPK_FORK_CAPABILITIES_SECTION} sections, expected exactly one`, + ]; + } + const capability = sections[0]; + if (capability.byteLength !== 2) { + return [ + `${WPK_FORK_CAPABILITIES_SECTION} has ${capability.byteLength} bytes, expected 2`, + ]; + } + if (capability[0] !== WPK_FORK_CAPABILITIES_VERSION) { + return [ + `${WPK_FORK_CAPABILITIES_SECTION} version ${capability[0]} is unsupported`, + ]; + } + const flags = capability[1]!; + if ((flags & ~WPK_FORK_CAP_KNOWN_MASK) !== 0) { + return [ + `${WPK_FORK_CAPABILITIES_SECTION} has unknown flags 0x${flags.toString(16)}`, + ]; + } + if ((flags & WPK_FORK_CAP_REQUIRED_FLAGS) !== WPK_FORK_CAP_REQUIRED_FLAGS) { + return [ + `${WPK_FORK_CAPABILITIES_SECTION} flags 0x${flags.toString(16)} omit required activation-state safety flags 0x${WPK_FORK_CAP_REQUIRED_FLAGS.toString(16)}`, + ]; + } + return []; +} + +function validateForkUnwindTransport(facts: WasmForkArtifactFacts): string[] { + const failures: string[] = []; + const identity = `${FORK_UNWIND_TAG_IMPORT_MODULE}.${FORK_UNWIND_TAG_IMPORT_NAME}`; + const tags = facts.tagImports.get(identity); + if (!tags) { + failures.push(`missing required private fork-unwind tag import ${identity}`); + } else if (tags.length !== 1) { + failures.push(`duplicate private fork-unwind tag import ${identity}`); + } else if (tags[0]!.params.length !== 0 || tags[0]!.results.length !== 0) { + failures.push(`private fork-unwind tag ${identity} must have an empty payload`); + } + + if (facts.unwindTransportDescriptors.length === 0) { + failures.push(`missing required ${FORK_UNWIND_TRANSPORT_SECTION} descriptor`); + } else if (facts.unwindTransportDescriptors.length !== 1) { + failures.push( + `has ${facts.unwindTransportDescriptors.length} ${FORK_UNWIND_TRANSPORT_SECTION} descriptors, expected exactly one`, + ); + } else { + const descriptor = facts.unwindTransportDescriptors[0]!; + if ( + descriptor.length !== 2 + || descriptor[0] !== FORK_UNWIND_TRANSPORT_VERSION + || descriptor[1] !== FORK_UNWIND_TRANSPORT_PAYLOAD_ARITY + ) { + failures.push( + `${FORK_UNWIND_TRANSPORT_SECTION} must be [${ + FORK_UNWIND_TRANSPORT_VERSION + }, ${FORK_UNWIND_TRANSPORT_PAYLOAD_ARITY}]`, + ); + } + } + return failures; +} + +function validateForkModuleStateDescriptor( + descriptors: readonly Uint8Array[], + expectedPointerWidth: number | null, +): string[] { + if (descriptors.length === 0) { + return [`missing required ${WPK_FORK_MODULE_STATE_FORMAT_SECTION} descriptor`]; + } + if (descriptors.length !== 1) { + return [ + `has ${descriptors.length} ${WPK_FORK_MODULE_STATE_FORMAT_SECTION} descriptors, expected exactly one`, + ]; + } + const bytes = descriptors[0]!; + if (bytes.byteLength !== WPK_FORK_MODULE_STATE_DESCRIPTOR_SIZE) { + return [ + `${WPK_FORK_MODULE_STATE_FORMAT_SECTION} has ${bytes.byteLength} bytes, expected ${WPK_FORK_MODULE_STATE_DESCRIPTOR_SIZE}`, + ]; + } + if (!WPK_FORK_MODULE_STATE_FORMAT_MAGIC.every((byte, index) => bytes[index] === byte)) { + return [`${WPK_FORK_MODULE_STATE_FORMAT_SECTION} has invalid magic`]; + } + const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); + const version = view.getUint16(4, true); + const declaredSize = view.getUint16(6, true); + const pointerWidth = view.getUint8(8); + const pointerFormat = WPK_FORK_MODULE_STATE_POINTER_WIDTHS.find( + ({ bytes }) => bytes === pointerWidth, + ); + const alignment = view.getUint8(9); + const flags = view.getUint16(10, true); + const arenaVersion = view.getUint16(12, true); + const recordVersion = view.getUint16(14, true); + const rootWord = view.getUint32(16, true); + const reserved = view.getUint32(20, true); + const failures: string[] = []; + if (version !== WPK_FORK_MODULE_STATE_FORMAT_VERSION) { + failures.push(`${WPK_FORK_MODULE_STATE_FORMAT_SECTION} version ${version} is unsupported`); + } + if (declaredSize !== WPK_FORK_MODULE_STATE_DESCRIPTOR_SIZE) { + failures.push(`${WPK_FORK_MODULE_STATE_FORMAT_SECTION} declares size ${declaredSize}`); + } + if (!pointerFormat) { + failures.push(`${WPK_FORK_MODULE_STATE_FORMAT_SECTION} pointer width ${pointerWidth} is unsupported`); + } else if (expectedPointerWidth !== null && pointerWidth !== expectedPointerWidth) { + failures.push( + `${WPK_FORK_MODULE_STATE_FORMAT_SECTION} pointer width ${pointerWidth} does not match linked frames ${expectedPointerWidth}`, + ); + } + if (alignment !== WPK_FORK_MODULE_STATE_RECORD_ALIGNMENT) { + failures.push(`${WPK_FORK_MODULE_STATE_FORMAT_SECTION} alignment ${alignment} is unsupported`); + } + if (flags !== WPK_FORK_MODULE_STATE_REQUIRED_FLAGS) { + failures.push( + `${WPK_FORK_MODULE_STATE_FORMAT_SECTION} flags 0x${flags.toString(16)} do not equal required flags 0x${WPK_FORK_MODULE_STATE_REQUIRED_FLAGS.toString(16)}`, + ); + } + if (arenaVersion !== WPK_FORK_MODULE_STATE_ARENA_VERSION) { + failures.push(`${WPK_FORK_MODULE_STATE_FORMAT_SECTION} arena version ${arenaVersion} is unsupported`); + } + if (recordVersion !== WPK_FORK_MODULE_STATE_RECORD_VERSION) { + failures.push(`${WPK_FORK_MODULE_STATE_FORMAT_SECTION} record version ${recordVersion} is unsupported`); + } + if (rootWord !== WPK_FORK_MODULE_STATE_ROOT_POINTER_WORD_OFFSET) { + failures.push(`${WPK_FORK_MODULE_STATE_FORMAT_SECTION} root word ${rootWord} is unsupported`); + } + if (reserved !== 0) { + failures.push(`${WPK_FORK_MODULE_STATE_FORMAT_SECTION} reserved field is nonzero`); + } + return failures; +} + +function validateForkExceptionCodecDescriptor( + descriptors: readonly Uint8Array[], +): string[] { + if (descriptors.length === 0) { + return [`missing required ${WPK_FORK_EXCEPTION_CODEC_SECTION} descriptor`]; + } + if (descriptors.length !== 1) { + return [ + `has ${descriptors.length} ${WPK_FORK_EXCEPTION_CODEC_SECTION} descriptors, expected exactly one`, + ]; + } + const bytes = descriptors[0]!; + if (bytes.byteLength < WPK_FORK_EXCEPTION_CODEC_HEADER_SIZE) { + return [`${WPK_FORK_EXCEPTION_CODEC_SECTION} descriptor is truncated`]; + } + const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); + const failures: string[] = []; + if (view.getUint8(0) !== WPK_FORK_EXCEPTION_CODEC_VERSION) { + failures.push( + `${WPK_FORK_EXCEPTION_CODEC_SECTION} version ${view.getUint8(0)} is unsupported`, + ); + } + if (view.getUint8(1) !== 0 || view.getUint16(2, true) !== 0) { + failures.push(`${WPK_FORK_EXCEPTION_CODEC_SECTION} reserved fields are nonzero`); + } + const count = view.getUint32(4, true); + const expectedSize = WPK_FORK_EXCEPTION_CODEC_HEADER_SIZE + + count * WPK_FORK_EXCEPTION_CODEC_TAG_RECORD_SIZE; + if (!Number.isSafeInteger(expectedSize) || bytes.byteLength !== expectedSize) { + failures.push( + `${WPK_FORK_EXCEPTION_CODEC_SECTION} has ${bytes.byteLength} bytes, expected ${expectedSize}`, + ); + return failures; + } + + const layouts = new Set(); + for (let index = 0; index < count; index++) { + const offset = WPK_FORK_EXCEPTION_CODEC_HEADER_SIZE + + index * WPK_FORK_EXCEPTION_CODEC_TAG_RECORD_SIZE; + const ordinal = view.getUint32(offset, true); + const layoutId = view.getUint32(offset + 4, true); + if (ordinal !== index) { + failures.push( + `${WPK_FORK_EXCEPTION_CODEC_SECTION} tag ordinal ${ordinal} is noncanonical at ${index}`, + ); + } + if (layoutId > 0x7fff_ffff || layouts.has(layoutId)) { + failures.push( + `${WPK_FORK_EXCEPTION_CODEC_SECTION} layout id ${layoutId} is invalid or duplicated`, + ); + } + layouts.add(layoutId); + // The remaining u32 fields are deliberately shape-neutral byte/reference + // counts. Any tag payload is legal when the instrumenter can emit its + // recursive reference recipe; the guard validates format, not user shape. + } + return failures; +} + +const FORK_IMPORTED_GLOBAL_TYPE_CODES = new Set([ + WPK_FORK_MODULE_STATE_GLOBAL_TYPE_I32, + WPK_FORK_MODULE_STATE_GLOBAL_TYPE_I64, + WPK_FORK_MODULE_STATE_GLOBAL_TYPE_F32, + WPK_FORK_MODULE_STATE_GLOBAL_TYPE_F64, + WPK_FORK_MODULE_STATE_GLOBAL_TYPE_V128, + WPK_FORK_MODULE_STATE_GLOBAL_TYPE_FUNCREF, + WPK_FORK_MODULE_STATE_GLOBAL_TYPE_EXTERNREF, + WPK_FORK_MODULE_STATE_GLOBAL_TYPE_EXNREF, + WPK_FORK_MODULE_STATE_GLOBAL_TYPE_ANYREF, +]); + +interface ForkImportedGlobalRecord { + ownerId: number; + typeCode: number; + flags: number; + importOrdinal: number; + module: string; + name: string; +} + +function importedGlobalNeedsRecipe(global: WasmGlobalImportType): boolean { + return !( + global.module === WPK_FORK_EXCEPTION_CODEC_IMPORT_MODULE + && ( + global.name === WPK_FORK_EXCEPTION_IMPORT_ACTIVATION + || global.name === "__channel_base" + // This immutable control address is reconstructed by each host Worker + // from the ABI-defined process channel layout. It is not guest module + // state and must not be serialized as an imported-global recipe. + || global.name === "__wpk_fork_module_state_table_generation_addr" + ) + ); +} + +function validateForkImportedGlobalsDescriptor( + facts: WasmForkArtifactFacts, +): string[] { + const descriptors = facts.importedGlobalsDescriptors; + if (descriptors.length === 0) { + return [`missing required ${WPK_FORK_IMPORTED_GLOBALS_SECTION} descriptor`]; + } + if (descriptors.length !== 1) { + return [ + `has ${descriptors.length} ${WPK_FORK_IMPORTED_GLOBALS_SECTION} descriptors, expected exactly one`, + ]; + } + const bytes = descriptors[0]!; + if (bytes.byteLength < WPK_FORK_IMPORTED_GLOBALS_HEADER_SIZE) { + return [`${WPK_FORK_IMPORTED_GLOBALS_SECTION} descriptor is truncated`]; + } + const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); + const failures: string[] = []; + if (!WPK_FORK_IMPORTED_GLOBALS_MAGIC.every((byte, index) => bytes[index] === byte)) { + failures.push(`${WPK_FORK_IMPORTED_GLOBALS_SECTION} has invalid magic`); + } + if (view.getUint16(4, true) !== WPK_FORK_IMPORTED_GLOBALS_VERSION) { + failures.push( + `${WPK_FORK_IMPORTED_GLOBALS_SECTION} version ${view.getUint16(4, true)} is unsupported`, + ); + } + if (view.getUint16(6, true) !== WPK_FORK_IMPORTED_GLOBALS_HEADER_SIZE) { + failures.push(`${WPK_FORK_IMPORTED_GLOBALS_SECTION} declares an invalid header size`); + } + const count = view.getUint32(8, true); + if (view.getUint32(12, true) !== 0) { + failures.push(`${WPK_FORK_IMPORTED_GLOBALS_SECTION} reserved field is nonzero`); + } + + const owners = new Set(); + const importOrdinals = new Set(); + const decoder = new TextDecoder("utf-8", { fatal: true }); + const records: ForkImportedGlobalRecord[] = []; + let previousImportOrdinal = -1; + let offset = WPK_FORK_IMPORTED_GLOBALS_HEADER_SIZE; + for (let index = 0; index < count; index++) { + if (offset + WPK_FORK_IMPORTED_GLOBALS_RECORD_HEADER_SIZE > bytes.byteLength) { + failures.push( + `${WPK_FORK_IMPORTED_GLOBALS_SECTION} record ${index} header is truncated`, + ); + return failures; + } + const recordSize = view.getUint32(offset, true); + const ownerId = view.getUint32(offset + 4, true); + const typeCode = view.getUint8(offset + 8); + const flags = view.getUint8(offset + 9); + const moduleLength = view.getUint32(offset + 12, true); + const nameLength = view.getUint32(offset + 16, true); + const importOrdinal = view.getUint32(offset + 20, true); + const expectedSize = WPK_FORK_IMPORTED_GLOBALS_RECORD_HEADER_SIZE + + moduleLength + + nameLength; + if ( + !Number.isSafeInteger(expectedSize) + || recordSize !== expectedSize + || recordSize < WPK_FORK_IMPORTED_GLOBALS_RECORD_HEADER_SIZE + || offset + recordSize > bytes.byteLength + ) { + failures.push( + `${WPK_FORK_IMPORTED_GLOBALS_SECTION} record ${index} has invalid bounds`, + ); + return failures; + } + if (ownerId === 0 || owners.has(ownerId)) { + failures.push( + `${WPK_FORK_IMPORTED_GLOBALS_SECTION} record ${index} has invalid or duplicated owner ${ownerId}`, + ); + } + owners.add(ownerId); + if (!FORK_IMPORTED_GLOBAL_TYPE_CODES.has(typeCode)) { + failures.push( + `${WPK_FORK_IMPORTED_GLOBALS_SECTION} record ${index} has unknown value type ${typeCode}`, + ); + } + if ((flags & ~WPK_FORK_IMPORTED_GLOBALS_KNOWN_FLAGS) !== 0) { + failures.push( + `${WPK_FORK_IMPORTED_GLOBALS_SECTION} record ${index} has unknown flags 0x${flags.toString(16)}`, + ); + } + if (view.getUint16(offset + 10, true) !== 0) { + failures.push( + `${WPK_FORK_IMPORTED_GLOBALS_SECTION} record ${index} reserved fields are nonzero`, + ); + } + if ( + importOrdinals.has(importOrdinal) + || importOrdinal <= previousImportOrdinal + ) { + failures.push( + `${WPK_FORK_IMPORTED_GLOBALS_SECTION} record ${index} has duplicated or unordered import ordinal`, + ); + } + importOrdinals.add(importOrdinal); + previousImportOrdinal = importOrdinal; + const namesOffset = offset + WPK_FORK_IMPORTED_GLOBALS_RECORD_HEADER_SIZE; + try { + const module = decoder.decode( + bytes.subarray(namesOffset, namesOffset + moduleLength), + ); + const name = decoder.decode( + bytes.subarray( + namesOffset + moduleLength, + namesOffset + moduleLength + nameLength, + ), + ); + records.push({ + ownerId, + typeCode, + flags, + importOrdinal, + module, + name, + }); + } catch { + failures.push( + `${WPK_FORK_IMPORTED_GLOBALS_SECTION} record ${index} contains invalid UTF-8`, + ); + } + offset += recordSize; + } + if (offset !== bytes.byteLength) { + failures.push(`${WPK_FORK_IMPORTED_GLOBALS_SECTION} has trailing bytes`); + } + + const globalImports = [...facts.globalImports.values()].flat(); + const globalImportsByIndex = new Map( + globalImports.map((global) => [global.index, global]), + ); + const matchedImportIndices = new Set(); + for (const record of records) { + const catalogName = + `${WPK_FORK_GLOBAL_CATALOG_EXPORT_PREFIX}${record.ownerId}`; + const catalog = facts.exports.get(catalogName); + if (!catalog || catalog.length !== 1 || catalog[0]!.kind !== 3) { + failures.push( + `${WPK_FORK_IMPORTED_GLOBALS_SECTION} owner ${record.ownerId} lacks exactly one global catalog export ${catalogName}`, + ); + continue; + } + const imported = globalImportsByIndex.get(catalog[0]!.index); + if (!imported || !importedGlobalNeedsRecipe(imported)) { + failures.push( + `${WPK_FORK_IMPORTED_GLOBALS_SECTION} owner ${record.ownerId} does not identify a reconstructible imported global`, + ); + continue; + } + if ( + imported.module !== record.module + || imported.name !== record.name + || imported.importOrdinal !== record.importOrdinal + || imported.recipeTypeCode !== record.typeCode + || imported.mutable !== + ((record.flags & WPK_FORK_IMPORTED_GLOBALS_FLAG_MUTABLE) !== 0) + || imported.shared !== + ((record.flags & WPK_FORK_IMPORTED_GLOBALS_FLAG_SHARED) !== 0) + ) { + failures.push( + `${WPK_FORK_IMPORTED_GLOBALS_SECTION} owner ${record.ownerId} does not match its imported global declaration`, + ); + continue; + } + if (matchedImportIndices.has(imported.index)) { + failures.push( + `${WPK_FORK_IMPORTED_GLOBALS_SECTION} repeats imported global index ${imported.index}`, + ); + continue; + } + matchedImportIndices.add(imported.index); + } + + for (const imported of globalImports) { + if ( + importedGlobalNeedsRecipe(imported) + && !matchedImportIndices.has(imported.index) + ) { + failures.push( + `${WPK_FORK_IMPORTED_GLOBALS_SECTION} omits imported global ` + + `${imported.module}.${imported.name} at index ${imported.index}`, + ); + } + } + + for (const [name, exports] of facts.exports) { + if (!name.startsWith(WPK_FORK_GLOBAL_CATALOG_EXPORT_PREFIX)) continue; + const suffix = name.slice(WPK_FORK_GLOBAL_CATALOG_EXPORT_PREFIX.length); + const owner = Number(suffix); + if ( + !/^[1-9][0-9]*$/.test(suffix) + || !Number.isSafeInteger(owner) + || owner > 0xffff_ffff + || exports.length !== 1 + || exports[0]!.kind !== 3 + ) { + failures.push(`malformed reserved fork global catalog export ${name}`); + } + } + return failures; +} + +const FORK_IMPORTED_TABLE_TYPE_CODES = new Set([ + WPK_FORK_MODULE_STATE_GLOBAL_TYPE_FUNCREF, + WPK_FORK_MODULE_STATE_GLOBAL_TYPE_EXTERNREF, + WPK_FORK_MODULE_STATE_GLOBAL_TYPE_EXNREF, + WPK_FORK_MODULE_STATE_GLOBAL_TYPE_ANYREF, +]); + +interface ForkImportedTableRecord { + ownerId: number; + typeCode: number; + flags: number; + importOrdinal: number; + module: string; + name: string; +} + +function importedTableNeedsRecipe(table: WasmTableImportType): boolean { + return !WPK_FORK_REQUIRED_TABLE_IMPORTS.some( + ({ module, name }) => table.module === module && table.name === name, + ); +} + +/** + * Validate the pre-instantiation table-identity recipe one declaration at a + * time. + * + * WHY: the same import-object property may feed several Wasm table imports, + * and an imported table may be shared by several module activations. Names + * alone cannot prove which declaration owns which catalog export; the full + * import ordinal and exact table index make that identity deterministic before + * any child continuation executes. + */ +function validateForkImportedTablesDescriptor( + facts: WasmForkArtifactFacts, +): string[] { + const descriptors = facts.importedTablesDescriptors; + if (descriptors.length === 0) { + return [`missing required ${WPK_FORK_IMPORTED_TABLES_SECTION} descriptor`]; + } + if (descriptors.length !== 1) { + return [ + `has ${descriptors.length} ${WPK_FORK_IMPORTED_TABLES_SECTION} descriptors, expected exactly one`, + ]; + } + const bytes = descriptors[0]!; + if (bytes.byteLength < WPK_FORK_IMPORTED_TABLES_HEADER_SIZE) { + return [`${WPK_FORK_IMPORTED_TABLES_SECTION} descriptor is truncated`]; + } + const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); + const failures: string[] = []; + if (!WPK_FORK_IMPORTED_TABLES_MAGIC.every((byte, index) => bytes[index] === byte)) { + failures.push(`${WPK_FORK_IMPORTED_TABLES_SECTION} has invalid magic`); + } + if (view.getUint16(4, true) !== WPK_FORK_IMPORTED_TABLES_VERSION) { + failures.push( + `${WPK_FORK_IMPORTED_TABLES_SECTION} version ${view.getUint16(4, true)} is unsupported`, + ); + } + if (view.getUint16(6, true) !== WPK_FORK_IMPORTED_TABLES_HEADER_SIZE) { + failures.push(`${WPK_FORK_IMPORTED_TABLES_SECTION} declares an invalid header size`); + } + const count = view.getUint32(8, true); + if (view.getUint32(12, true) !== 0) { + failures.push(`${WPK_FORK_IMPORTED_TABLES_SECTION} reserved field is nonzero`); + } + + const owners = new Set(); + const importOrdinals = new Set(); + const decoder = new TextDecoder("utf-8", { fatal: true }); + const records: ForkImportedTableRecord[] = []; + let previousImportOrdinal = -1; + let offset = WPK_FORK_IMPORTED_TABLES_HEADER_SIZE; + for (let index = 0; index < count; index++) { + if (offset + WPK_FORK_IMPORTED_TABLES_RECORD_HEADER_SIZE > bytes.byteLength) { + failures.push( + `${WPK_FORK_IMPORTED_TABLES_SECTION} record ${index} header is truncated`, + ); + return failures; + } + const recordSize = view.getUint32(offset, true); + const ownerId = view.getUint32(offset + 4, true); + const typeCode = view.getUint8(offset + 8); + const flags = view.getUint8(offset + 9); + const moduleLength = view.getUint32(offset + 12, true); + const nameLength = view.getUint32(offset + 16, true); + const importOrdinal = view.getUint32(offset + 20, true); + const expectedSize = WPK_FORK_IMPORTED_TABLES_RECORD_HEADER_SIZE + + moduleLength + + nameLength; + if ( + !Number.isSafeInteger(expectedSize) + || recordSize !== expectedSize + || recordSize < WPK_FORK_IMPORTED_TABLES_RECORD_HEADER_SIZE + || offset + recordSize > bytes.byteLength + ) { + failures.push( + `${WPK_FORK_IMPORTED_TABLES_SECTION} record ${index} has invalid bounds`, + ); + return failures; + } + if (ownerId === 0 || owners.has(ownerId)) { + failures.push( + `${WPK_FORK_IMPORTED_TABLES_SECTION} record ${index} has invalid or duplicated owner ${ownerId}`, + ); + } + owners.add(ownerId); + if (!FORK_IMPORTED_TABLE_TYPE_CODES.has(typeCode)) { + failures.push( + `${WPK_FORK_IMPORTED_TABLES_SECTION} record ${index} has unknown element type ${typeCode}`, + ); + } + if ((flags & ~WPK_FORK_IMPORTED_TABLES_KNOWN_FLAGS) !== 0) { + failures.push( + `${WPK_FORK_IMPORTED_TABLES_SECTION} record ${index} has unknown flags 0x${flags.toString(16)}`, + ); + } + if (view.getUint16(offset + 10, true) !== 0) { + failures.push( + `${WPK_FORK_IMPORTED_TABLES_SECTION} record ${index} reserved fields are nonzero`, + ); + } + if ( + importOrdinals.has(importOrdinal) + || importOrdinal <= previousImportOrdinal + ) { + failures.push( + `${WPK_FORK_IMPORTED_TABLES_SECTION} record ${index} has duplicated or unordered import ordinal`, + ); + } + importOrdinals.add(importOrdinal); + previousImportOrdinal = importOrdinal; + const namesOffset = offset + WPK_FORK_IMPORTED_TABLES_RECORD_HEADER_SIZE; + try { + const module = decoder.decode( + bytes.subarray(namesOffset, namesOffset + moduleLength), + ); + const name = decoder.decode( + bytes.subarray( + namesOffset + moduleLength, + namesOffset + moduleLength + nameLength, + ), + ); + records.push({ + ownerId, + typeCode, + flags, + importOrdinal, + module, + name, + }); + } catch { + failures.push( + `${WPK_FORK_IMPORTED_TABLES_SECTION} record ${index} contains invalid UTF-8`, + ); + } + offset += recordSize; + } + if (offset !== bytes.byteLength) { + failures.push(`${WPK_FORK_IMPORTED_TABLES_SECTION} has trailing bytes`); + } + + const tableImports = [...facts.tableImports.values()].flat(); + const tableImportsByIndex = new Map( + tableImports.map((table) => [table.index, table]), + ); + const matchedImportIndices = new Set(); + for (const record of records) { + const catalogName = + `${WPK_FORK_TABLE_CATALOG_EXPORT_PREFIX}${record.ownerId}`; + const catalog = facts.exports.get(catalogName); + if (!catalog || catalog.length !== 1 || catalog[0]!.kind !== 1) { + failures.push( + `${WPK_FORK_IMPORTED_TABLES_SECTION} owner ${record.ownerId} lacks exactly one table catalog export ${catalogName}`, + ); + continue; + } + const imported = tableImportsByIndex.get(catalog[0]!.index); + if (!imported || !importedTableNeedsRecipe(imported)) { + failures.push( + `${WPK_FORK_IMPORTED_TABLES_SECTION} owner ${record.ownerId} does not identify a reconstructible imported table`, + ); + continue; + } + if ( + imported.module !== record.module + || imported.name !== record.name + || imported.importOrdinal !== record.importOrdinal + || imported.recipeTypeCode !== record.typeCode + || imported.table64 !== + ((record.flags & WPK_FORK_IMPORTED_TABLES_FLAG_TABLE64) !== 0) + ) { + failures.push( + `${WPK_FORK_IMPORTED_TABLES_SECTION} owner ${record.ownerId} does not match its imported table declaration`, + ); + continue; + } + if (matchedImportIndices.has(imported.index)) { + failures.push( + `${WPK_FORK_IMPORTED_TABLES_SECTION} repeats imported table index ${imported.index}`, + ); + continue; + } + matchedImportIndices.add(imported.index); + } + + for (const imported of tableImports) { + if ( + importedTableNeedsRecipe(imported) + && !matchedImportIndices.has(imported.index) + ) { + failures.push( + `${WPK_FORK_IMPORTED_TABLES_SECTION} omits imported table ` + + `${imported.module}.${imported.name} at index ${imported.index}`, + ); + } + } + + for (const [name, exports] of facts.exports) { + if (!name.startsWith(WPK_FORK_TABLE_CATALOG_EXPORT_PREFIX)) continue; + const suffix = name.slice(WPK_FORK_TABLE_CATALOG_EXPORT_PREFIX.length); + const owner = Number(suffix); + if ( + !/^[1-9][0-9]*$/.test(suffix) + || !Number.isSafeInteger(owner) + || owner > 0xffff_ffff + || exports.length !== 1 + || exports[0]!.kind !== 1 + ) { + failures.push(`malformed reserved fork table catalog export ${name}`); + } + } + return failures; +} + +type ForkAbiValueType = + | "ptr" + | "i32" + | "i64" + | "anyref" + | "exnref" + | "externref" + | "funcref"; + function expectedWasmValueType( - value: "ptr" | "i32", + value: ForkAbiValueType, pointerWidth: number, ): number { - if (value === "i32") return 0x7f; - return pointerWidth === 8 ? 0x7e : 0x7f; + switch (value) { + case "ptr": + return pointerWidth === 8 ? 0x7e : 0x7f; + case "i32": + return 0x7f; + case "i64": + return 0x7e; + case "anyref": + return 0x6e; + case "exnref": + return 0x69; + case "externref": + return 0x6f; + case "funcref": + return 0x70; + } } function signatureMatches( actual: WasmFunctionSignature, - params: readonly ("ptr" | "i32")[], - results: readonly ("ptr" | "i32")[], + params: readonly ForkAbiValueType[], + results: readonly ForkAbiValueType[], pointerWidth: number, ): boolean { return actual.params.length === params.length && @@ -536,24 +1812,171 @@ function signatureMatches( } function signatureText( - params: readonly ("ptr" | "i32")[], - results: readonly ("ptr" | "i32")[], + params: readonly ForkAbiValueType[], + results: readonly ForkAbiValueType[], pointerWidth: number, ): string { - const render = (value: "ptr" | "i32") => - value === "ptr" ? (pointerWidth === 8 ? "i64" : "i32") : "i32"; + const render = (value: ForkAbiValueType) => + value === "ptr" ? (pointerWidth === 8 ? "i64" : "i32") : value; return `(${params.map(render).join(", ")}) -> (${results.map(render).join(", ")})`; } +function validateForkActivationImport(facts: WasmForkArtifactFacts): string[] { + const identity = + `${WPK_FORK_EXCEPTION_CODEC_IMPORT_MODULE}.${WPK_FORK_EXCEPTION_IMPORT_ACTIVATION}`; + const imports = facts.globalImports.get(identity); + if (!imports) { + return [`missing required immutable exception-codec activation import ${identity}`]; + } + if (imports.length !== 1) { + return [`duplicate exception-codec activation import ${identity}`]; + } + if (imports[0]!.valueType !== 0x7f || imports[0]!.mutable) { + return [`exception-codec activation import ${identity} must be immutable i32`]; + } + return []; +} + +function validateForkTableImports(facts: WasmForkArtifactFacts): string[] { + const failures: string[] = []; + for (const requirement of WPK_FORK_REQUIRED_TABLE_IMPORTS) { + const identity = `${requirement.module}.${requirement.name}`; + const imports = facts.tableImports.get(identity); + if (!imports) { + failures.push(`missing required ABI 43 fork-runtime table import ${identity}`); + continue; + } + if (imports.length !== 1) { + failures.push(`duplicate ABI 43 fork-runtime table import ${identity}`); + continue; + } + const actual = imports[0]!; + const expectedElement = expectedWasmValueType(requirement.element, 4); + if ( + actual.elementType !== expectedElement + || actual.table64 !== requirement.table64 + || actual.minimum !== requirement.minimum + || actual.maximum !== requirement.maximum + ) { + failures.push( + `ABI 43 fork-runtime table import ${identity} has the wrong type or limits`, + ); + } + } + return failures; +} + +function validateForkStaticRootCatalog( + facts: WasmForkArtifactFacts, +): string[] { + if (facts.staticRootDescriptors.length === 0) { + return [`missing required ${FORK_STATIC_ROOT_CATALOG_SECTION} descriptor`]; + } + if (facts.staticRootDescriptors.length !== 1) { + return [ + `has ${facts.staticRootDescriptors.length} ${FORK_STATIC_ROOT_CATALOG_SECTION} descriptors, expected exactly one`, + ]; + } + const descriptor = facts.staticRootDescriptors[0]!; + if (descriptor.byteLength !== FORK_STATIC_ROOT_CATALOG_HEADER_SIZE) { + return [ + `${FORK_STATIC_ROOT_CATALOG_SECTION} has ${descriptor.byteLength} bytes, expected ${FORK_STATIC_ROOT_CATALOG_HEADER_SIZE}`, + ]; + } + const failures: string[] = []; + if ( + FORK_STATIC_ROOT_CATALOG_MAGIC.some( + (byte, index) => descriptor[index] !== byte, + ) + ) { + failures.push(`${FORK_STATIC_ROOT_CATALOG_SECTION} has invalid magic`); + } + const view = new DataView( + descriptor.buffer, + descriptor.byteOffset, + descriptor.byteLength, + ); + if (view.getUint16(4, true) !== FORK_STATIC_ROOT_CATALOG_VERSION) { + failures.push( + `${FORK_STATIC_ROOT_CATALOG_SECTION} version ${view.getUint16(4, true)} is unsupported`, + ); + } + if (view.getUint16(6, true) !== FORK_STATIC_ROOT_CATALOG_HEADER_SIZE) { + failures.push( + `${FORK_STATIC_ROOT_CATALOG_SECTION} declares an invalid header size`, + ); + } + const count = view.getUint32(8, true); + const exports = facts.tableExports.get(FORK_STATIC_ROOT_CATALOG_EXPORT); + if (!exports || exports.length !== 1) { + failures.push( + `missing exactly one table export ${FORK_STATIC_ROOT_CATALOG_EXPORT}`, + ); + return failures; + } + const importedTableCount = [...facts.tableImports.values()] + .reduce((total, entries) => total + entries.length, 0); + const tableIndex = exports[0]!; + const table = facts.tables[tableIndex]; + if (tableIndex < importedTableCount || !table) { + failures.push( + `${FORK_STATIC_ROOT_CATALOG_EXPORT} must export a module-local table`, + ); + return failures; + } + if ( + table.elementType !== 0x6e + || table.table64 + || table.minimum !== count + || table.maximum !== count + ) { + failures.push( + `${FORK_STATIC_ROOT_CATALOG_EXPORT} must be a fixed table32 anyref catalog of length ${count}`, + ); + } + return failures; +} + function describeForkArtifactContractFailures( facts: WasmForkArtifactFacts, ): string[] { const failures: string[] = []; + if (facts.nativeStartCount !== 0) { + // WHY: staged dlopen may instantiate this module while a loader import is + // active. The transform must defer the source start function to the + // explicit bootstrap so guest Wasm cannot reenter that import. + failures.push( + `ABI 43 fork artifact retains ${facts.nativeStartCount} native Wasm start ` + + `section${facts.nativeStartCount === 1 ? "" : "s"}; rebuild and ` + + "reinstrument it so initialization is owned by " + + "wpk_fork_module_bootstrap", + ); + } + if (facts.functionImports.has("env.__wasm_dlopen")) { + // WHY: this host import can synchronously enter side-module Wasm before + // returning. ABI 43 instrumentation lowers every valid occurrence to the + // staged prepare/next/commit protocol, so retaining it proves that the + // activation-state capability was copied or emitted by an incomplete + // transform. + failures.push( + "ABI 43 fork artifact retains reentrant env.__wasm_dlopen; " + + "rebuild and reinstrument it with the staged loader lowering", + ); + } + failures.push(...validateForkCapabilities(facts.forkCapabilities)); + failures.push( + ...validateForkExceptionCodecDescriptor(facts.exceptionCodecDescriptors), + ...validateForkImportedGlobalsDescriptor(facts), + ...validateForkImportedTablesDescriptor(facts), + ...validateForkActivationImport(facts), + ...validateForkTableImports(facts), + ...validateForkStaticRootCatalog(facts), + ); for (const requirement of WPK_FORK_REQUIRED_EXPORTS) { const signatures = facts.functionExports.get(requirement.name); if (!signatures) continue; if (signatures.length !== 1) { - failures.push(`duplicate ABI 42 wasm-fork-instrument export ${requirement.name}`); + failures.push(`duplicate ABI 43 wasm-fork-instrument export ${requirement.name}`); } } const missingExports = WPK_FORK_REQUIRED_EXPORTS @@ -579,25 +2002,40 @@ function describeForkArtifactContractFailures( failures.push(error instanceof Error ? error.message : String(error)); } } + failures.push( + ...validateForkModuleStateDescriptor( + facts.moduleStateDescriptors, + pointerWidth, + ), + ); const presentFrameImports = WPK_FORK_REQUIRED_IMPORTS.filter(({ module, name }) => facts.functionImports.has(`${module}.${name}`) ); + const unwindTagIdentity = + `${FORK_UNWIND_TAG_IMPORT_MODULE}.${FORK_UNWIND_TAG_IMPORT_NAME}`; const requiresFrameImports = facts.importsKernelFork || presentFrameImports.length > 0; + const requiresUnwindTransport = + requiresFrameImports + || facts.tagImports.has(unwindTagIdentity) + || facts.unwindTransportDescriptors.length > 0; + if (requiresUnwindTransport) { + failures.push(...validateForkUnwindTransport(facts)); + } if (requiresFrameImports) { const missingImports = WPK_FORK_REQUIRED_IMPORTS .filter(({ module, name }) => !facts.functionImports.has(`${module}.${name}`)) .map(({ module, name }) => `${module}.${name}`); if (missingImports.length > 0) { failures.push( - `incomplete ABI 42 linked-frame imports; missing ${missingImports.join(", ")}`, + `incomplete ABI 43 fork-runtime imports; missing ${missingImports.join(", ")}`, ); } for (const requirement of WPK_FORK_REQUIRED_IMPORTS) { const identity = `${requirement.module}.${requirement.name}`; const signatures = facts.functionImports.get(identity); if (signatures && signatures.length !== 1) { - failures.push(`duplicate ABI 42 linked-frame import ${identity}`); + failures.push(`duplicate ABI 43 fork-runtime import ${identity}`); } } } @@ -605,12 +2043,12 @@ function describeForkArtifactContractFailures( if (pointerWidth !== null) { if (facts.memoryPointerWidths.length !== 1) { failures.push( - `ABI 42 fork instrumentation requires exactly one module memory, found ${facts.memoryPointerWidths.length}`, + `ABI 43 fork instrumentation requires exactly one module memory, found ${facts.memoryPointerWidths.length}`, ); } else if (facts.memoryPointerWidths[0] !== pointerWidth) { const article = pointerWidth === 8 ? "an" : "a"; failures.push( - `ABI 42 linked-frame descriptor declares ${article} ${pointerWidth}-byte pointer but the module memory uses ${facts.memoryPointerWidths[0]}-byte addresses`, + `ABI 43 linked-frame descriptor declares ${article} ${pointerWidth}-byte pointer but the module memory uses ${facts.memoryPointerWidths[0]}-byte addresses`, ); } for (const requirement of WPK_FORK_REQUIRED_EXPORTS) { @@ -625,7 +2063,7 @@ function describeForkArtifactContractFailures( ) ) { failures.push( - `ABI 42 wasm-fork-instrument export ${requirement.name} has the wrong signature; expected ${ + `ABI 43 wasm-fork-instrument export ${requirement.name} has the wrong signature; expected ${ signatureText(requirement.params, requirement.results, pointerWidth) }`, ); @@ -645,7 +2083,7 @@ function describeForkArtifactContractFailures( ) ) { failures.push( - `ABI 42 linked-frame import ${identity} has the wrong signature; expected ${ + `ABI 43 fork-runtime import ${identity} has the wrong signature; expected ${ signatureText(requirement.params, requirement.results, pointerWidth) }`, ); @@ -657,6 +2095,61 @@ function describeForkArtifactContractFailures( return failures; } +/** + * Validate the complete ABI-epoch fork contract without compiling or running + * the artifact. This is shared by program admission and the dynamic linker so + * a side module cannot defer a malformed reconstruction recipe until replay. + */ +export function describeWasmForkArtifactContractFailures( + programBytes: ArrayBuffer, +): string[] { + try { + return describeForkArtifactContractFailures( + readWasmForkArtifactFacts(programBytes), + ); + } catch (error) { + return [ + `cannot validate ABI 43 fork-artifact contract: ${ + error instanceof Error ? error.message : String(error) + }`, + ]; + } +} + +/** + * Return exact function-import identities, ordinals, and binary signatures. + * + * WebAssembly.Module.imports() omits function types. Fork-safe host-import + * routing needs the artifact-declared signature so an owner descriptor cannot + * accidentally reinterpret the same scalar words under a different type. + */ +export function readWasmFunctionImports( + programBytes: ArrayBuffer, +): readonly WasmFunctionImportType[] { + return Object.freeze( + readWasmForkArtifactFacts(programBytes).functionImportEntries.map( + (entry) => + Object.freeze({ + ...entry, + signature: Object.freeze({ + params: Object.freeze([...entry.signature.params]), + results: Object.freeze([...entry.signature.results]), + paramTypes: Object.freeze( + entry.signature.paramTypes.map((type) => + Object.freeze({ ...type }) + ), + ), + resultTypes: Object.freeze( + entry.signature.resultTypes.map((type) => + Object.freeze({ ...type }) + ), + ), + }), + }), + ), + ); +} + /** * Return import names in `module.field` form. This is intentionally a small * section parser rather than `new WebAssembly.Module(...)` so release/resolver @@ -688,16 +2181,16 @@ export function readWasmImportNames(programBytes: ArrayBuffer): string[] { if (kind === 0) { const [, n] = readULEB128(src, pos); pos += n; } else if (kind === 1) { - pos++; - const flags = src[pos++]; - const [, minBytes] = readULEB128(src, pos); pos += minBytes; - if (flags & 1) { const [, maxBytes] = readULEB128(src, pos); pos += maxBytes; } + pos = readWasmValueType(src, pos, "table import type").next; + pos = readLimits(src, pos).next; } else if (kind === 2) { - const flags = src[pos++]; - const [, minBytes] = readULEB128(src, pos); pos += minBytes; - if (flags & 1) { const [, maxBytes] = readULEB128(src, pos); pos += maxBytes; } + pos = readLimits(src, pos).next; } else if (kind === 3) { - pos += 2; + pos = readWasmValueType(src, pos, "global import type").next; + pos++; + } else if (kind === 4) { + pos++; // tag attribute + const [, typeBytes] = readULEB128(src, pos); pos += typeBytes; } } break; @@ -774,7 +2267,10 @@ export function wasmHasCompleteForkInstrumentation(programBytes: ArrayBuffer): b const facts = readWasmForkArtifactFacts(programBytes); const hasForkSurface = WPK_FORK_REQUIRED_EXPORTS.some(({ name }) => facts.functionExports.has(name) - ) || facts.linkedFrameDescriptors.length > 0; + ) || facts.linkedFrameDescriptors.length > 0 + || facts.exceptionCodecDescriptors.length > 0 + || facts.importedGlobalsDescriptors.length > 0 + || facts.importedTablesDescriptors.length > 0; return hasForkSurface && describeForkArtifactContractFailures(facts).length === 0; } catch { return false; @@ -797,14 +2293,15 @@ export function describeWasmArtifactPolicyFailures( } = {}, ): string[] { const failures: string[] = []; + let declaredAbi: number | null = null; if (wasmContainsLegacyAsyncify(programBytes)) { failures.push("contains asyncify_"); } if (options.expectedAbi !== undefined && options.expectedAbi !== null) { - const abi = extractAbiVersion(programBytes); - if (abi !== null && abi !== options.expectedAbi) { - failures.push(`ABI ${abi}, expected ${options.expectedAbi}`); + declaredAbi = extractAbiVersion(programBytes); + if (declaredAbi !== null && declaredAbi !== options.expectedAbi) { + failures.push(`ABI ${declaredAbi}, expected ${options.expectedAbi}`); } } @@ -825,10 +2322,50 @@ export function describeWasmArtifactPolicyFailures( const descriptorCount = customSections.filter((name) => name === WPK_FORK_LINKED_FRAME_FORMAT_SECTION ).length; + const capabilityCount = customSections.filter((name) => + name === WPK_FORK_CAPABILITIES_SECTION + ).length; + const moduleStateDescriptorCount = customSections.filter((name) => + name === WPK_FORK_MODULE_STATE_FORMAT_SECTION + ).length; + const exceptionCodecDescriptorCount = customSections.filter((name) => + name === WPK_FORK_EXCEPTION_CODEC_SECTION + ).length; + const importedGlobalsDescriptorCount = customSections.filter((name) => + name === WPK_FORK_IMPORTED_GLOBALS_SECTION + ).length; + const importedTablesDescriptorCount = customSections.filter((name) => + name === WPK_FORK_IMPORTED_TABLES_SECTION + ).length; + const unwindTransportCount = customSections.filter((name) => + name === FORK_UNWIND_TRANSPORT_SECTION + ).length; + const hasUnwindTagImport = importNames.includes( + `${FORK_UNWIND_TAG_IMPORT_MODULE}.${FORK_UNWIND_TAG_IMPORT_NAME}`, + ); const hasForkArtifactSurface = - presentWpkExports.length > 0 || presentWpkImports.length > 0 || descriptorCount > 0; + presentWpkExports.length > 0 || presentWpkImports.length > 0 || + descriptorCount > 0 || capabilityCount > 0 || + moduleStateDescriptorCount > 0 || exceptionCodecDescriptorCount > 0 || + importedGlobalsDescriptorCount > 0 || importedTablesDescriptorCount > 0 || + unwindTransportCount > 0 || + hasUnwindTagImport; + if ( + options.expectedAbi !== undefined && + options.expectedAbi !== null && + hasForkArtifactSurface && + declaredAbi === null + ) { + // WHY: the safety bit names an ABI-epoch contract. Without the program's + // ABI marker, copied capability metadata could make an ABI 42 transform + // look safe to an ABI 43 host. + failures.push( + `ABI ${options.expectedAbi} fork artifact is missing __abi_version; ` + + "the activation-state capability epoch cannot be verified", + ); + } if (options.forbidForkInstrumentation && hasForkArtifactSurface) { - failures.push("contains ABI 42 wasm-fork-instrument metadata, imports, or exports"); + failures.push("contains ABI 43 wasm-fork-instrument metadata, imports, or exports"); } const requireForkInstrumentation = @@ -843,7 +2380,7 @@ export function describeWasmArtifactPolicyFailures( ); } catch (error) { failures.push( - `cannot validate ABI 42 fork-artifact contract: ${ + `cannot validate ABI 43 fork-artifact contract: ${ error instanceof Error ? error.message : String(error) }`, ); @@ -860,7 +2397,7 @@ export function describeWasmArtifactPolicyFailures( */ function readGlobalInitAddr(src: Uint8Array, pos: number): bigint | null { // valtype + mut + init expr (terminated by 0x0B) - pos++; // valtype + pos = readWasmValueType(src, pos, "global type").next; pos++; // mut const opcode = src[pos++]; if (opcode === 0x41) { @@ -882,7 +2419,8 @@ function readGlobalInitAddr(src: Uint8Array, pos: number): bigint | null { * expression ends at the first 0x0B (end) opcode. */ function skipGlobalEntry(src: Uint8Array, pos: number): number { - pos += 2; // valtype + mut + pos = readWasmValueType(src, pos, "global type").next; + pos++; // mutability while (src[pos] !== 0x0B) pos++; return pos + 1; // skip the end opcode } @@ -1054,7 +2592,11 @@ function extractI32ConstFunctionExport( pos += localGroupsBytes; for (let i = 0; i < localGroups; i++) { const [, n] = readULEB128(src, pos); pos += n; // count - pos++; // valtype + try { + pos = readWasmValueType(src, pos, `function local group ${i}`).next; + } catch { + return null; + } if (pos > bodyEnd) return null; } return pos; @@ -1173,48 +2715,41 @@ export function detectPtrWidth(programBytes: ArrayBuffer): 4 | 8 { const src = new Uint8Array(programBytes); if (src.length < 8) return 4; - function readLEB128(buf: Uint8Array, off: number): [number, number] { - let result = 0, shift = 0, pos = off; - for (;;) { - const byte = buf[pos++]; - result |= (byte & 0x7f) << shift; - if ((byte & 0x80) === 0) break; - shift += 7; - } - return [result, pos - off]; - } - // Skip magic + version (8 bytes) let offset = 8; while (offset < src.length) { const sectionId = src[offset]; - const [sectionSize, sizeBytes] = readLEB128(src, offset + 1); + const [sectionSize, sizeBytes] = readULEB128(src, offset + 1); const contentOffset = offset + 1 + sizeBytes; if (sectionId === 2) { // Import section — look for memory imports let pos = contentOffset; - const [importCount, countBytes] = readLEB128(src, pos); + const [importCount, countBytes] = readULEB128(src, pos); pos += countBytes; for (let i = 0; i < importCount; i++) { - const [modLen, modLenBytes] = readLEB128(src, pos); pos += modLenBytes + modLen; - const [fieldLen, fieldLenBytes] = readLEB128(src, pos); pos += fieldLenBytes + fieldLen; + const [modLen, modLenBytes] = readULEB128(src, pos); pos += modLenBytes + modLen; + const [fieldLen, fieldLenBytes] = readULEB128(src, pos); pos += fieldLenBytes + fieldLen; const kind = src[pos++]; if (kind === 2) { - // Memory import: flags byte, then limits - const flags = src[pos]; - if (flags & 0x04) return 8; // memory64 bit set + const limits = readLimits(src, pos); + if ((limits.flags & 0x04) !== 0) return 8; return 4; } // Skip non-memory imports - if (kind === 0) { const [, n] = readLEB128(src, pos); pos += n; } + if (kind === 0) { const [, n] = readULEB128(src, pos); pos += n; } else if (kind === 1) { - pos++; // ref type - const f = src[pos++]; - const [, n] = readLEB128(src, pos); pos += n; - if (f & 1) { const [, n2] = readLEB128(src, pos); pos += n2; } + pos = readWasmValueType(src, pos, "table import type").next; + pos = readLimits(src, pos).next; + } + else if (kind === 3) { + pos = readWasmValueType(src, pos, "global import type").next; + pos++; + } else if (kind === 4) { + pos++; // tag attribute + const [, typeBytes] = readULEB128(src, pos); + pos += typeBytes; } - else if (kind === 3) { pos += 2; } // global: type + mutability } break; } diff --git a/host/src/dylink-fork-archive.ts b/host/src/dylink-fork-archive.ts new file mode 100644 index 0000000000..d708a29d7d --- /dev/null +++ b/host/src/dylink-fork-archive.ts @@ -0,0 +1,2152 @@ +import type { + DylinkForkLibraryState, + DylinkForkState, + DylinkForkTransactionState, + DylinkInitializationStage, +} from "./dylink"; +import { computeForkModuleTemplateIdSync } from "./fork-module-state"; + +const ARCHIVE_MAGIC = 0x414c_464b; // "KFLA" in little-endian memory. +const ARCHIVE_VERSION = 4; +const ARCHIVE_HEADER_SIZE = 104; +const MODULE_MAGIC = 0x4d4c_464b; // "KFLM" in little-endian memory. +const MODULE_VERSION = 5; +const MODULE_HEADER_SIZE = 136; +const MODULE_DIGEST_OFFSET = 72; +const MODULE_DIGEST_SIZE = 32; +const MODULE_ALLOCATION_SIZE = 32; +const MODULE_FLAG_INITIALIZING = 1; +const MODULE_FLAG_GLOBAL = 1 << 1; +const MODULE_FLAG_COMMITTED_GLOBAL_ROOT = 1 << 2; +const MODULE_FLAG_KNOWN_MASK = + MODULE_FLAG_INITIALIZING + | MODULE_FLAG_GLOBAL + | MODULE_FLAG_COMMITTED_GLOBAL_ROOT; +const TRANSACTION_MAGIC = 0x544c_464b; // "KFLT" in little-endian memory. +const TRANSACTION_VERSION = 2; +const TRANSACTION_HEADER_SIZE = 80; +const TRANSACTION_DIGEST_OFFSET = 40; +const TRANSACTION_FLAG_GLOBAL = 1; +const TABLE_PATCH_MAGIC = 0x504a_464b; // "KFJP" in little-endian memory. +const TABLE_PATCH_VERSION = 1; +const TABLE_PATCH_HEADER_SIZE = 64; +const TABLE_PATCH_RUN_SIZE = 24; +const MAX_TABLE_PATCH_RECORDS = 256; +const MAX_TABLE_PATCH_BYTES = 1024 * 1024; +const FIRST_DYLINK_HANDLE = 2; +const EXHAUSTED_DYLINK_HANDLE = 0x1_0000_0000; +const MAX_EXACT_GENERATION = Number.MAX_SAFE_INTEGER; + +export interface DylinkForkArchiveSnapshot extends DylinkForkState { + /** + * Monotonic publication generation. + * + * Zero means that no archive has ever been published. A Worker may compare + * this scalar before parsing module records; a changed value requires a + * complete validated read before it can execute a table function installed + * by dlopen. + */ + readonly generation: number; + /** Sealed table-only KFMS arena, or zero before the first mutation. */ + readonly tableStateRoot: number; + /** Generation represented by `tableStateRoot`, or zero when it is absent. */ + readonly tableCheckpointGeneration: number; + /** Ordered funcref patches published after the current checkpoint. */ + readonly tablePatches: readonly DylinkForkTablePatch[]; +} + +export interface DylinkForkTablePatchRun { + readonly length: number; + /** Null has no function coordinate. */ + readonly function: + | null + | Readonly<{ + activationId: number; + ordinal: number; + }>; +} + +export interface DylinkForkTablePatch { + /** Assigned atomically by the archive at publication. */ + readonly generation?: number; + readonly activationId: number; + readonly ownerId: number; + readonly start: number; + readonly tableLength: number; + readonly runs: readonly DylinkForkTablePatchRun[]; +} + +export interface DylinkForkArchiveAllocation { + readonly address: number; + readonly size: number; +} + +export type DylinkForkArchiveAllocate = ( + size: number, +) => DylinkForkArchiveAllocation; + +export type DylinkForkArchiveDeallocate = ( + allocation: DylinkForkArchiveAllocation, +) => void; + +export interface DylinkForkGenerationFence { + read(): number; + write(generation: number): void; +} + +export interface DylinkForkTablePublication { + readonly snapshot: DylinkForkArchiveSnapshot; + readonly previousTableStateRoot: number; +} + +export interface DylinkForkTablePatchPublication { + readonly snapshot: DylinkForkArchiveSnapshot; +} + +/** + * Per-Worker generation gate for deterministic module/table recipes. + * + * The callback instantiates missing side modules into that Worker's own table + * and activation catalog. No function object crosses the Worker boundary. + */ +export class DylinkForkTableReplica { + private appliedGeneration = 0; + + constructor( + private readonly archive: DylinkForkArchive, + private readonly materialize: ( + snapshot: DylinkForkArchiveSnapshot, + previousGeneration: number, + ) => void, + private readonly label: string, + ) {} + + generation(): number { + return this.appliedGeneration; + } + + /** + * Advance the Worker that encoded the just-published state without + * reconstructing typed references back into their source Table. + */ + adoptPublishedGeneration(generation: number): void { + if ( + !Number.isSafeInteger(generation) + || generation < this.appliedGeneration + ) { + throw new RangeError( + `${this.label}: cannot adopt dylink generation ${String(generation)}`, + ); + } + this.appliedGeneration = generation; + } + + reconcile(): boolean { + const published = this.archive.generation(); + if (published === this.appliedGeneration) return false; + if (published < this.appliedGeneration) { + throw new Error( + `${this.label}: dylink archive generation moved backward from ` + + `${this.appliedGeneration} to ${published}`, + ); + } + const snapshot = this.archive.read(); + if (snapshot.generation !== published) { + throw new Error( + `${this.label}: dylink archive changed while its reader lock was held`, + ); + } + this.materialize(snapshot, this.appliedGeneration); + // Publish locally only after every fresh function object is installed. + this.appliedGeneration = snapshot.generation; + return true; + } +} + +interface IndexedModule { + readonly allocation: DylinkForkArchiveAllocation; + readonly state: DylinkForkLibraryState; +} + +interface IndexedTablePatch { + readonly allocation: DylinkForkArchiveAllocation; + readonly patch: DylinkForkTablePatch & { readonly generation: number }; +} + +interface IndexedTransaction { + readonly allocation: DylinkForkArchiveAllocation; + readonly state: DylinkForkTransactionState; +} + +function align8(value: number): number { + const aligned = Math.ceil(value / 8) * 8; + if (!Number.isSafeInteger(aligned)) { + throw new RangeError("dylink fork archive size exceeds exact host integers"); + } + return aligned; +} + +function canonicalProviderDependencies( + state: DylinkForkLibraryState, +): string[] { + const dependencies = [...(state.providerDependencies ?? [])].sort(); + const seen = new Set(); + for (const dependency of dependencies) { + if ( + typeof dependency !== "string" + || dependency.length === 0 + || dependency === state.name + || seen.has(dependency) + ) { + throw new Error( + `${state.name}: invalid or duplicate runtime provider ${String(dependency)}`, + ); + } + seen.add(dependency); + } + return dependencies; +} + +function canonicalMemoryAllocations( + state: DylinkForkLibraryState, +): NonNullable[number][] { + const allocations = [...(state.allocations ?? [])] + .map((allocation) => ({ ...allocation })) + .sort( + (left, right) => + left.mappingAddress - right.mappingAddress + || left.address - right.address, + ); + let previousMappingEnd = 0; + for (const [index, allocation] of allocations.entries()) { + checkedAddress( + allocation.address, + `${state.name}: allocation ${index} address`, + ); + checkedAddress( + allocation.size, + `${state.name}: allocation ${index} size`, + ); + checkedAddress( + allocation.mappingAddress, + `${state.name}: allocation ${index} mapping address`, + ); + checkedAddress( + allocation.mappingSize, + `${state.name}: allocation ${index} mapping size`, + ); + const logicalEnd = allocation.address + allocation.size; + const mappingEnd = allocation.mappingAddress + allocation.mappingSize; + if ( + !Number.isSafeInteger(logicalEnd) + || !Number.isSafeInteger(mappingEnd) + || allocation.address < allocation.mappingAddress + || logicalEnd > mappingEnd + ) { + throw new RangeError( + `${state.name}: allocation ${index} escapes its process mapping`, + ); + } + if (allocation.mappingAddress < previousMappingEnd) { + throw new Error(`${state.name}: process allocation mappings overlap`); + } + previousMappingEnd = mappingEnd; + } + return allocations; +} + +function encodeProviderDependencies( + state: DylinkForkLibraryState, +): Readonly<{ + bytes: Uint8Array; + count: number; +}> { + const encoded = canonicalProviderDependencies(state).map((dependency) => { + const bytes = new TextEncoder().encode(dependency); + checkedU32(bytes.length, `${state.name}: runtime provider name length`); + return bytes; + }); + const size = encoded.reduce((total, bytes) => { + const next = total + 4 + bytes.length; + if (!Number.isSafeInteger(next) || next > 0xffff_ffff) { + throw new RangeError( + `${state.name}: runtime provider archive is too large`, + ); + } + return next; + }, 0); + const bytes = new Uint8Array(size); + const view = new DataView(bytes.buffer); + let cursor = 0; + for (const name of encoded) { + view.setUint32(cursor, name.length, true); + cursor += 4; + bytes.set(name, cursor); + cursor += name.length; + } + return { bytes, count: encoded.length }; +} + +function decodeProviderDependencies( + bytes: Uint8Array, + count: number, + context: string, +): string[] { + checkedU32(count, `${context} runtime provider count`); + const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); + const dependencies: string[] = []; + let cursor = 0; + for (let index = 0; index < count; index++) { + if (cursor > bytes.length - 4) { + throw new Error(`${context}: truncated runtime provider metadata`); + } + const length = view.getUint32(cursor, true); + cursor += 4; + if (length === 0 || cursor > bytes.length - length) { + throw new Error(`${context}: invalid runtime provider name length`); + } + let dependency: string; + try { + dependency = new TextDecoder("utf-8", { fatal: true }).decode( + bytes.subarray(cursor, cursor + length), + ); + } catch { + throw new Error(`${context}: invalid UTF-8 runtime provider name`); + } + if ( + dependencies.length > 0 + && dependencies[dependencies.length - 1]! >= dependency + ) { + throw new Error(`${context}: noncanonical runtime provider ordering`); + } + dependencies.push(dependency); + cursor += length; + } + if (cursor !== bytes.length) { + throw new Error(`${context}: noncanonical runtime provider metadata`); + } + return dependencies; +} + +function equalBytes(left: Readonly, right: Readonly): boolean { + return left.length === right.length + && left.every((byte, index) => byte === right[index]); +} + +function checkedU32(value: number, context: string, allowZero = true): number { + if ( + !Number.isInteger(value) + || value < (allowZero ? 0 : 1) + || value > 0xffff_ffff + ) { + throw new RangeError(`${context} is not ${allowZero ? "a" : "a nonzero"} u32`); + } + return value; +} + +function checkedAddress(value: number, context: string, allowZero = false): number { + if (!Number.isSafeInteger(value) || value < (allowZero ? 0 : 1)) { + throw new RangeError(`${context} is not an exact positive address`); + } + return value; +} + +function checkedExactNonnegative(value: number, context: string): number { + if (!Number.isSafeInteger(value) || value < 0) { + throw new RangeError(`${context} is not an exact non-negative integer`); + } + return value; +} + +function checkedNextHandle(value: number): number { + if ( + !Number.isSafeInteger(value) + || value < FIRST_DYLINK_HANDLE + || value > EXHAUSTED_DYLINK_HANDLE + ) { + throw new RangeError(`dylink fork archive next handle ${String(value)} is invalid`); + } + return value; +} + +function initializationStageCode(stage: DylinkInitializationStage): number { + switch (stage) { + case "bootstrap": return 1; + case "relocations": return 2; + case "constructors": return 3; + } +} + +function decodeInitializationStage( + code: number, + context: string, +): DylinkInitializationStage { + switch (code) { + case 1: return "bootstrap"; + case 2: return "relocations"; + case 3: return "constructors"; + default: throw new Error(`${context}: invalid initialization stage ${code}`); + } +} + +/** + * Versioned, bounded-by-live-closure dylink state copied through process memory. + * + * The JavaScript object graph is only an index/cache. Durable state is the + * header and module records in linear memory, which a pthread or fresh process + * worker can validate and adopt independently. + */ +export class DylinkForkArchive { + private headerAddress = 0; + private indexed = false; + private modules = new Map(); + private transactions = new Map(); + private tablePatchRecords: IndexedTablePatch[] = []; + private state: DylinkForkArchiveSnapshot = { + generation: 0, + tableStateRoot: 0, + tableCheckpointGeneration: 0, + tablePatches: [], + nextHandle: FIRST_DYLINK_HANDLE, + libraries: [], + transactions: [], + }; + + constructor( + private readonly memory: WebAssembly.Memory, + private readonly ptrWidth: 4 | 8, + private readonly readHead: () => number, + private readonly writeHead: (address: number) => void, + private readonly allocate: DylinkForkArchiveAllocate, + private readonly deallocate: DylinkForkArchiveDeallocate, + private readonly label: string, + private readonly generationFence?: DylinkForkGenerationFence, + ) { + if (ptrWidth !== 4 && ptrWidth !== 8) { + throw new RangeError(`${label}: invalid archive pointer width ${ptrWidth}`); + } + } + + /** + * Return the current publication generation without walking module records. + * + * Callers still hold the process archive reader lock while acting on the + * result. The scalar is a fast-path hint, not permission to consume a + * concurrently changing archive. + */ + generation(): number { + const head = this.readHead(); + const fenced = this.generationFence?.read(); + if (head === 0) { + if (fenced !== undefined && fenced !== 0) { + throw new Error(`${this.label}: published generation has no archive header`); + } + return 0; + } + this.checkedRange(head, ARCHIVE_HEADER_SIZE, "archive header"); + return fenced ?? this.readGeneration(head); + } + + /** Validate and return an owned snapshot of the copied archive. */ + read(): DylinkForkArchiveSnapshot { + this.refreshIndex(); + return this.copyState(this.state); + } + + /** + * Publish the exact compact live linker state. + * + * Callers serialize this operation with process dlopen/fork arbitration. + * New records are fully initialized before the header points at them; stale + * records become unreachable before their mappings are released. + */ + sync(nextState: DylinkForkState): DylinkForkArchiveSnapshot { + const owned = this.validateState(nextState); + this.refreshIndex(); + this.ensureHeader(owned.nextHandle); + const generation = this.nextGeneration(this.state.generation); + + const target: IndexedModule[] = []; + const targetNames = new Set(); + const replaced: IndexedModule[] = []; + for (const library of owned.libraries) { + targetNames.add(library.name); + const current = this.modules.get(library.name); + if (current) { + this.requireImmutableMatch(current.state, library); + const currentProviders = + canonicalProviderDependencies(current.state); + const nextProviders = canonicalProviderDependencies(library); + if ( + currentProviders.length === nextProviders.length + && currentProviders.every( + (dependency, index) => dependency === nextProviders[index], + ) + ) { + this.writeMutableState(current.allocation.address, library); + target.push({ + allocation: current.allocation, + state: library, + }); + } else { + // Constructor dlsym can add a runtime-provider edge between archive + // generations. Publish a complete replacement record; never resize a + // reachable record beneath pthread readers. + target.push(this.allocateModule(library)); + replaced.push(current); + } + } else { + target.push(this.allocateModule(library)); + } + } + + for (let index = 0; index < target.length; index++) { + this.writeU64( + target[index]!.allocation.address + 8, + target[index + 1]?.allocation.address ?? 0, + ); + } + + const transactionTarget: IndexedTransaction[] = []; + const transactionTokens = new Set(); + for (const transaction of owned.transactions ?? []) { + transactionTokens.add(transaction.token); + const current = this.transactions.get(transaction.token); + if (current) { + if ( + current.state.name !== transaction.name + || current.state.globalVisibility !== transaction.globalVisibility + || !equalBytes(current.state.moduleBytes, transaction.moduleBytes) + ) { + throw new Error( + `${this.label}: staged transaction ${transaction.token} changed identity`, + ); + } + transactionTarget.push({ + allocation: current.allocation, + state: transaction, + }); + } else { + transactionTarget.push(this.allocateTransaction(transaction)); + } + } + for (let index = 0; index < transactionTarget.length; index++) { + this.writeU64( + transactionTarget[index]!.allocation.address + 8, + transactionTarget[index + 1]?.allocation.address ?? 0, + ); + } + + const view = new DataView(this.memory.buffer); + this.writeU64(this.headerAddress + 16, owned.nextHandle); + view.setUint32(this.headerAddress + 24, target.length, true); + view.setUint32(this.headerAddress + 28, 0, true); + this.writeU64( + this.headerAddress + 32, + target[0]?.allocation.address ?? 0, + ); + view.setUint32(this.headerAddress + 88, transactionTarget.length, true); + view.setUint32(this.headerAddress + 92, 0, true); + this.writeU64( + this.headerAddress + 96, + transactionTarget[0]?.allocation.address ?? 0, + ); + // WHY: generation is the publication fence consumed by other Workers. + // Write it only after every reachable record and header field is complete; + // otherwise a pthread could observe "new" and instantiate a half-written + // function recipe graph. + this.writeGeneration(this.headerAddress, generation); + this.generationFence?.write(generation); + + const stale = [...this.modules.values()].filter( + ({ state }) => !targetNames.has(state.name), + ); + const staleTransactions = [...this.transactions.values()].filter( + ({ state }) => !transactionTokens.has(state.token), + ); + this.modules = new Map(target.map((entry) => [entry.state.name, entry])); + this.transactions = new Map( + transactionTarget.map((entry) => [entry.state.token, entry]), + ); + this.state = { + generation, + tableStateRoot: this.state.tableStateRoot, + tableCheckpointGeneration: this.state.tableCheckpointGeneration, + tablePatches: this.state.tablePatches, + nextHandle: owned.nextHandle, + libraries: target.map(({ state }) => state), + transactions: transactionTarget.map(({ state }) => state), + }; + for (const entry of stale) this.deallocate(entry.allocation); + for (const entry of replaced) this.deallocate(entry.allocation); + for (const entry of staleTransactions) this.deallocate(entry.allocation); + return this.copyState(this.state); + } + + /** + * Publish a sealed typed table snapshot while the process writer lock is held. + */ + publishTableState(tableStateRoot: number): DylinkForkTablePublication { + checkedAddress(tableStateRoot, `${this.label}: table-state root`); + this.refreshIndex(); + this.ensureHeader(this.state.nextHandle); + const previousTableStateRoot = this.state.tableStateRoot; + const generation = this.nextGeneration(this.state.generation); + const stalePatches = this.tablePatchRecords; + this.writeU64(this.headerAddress + 48, tableStateRoot); + this.writeU64(this.headerAddress + 56, 0); + this.writeU64(this.headerAddress + 64, 0); + const view = new DataView(this.memory.buffer); + view.setUint32(this.headerAddress + 72, 0, true); + view.setUint32(this.headerAddress + 76, 0, true); + this.writeU64(this.headerAddress + 80, generation); + this.writeGeneration(this.headerAddress, generation); + // This fixed shared-memory word is the Wasm fast-path fence. It must be + // last so a changed value always names a complete header and sealed arena. + this.generationFence?.write(generation); + this.state = { + ...this.state, + generation, + tableStateRoot, + tableCheckpointGeneration: generation, + tablePatches: [], + }; + this.tablePatchRecords = []; + for (const record of stalePatches) { + this.deallocate(record.allocation); + } + return { + snapshot: this.copyState(this.state), + previousTableStateRoot, + }; + } + + /** + * Whether one deterministic funcref patch fits before bounded compaction. + */ + canPublishTablePatch(patch: DylinkForkTablePatch): boolean { + this.refreshIndex(); + const owned = this.validateTablePatch(patch); + const size = this.tablePatchSize(owned); + return ( + this.tablePatchRecords.length < MAX_TABLE_PATCH_RECORDS + && this.tablePatchBytes() + size <= MAX_TABLE_PATCH_BYTES + ); + } + + /** + * Append one deterministic funcref patch under the process writer lock. + * + * A caller that receives `false` from `canPublishTablePatch` first publishes + * a full typed checkpoint. This keeps retained journal memory bounded while + * making the common mutation proportional only to its changed range. + */ + publishTablePatch( + patch: DylinkForkTablePatch, + ): DylinkForkTablePatchPublication { + const owned = this.validateTablePatch(patch); + this.refreshIndex(); + this.ensureHeader(this.state.nextHandle); + const size = this.tablePatchSize(owned); + if ( + this.tablePatchRecords.length >= MAX_TABLE_PATCH_RECORDS + || this.tablePatchBytes() + size > MAX_TABLE_PATCH_BYTES + ) { + throw new Error(`${this.label}: table patch journal requires compaction`); + } + const generation = this.nextGeneration(this.state.generation); + const record = this.allocateTablePatch(owned, generation); + const previous = this.tablePatchRecords.at(-1); + if (previous) { + this.writeU64(previous.allocation.address + 8, record.allocation.address); + } else { + this.writeU64(this.headerAddress + 56, record.allocation.address); + } + this.writeU64(this.headerAddress + 64, record.allocation.address); + const view = new DataView(this.memory.buffer); + view.setUint32( + this.headerAddress + 72, + this.tablePatchRecords.length + 1, + true, + ); + view.setUint32( + this.headerAddress + 76, + this.tablePatchBytes() + size, + true, + ); + this.writeGeneration(this.headerAddress, generation); + this.generationFence?.write(generation); + this.tablePatchRecords = [...this.tablePatchRecords, record]; + this.state = { + ...this.state, + generation, + tablePatches: [...this.state.tablePatches, record.patch], + }; + return { snapshot: this.copyState(this.state) }; + } + + private refreshIndex(): void { + const head = this.readHead(); + if (head === 0) { + if (this.headerAddress !== 0 || this.state.generation !== 0) { + this.resetIndex(); + } + this.indexed = true; + return; + } + if ( + this.indexed + && this.headerAddress === head + && this.state.generation === this.generation() + ) { + return; + } + this.resetIndex(); + this.ensureIndexed(); + } + + private resetIndex(): void { + this.headerAddress = 0; + this.indexed = false; + this.modules = new Map(); + this.transactions = new Map(); + this.tablePatchRecords = []; + this.state = { + generation: 0, + tableStateRoot: 0, + tableCheckpointGeneration: 0, + tablePatches: [], + nextHandle: FIRST_DYLINK_HANDLE, + libraries: [], + transactions: [], + }; + } + + private ensureIndexed(): void { + if (this.indexed) return; + const head = this.readHead(); + if (head === 0) { + this.indexed = true; + return; + } + this.headerAddress = this.checkedRange( + head, + ARCHIVE_HEADER_SIZE, + "archive header", + ); + const view = new DataView(this.memory.buffer); + if (view.getUint32(head, true) !== ARCHIVE_MAGIC) { + throw new Error(`${this.label}: invalid dylink fork archive magic`); + } + if (view.getUint16(head + 4, true) !== ARCHIVE_VERSION) { + throw new Error(`${this.label}: unsupported dylink fork archive version`); + } + if (view.getUint16(head + 6, true) !== ARCHIVE_HEADER_SIZE) { + throw new Error(`${this.label}: invalid dylink fork archive header size`); + } + if (view.getUint8(head + 8) !== this.ptrWidth) { + throw new Error(`${this.label}: dylink fork archive pointer-width mismatch`); + } + for (let offset = 9; offset < 16; offset++) { + if (view.getUint8(head + offset) !== 0) { + throw new Error(`${this.label}: nonzero dylink fork archive header reserved byte`); + } + } + const nextHandle = checkedNextHandle(this.readU64(head + 16, "next handle")); + const count = view.getUint32(head + 24, true); + const maximumPhysicalModules = Math.floor( + (this.memory.buffer.byteLength - ARCHIVE_HEADER_SIZE) + / MODULE_HEADER_SIZE, + ); + if (count > maximumPhysicalModules) { + throw new RangeError( + `${this.label}: dylink fork archive module count exceeds its memory geometry`, + ); + } + if (view.getUint32(head + 28, true) !== 0) { + throw new Error(`${this.label}: nonzero dylink fork archive header flags`); + } + let cursor = this.readU64(head + 32, "first module"); + const generation = this.readGeneration(head); + if (generation === 0) { + throw new Error(`${this.label}: unpublished dylink fork archive`); + } + const fenced = this.generationFence?.read(); + if (fenced !== undefined && fenced !== generation) { + throw new Error( + `${this.label}: dylink archive generation does not match its publication fence`, + ); + } + const tableStateRoot = this.readU64(head + 48, "table-state root"); + if (tableStateRoot !== 0) { + this.checkedRange(tableStateRoot, 1, "table-state root"); + } + let tablePatchCursor = this.readU64(head + 56, "first table patch"); + const tablePatchTail = this.readU64(head + 64, "last table patch"); + const tablePatchCount = view.getUint32(head + 72, true); + const declaredTablePatchBytes = view.getUint32(head + 76, true); + const tableCheckpointGeneration = this.readU64( + head + 80, + "table checkpoint generation", + ); + const transactionCount = view.getUint32(head + 88, true); + if (view.getUint32(head + 92, true) !== 0) { + throw new Error(`${this.label}: nonzero dylink transaction flags`); + } + let transactionCursor = this.readU64( + head + 96, + "first staged transaction", + ); + const maximumPhysicalTransactions = Math.floor( + (this.memory.buffer.byteLength - ARCHIVE_HEADER_SIZE) + / TRANSACTION_HEADER_SIZE, + ); + if (transactionCount > maximumPhysicalTransactions) { + throw new RangeError( + `${this.label}: staged transaction count exceeds its memory geometry`, + ); + } + if ((transactionCount === 0) !== (transactionCursor === 0)) { + throw new Error( + `${this.label}: staged transaction count/head mismatch`, + ); + } + if ( + tableCheckpointGeneration > generation + || (tableStateRoot === 0) !== (tableCheckpointGeneration === 0) + ) { + throw new Error(`${this.label}: inconsistent table checkpoint`); + } + if ( + tablePatchCount > MAX_TABLE_PATCH_RECORDS + || declaredTablePatchBytes > MAX_TABLE_PATCH_BYTES + ) { + throw new RangeError(`${this.label}: table patch journal is too large`); + } + if ( + (tablePatchCount === 0) + !== (tablePatchCursor === 0 && tablePatchTail === 0) + ) { + throw new Error(`${this.label}: table patch count/head/tail mismatch`); + } + if ((count === 0) !== (cursor === 0)) { + throw new Error(`${this.label}: dylink fork archive count/head mismatch`); + } + + const intervals: Array<{ start: number; end: number }> = [{ + start: head, + end: head + ARCHIVE_HEADER_SIZE, + }]; + const seenAddresses = new Set(); + const seenNames = new Set(); + const seenActivations = new Set(); + const seenHandles = new Set(); + const libraries: DylinkForkLibraryState[] = []; + const modules = new Map(); + for (let ordinal = 0; ordinal < count; ordinal++) { + if (cursor === 0 || seenAddresses.has(cursor)) { + throw new Error(`${this.label}: cyclic or truncated dylink fork archive`); + } + seenAddresses.add(cursor); + const decoded = this.readModule(cursor, ordinal, intervals); + if (seenNames.has(decoded.state.name)) { + throw new Error(`${this.label}: duplicate archived module ${decoded.state.name}`); + } + seenNames.add(decoded.state.name); + if (decoded.state.activationId !== undefined) { + if (seenActivations.has(decoded.state.activationId)) { + throw new Error( + `${this.label}: duplicate archived activation ${decoded.state.activationId}`, + ); + } + seenActivations.add(decoded.state.activationId); + } + if (decoded.state.handle !== undefined) { + if (seenHandles.has(decoded.state.handle)) { + throw new Error( + `${this.label}: duplicate archived handle ${decoded.state.handle}`, + ); + } + if (decoded.state.handle >= nextHandle) { + throw new Error( + `${this.label}: archived handle ${decoded.state.handle} reaches next handle`, + ); + } + seenHandles.add(decoded.state.handle); + } + libraries.push(decoded.state); + modules.set(decoded.state.name, decoded); + cursor = this.readU64(cursor + 8, `module ${ordinal} next`); + } + if (cursor !== 0) { + throw new Error(`${this.label}: dylink fork archive has more records than declared`); + } + const transactionRecords: IndexedTransaction[] = []; + const seenTransactionTokens = new Set(); + for (let ordinal = 0; ordinal < transactionCount; ordinal++) { + if ( + transactionCursor === 0 + || seenAddresses.has(transactionCursor) + ) { + throw new Error( + `${this.label}: cyclic or truncated staged transaction archive`, + ); + } + seenAddresses.add(transactionCursor); + const decoded = this.readTransaction( + transactionCursor, + ordinal, + intervals, + ); + if (seenTransactionTokens.has(decoded.state.token)) { + throw new Error( + `${this.label}: duplicate staged transaction ${decoded.state.token}`, + ); + } + seenTransactionTokens.add(decoded.state.token); + transactionRecords.push(decoded); + transactionCursor = this.readU64( + transactionCursor + 8, + `staged transaction ${ordinal} next`, + ); + } + if (transactionCursor !== 0) { + throw new Error( + `${this.label}: staged transaction archive has extra records`, + ); + } + const tablePatchRecords: IndexedTablePatch[] = []; + let previousPatchGeneration = tableCheckpointGeneration; + let tablePatchBytes = 0; + for (let ordinal = 0; ordinal < tablePatchCount; ordinal++) { + if ( + tablePatchCursor === 0 + || seenAddresses.has(tablePatchCursor) + ) { + throw new Error(`${this.label}: cyclic or truncated table patch journal`); + } + seenAddresses.add(tablePatchCursor); + const decoded = this.readTablePatch( + tablePatchCursor, + ordinal, + intervals, + ); + const patchGeneration = decoded.patch.generation; + if ( + patchGeneration <= previousPatchGeneration + || patchGeneration > generation + ) { + throw new Error( + `${this.label}: table patch ${ordinal} has non-monotonic generation`, + ); + } + previousPatchGeneration = patchGeneration; + tablePatchBytes += decoded.allocation.size; + if (tablePatchBytes > MAX_TABLE_PATCH_BYTES) { + throw new RangeError(`${this.label}: table patch journal byte count overflow`); + } + tablePatchRecords.push(decoded); + tablePatchCursor = this.readU64( + tablePatchCursor + 8, + `table patch ${ordinal} next`, + ); + } + if (tablePatchCursor !== 0) { + throw new Error(`${this.label}: table patch journal has extra records`); + } + if ( + (tablePatchRecords.at(-1)?.allocation.address ?? 0) !== tablePatchTail + || tablePatchBytes !== declaredTablePatchBytes + ) { + throw new Error(`${this.label}: table patch tail/byte count mismatch`); + } + for (const library of libraries) { + for (const allocation of library.allocations ?? []) { + const start = allocation.mappingAddress; + const end = start + allocation.mappingSize; + if ( + intervals.some( + (interval) => start < interval.end && interval.start < end, + ) + ) { + throw new Error( + `${this.label}: ${library.name} process mapping overlaps archive storage`, + ); + } + } + } + const validatedState = this.validateState({ + nextHandle, + libraries, + transactions: transactionRecords.map(({ state }) => state), + }); + this.modules = modules; + this.transactions = new Map( + transactionRecords.map((entry) => [entry.state.token, entry]), + ); + this.tablePatchRecords = tablePatchRecords; + this.state = { + generation, + tableStateRoot, + tableCheckpointGeneration, + tablePatches: tablePatchRecords.map(({ patch }) => patch), + nextHandle: validatedState.nextHandle, + libraries: validatedState.libraries, + transactions: validatedState.transactions ?? [], + }; + this.indexed = true; + } + + private ensureHeader(nextHandle: number): void { + if (this.headerAddress !== 0) return; + const allocation = this.allocate(ARCHIVE_HEADER_SIZE); + if (allocation.size !== ARCHIVE_HEADER_SIZE) { + throw new Error(`${this.label}: archive allocator changed the header size`); + } + const address = this.checkedRange( + allocation.address, + allocation.size, + "new archive header", + ); + const bytes = new Uint8Array(this.memory.buffer, address, allocation.size); + bytes.fill(0); + const view = new DataView(this.memory.buffer); + view.setUint32(address, ARCHIVE_MAGIC, true); + view.setUint16(address + 4, ARCHIVE_VERSION, true); + view.setUint16(address + 6, ARCHIVE_HEADER_SIZE, true); + view.setUint8(address + 8, this.ptrWidth); + this.writeU64(address + 16, nextHandle); + this.headerAddress = address; + // The zero generation keeps this header explicitly unpublished until + // sync() has linked every module record and performs the final release. + this.writeHead(address); + } + + private validateTablePatch( + patch: DylinkForkTablePatch, + ): DylinkForkTablePatch { + if (patch.generation !== undefined) { + throw new Error(`${this.label}: caller assigned a table patch generation`); + } + const activationId = checkedU32( + patch.activationId, + `${this.label}: table patch activation`, + ); + const ownerId = checkedU32( + patch.ownerId, + `${this.label}: table patch owner`, + false, + ); + const start = checkedExactNonnegative( + patch.start, + `${this.label}: table patch start`, + ); + const tableLength = checkedExactNonnegative( + patch.tableLength, + `${this.label}: table patch length`, + ); + if (!Array.isArray(patch.runs) || patch.runs.length === 0) { + throw new Error(`${this.label}: table patch has no runs`); + } + checkedU32( + patch.runs.length, + `${this.label}: table patch run count`, + false, + ); + let changed = 0; + const runs = patch.runs.map((run, ordinal) => { + const length = checkedExactNonnegative( + run.length, + `${this.label}: table patch run ${ordinal} length`, + ); + if (length === 0) { + throw new RangeError(`${this.label}: table patch run ${ordinal} is empty`); + } + changed += length; + if (!Number.isSafeInteger(changed)) { + throw new RangeError(`${this.label}: table patch range is too large`); + } + if (run.function === null) { + return Object.freeze({ length, function: null }); + } + if ( + typeof run.function !== "object" + || run.function === null + ) { + throw new TypeError( + `${this.label}: table patch run ${ordinal} has no function recipe`, + ); + } + return Object.freeze({ + length, + function: Object.freeze({ + activationId: checkedU32( + run.function.activationId, + `${this.label}: table patch run ${ordinal} activation`, + ), + ordinal: checkedU32( + run.function.ordinal, + `${this.label}: table patch run ${ordinal} function ordinal`, + ), + }), + }); + }); + if (start + changed > tableLength) { + throw new RangeError(`${this.label}: table patch exceeds final table length`); + } + return Object.freeze({ + activationId, + ownerId, + start, + tableLength, + runs: Object.freeze(runs), + }); + } + + private tablePatchSize(patch: DylinkForkTablePatch): number { + return TABLE_PATCH_HEADER_SIZE + patch.runs.length * TABLE_PATCH_RUN_SIZE; + } + + private tablePatchBytes(): number { + return this.tablePatchRecords.reduce( + (total, record) => total + record.allocation.size, + 0, + ); + } + + private allocateTablePatch( + patch: DylinkForkTablePatch, + generation: number, + ): IndexedTablePatch { + const totalSize = this.tablePatchSize(patch); + const allocation = this.allocate(totalSize); + if (allocation.size !== totalSize) { + throw new Error(`${this.label}: archive allocator changed a table patch size`); + } + const address = this.checkedRange( + allocation.address, + allocation.size, + "new table patch", + ); + const bytes = new Uint8Array(this.memory.buffer, address, totalSize); + bytes.fill(0); + const view = new DataView(this.memory.buffer); + view.setUint32(address, TABLE_PATCH_MAGIC, true); + view.setUint16(address + 4, TABLE_PATCH_VERSION, true); + view.setUint16(address + 6, TABLE_PATCH_HEADER_SIZE, true); + this.writeU64(address + 16, totalSize); + this.writeU64(address + 24, generation); + view.setUint32(address + 32, patch.activationId, true); + view.setUint32(address + 36, patch.ownerId, true); + this.writeU64(address + 40, patch.start); + this.writeU64(address + 48, patch.tableLength); + view.setUint32(address + 56, patch.runs.length, true); + for (const [ordinal, run] of patch.runs.entries()) { + const offset = address + TABLE_PATCH_HEADER_SIZE + + ordinal * TABLE_PATCH_RUN_SIZE; + this.writeU64(offset, run.length); + view.setUint32(offset + 8, run.function === null ? 0 : 1, true); + view.setUint32(offset + 12, run.function?.activationId ?? 0, true); + view.setUint32(offset + 16, run.function?.ordinal ?? 0, true); + view.setUint32(offset + 20, 0, true); + } + return { + allocation, + patch: Object.freeze({ + ...this.copyTablePatch(patch), + generation, + }), + }; + } + + private readTablePatch( + address: number, + ordinal: number, + intervals: Array<{ start: number; end: number }>, + ): IndexedTablePatch { + this.checkedRange( + address, + TABLE_PATCH_HEADER_SIZE, + `table patch ${ordinal} header`, + ); + const view = new DataView(this.memory.buffer); + if (view.getUint32(address, true) !== TABLE_PATCH_MAGIC) { + throw new Error(`${this.label}: table patch ${ordinal} has invalid magic`); + } + if (view.getUint16(address + 4, true) !== TABLE_PATCH_VERSION) { + throw new Error(`${this.label}: table patch ${ordinal} has unsupported version`); + } + if (view.getUint16(address + 6, true) !== TABLE_PATCH_HEADER_SIZE) { + throw new Error(`${this.label}: table patch ${ordinal} has invalid header size`); + } + const runCount = view.getUint32(address + 56, true); + const totalSize = this.readU64( + address + 16, + `table patch ${ordinal} allocation size`, + ); + const expectedSize = + TABLE_PATCH_HEADER_SIZE + runCount * TABLE_PATCH_RUN_SIZE; + if (totalSize !== expectedSize) { + throw new Error(`${this.label}: table patch ${ordinal} has invalid size`); + } + this.checkedRange(address, totalSize, `table patch ${ordinal}`); + const end = address + totalSize; + if (intervals.some((interval) => address < interval.end && interval.start < end)) { + throw new Error(`${this.label}: table patch ${ordinal} overlaps an archive record`); + } + intervals.push({ start: address, end }); + const generation = this.readU64( + address + 24, + `table patch ${ordinal} generation`, + ); + if (generation === 0 || view.getUint32(address + 60, true) !== 0) { + throw new Error(`${this.label}: table patch ${ordinal} has invalid metadata`); + } + const runs: DylinkForkTablePatchRun[] = []; + for (let index = 0; index < runCount; index++) { + const offset = address + TABLE_PATCH_HEADER_SIZE + + index * TABLE_PATCH_RUN_SIZE; + const length = this.readU64( + offset, + `table patch ${ordinal} run ${index} length`, + ); + const kind = view.getUint32(offset + 8, true); + const activationId = view.getUint32(offset + 12, true); + const functionOrdinal = view.getUint32(offset + 16, true); + if (view.getUint32(offset + 20, true) !== 0 || (kind !== 0 && kind !== 1)) { + throw new Error(`${this.label}: table patch ${ordinal} run ${index} is invalid`); + } + if (kind === 0 && (activationId !== 0 || functionOrdinal !== 0)) { + throw new Error( + `${this.label}: null table patch run ${index} has a function coordinate`, + ); + } + runs.push({ + length, + function: kind === 0 + ? null + : { activationId, ordinal: functionOrdinal }, + }); + } + const patch = this.validateTablePatch({ + activationId: view.getUint32(address + 32, true), + ownerId: view.getUint32(address + 36, true), + start: this.readU64(address + 40, `table patch ${ordinal} start`), + tableLength: this.readU64( + address + 48, + `table patch ${ordinal} table length`, + ), + runs, + }); + return { + allocation: { address, size: totalSize }, + patch: Object.freeze({ + ...this.copyTablePatch(patch), + generation, + }), + }; + } + + private allocateTransaction( + state: DylinkForkTransactionState, + ): IndexedTransaction { + const name = new TextEncoder().encode(state.name); + const nameAligned = align8(name.length); + const totalSize = + TRANSACTION_HEADER_SIZE + nameAligned + state.moduleBytes.length; + const allocation = this.allocate(totalSize); + if (allocation.size !== totalSize) { + throw new Error( + `${this.label}: archive allocator changed a transaction record size`, + ); + } + const address = this.checkedRange( + allocation.address, + allocation.size, + `new staged transaction ${state.token}`, + ); + const bytes = new Uint8Array(this.memory.buffer, address, totalSize); + bytes.fill(0); + const view = new DataView(this.memory.buffer); + view.setUint32(address, TRANSACTION_MAGIC, true); + view.setUint16(address + 4, TRANSACTION_VERSION, true); + view.setUint16(address + 6, TRANSACTION_HEADER_SIZE, true); + this.writeU64(address + 16, totalSize); + view.setUint32(address + 24, state.token, true); + view.setUint32(address + 28, name.length, true); + view.setUint32(address + 32, state.moduleBytes.length, true); + view.setUint32( + address + 36, + state.globalVisibility ? TRANSACTION_FLAG_GLOBAL : 0, + true, + ); + bytes.set( + computeForkModuleTemplateIdSync(state.moduleBytes), + TRANSACTION_DIGEST_OFFSET, + ); + bytes.set(name, TRANSACTION_HEADER_SIZE); + bytes.set( + state.moduleBytes, + TRANSACTION_HEADER_SIZE + nameAligned, + ); + return { + allocation, + state: this.copyTransaction(state), + }; + } + + private readTransaction( + address: number, + ordinal: number, + intervals: Array<{ start: number; end: number }>, + ): IndexedTransaction { + this.checkedRange( + address, + TRANSACTION_HEADER_SIZE, + `staged transaction ${ordinal} header`, + ); + const view = new DataView(this.memory.buffer); + if (view.getUint32(address, true) !== TRANSACTION_MAGIC) { + throw new Error( + `${this.label}: staged transaction ${ordinal} has invalid magic`, + ); + } + if (view.getUint16(address + 4, true) !== TRANSACTION_VERSION) { + throw new Error( + `${this.label}: staged transaction ${ordinal} has unsupported version`, + ); + } + if (view.getUint16(address + 6, true) !== TRANSACTION_HEADER_SIZE) { + throw new Error( + `${this.label}: staged transaction ${ordinal} has invalid header size`, + ); + } + const allocationSize = this.readU64( + address + 16, + `staged transaction ${ordinal} allocation size`, + ); + const nameLength = view.getUint32(address + 28, true); + const bytesLength = view.getUint32(address + 32, true); + const expectedSize = + TRANSACTION_HEADER_SIZE + align8(nameLength) + bytesLength; + const flags = view.getUint32(address + 36, true); + if ( + allocationSize !== expectedSize + || (flags & ~TRANSACTION_FLAG_GLOBAL) !== 0 + ) { + throw new Error( + `${this.label}: staged transaction ${ordinal} has invalid metadata`, + ); + } + this.checkedRange( + address, + allocationSize, + `staged transaction ${ordinal}`, + ); + const end = address + allocationSize; + if ( + intervals.some((interval) => + address < interval.end && interval.start < end + ) + ) { + throw new Error( + `${this.label}: staged transaction ${ordinal} overlaps an archive record`, + ); + } + intervals.push({ start: address, end }); + const nameBytes = new Uint8Array( + this.memory.buffer, + address + TRANSACTION_HEADER_SIZE, + nameLength, + ); + let name: string; + try { + name = new TextDecoder("utf-8", { fatal: true }).decode( + new Uint8Array(nameBytes), + ); + } catch { + throw new Error( + `${this.label}: staged transaction ${ordinal} has invalid UTF-8 name`, + ); + } + const moduleBytes = new Uint8Array( + new Uint8Array( + this.memory.buffer, + address + TRANSACTION_HEADER_SIZE + align8(nameLength), + bytesLength, + ), + ); + const expectedDigest = new Uint8Array( + this.memory.buffer, + address + TRANSACTION_DIGEST_OFFSET, + MODULE_DIGEST_SIZE, + ); + if ( + !equalBytes( + expectedDigest, + computeForkModuleTemplateIdSync(moduleBytes), + ) + ) { + throw new Error( + `${this.label}: staged transaction ${ordinal} failed SHA-256 validation`, + ); + } + const state = this.validateTransaction({ + token: view.getUint32(address + 24, true), + name, + moduleBytes, + globalVisibility: (flags & TRANSACTION_FLAG_GLOBAL) !== 0, + }); + return { + allocation: { address, size: allocationSize }, + state, + }; + } + + private allocateModule(state: DylinkForkLibraryState): IndexedModule { + const name = new TextEncoder().encode(state.name); + const nameAligned = align8(name.length); + const moduleBytesAligned = align8(state.moduleBytes.length); + const providers = encodeProviderDependencies(state); + const providerBytesAligned = align8(providers.bytes.length); + const allocations = canonicalMemoryAllocations(state); + const allocationBytesLength = + allocations.length * MODULE_ALLOCATION_SIZE; + checkedU32( + allocationBytesLength, + `${state.name}: process allocation archive size`, + ); + const totalSize = + MODULE_HEADER_SIZE + + nameAligned + + moduleBytesAligned + + providerBytesAligned + + allocationBytesLength; + const allocation = this.allocate(totalSize); + if (allocation.size !== totalSize) { + throw new Error(`${this.label}: archive allocator changed a module record size`); + } + const address = this.checkedRange( + allocation.address, + allocation.size, + `new module ${state.name}`, + ); + const bytes = new Uint8Array(this.memory.buffer, address, totalSize); + bytes.fill(0); + const view = new DataView(this.memory.buffer); + view.setUint32(address, MODULE_MAGIC, true); + view.setUint16(address + 4, MODULE_VERSION, true); + view.setUint16(address + 6, MODULE_HEADER_SIZE, true); + this.writeU64(address + 16, totalSize); + this.writeU64(address + 24, state.memoryBase); + this.writeU64(address + 32, state.tableBase); + this.writeU64(address + 40, state.tlsBase ?? 0); + view.setUint32(address + 48, state.activationId ?? 0, true); + view.setUint32(address + 52, state.handle ?? 0, true); + view.setUint32(address + 56, state.refCount ?? 0, true); + view.setUint32(address + 60, name.length, true); + view.setUint32(address + 64, state.moduleBytes.length, true); + view.setUint32( + address + 68, + (state.initialization === undefined ? 0 : MODULE_FLAG_INITIALIZING) + | (state.globalVisibility ? MODULE_FLAG_GLOBAL : 0) + | ( + state.committedGlobalRoot + ? MODULE_FLAG_COMMITTED_GLOBAL_ROOT + : 0 + ), + true, + ); + bytes.set( + computeForkModuleTemplateIdSync(state.moduleBytes), + MODULE_DIGEST_OFFSET, + ); + bytes.set(name, MODULE_HEADER_SIZE); + bytes.set(state.moduleBytes, MODULE_HEADER_SIZE + nameAligned); + bytes.set( + providers.bytes, + MODULE_HEADER_SIZE + nameAligned + moduleBytesAligned, + ); + const allocationOffset = + address + + MODULE_HEADER_SIZE + + nameAligned + + moduleBytesAligned + + providerBytesAligned; + for (const [index, allocation] of allocations.entries()) { + const offset = allocationOffset + index * MODULE_ALLOCATION_SIZE; + this.writeU64(offset, allocation.address); + this.writeU64(offset + 8, allocation.size); + this.writeU64(offset + 16, allocation.mappingAddress); + this.writeU64(offset + 24, allocation.mappingSize); + } + view.setUint32( + address + 104, + state.initialization?.transactionToken ?? 0, + true, + ); + view.setUint32( + address + 108, + state.initialization === undefined + ? 0 + : initializationStageCode(state.initialization.stage), + true, + ); + this.writeU64( + address + 112, + state.initialization?.tableIndex ?? 0, + ); + view.setUint32(address + 120, providers.bytes.length, true); + view.setUint32(address + 124, providers.count, true); + view.setUint32(address + 128, allocationBytesLength, true); + view.setUint32(address + 132, allocations.length, true); + return { + allocation, + state: this.copyLibrary(state), + }; + } + + private readModule( + address: number, + ordinal: number, + intervals: Array<{ start: number; end: number }>, + ): IndexedModule { + this.checkedRange(address, MODULE_HEADER_SIZE, `module ${ordinal} header`); + const view = new DataView(this.memory.buffer); + if (view.getUint32(address, true) !== MODULE_MAGIC) { + throw new Error(`${this.label}: module ${ordinal} has invalid archive magic`); + } + if (view.getUint16(address + 4, true) !== MODULE_VERSION) { + throw new Error(`${this.label}: module ${ordinal} has unsupported archive version`); + } + if (view.getUint16(address + 6, true) !== MODULE_HEADER_SIZE) { + throw new Error(`${this.label}: module ${ordinal} has invalid archive header size`); + } + const allocationSize = this.readU64( + address + 16, + `module ${ordinal} allocation size`, + ); + const nameLength = view.getUint32(address + 60, true); + const bytesLength = view.getUint32(address + 64, true); + const providerBytesLength = view.getUint32(address + 120, true); + const providerCount = view.getUint32(address + 124, true); + const allocationBytesLength = view.getUint32(address + 128, true); + const allocationCount = view.getUint32(address + 132, true); + if ( + allocationBytesLength + !== allocationCount * MODULE_ALLOCATION_SIZE + ) { + throw new Error( + `${this.label}: module ${ordinal} has noncanonical allocation metadata`, + ); + } + const expectedSize = + MODULE_HEADER_SIZE + + align8(nameLength) + + align8(bytesLength) + + align8(providerBytesLength) + + allocationBytesLength; + if (allocationSize !== expectedSize) { + throw new Error(`${this.label}: module ${ordinal} has noncanonical allocation size`); + } + this.checkedRange(address, allocationSize, `module ${ordinal} allocation`); + const end = address + allocationSize; + if (intervals.some((interval) => address < interval.end && interval.start < end)) { + throw new Error(`${this.label}: module ${ordinal} overlaps another archive record`); + } + intervals.push({ start: address, end }); + const flags = view.getUint32(address + 68, true); + if ((flags & ~MODULE_FLAG_KNOWN_MASK) !== 0) { + throw new Error(`${this.label}: module ${ordinal} has unknown archive flags`); + } + const nameBytes = new Uint8Array( + this.memory.buffer, + address + MODULE_HEADER_SIZE, + nameLength, + ); + let name: string; + try { + name = new TextDecoder("utf-8", { fatal: true }).decode( + new Uint8Array(nameBytes), + ); + } catch { + throw new Error(`${this.label}: module ${ordinal} has invalid UTF-8 name`); + } + if (name.length === 0) { + throw new Error(`${this.label}: module ${ordinal} has an empty name`); + } + const moduleBytes = new Uint8Array( + new Uint8Array( + this.memory.buffer, + address + MODULE_HEADER_SIZE + align8(nameLength), + bytesLength, + ), + ); + const providerBytes = new Uint8Array( + new Uint8Array( + this.memory.buffer, + address + + MODULE_HEADER_SIZE + + align8(nameLength) + + align8(bytesLength), + providerBytesLength, + ), + ); + const providerDependencies = decodeProviderDependencies( + providerBytes, + providerCount, + `${this.label}: module ${name}`, + ); + const allocationOffset = + address + + MODULE_HEADER_SIZE + + align8(nameLength) + + align8(bytesLength) + + align8(providerBytesLength); + const allocations = Array.from( + { length: allocationCount }, + (_, index) => { + const offset = allocationOffset + index * MODULE_ALLOCATION_SIZE; + return { + address: this.readU64( + offset, + `${name} allocation ${index} address`, + ), + size: this.readU64( + offset + 8, + `${name} allocation ${index} size`, + ), + mappingAddress: this.readU64( + offset + 16, + `${name} allocation ${index} mapping address`, + ), + mappingSize: this.readU64( + offset + 24, + `${name} allocation ${index} mapping size`, + ), + }; + }, + ); + const expectedDigest = new Uint8Array( + this.memory.buffer, + address + MODULE_DIGEST_OFFSET, + MODULE_DIGEST_SIZE, + ); + if (!equalBytes(expectedDigest, computeForkModuleTemplateIdSync(moduleBytes))) { + throw new Error(`${this.label}: module ${name} failed archive SHA-256 validation`); + } + const activationId = view.getUint32(address + 48, true); + const handle = view.getUint32(address + 52, true); + const refCount = view.getUint32(address + 56, true); + if ((handle === 0) !== (refCount === 0)) { + throw new Error(`${this.label}: module ${name} has inconsistent handle/refcount`); + } + const initializing = (flags & MODULE_FLAG_INITIALIZING) !== 0; + const transactionToken = view.getUint32(address + 104, true); + const stageCode = view.getUint32(address + 108, true); + const initializationTableIndex = this.readU64( + address + 112, + `${name} initialization table index`, + ); + if ( + initializing + !== ( + transactionToken !== 0 + && stageCode !== 0 + && initializationTableIndex !== 0 + ) + ) { + throw new Error( + `${this.label}: module ${name} has inconsistent initialization metadata`, + ); + } + const state: DylinkForkLibraryState = { + name, + moduleBytes, + memoryBase: this.readU64(address + 24, `${name} memory base`), + tableBase: this.readU64(address + 32, `${name} table base`), + activationId: activationId === 0 ? undefined : activationId, + tlsBase: this.optionalPositiveU64(address + 40, `${name} TLS base`), + globalVisibility: (flags & MODULE_FLAG_GLOBAL) !== 0, + committedGlobalRoot: + (flags & MODULE_FLAG_COMMITTED_GLOBAL_ROOT) !== 0 + ? true + : undefined, + ...(providerDependencies.length === 0 + ? {} + : { providerDependencies }), + ...(allocations.length === 0 ? {} : { allocations }), + handle: handle === 0 ? undefined : handle, + refCount: refCount === 0 ? undefined : refCount, + ...(initializing + ? { + initialization: { + transactionToken, + stage: decodeInitializationStage( + stageCode, + `${this.label}: module ${name}`, + ), + tableIndex: initializationTableIndex, + }, + } + : {}), + }; + this.validateLibrary(state, Number.MAX_SAFE_INTEGER); + return { + allocation: { address, size: allocationSize }, + state, + }; + } + + private validateState(state: DylinkForkState): DylinkForkState { + const nextHandle = checkedNextHandle(state.nextHandle); + checkedU32( + state.libraries.length, + `${this.label}: live module count`, + ); + const names = new Set(); + const activations = new Set(); + const handles = new Set(); + const libraries = state.libraries.map((library) => { + this.validateLibrary(library, nextHandle); + if (names.has(library.name)) { + throw new Error(`${this.label}: duplicate live module ${library.name}`); + } + names.add(library.name); + if (library.activationId !== undefined) { + if (activations.has(library.activationId)) { + throw new Error( + `${this.label}: duplicate live activation ${library.activationId}`, + ); + } + activations.add(library.activationId); + } + if (library.handle !== undefined) { + if (handles.has(library.handle)) { + throw new Error(`${this.label}: duplicate live handle ${library.handle}`); + } + handles.add(library.handle); + } + return this.copyLibrary(library); + }); + const ownedMappings = libraries + .flatMap((library) => + (library.allocations ?? []).map((allocation) => ({ + name: library.name, + start: allocation.mappingAddress, + end: allocation.mappingAddress + allocation.mappingSize, + })) + ) + .sort((left, right) => left.start - right.start); + for (let index = 1; index < ownedMappings.length; index++) { + const previous = ownedMappings[index - 1]!; + const current = ownedMappings[index]!; + if (current.start < previous.end) { + throw new Error( + `${this.label}: ${previous.name} and ${current.name} own overlapping mappings`, + ); + } + } + for (const library of libraries) { + for (const dependency of library.providerDependencies ?? []) { + if (!names.has(dependency)) { + throw new Error( + `${this.label}: ${library.name} names absent runtime provider ` + + dependency, + ); + } + } + } + const transactionTokens = new Set(); + const transactions = (state.transactions ?? []).map((transaction) => { + const owned = this.validateTransaction(transaction); + if (transactionTokens.has(owned.token)) { + throw new Error( + `${this.label}: duplicate staged transaction ${owned.token}`, + ); + } + transactionTokens.add(owned.token); + return owned; + }); + const initializationCounts = new Map(); + for (const library of libraries) { + const initialization = library.initialization; + if (!initialization) continue; + if (!transactionTokens.has(initialization.transactionToken)) { + throw new Error( + `${this.label}: ${library.name} names absent staged transaction ` + + `${initialization.transactionToken}`, + ); + } + initializationCounts.set( + initialization.transactionToken, + (initializationCounts.get(initialization.transactionToken) ?? 0) + 1, + ); + } + for (const transaction of transactions) { + if (initializationCounts.get(transaction.token) !== 1) { + throw new Error( + `${this.label}: staged transaction ${transaction.token} must own ` + + "exactly one issued initialization entry", + ); + } + } + return { + nextHandle, + libraries, + ...(transactions.length === 0 ? {} : { transactions }), + }; + } + + private validateTransaction( + state: DylinkForkTransactionState, + ): DylinkForkTransactionState { + const token = checkedU32( + state.token, + `${this.label}: staged transaction token`, + false, + ); + if (typeof state.name !== "string" || state.name.length === 0) { + throw new TypeError( + `${this.label}: staged transaction ${token} has an empty name`, + ); + } + if ( + !(state.moduleBytes instanceof Uint8Array) + || state.moduleBytes.length === 0 + ) { + throw new TypeError( + `${this.label}: staged transaction ${token} has no module bytes`, + ); + } + if (typeof state.globalVisibility !== "boolean") { + throw new TypeError( + `${this.label}: staged transaction ${token} has invalid visibility`, + ); + } + return this.copyTransaction({ + token, + name: state.name, + moduleBytes: state.moduleBytes, + globalVisibility: state.globalVisibility, + }); + } + + private validateLibrary( + state: DylinkForkLibraryState, + nextHandle: number, + ): void { + if (typeof state.name !== "string" || state.name.length === 0) { + throw new TypeError(`${this.label}: live module name is empty`); + } + if (!(state.moduleBytes instanceof Uint8Array) || state.moduleBytes.length === 0) { + throw new TypeError(`${this.label}: ${state.name} has no owned module bytes`); + } + if (typeof state.globalVisibility !== "boolean") { + throw new TypeError(`${this.label}: ${state.name} has invalid visibility`); + } + if (state.committedGlobalRoot && !state.globalVisibility) { + throw new Error( + `${this.label}: ${state.name} is a committed GLOBAL root but is LOCAL`, + ); + } + canonicalProviderDependencies(state); + const allocations = canonicalMemoryAllocations(state); + for (const [index, allocation] of allocations.entries()) { + if ( + allocation.mappingAddress + > this.memory.buffer.byteLength - allocation.mappingSize + ) { + throw new RangeError( + `${this.label}: ${state.name} allocation ${index} escapes linear memory`, + ); + } + } + checkedAddress(state.memoryBase, `${state.name} memory base`, true); + checkedAddress(state.tableBase, `${state.name} table base`, true); + if (state.activationId !== undefined) { + checkedU32(state.activationId, `${state.name} activation id`, false); + } + if (state.tlsBase !== undefined) { + checkedAddress(state.tlsBase, `${state.name} TLS base`); + } + const hasHandle = state.handle !== undefined; + if (hasHandle !== (state.refCount !== undefined)) { + throw new Error(`${this.label}: ${state.name} handle/refcount presence differs`); + } + if (hasHandle) { + const handle = checkedU32(state.handle!, `${state.name} handle`, false); + if (handle < FIRST_DYLINK_HANDLE || handle >= nextHandle) { + throw new RangeError(`${this.label}: ${state.name} handle ${handle} is out of range`); + } + checkedU32(state.refCount!, `${state.name} refcount`, false); + } + if (state.initialization !== undefined) { + checkedU32( + state.initialization.transactionToken, + `${state.name} initialization transaction`, + false, + ); + initializationStageCode(state.initialization.stage); + checkedAddress( + state.initialization.tableIndex, + `${state.name} initialization table index`, + ); + if (hasHandle) { + throw new Error( + `${this.label}: initializing module ${state.name} already has a handle`, + ); + } + } + } + + private requireImmutableMatch( + current: DylinkForkLibraryState, + next: DylinkForkLibraryState, + ): void { + const currentAllocations = canonicalMemoryAllocations(current); + const nextAllocations = canonicalMemoryAllocations(next); + if ( + current.memoryBase !== next.memoryBase + || current.tableBase !== next.tableBase + || current.activationId !== next.activationId + || currentAllocations.length !== nextAllocations.length + || currentAllocations.some((allocation, index) => { + const expected = nextAllocations[index]; + return ( + expected === undefined + || allocation.address !== expected.address + || allocation.size !== expected.size + || allocation.mappingAddress !== expected.mappingAddress + || allocation.mappingSize !== expected.mappingSize + ); + }) + || ( + current.tlsBase !== next.tlsBase + && current.initialization === undefined + ) + || !equalBytes(current.moduleBytes, next.moduleBytes) + ) { + throw new Error( + `${this.label}: live module ${next.name} changed immutable archive identity`, + ); + } + } + + private writeMutableState( + address: number, + state: DylinkForkLibraryState, + ): void { + const view = new DataView(this.memory.buffer); + this.writeU64(address + 40, state.tlsBase ?? 0); + view.setUint32(address + 52, state.handle ?? 0, true); + view.setUint32(address + 56, state.refCount ?? 0, true); + view.setUint32( + address + 68, + (state.initialization === undefined ? 0 : MODULE_FLAG_INITIALIZING) + | (state.globalVisibility ? MODULE_FLAG_GLOBAL : 0) + | ( + state.committedGlobalRoot + ? MODULE_FLAG_COMMITTED_GLOBAL_ROOT + : 0 + ), + true, + ); + view.setUint32( + address + 104, + state.initialization?.transactionToken ?? 0, + true, + ); + view.setUint32( + address + 108, + state.initialization === undefined + ? 0 + : initializationStageCode(state.initialization.stage), + true, + ); + this.writeU64( + address + 112, + state.initialization?.tableIndex ?? 0, + ); + } + + private checkedRange(address: number, size: number, context: string): number { + checkedAddress(address, `${this.label}: ${context}`); + if ( + !Number.isSafeInteger(size) + || size <= 0 + || address > this.memory.buffer.byteLength - size + ) { + throw new RangeError(`${this.label}: ${context} escapes linear memory`); + } + return address; + } + + private readU64(address: number, context: string): number { + const value = new DataView(this.memory.buffer).getBigUint64(address, true); + if (value > BigInt(Number.MAX_SAFE_INTEGER)) { + throw new RangeError(`${this.label}: ${context} exceeds exact host integers`); + } + return Number(value); + } + + private optionalPositiveU64( + address: number, + context: string, + ): number | undefined { + const value = this.readU64(address, context); + return value === 0 ? undefined : checkedAddress(value, context); + } + + private writeU64(address: number, value: number): void { + if (!Number.isSafeInteger(value) || value < 0) { + throw new RangeError(`${this.label}: cannot archive inexact u64 ${String(value)}`); + } + new DataView(this.memory.buffer).setBigUint64(address, BigInt(value), true); + } + + private nextGeneration(current: number): number { + if ( + !Number.isSafeInteger(current) + || current < 0 + || current >= MAX_EXACT_GENERATION + ) { + throw new RangeError(`${this.label}: dylink archive generation is exhausted`); + } + return current + 1; + } + + private readGeneration(header: number): number { + const address = header + 40; + let value: bigint; + if ( + typeof SharedArrayBuffer !== "undefined" + && this.memory.buffer instanceof SharedArrayBuffer + ) { + value = Atomics.load(new BigUint64Array(this.memory.buffer, address, 1), 0); + } else { + value = new DataView(this.memory.buffer).getBigUint64(address, true); + } + if (value > BigInt(MAX_EXACT_GENERATION)) { + throw new RangeError(`${this.label}: dylink archive generation is inexact`); + } + return Number(value); + } + + private writeGeneration(header: number, generation: number): void { + if ( + !Number.isSafeInteger(generation) + || generation <= 0 + || generation > MAX_EXACT_GENERATION + ) { + throw new RangeError(`${this.label}: invalid dylink archive generation`); + } + const address = header + 40; + if ( + typeof SharedArrayBuffer !== "undefined" + && this.memory.buffer instanceof SharedArrayBuffer + ) { + Atomics.store( + new BigUint64Array(this.memory.buffer, address, 1), + 0, + BigInt(generation), + ); + } else { + new DataView(this.memory.buffer).setBigUint64( + address, + BigInt(generation), + true, + ); + } + } + + private copyLibrary(state: DylinkForkLibraryState): DylinkForkLibraryState { + const providerDependencies = canonicalProviderDependencies(state); + const allocations = canonicalMemoryAllocations(state); + return { + name: state.name, + moduleBytes: new Uint8Array(state.moduleBytes), + memoryBase: state.memoryBase, + tableBase: state.tableBase, + ...(state.activationId === undefined + ? {} + : { activationId: state.activationId }), + ...(state.tlsBase === undefined ? {} : { tlsBase: state.tlsBase }), + globalVisibility: state.globalVisibility, + ...(state.committedGlobalRoot + ? { committedGlobalRoot: true } + : {}), + ...(providerDependencies.length === 0 + ? {} + : { providerDependencies }), + ...(allocations.length === 0 ? {} : { allocations }), + ...(state.handle === undefined ? {} : { handle: state.handle }), + ...(state.refCount === undefined ? {} : { refCount: state.refCount }), + ...(state.initialization === undefined + ? {} + : { + initialization: { + transactionToken: state.initialization.transactionToken, + stage: state.initialization.stage, + tableIndex: state.initialization.tableIndex, + }, + }), + }; + } + + private copyTransaction( + state: DylinkForkTransactionState, + ): DylinkForkTransactionState { + return { + token: state.token, + name: state.name, + moduleBytes: new Uint8Array(state.moduleBytes), + globalVisibility: state.globalVisibility, + }; + } + + private copyTablePatch( + patch: DylinkForkTablePatch, + ): DylinkForkTablePatch { + return { + ...(patch.generation === undefined + ? {} + : { generation: patch.generation }), + activationId: patch.activationId, + ownerId: patch.ownerId, + start: patch.start, + tableLength: patch.tableLength, + runs: patch.runs.map((run) => ({ + length: run.length, + function: run.function === null + ? null + : { + activationId: run.function.activationId, + ordinal: run.function.ordinal, + }, + })), + }; + } + + private copyState( + state: DylinkForkArchiveSnapshot, + ): DylinkForkArchiveSnapshot { + return { + generation: state.generation, + tableStateRoot: state.tableStateRoot, + tableCheckpointGeneration: state.tableCheckpointGeneration, + tablePatches: state.tablePatches.map((patch) => + this.copyTablePatch(patch) + ), + nextHandle: state.nextHandle, + libraries: state.libraries.map((library) => this.copyLibrary(library)), + ...(state.transactions === undefined || state.transactions.length === 0 + ? {} + : { + transactions: state.transactions.map((transaction) => + this.copyTransaction(transaction) + ), + }), + }; + } +} diff --git a/host/src/dylink.ts b/host/src/dylink.ts index 2f2d1171bc..c156a55e1b 100644 --- a/host/src/dylink.ts +++ b/host/src/dylink.ts @@ -8,17 +8,26 @@ import { ABI_VERSION, + WPK_FORK_CAPABILITIES_SECTION, + WPK_FORK_CAPABILITIES_VERSION, + WPK_FORK_CAP_ACTIVATION_STATE_SAFE, + WPK_FORK_CAP_DYLINK_MAIN, + WPK_FORK_CAP_KNOWN_MASK, + WPK_FORK_CAP_SIDE_ENTRY, WPK_FORK_REQUIRED_EXPORTS, WPK_FORK_REQUIRED_IMPORTS, } from "./generated/abi"; import { - ContinuationAllocationError, - invokeForkContinuationBegin, - LinkedForkContinuation, - readLinkedFrameFormat, - type ContinuationAllocate, - type ContinuationDeallocate, -} from "./fork-continuation"; + describeWasmForkArtifactContractFailures, + extractAbiVersion, + readWasmFunctionImports, + type WasmFunctionImportType, +} from "./constants"; +import { + FORK_UNWIND_TAG_IMPORT_MODULE, + FORK_UNWIND_TAG_IMPORT_NAME, + requireForkUnwindTag, +} from "./fork-unwind-transport"; // dylink.0 sub-section types const WASM_DYLINK_MEM_INFO = 1; @@ -33,18 +42,20 @@ const WASM_DYLINK_FLAG_WEAK = 0x02; export const SIDE_MODULE_FORK_EXPORTS = WPK_FORK_REQUIRED_EXPORTS.map( ({ name }) => name, ); +const SIDE_MODULE_FORK_EXPORT_SET: ReadonlySet = + new Set(SIDE_MODULE_FORK_EXPORTS); -export const FORK_CAPABILITIES_SECTION = "kandelo.wpk_fork.capabilities"; -export const FORK_CAPABILITIES_VERSION = 1; -export const FORK_CAP_SIDE_ENTRY = 1 << 0; -export const FORK_CAP_DYLINK_MAIN = 1 << 1; -const FORK_CAP_KNOWN_MASK = FORK_CAP_SIDE_ENTRY | FORK_CAP_DYLINK_MAIN; -export const FORK_CAPABILITIES_REQUIRED_ABI = 17; +function isForkRuntimeExport(name: string): boolean { + return SIDE_MODULE_FORK_EXPORT_SET.has(name); +} -const WPK_FORK_NORMAL = 0; -const WPK_FORK_UNWINDING = 1; -const WPK_FORK_REWINDING = 2; -const WPK_FORK_ABORT_UNWINDING = 3; +export const FORK_CAPABILITIES_SECTION = WPK_FORK_CAPABILITIES_SECTION; +export const FORK_CAPABILITIES_VERSION = WPK_FORK_CAPABILITIES_VERSION; +export const FORK_CAP_SIDE_ENTRY = WPK_FORK_CAP_SIDE_ENTRY; +export const FORK_CAP_DYLINK_MAIN = WPK_FORK_CAP_DYLINK_MAIN; +export const FORK_CAP_ACTIVATION_STATE_SAFE = WPK_FORK_CAP_ACTIVATION_STATE_SAFE; +const FORK_CAP_KNOWN_MASK = WPK_FORK_CAP_KNOWN_MASK; +export const FORK_CAPABILITIES_REQUIRED_ABI = 17; export interface ForkInstrumentCapabilityClaim { /** False for an ABI-16 artifact built before role markers were introduced. */ @@ -312,6 +323,39 @@ function requireWasmAddress( return value; } +function copyForkMemoryAllocation( + allocation: DylinkForkMemoryAllocation, + context: string, +): DylinkForkMemoryAllocation { + const fields = [ + ["address", allocation.address], + ["size", allocation.size], + ["mapping address", allocation.mappingAddress], + ["mapping size", allocation.mappingSize], + ] as const; + for (const [field, value] of fields) { + if (!Number.isSafeInteger(value) || value <= 0) { + throw new RangeError(`${context}: ${field} is not an exact positive integer`); + } + } + const logicalEnd = allocation.address + allocation.size; + const mappingEnd = allocation.mappingAddress + allocation.mappingSize; + if ( + !Number.isSafeInteger(logicalEnd) + || !Number.isSafeInteger(mappingEnd) + || allocation.address < allocation.mappingAddress + || logicalEnd > mappingEnd + ) { + throw new RangeError(`${context}: logical allocation escapes its process mapping`); + } + return Object.freeze({ + address: allocation.address, + size: allocation.size, + mappingAddress: allocation.mappingAddress, + mappingSize: allocation.mappingSize, + }); +} + function tableAddress( table: WebAssembly.Table, value: number, @@ -367,6 +411,17 @@ function growMemory(memory: WebAssembly.Memory, delta: number, ptrWidth: 4 | 8): /** * Shared library instance loaded into a process's address space. */ +export interface DylinkForkMemoryAllocation { + /** Aligned address returned to the side module. */ + readonly address: number; + /** Logical byte count requested by the side module. */ + readonly size: number; + /** Exact mmap base owned by the process allocator. */ + readonly mappingAddress: number; + /** Exact mmap byte count that must be passed to munmap. */ + readonly mappingSize: number; +} + export interface LoadedSharedLibrary { /** Wasm module instance */ instance: WebAssembly.Instance; @@ -380,43 +435,169 @@ export interface LoadedSharedLibrary { metadata: DylinkMetadata; /** Path/name of the library */ name: string; - /** Fork save buffer for an instrumented side module importing env.fork. */ - forkBufAddr?: number; - forkContinuation?: LinkedForkContinuation; + /** Immutable loader-owned snapshot used by dependency-first fork archives. */ + moduleBytes: Uint8Array; + /** Stable process activation coordinate persisted in the dlopen archive. */ + activationId?: number; + /** + * Exact process-table entries whose callable values belong to this module. + * + * Table length cannot shrink, so final unload clears these slots to null. + * Gaps stay addressable and a later archive preserves their positions. + */ + ownedTableEntries: readonly number[]; + /** GOT cells consumed by this module, with their exact symbol kind. */ + gotImports: readonly Readonly<{ + name: string; + kind: "mem" | "func"; + }>[]; + /** Release the registered activation exactly once on final unload. */ + unregisterForkActivation?: () => void; /** Thread-local-storage base captured from the parent instance. */ tlsBase?: number; - /** Whether this module can originate a coordinated env.fork unwind. */ - forkCapable: boolean; - /** Function/GOT.func imports used for conservative cross-side isolation. */ - functionImports: ReadonlySet; - /** Function exports visible to later side modules. */ - functionExports: ReadonlySet; - /** Dynamic lookup from a side module defeats static cross-side isolation. */ - importsDynamicLookup: boolean; + /** Provisional objects are visible to nested loader transactions. */ + loadState?: "initializing" | "loaded"; + /** Whether this object contributes exports to the RTLD_DEFAULT scope. */ + globalVisibility: boolean; + /** + * True when a completed RTLD_GLOBAL dlopen selected this object as its root. + * + * WHY: an outer constructor can promote a pre-existing LOCAL closure and + * then fail after a nested, independently committed GLOBAL open. Rollback + * must undo only the outer promotion and reapply the surviving root. + */ + committedGlobalRoot?: boolean; + /** Present only while libc owns an issued loader entry. */ + initialization?: Readonly<{ + transactionToken: number; + stage: DylinkInitializationStage; + tableIndex: number; + }>; + /** Other side modules whose symbols this instance captured while linking. */ + providerDependencies?: ReadonlySet; + /** Process mappings owned until rollback or final unload. */ + allocations?: readonly DylinkForkMemoryAllocation[]; + /** Standalone-linker heap high-water mark owned by this object. */ + heapReservationEnd?: number; } -export interface SideModuleForkState { - name: string; - instance: WebAssembly.Instance; - forkBufAddr: number; - continuation: LinkedForkContinuation; +export interface DylinkForkActivationRequest { + readonly name: string; + readonly module: WebAssembly.Module; + readonly moduleBytes: Uint8Array; + /** Exact archived coordinate in a fresh fork child; absent in the parent. */ + readonly replayActivationId?: number; } /** - * Process-worker coordination for the one supported side-module fork shape: - * a main-module call_indirect directly invokes one instrumented side module. - * The loader rejects statically visible side-to-side linkage and side-owned - * dlopen/dlsym around a fork-capable module. Opaque callbacks passed through - * main memory or the shared table cannot yet be attributed to a module at - * runtime and remain an explicit unsupported residual. + * One pre-instantiation reservation from the process activation coordinator. + * + * `env` owns every fork/frame/module/reference/exception/GC import, including + * `fork` itself. The loader only binds those values; it does not keep a + * module-local continuation or infer which activation is currently active. */ -export interface SideModuleForkSupport { - setActiveFork: (state: SideModuleForkState) => void; - clearActiveFork: (state: SideModuleForkState) => void; - /** Invoke the immutable main-module fork trampoline and verify its state. */ - invokeMainFork: (expectedStateAfter: 0 | 1 | readonly (0 | 1)[]) => number; - /** Put the already-unwinding main image into allocation-failure replay. */ - beginMainAbort: (errno: number) => void; +export interface PreparedDylinkForkActivation { + readonly activationId: number; + readonly env: Readonly>; + /** + * Wrap the loader's final lazy import object immediately before + * instantiation. Imported-global/table ownership observes the engine's exact + * property reads, including duplicate `(module, name)` declarations. + */ + wrapImports(imports: WebAssembly.Imports): WebAssembly.Imports; + register(instance: WebAssembly.Instance): void; + unregister(): void; +} + +export interface DylinkForkActivationOwner { + prepare(request: DylinkForkActivationRequest): PreparedDylinkForkActivation; +} + +/** Compact live linker state persisted by the process fork archive. */ +export interface DylinkForkLibraryState { + readonly name: string; + /** Loader-owned immutable-by-contract artifact snapshot. */ + readonly moduleBytes: Readonly; + readonly memoryBase: number; + readonly tableBase: number; + readonly activationId?: number; + readonly tlsBase?: number; + readonly globalVisibility: boolean; + readonly committedGlobalRoot?: boolean; + /** + * Runtime symbol providers captured outside immutable DT_NEEDED edges. + * + * Constructor dlsym calls are not re-executed in a fresh child, so their + * lifetime edges must be explicit reconstruction data. + */ + readonly providerDependencies?: readonly string[]; + /** + * Exact allocator ownership copied into a fork child. + * + * The child's linear memory and kernel mmap map are copied, but its Worker + * has fresh JavaScript bookkeeping. These recipes reconnect the two without + * issuing a second mmap or guessing the allocator's alignment padding. + */ + readonly allocations?: readonly DylinkForkMemoryAllocation[]; + /** Absent when the module is live only as a NEEDED dependency. */ + readonly handle?: number; + /** Present exactly when `handle` is present. */ + readonly refCount?: number; + /** Durable continuation point for one libc-driven initialization call. */ + readonly initialization?: Readonly<{ + transactionToken: number; + stage: DylinkInitializationStage; + tableIndex: number; + }>; +} + +export interface DylinkForkTransactionState { + readonly token: number; + readonly name: string; + readonly moduleBytes: Readonly; + readonly globalVisibility: boolean; +} + +export interface DylinkForkState { + readonly nextHandle: number; + /** Dependency-first `loadedLibraries` insertion order. */ + readonly libraries: readonly DylinkForkLibraryState[]; + /** Outer-to-inner loader transactions stopped in ordinary Wasm calls. */ + readonly transactions?: readonly DylinkForkTransactionState[]; +} + +export interface DylinkForkPublishedState extends DylinkForkState { + /** Monotonic archive publication observed under the process reader lock. */ + readonly generation: number; +} + +interface PendingDlopenTransaction { + readonly token: number; + readonly name: string; + readonly moduleBytes: Uint8Array; + readonly globalVisibility: boolean; + readonly steps: Generator< + DylinkInitializationStep, + LoadedSharedLibrary, + DylinkForkLibraryState | undefined + >; + /** + * Mutable view captured by `loadSharedLibrarySyncSteps`. + * + * A replica may first observe a dependency initializer and only learn the + * root module's exact layout in a later archive generation. Updating this + * map lets the suspended generator consume that later reconstruction recipe + * without allocating a Worker-local layout. + */ + readonly replayModules?: Map; + readonly initialLibraries: ReadonlySet; + readonly initialVisibility: ReadonlyMap; + readonly ownedLibraries: Set; + readonly initialHeapPointer?: number; + tableIndex?: number; + awaitingCompletion: boolean; + currentStep?: DylinkInitializationStep; + loaded?: LoadedSharedLibrary; } /** @@ -428,10 +609,9 @@ export interface SideModuleForkSupport { * null gaps up to that base but rejects a child table that already grew * past it (an interleaved dlsym, future GOT preallocation, etc.). * - `options.loadedLibraries` must NOT already contain `name`. Replay - * does not refresh existing entries; a duplicate would be silently - * deduped and return a handle whose memoryBase may not match. - * - The library must have no `dylink.0` NEEDED deps. Dep replay is not - * yet plumbed; `loadSharedLibrarySync` throws if you try. + * rejects duplicate module-load records before mutating linker state. + * - Every `dylink.0` NEEDED dependency must already have been replayed from + * its own earlier archive entry. */ export interface DylinkReplayOptions { /** Memory base returned by the parent's allocator. Data relocations in @@ -440,13 +620,23 @@ export interface DylinkReplayOptions { memoryBase: number; /** Exact table base observed in the parent, including failed-load gaps. */ tableBase: number; - /** Exact side-module save buffer copied from the fork parent. */ - forkBufAddr?: number; + /** Exact stable activation coordinate copied from the fork parent. */ + activationId?: number; /** Exact mutable `__tls_base` value from the fork parent. The child memory * already contains the parent's live TLS bytes, so replay restores only * this instance-local global and deliberately does not call * `__wasm_init_tls`, which would reset those bytes to the initial image. */ tlsBase?: number; + /** Rebuild a generator stopped before this direct libc table call. */ + initializationStage?: DylinkInitializationStage; + /** Exact RTLD visibility of the parent object. */ + globalVisibility?: boolean; + /** Whether this object is the root of a committed RTLD_GLOBAL open. */ + committedGlobalRoot?: boolean; + /** Exact runtime provider edges already established in the parent. */ + providerDependencies?: readonly string[]; + /** Exact live mapping ownership copied from the parent process. */ + allocations?: readonly DylinkForkMemoryAllocation[]; } /** @@ -463,16 +653,31 @@ export interface LoadSharedLibraryOptions { heapPointer?: { value: number }; /** Allocate side-module linear-memory data in the process address space */ allocateMemory?: (size: number, align: number) => number; + /** + * Describe the exact mapping behind an aligned allocateMemory result. + * + * Process Workers use this to persist raw mmap ownership for fresh-Worker + * replay. Embedders whose allocator/deallocator use the logical range may + * omit it. + */ + describeMemoryAllocation?: ( + address: number, + size: number, + ) => Readonly<{ mappingAddress: number; mappingSize: number }>; + /** Adopt copied mapping ownership without allocating new process memory. */ + adoptMemoryAllocation?: (allocation: DylinkForkMemoryAllocation) => void; + /** Drop Worker-local ownership after another pthread published the unload. */ + forgetMemoryAllocation?: (allocation: DylinkForkMemoryAllocation) => void; /** Release a successful allocateMemory result when loading rolls back. */ deallocateMemory?: (addr: number, size: number) => void; - /** Page-granular process mapping used only for linked continuation chunks. */ - allocateContinuation?: ContinuationAllocate; - /** Release one inherited or parent-owned continuation mapping. */ - deallocateContinuation?: ContinuationDeallocate; /** Global symbol table: name → function or WebAssembly.Global */ globalSymbols: Map; + /** Defining side module for each global symbol; absent means the main image. */ + globalSymbolOwners?: Map; /** GOT entries: symbol name → mutable pointer-width WebAssembly.Global */ got: Map; + /** Internal exact type of every live GOT cell. */ + gotKinds?: Map; /** Already-loaded libraries for dedup and dependency resolution */ loadedLibraries: Map; /** @@ -487,18 +692,218 @@ export interface LoadSharedLibraryOptions { * matching payload type, so this must not be allocated per dlopen. */ cppExceptionTag?: WebAssembly.Tag; + /** + * Private unwind transport shared by the main image and every instrumented + * side module in this process Worker. + */ + forkUnwindTag?: WebAssembly.Tag; /** Process pointer width, which also determines the __c_longjmp payload. */ ptrWidth?: 4 | 8; - /** Immutable symbol names exported by the main module. */ - mainModuleSymbols?: ReadonlySet; - /** Present only in a process worker that can drive side-module unwind. */ - sideModuleFork?: SideModuleForkSupport; - /** Precise rebuild/boundary diagnostic when sideModuleFork is unavailable. */ - sideModuleForkUnavailableReason?: string; - /** Callback to locate and read a library file by name (async version) */ - resolveLibrary?: (name: string) => Promise; - /** Callback to locate and read a library file by name (sync version) */ - resolveLibrarySync?: (name: string) => Uint8Array | null; + /** Process owner for every ABI-43 side-module activation. */ + forkActivationOwner?: DylinkForkActivationOwner; + /** Precise rebuild/boundary diagnostic when the owner is unavailable. */ + forkActivationOwnerUnavailableReason?: string; + /** + * Journal host-created function-table entries (currently dlsym of a main + * export) into the same activation-owned sparse table state as Wasm writes. + */ + onTableMutation?: ( + table: WebAssembly.Table, + firstIndex: number, + length: number, + ) => void; + /** + * Route the exact final function import through the process Worker owner. + * This runs at Proxy property resolution so duplicate declarations and + * activation-owned exception identities are not collapsed eagerly. + */ + routeFunctionImport?: ( + imported: WasmFunctionImportType, + localImplementation: CallableFunction, + ) => CallableFunction; + /** Callback to locate a dependency relative to its requesting object. */ + resolveLibrary?: ( + name: string, + requester?: string, + ) => Promise; + /** Synchronous dependency resolver used by guest dlopen(). */ + resolveLibrarySync?: ( + name: string, + requester?: string, + ) => Uint8Array | null; +} + +interface DylinkLoadContext { + readonly ownedLibraries: Set; +} + +function symbolOwners( + options: LoadSharedLibraryOptions, +): Map { + options.globalSymbolOwners ??= new Map( + Array.from(options.globalSymbols.keys(), (name) => [name, undefined]), + ); + return options.globalSymbolOwners; +} + +function functionTableIndex( + options: LoadSharedLibraryOptions, + fn: Function, +): number | undefined { + const length = tableLength(options.table); + for (let index = 0; index < length; index++) { + if (getTableEntry(options.table, index) === fn) return index; + } + return undefined; +} + +function isPublicDylinkExport( + name: string, + value: WebAssembly.ExportValue, +): value is Function | WebAssembly.Global { + return ( + !name.startsWith("__") + && !isForkRuntimeExport(name) + && ( + typeof value === "function" + || value instanceof WebAssembly.Global + ) + ); +} + +function publishGlobalLibrarySymbols( + library: LoadedSharedLibrary, + options: LoadSharedLibraryOptions, +): void { + if (!library.globalVisibility) return; + const owners = symbolOwners(options); + for (const [name, value] of Object.entries(library.exports)) { + if ( + !isPublicDylinkExport(name, value) + || options.globalSymbols.has(name) + ) { + continue; + } + options.globalSymbols.set(name, value); + owners.set(name, library.name); + } +} + +function promoteLibraryGlobal( + library: LoadedSharedLibrary, + options: LoadSharedLibraryOptions, + visited = new Set(), +): void { + if (visited.has(library.name)) return; + visited.add(library.name); + for (const dependencyName of library.metadata.neededDynlibs) { + const dependency = options.loadedLibraries.get(dependencyName); + if (!dependency) { + throw new Error( + `${library.name}: loaded dependency ${dependencyName} is missing during promotion`, + ); + } + promoteLibraryGlobal(dependency, options, visited); + } + library.globalVisibility = true; + publishGlobalLibrarySymbols(library, options); +} + +function appendDependencyScope( + scope: LoadedSharedLibrary[], + roots: readonly LoadedSharedLibrary[], + options: LoadSharedLibraryOptions, +): void { + const seen = new Set(scope.map((library) => library.name)); + const queue = [...roots]; + for (let index = 0; index < queue.length; index++) { + const library = queue[index]!; + if (seen.has(library.name)) continue; + seen.add(library.name); + scope.push(library); + for (const dependencyName of library.metadata.neededDynlibs) { + const dependency = options.loadedLibraries.get(dependencyName); + if (!dependency) { + throw new Error( + `${library.name}: loaded dependency ${dependencyName} is missing`, + ); + } + if (!seen.has(dependency.name)) queue.push(dependency); + } + } +} + +function runtimeDependencyNames( + library: LoadedSharedLibrary, +): ReadonlySet { + const dependencies = new Set([ + ...library.metadata.neededDynlibs, + ...(library.providerDependencies ?? []), + ]); + dependencies.delete(library.name); + return dependencies; +} + +function scopedSymbol( + options: LoadSharedLibraryOptions, + dependencyScope: readonly LoadedSharedLibrary[], + name: string, +): Readonly<{ + value: Function | WebAssembly.Global; + owner?: string; +}> | undefined { + const global = options.globalSymbols.get(name); + if (global !== undefined) { + return { value: global, owner: symbolOwners(options).get(name) }; + } + for (const dependency of dependencyScope) { + const value = dependency.exports[name]; + if ( + typeof value === "function" + || value instanceof WebAssembly.Global + ) { + return { value, owner: dependency.name }; + } + } + return undefined; +} + +function refreshGlobalGotEntries(options: LoadSharedLibraryOptions): void { + const ptrWidth = options.ptrWidth ?? 4; + for (const [name, entry] of options.got) { + const kind = options.gotKinds?.get(name); + if (!kind) continue; + const symbol = options.globalSymbols.get(name); + if (kind === "mem" && symbol instanceof WebAssembly.Global) { + entry.value = requireWasmAddress( + symbol.value as WasmAddress, + ptrWidth, + `GOT.mem.${name}`, + ); + continue; + } + if (kind === "func" && typeof symbol === "function") { + const index = functionTableIndex(options, symbol); + entry.value = wasmAddress( + index ?? 0, + ptrWidth, + `GOT.func.${name}`, + ); + continue; + } + entry.value = wasmAddress(0, ptrWidth, `unresolved GOT.${kind}.${name}`); + } +} + +function requireNonzeroU32(value: number, context: string): number { + if ( + !Number.isInteger(value) + || value <= 0 + || value > 0xffff_ffff + ) { + throw new RangeError(`${context} is not a nonzero u32`); + } + return value; } type TagConstructor = new ( @@ -585,68 +990,24 @@ function resolveCppExceptionTag(options: LoadSharedLibraryOptions): WebAssembly. return options.cppExceptionTag; } -const SIDE_DYNAMIC_LOOKUP_IMPORTS = new Set([ - "__wasm_dlopen", - "__wasm_dlsym", - "dlopen", - "dlsym", -]); - -function intersectSideSymbols( - imports: ReadonlySet, - exports: ReadonlySet, - mainSymbols: ReadonlySet, -): string[] { - return Array.from(imports) - .filter((name) => !mainSymbols.has(name) && exports.has(name)) - .sort(); -} - -/** - * The current two-module unwind protocol supports main -> one side module. - * It cannot serialize an intervening side-module frame. Preserve ordinary - * independent multi-extension loading, but reject statically visible - * side-to-side linkage and side-originated dynamic lookup whenever either - * participant can fork. Function pointers passed opaquely through main memory - * remain a documented residual until the runtime has module activation hooks. - */ -function enforceDirectMainSideForkBoundary( - name: string, - forkCapable: boolean, - functionImports: ReadonlySet, - functionExports: ReadonlySet, - importsDynamicLookup: boolean, - options: LoadSharedLibraryOptions, -): void { - const mainSymbols = options.mainModuleSymbols ?? new Set(); - for (const loaded of options.loadedLibraries.values()) { - if (!forkCapable && !loaded.forkCapable) continue; - - if (importsDynamicLookup || loaded.importsDynamicLookup) { - throw new Error( - `${name}: fork-capable side modules cannot coexist with side-originated ` + - `dlopen/dlsym; only a direct main-module-to-side fork path is supported`, - ); - } +export type DylinkInitializationStage = + | "bootstrap" + | "relocations" + | "constructors"; - const newToLoaded = intersectSideSymbols( - functionImports, - loaded.functionExports, - mainSymbols, - ); - const loadedToNew = intersectSideSymbols( - loaded.functionImports, - functionExports, - mainSymbols, - ); - const crossSymbols = [...newToLoaded, ...loadedToNew]; - if (crossSymbols.length > 0) { - throw new Error( - `${name}: fork-capable side-module nesting through ${loaded.name} is unsupported ` + - `(cross-side symbols: ${Array.from(new Set(crossSymbols)).join(", ")})`, - ); - } - } +export interface DylinkInitializationStep { + readonly libraryName: string; + readonly stage: DylinkInitializationStage; + /** Exact module identity that must be publishable before `invoke` runs. */ + readonly forkState: DylinkForkLibraryState; + /** + * All loader-controlled guest entries have the canonical `() -> ()` shape. + * + * A process loader may install this exact function in its shared table and + * let libc invoke it as an ordinary Wasm call. Standalone embedders drive + * the same state machine synchronously. + */ + readonly invoke: () => void; } /** @@ -654,54 +1015,68 @@ function enforceDirectMainSideForkBoundary( * side module into the process address space. Used by both async and sync * entry points. */ -function instantiateSharedLibrary( +function* instantiateSharedLibrarySteps( name: string, wasmBytes: Uint8Array, metadata: DylinkMetadata, options: LoadSharedLibraryOptions, replay?: DylinkReplayOptions, -): LoadedSharedLibrary { + loadContext?: DylinkLoadContext, + globalVisibility = true, + dependencyScope: readonly LoadedSharedLibrary[] = [], +): Generator< + DylinkInitializationStep, + LoadedSharedLibrary, + DylinkForkLibraryState | undefined +> { validateLongjmpConfiguration(options); const ptrWidth = options.ptrWidth ?? 4; const pointerGlobalType = ptrWidth === 8 ? "i64" : "i32"; const module = new WebAssembly.Module(wasmBytes as unknown as BufferSource); const moduleImports = WebAssembly.Module.imports(module); const moduleExports = WebAssembly.Module.exports(module); + const moduleExportKinds = new Map( + moduleExports.map((moduleExport) => [ + moduleExport.name, + moduleExport.kind, + ]), + ); + const functionImports = readWasmFunctionImports( + wasmBytes.buffer.slice( + wasmBytes.byteOffset, + wasmBytes.byteOffset + wasmBytes.byteLength, + ) as ArrayBuffer, + ); + const functionImportsByName = new Map(); + for (const imported of functionImports) { + const key = `${imported.module.length}:${imported.module}${imported.name}`; + const entries = functionImportsByName.get(key) ?? []; + entries.push(imported); + functionImportsByName.set(key, entries); + } + const functionImportReads = new Map(); const importsFork = moduleImports.some((imp) => imp.module === "env" && imp.name === "fork" && imp.kind === "function" ); - const linkedFrameImportNames = WPK_FORK_REQUIRED_IMPORTS + const requiredForkFunctionImportNames = WPK_FORK_REQUIRED_IMPORTS .filter(({ module }) => module === "env") .map(({ name }) => name); - const linkedFrameImportCount = linkedFrameImportNames.filter((importName) => - moduleImports.some((imp) => - imp.module === "env" && imp.name === importName && imp.kind === "function" - ) - ).length; + const requiredForkFunctionImportCount = + requiredForkFunctionImportNames.filter((importName) => + moduleImports.some((imp) => + imp.module === "env" && imp.name === importName && imp.kind === "function" + ) + ).length; const presentForkExports = SIDE_MODULE_FORK_EXPORTS.filter((exportName) => moduleExports.some((exp) => exp.kind === "function" && exp.name === exportName) ); const hasCompleteForkInstrumentation = presentForkExports.length === SIDE_MODULE_FORK_EXPORTS.length; const forkCapabilityClaim = readForkInstrumentCapabilityClaim(module); - const claimsSideEntry = - forkCapabilityClaim.present - && (forkCapabilityClaim.flags & FORK_CAP_SIDE_ENTRY) !== 0; const sideEntryAvailable = forkInstrumentRoleAvailable( forkCapabilityClaim, FORK_CAP_SIDE_ENTRY, ); - const functionImports = new Set( - moduleImports - .filter((imp) => - (imp.module === "env" && imp.kind === "function") - || imp.module === "GOT.func" - ) - .map((imp) => imp.name), - ); - const functionExports = new Set( - moduleExports.filter((exp) => exp.kind === "function").map((exp) => exp.name), - ); const importedFunctionCount = moduleImports.filter((imp) => imp.kind === "function").length; const definedFunctionExports = readDefinedFunctionExports( wasmBytes, @@ -722,11 +1097,6 @@ function instantiateSharedLibrary( ) .map((imp) => imp.name), ); - const importsDynamicLookup = moduleImports.some((imp) => - imp.module === "env" - && imp.kind === "function" - && SIDE_DYNAMIC_LOOKUP_IMPORTS.has(imp.name) - ); const importsLongjmpTag = moduleImports.some((imp) => imp.module === "env" && imp.name === "__c_longjmp" @@ -741,6 +1111,11 @@ function instantiateSharedLibrary( const cppExceptionTag = importsCppExceptionTag ? resolveCppExceptionTag(options) : undefined; + const importsForkUnwindTag = moduleImports.some((imp) => + imp.module === FORK_UNWIND_TAG_IMPORT_MODULE + && imp.name === FORK_UNWIND_TAG_IMPORT_NAME + && (imp.kind as string) === "tag" + ); if (presentForkExports.length > 0 && !hasCompleteForkInstrumentation) { const missing = SIDE_MODULE_FORK_EXPORTS.filter((exportName) => @@ -750,6 +1125,51 @@ function instantiateSharedLibrary( `${name}: incomplete wasm-fork-instrument exports; missing ${missing.join(", ")}`, ); } + if (options.forkActivationOwner && !hasCompleteForkInstrumentation) { + throw new Error( + `${name}: fork-capable process requires complete ABI 43 side-boundary ` + + "instrumentation; rebuild the side module with wasm-fork-instrument", + ); + } + if ( + hasCompleteForkInstrumentation && + ( + !forkCapabilityClaim.present || + (forkCapabilityClaim.flags & FORK_CAP_ACTIVATION_STATE_SAFE) === 0 + ) + ) { + throw new Error( + `${name}: wasm-fork-instrument artifact lacks the ABI 43 ` + + "activation-state-safe capability; rebuild the side module", + ); + } + if (hasCompleteForkInstrumentation) { + const artifactBytes = wasmBytes.buffer.slice( + wasmBytes.byteOffset, + wasmBytes.byteOffset + wasmBytes.byteLength, + ) as ArrayBuffer; + const declaredAbi = extractAbiVersion(artifactBytes); + if (declaredAbi === null) { + throw new Error( + `${name}: ABI 43 fork-instrumented side module is missing __abi_version; ` + + "the activation-state capability epoch cannot be verified", + ); + } + if (declaredAbi !== ABI_VERSION) { + throw new Error( + `${name}: fork-instrumented side module declares ABI ${declaredAbi}, ` + + `but the host requires ABI ${ABI_VERSION}`, + ); + } + const contractFailures = + describeWasmForkArtifactContractFailures(artifactBytes); + if (contractFailures.length > 0) { + throw new Error( + `${name}: invalid ABI 43 fork reconstruction contract: ` + + contractFailures.join("; "), + ); + } + } if (importsFork && !hasCompleteForkInstrumentation) { throw new Error( `${name}: env.fork requires complete side-module instrumentation; ` + @@ -759,54 +1179,143 @@ function instantiateSharedLibrary( if (importsFork && !sideEntryAvailable) { throw new Error( `${name}: env.fork requires the versioned side-entry capability; ` + - "rebuild with the current wasm-fork-instrument --entry env.fork", + "rebuild with the current wasm-fork-instrument --entry env.fork", ); } - if (linkedFrameImportCount !== 0 && linkedFrameImportCount !== linkedFrameImportNames.length) { + if ( + options.forkActivationOwner + && hasCompleteForkInstrumentation + && !sideEntryAvailable + ) { + throw new Error( + `${name}: fork-capable process requires the versioned side-entry ` + + "boundary capability; rebuild the side module", + ); + } + if ( + requiredForkFunctionImportCount !== 0 + && requiredForkFunctionImportCount !== requiredForkFunctionImportNames.length + ) { throw new Error(`${name}: incomplete linked fork instrumentation imports; rebuild the module`); } - if (importsFork && linkedFrameImportCount !== linkedFrameImportNames.length) { - throw new Error(`${name}: env.fork requires ABI 42 linked continuation imports`); + if ( + importsFork + && requiredForkFunctionImportCount !== requiredForkFunctionImportNames.length + ) { + throw new Error(`${name}: env.fork requires ABI 43 linked continuation imports`); + } + if (hasCompleteForkInstrumentation && !options.forkActivationOwner) { + throw new Error( + `${name}: fork activation cannot be coordinated: ` + + (options.forkActivationOwnerUnavailableReason + ?? "ABI 43 side modules require a process activation owner"), + ); } - if (claimsSideEntry && !importsFork) { - throw new Error(`${name}: side-entry capability is present without an env.fork import`); + if ( + hasCompleteForkInstrumentation + && replay !== undefined + && replay.activationId === undefined + ) { + throw new Error(`${name}: fork replay is missing its archived activation id`); } - if (importsFork && !options.sideModuleFork) { + const replayActivationId = replay?.activationId === undefined + ? undefined + : requireNonzeroU32( + replay.activationId, + `${name}: archived activation id`, + ); + if (!hasCompleteForkInstrumentation && replay?.activationId !== undefined) { throw new Error( - `${name}: env.fork cannot be coordinated: ` + - (options.sideModuleForkUnavailableReason - ?? "side-module fork requires a process-worker unwind coordinator"), + `${name}: fork replay supplied an activation id for an uninstrumented module`, ); } - enforceDirectMainSideForkBoundary( - name, - importsFork, - functionImports, - functionExports, - importsDynamicLookup, - options, - ); const tableRollbackBase = tableLength(options.table); const heapRollbackValue = options.heapPointer?.value; const symbolRollback = new Map(options.globalSymbols); + const owners = symbolOwners(options); + const ownerRollback = new Map(owners); const gotRollback = new Map( Array.from(options.got, ([symbol, global]) => [ symbol, { global, value: global.value }, ] as const), ); - const allocations: Array<{ addr: number; size: number }> = []; + const gotKinds = options.gotKinds ??= new Map(); + const gotKindsRollback = new Map(gotKinds); + const allocations: DylinkForkMemoryAllocation[] = []; + const ownedTableEntries = new Set(); + const gotImports = new Map(); + const localGot = new Map(); + const providerDependencies = new Set( + replay?.providerDependencies ?? [], + ); + const recordProvider = (owner: string | undefined): void => { + if (owner !== undefined && owner !== name) { + providerDependencies.add(owner); + } + }; + let preparedActivation: PreparedDylinkForkActivation | undefined; + let preparedActivationId: number | undefined; + let provisionalLibrary: LoadedSharedLibrary | undefined; + let forkActivationReleased = false; + const unregisterForkActivation = (): void => { + if (!preparedActivation || forkActivationReleased) return; + // WHY: registration may have partially succeeded before throwing. The + // owner's teardown is the only authority that can release the activation + // ID, resume catalog, typed roots, and continuation binding atomically. + forkActivationReleased = true; + preparedActivation.unregister(); + }; const allocate = (size: number, align: number): number => { if (!options.allocateMemory) { throw new Error(`${name}: no side-module memory allocator configured`); } - const addr = options.allocateMemory(size, align); - allocations.push({ addr, size }); - return addr; + const address = options.allocateMemory(size, align); + const described = options.describeMemoryAllocation?.(address, size); + allocations.push(copyForkMemoryAllocation({ + address, + size, + mappingAddress: described?.mappingAddress ?? address, + mappingSize: described?.mappingSize ?? size, + }, `${name}: allocated side-module memory`)); + return address; }; try { + if (hasCompleteForkInstrumentation) { + preparedActivation = options.forkActivationOwner!.prepare({ + name, + module, + moduleBytes: wasmBytes, + replayActivationId, + }); + if ( + !preparedActivation + || typeof preparedActivation !== "object" + || typeof preparedActivation.wrapImports !== "function" + || typeof preparedActivation.register !== "function" + || typeof preparedActivation.unregister !== "function" + || !preparedActivation.env + || typeof preparedActivation.env !== "object" + ) { + throw new TypeError(`${name}: activation owner returned an invalid preparation`); + } + preparedActivationId = requireNonzeroU32( + preparedActivation.activationId, + `${name}: prepared activation id`, + ); + if ( + replayActivationId !== undefined + && preparedActivationId !== replayActivationId + ) { + throw new Error( + `${name}: activation owner returned ${preparedActivationId}, ` + + `but replay requires ${replayActivationId}`, + ); + } + } + // Allocate memory region const memAlign = 1 << metadata.memoryAlign; let memoryBase = 0; @@ -815,6 +1324,47 @@ function instantiateSharedLibrary( // Reuse parent's memoryBase: data-reloc'd pointers baked into the // memcpy'd data section already encode (parentMemoryBase + offset). memoryBase = replay.memoryBase; + const archivedAllocations = (replay.allocations ?? []).map( + (allocation, index) => copyForkMemoryAllocation( + allocation, + `${name}: archived allocation ${index}`, + ), + ); + if ( + archivedAllocations.length !== 0 + && ( + archivedAllocations.length !== 1 + || archivedAllocations[0]!.address !== memoryBase + || archivedAllocations[0]!.size !== metadata.memorySize + ) + ) { + throw new Error( + `${name}: archived allocation does not match its dylink memory region`, + ); + } + if ( + options.adoptMemoryAllocation + && archivedAllocations.length === 0 + ) { + throw new Error( + `${name}: fork replay is missing process mapping ownership`, + ); + } + for (const allocation of archivedAllocations) { + if ( + allocation.mappingAddress + > options.memory.buffer.byteLength - allocation.mappingSize + ) { + throw new RangeError( + `${name}: archived process mapping escapes copied linear memory`, + ); + } + // WHY: fork copied both kernel mmap state and the bytes, but the new + // Worker has an empty JS allocator index. Adopt that ownership; do + // not allocate or zero a second region. + options.adoptMemoryAllocation?.(allocation); + allocations.push(allocation); + } } else if (options.allocateMemory) { memoryBase = allocate(metadata.memorySize, memAlign); const end = memoryBase + metadata.memorySize; @@ -845,6 +1395,8 @@ function instantiateSharedLibrary( // post-startup data via fork memcpy. new Uint8Array(options.memory.buffer, memoryBase, metadata.memorySize).fill(0); } + } else if ((replay?.allocations?.length ?? 0) !== 0) { + throw new Error(`${name}: zero-memory side module owns archived mappings`); } // Reproduce the parent's exact table base, including null gaps left by a @@ -865,25 +1417,10 @@ function instantiateSharedLibrary( } tableBase = replay.tableBase; } - if (metadata.tableSize > 0) growTable(options.table, metadata.tableSize); - - let sideForkBufAddr = 0; - let sideForkContinuation: LinkedForkContinuation | undefined; - if (importsFork) { - if (!options.allocateContinuation || !options.deallocateContinuation) { - throw new Error( - `${name}: linked continuations require process-mapping allocation and cleanup`, - ); - } - sideForkContinuation = new LinkedForkContinuation( - options.memory, - readLinkedFrameFormat(module), - options.allocateContinuation, - options.deallocateContinuation, - name, - ); - if (replay) { - sideForkBufAddr = replay.forkBufAddr ?? 0; + if (metadata.tableSize > 0) { + growTable(options.table, metadata.tableSize); + for (let index = 0; index < metadata.tableSize; index++) { + ownedTableEntries.add(tableBase + index); } } @@ -918,14 +1455,14 @@ function instantiateSharedLibrary( // function at runtime, the function must live in the shared // indirect_function_table and the GOT entry must hold its index. const tableIndexFor = (fn: Function): number => { + const existing = functionTableIndex(options, fn); + if (existing !== undefined) return existing; const tbl = options.table; const length = tableLength(tbl); - for (let i = 0; i < length; i++) { - if (getTableEntry(tbl, i) === fn) return i; - } const idx = length; growTable(tbl, 1); setTableEntry(tbl, idx, fn); + options.onTableMutation?.(tbl, idx, 1); return idx; }; @@ -933,19 +1470,88 @@ function instantiateSharedLibrary( symName: string, kind: "mem" | "func", ): WebAssembly.Global => { + const resolved = scopedSymbol(options, dependencyScope, symName); + recordProvider(resolved?.owner); + if ( + resolved + && ( + (kind === "mem" && !(resolved.value instanceof WebAssembly.Global)) + || (kind === "func" && typeof resolved.value !== "function") + ) + ) { + throw new Error( + `${name}: GOT.${kind} symbol ${symName} has the wrong kind`, + ); + } + const resolvedFunctionIndex = + kind === "func" && typeof resolved?.value === "function" + ? tableIndexFor(resolved.value) + : undefined; + const globallyResolved = + resolved !== undefined + && options.globalSymbols.get(symName) === resolved.value; + const localKey = `${kind}:${symName}`; + const selfExportKind = moduleExportKinds.get(symName); + const isSelfReference = + resolved === undefined + && ( + (kind === "mem" && selfExportKind === "global") + || (kind === "func" && selfExportKind === "function") + ); + + // A LOCAL dependency and a module's own interposable export must not + // acquire a process-global GOT cell. The importing instance owns this + // cell, and its exact provider is captured in the dependency closure. + if ((resolved && !globallyResolved) || isSelfReference) { + let localEntry = localGot.get(localKey); + if (!localEntry) { + let initial = wasmAddress( + 0, + ptrWidth, + `${name}: local GOT.${kind}.${symName}`, + ); + if (resolved) { + initial = kind === "mem" + ? requireWasmAddress( + (resolved.value as WebAssembly.Global).value as WasmAddress, + ptrWidth, + `${name}: local GOT.mem.${symName}`, + ) + : wasmAddress( + resolvedFunctionIndex!, + ptrWidth, + `${name}: local GOT.func.${symName}`, + ); + } + localEntry = new WebAssembly.Global( + { value: pointerGlobalType, mutable: true }, + initial, + ); + localGot.set(localKey, localEntry); + } + return localEntry; + } + + const knownKind = gotKinds.get(symName); + if (knownKind !== undefined && knownKind !== kind) { + throw new Error( + `${name}: GOT symbol ${symName} is both ${knownKind} and ${kind}`, + ); + } + gotKinds.set(symName, kind); + gotImports.set(symName, kind); let entry = options.got.get(symName); if (!entry) { let initial = wasmAddress(0, ptrWidth, `${name}: GOT.${kind}.${symName}`); - const sym = options.globalSymbols.get(symName); - if (kind === "mem" && sym instanceof WebAssembly.Global) { + if (kind === "mem" && resolved?.value instanceof WebAssembly.Global) { initial = requireWasmAddress( - sym.value as WasmAddress, + resolved.value.value as WasmAddress, ptrWidth, `${name}: GOT.mem.${symName}`, ); - } else if (kind === "func" && typeof sym === "function") { + } else if (resolvedFunctionIndex !== undefined) { initial = wasmAddress( - tableIndexFor(sym), + resolvedFunctionIndex, ptrWidth, `${name}: GOT.func.${symName}`, ); @@ -961,171 +1567,122 @@ function instantiateSharedLibrary( ptrWidth, `${name}: existing GOT.${kind}.${symName}`, ); + if (kind === "mem" && resolved?.value instanceof WebAssembly.Global) { + entry.value = requireWasmAddress( + resolved.value.value as WasmAddress, + ptrWidth, + `${name}: GOT.mem.${symName}`, + ); + } else if (resolvedFunctionIndex !== undefined) { + entry.value = wasmAddress( + resolvedFunctionIndex, + ptrWidth, + `${name}: GOT.func.${symName}`, + ); + } } return entry; }; let instance: WebAssembly.Instance | null = null; - let sideForkState: SideModuleForkState | null = null; - const forkState = (): number => { - if (!instance) throw new Error(`${name}: side-module fork before instantiation`); - return Number((instance.exports.wpk_fork_state as () => number)()); - }; - - const sideModuleForkImport = (): number => { - if (!instance || !options.sideModuleFork || !sideForkContinuation) { - throw new Error(`${name}: side-module fork coordinator is unavailable`); - } - const state = forkState(); - if (state === WPK_FORK_NORMAL) { - try { - sideForkBufAddr = Number(sideForkContinuation!.beginUnwind()); - } catch (error) { - if (error instanceof ContinuationAllocationError) return -error.errno; - throw error; - } - const loaded = options.loadedLibraries.get(name); - if (loaded) loaded.forkBufAddr = sideForkBufAddr; - invokeForkContinuationBegin( - instance.exports.wpk_fork_unwind_begin, - sideForkBufAddr, - ptrWidth, - `${name}: side-module linked fork unwind`, - ); - if (forkState() !== WPK_FORK_UNWINDING) { - throw new Error(`${name}: side-module fork failed to enter UNWINDING`); - } - const startedState: SideModuleForkState = { - name, - instance, - forkBufAddr: sideForkBufAddr, - continuation: sideForkContinuation!, - }; - sideForkState = startedState; - options.sideModuleFork.setActiveFork(startedState); - const result = options.sideModuleFork.invokeMainFork([ - WPK_FORK_NORMAL, - WPK_FORK_UNWINDING, - ]); - if (result < 0) { - // Main root allocation failed synchronously: no side activation has - // returned yet, so unwind the side control state without replay. - (instance.exports.wpk_fork_unwind_end as () => void)(); - sideForkContinuation!.cancelUnwindAndRelease(); - options.sideModuleFork.clearActiveFork(startedState); - const loaded = options.loadedLibraries.get(name); - if (loaded) loaded.forkBufAddr = undefined; - sideForkState = null; - } - return result; - } - - if (state === WPK_FORK_REWINDING) { - (instance.exports.wpk_fork_rewind_end as () => void)(); - sideForkContinuation!.finishReplayAndRelease(); - if (forkState() !== WPK_FORK_NORMAL) { - throw new Error(`${name}: side-module fork failed to finish REWINDING`); - } - // A fork child re-instantiates this module, so its closure cannot retain - // the parent's SideModuleForkState object. The worker reconstructs the - // active identity from the copied archive/buffer metadata; rebuild the - // same structural identity here before clearing it. - const completedState = sideForkState ?? { - name, - instance, - forkBufAddr: sideForkBufAddr, - continuation: sideForkContinuation!, - }; - const result = options.sideModuleFork.invokeMainFork(WPK_FORK_NORMAL); - options.sideModuleFork.clearActiveFork(completedState); - const loaded = options.loadedLibraries.get(name); - if (loaded) loaded.forkBufAddr = undefined; - sideForkState = null; - return result; - } - - if (state === WPK_FORK_ABORT_UNWINDING) { - const errno = sideForkContinuation!.abortErrno(); - (instance.exports.wpk_fork_abort_end as () => void)(); - sideForkContinuation!.finishAbortReplayAndRelease(); - const completedState = sideForkState; - if (!completedState) { - throw new Error(`${name}: side-module abort lost its active fork identity`); - } - const result = options.sideModuleFork.invokeMainFork(WPK_FORK_NORMAL); - options.sideModuleFork.clearActiveFork(completedState); - const loaded = options.loadedLibraries.get(name); - if (loaded) loaded.forkBufAddr = undefined; - sideForkState = null; - if (result !== -errno) { - throw new Error(`${name}: main/side continuation abort errno mismatch`); - } - return result; - } - - throw new Error(`${name}: env.fork reached in unexpected state ${state}`); + const routeFunctionImport = ( + moduleName: string, + importName: string, + value: WebAssembly.ImportValue | WebAssembly.Tag | undefined, + ): WebAssembly.ImportValue | WebAssembly.Tag | undefined => { + if (typeof value !== "function" || !options.routeFunctionImport) { + return value; + } + const key = `${moduleName.length}:${moduleName}${importName}`; + const entries = functionImportsByName.get(key); + if (!entries || entries.length === 0) return value; + const read = functionImportReads.get(key) ?? 0; + const imported = entries[Math.min(read, entries.length - 1)]!; + functionImportReads.set(key, read + 1); + return options.routeFunctionImport(imported, value); }; // Construct imports const imports: WebAssembly.Imports = { env: new Proxy({} as Record, { get(_target, prop: string) { + let value: WebAssembly.ImportValue | WebAssembly.Tag | undefined; switch (prop) { - case "memory": return options.memory; - case "__indirect_function_table": return options.table; - case "__memory_base": return memoryBaseGlobal; - case "__table_base": return tableBaseGlobal; - case "__stack_pointer": return options.stackPointer; - case "__c_longjmp": return longjmpTag; - case "__cpp_exception": return cppExceptionTag; - case "fork": - if (importsFork) return sideModuleForkImport; + case "memory": value = options.memory; break; + case "__indirect_function_table": value = options.table; break; + case "__memory_base": value = memoryBaseGlobal; break; + case "__table_base": value = tableBaseGlobal; break; + case "__stack_pointer": value = options.stackPointer; break; + case "__c_longjmp": value = longjmpTag; break; + case "__cpp_exception": value = cppExceptionTag; break; + case FORK_UNWIND_TAG_IMPORT_NAME: + if ( + preparedActivation + && Object.hasOwn(preparedActivation.env, prop) + ) { + value = preparedActivation.env[prop]; + } else if (hasCompleteForkInstrumentation) { + // WHY: an ABI 43 linked continuation has one process-level + // activation owner. Letting this tag fall back independently + // could bind unwind exceptions to a different realm than the + // owner's frame and exception reconstruction imports. + value = undefined; + } else { + value = importsForkUnwindTag + ? requireForkUnwindTag(options.forkUnwindTag, name) + : undefined; + } break; - case "__wpk_fork_frame_reserve": - if (importsFork) return (size: number | bigint) => { - const frame = sideForkContinuation!.reserveFrame(size); - if (frame === 0 || frame === 0n) { - const errno = sideForkContinuation!.abortErrno(); - options.sideModuleFork!.beginMainAbort(errno); - invokeForkContinuationBegin( - instance!.exports.wpk_fork_abort_begin, - sideForkBufAddr, - ptrWidth, - `${name}: side-module linked fork abort`, - ); + default: + if ( + preparedActivation + && Object.hasOwn(preparedActivation.env, prop) + ) { + value = preparedActivation.env[prop]; + } else if ( + hasCompleteForkInstrumentation + && (prop === "fork" || prop.startsWith("__wpk_fork_")) + ) { + // WHY: falling through to a process symbol would split + // ownership between the loader and coordinator. Missing + // activation imports fail before the side module executes. + value = undefined; + } else { + const resolved = scopedSymbol( + options, + dependencyScope, + prop, + ); + if (resolved !== undefined) { + recordProvider(resolved.owner); + value = resolved.value; + } else if (selfFunctionImports.has(prop)) { + value = (...args: unknown[]) => { + const fn = instance?.exports[prop]; + if (typeof fn !== "function") { + throw new Error(`${name}: self import env.${prop} is unavailable`); + } + return (fn as Function)(...args); + }; } - return frame; - }; - break; - case "__wpk_fork_frame_commit": - if (importsFork) return (payload: number | bigint) => - sideForkContinuation!.commitFrame(payload); - break; - case "__wpk_fork_frame_next": - if (importsFork) return (size: number | bigint) => - sideForkContinuation!.nextFrame(size); - break; - } - const sym = options.globalSymbols.get(prop); - if (sym !== undefined) return sym; - if (selfFunctionImports.has(prop)) { - return (...args: unknown[]) => { - const fn = instance?.exports[prop]; - if (typeof fn !== "function") { - throw new Error(`${name}: self import env.${prop} is unavailable`); } - return (fn as Function)(...args); - }; } - return undefined; + return routeFunctionImport("env", prop, value); }, has(_target, prop: string) { if (["memory", "__indirect_function_table", "__memory_base", "__table_base", "__stack_pointer", "__c_longjmp", "__cpp_exception"].includes(prop)) return true; - if (prop === "fork" && importsFork) return true; - if (linkedFrameImportNames.some((name) => name === prop) && importsFork) return true; - return options.globalSymbols.has(prop) || selfFunctionImports.has(prop); + if ( + preparedActivation + && Object.hasOwn(preparedActivation.env, prop) + ) return true; + if ( + hasCompleteForkInstrumentation + && (prop === "fork" || prop.startsWith("__wpk_fork_")) + ) return false; + return scopedSymbol(options, dependencyScope, prop) !== undefined + || selfFunctionImports.has(prop); }, }), "GOT.mem": new Proxy({} as Record, { @@ -1140,8 +1697,77 @@ function instantiateSharedLibrary( }), }; + // Imported global/table identity is observable only while WebAssembly + // lazily resolves this exact proxy graph. Give the process owner one + // synchronous wrapper boundary; eager enumeration would collapse duplicate + // `(module, name)` declarations and capture the wrong provider. + const instanceImports = preparedActivation + ? preparedActivation.wrapImports(imports) + : imports; + if (!instanceImports || typeof instanceImports !== "object") { + throw new TypeError(`${name}: activation owner returned invalid wrapped imports`); + } + // Instantiate synchronously after validating the side-module fork contract. - instance = new WebAssembly.Instance(module, imports); + instance = new WebAssembly.Instance(module, instanceImports); + preparedActivation?.register(instance); + const ownedModuleBytes = wasmBytes.slice(); + const initializationForkState = ( + tlsBase?: number, + ): DylinkForkLibraryState => ({ + name, + moduleBytes: ownedModuleBytes, + memoryBase, + tableBase, + activationId: preparedActivationId, + globalVisibility, + ...(allocations.length === 0 + ? {} + : { allocations: allocations.map((allocation) => ({ ...allocation })) }), + ...(tlsBase === undefined ? {} : { tlsBase }), + }); + provisionalLibrary = { + instance, + memoryBase, + tableBase, + exports: {}, + metadata, + name, + moduleBytes: ownedModuleBytes, + activationId: preparedActivationId, + ownedTableEntries: [], + gotImports: [], + unregisterForkActivation: preparedActivation + ? unregisterForkActivation + : undefined, + loadState: "initializing", + globalVisibility, + providerDependencies, + allocations, + heapReservationEnd: options.heapPointer?.value, + }; + options.loadedLibraries.set(name, provisionalLibrary); + loadContext?.ownedLibraries.add(provisionalLibrary); + let stateAfterBootstrap: DylinkForkLibraryState | undefined; + if ( + preparedActivation + && (!replay || replay.initializationStage !== undefined) + ) { + const bootstrap = instance.exports.wpk_fork_module_bootstrap; + if (typeof bootstrap !== "function") { + throw new Error(`${name}: fork activation is missing its module bootstrap`); + } + // WHY: activation registration must not itself enter guest code. Keeping + // bootstrap at the loader boundary lets the process/libc staged loader + // replace this direct call with a normal Wasm table call without + // changing activation ownership or registration ordering. + stateAfterBootstrap = yield { + libraryName: name, + stage: "bootstrap", + forkState: initializationForkState(), + invoke: bootstrap as () => void, + }; + } // A threaded wasm-ld side module initializes its mutable __tls_base from // __memory_base in the start function. Fork-child memory already carries @@ -1196,13 +1822,27 @@ function instantiateSharedLibrary( throw new Error(`${name}: exported __tls_base must be mutable for fork replay`); } if (replay) { - if (!Number.isSafeInteger(replay.tlsBase) || replay.tlsBase! <= 0) { + const replayTlsBase = stateAfterBootstrap?.tlsBase ?? replay.tlsBase; + if (replayTlsBase !== undefined) { + if (!Number.isSafeInteger(replayTlsBase) || replayTlsBase <= 0) { + throw new Error(`${name}: fork replay is missing a valid side-module TLS base`); + } + try { + tlsBaseExport.value = typeof initialRawTlsBase === "bigint" + ? BigInt(replayTlsBase) + : replayTlsBase; + } catch { + throw new Error(`${name}: exported __tls_base must be mutable for fork replay`); + } + } else if (replay.initializationStage !== "bootstrap") { throw new Error(`${name}: fork replay is missing a valid side-module TLS base`); } + // A child stopped inside bootstrap has no archived TLS value yet. + // Once its restored bootstrap call returns, the instance-local global + // is authoritative. A non-calling pthread replica instead receives the + // later archived value through the generator resume above. try { - tlsBaseExport.value = typeof initialRawTlsBase === "bigint" - ? BigInt(replay.tlsBase!) - : replay.tlsBase!; + tlsBaseExport.value = tlsBaseExport.value; } catch { throw new Error(`${name}: exported __tls_base must be mutable for fork replay`); } @@ -1271,79 +1911,144 @@ function instantiateSharedLibrary( relocatedExports[exportName] = exportValue; } } + provisionalLibrary.exports = relocatedExports; + provisionalLibrary.tlsBase = tlsBase; // Update GOT with this library's exports for (const [exportName, exportValue] of Object.entries(relocatedExports)) { - if (exportName.startsWith("__")) continue; + if (exportName.startsWith("__") || isForkRuntimeExport(exportName)) { + // WHY: these are activation-control entry points, not ELF-visible + // application symbols. Publishing them would put post-catalog + // instrumenter helpers into the mutable process table and manufacture + // reference state with no source-function reconstruction recipe. + continue; + } const alreadyDefined = options.globalSymbols.has(exportName); if (typeof exportValue === "function") { const tableIdx = tableLength(options.table); growTable(options.table, 1); setTableEntry(options.table, tableIdx, exportValue as unknown as Function); - - const gotEntry = options.got.get(exportName); - if (gotEntry && !alreadyDefined) { - gotEntry.value = wasmAddress( + ownedTableEntries.add(tableIdx); + // This write is performed by the host loader, outside generated + // table.set instrumentation. Attribute it to the shared table owner so + // fork captures the side function as an activation+ordinal recipe. + options.onTableMutation?.(options.table, tableIdx, 1); + + const localEntry = localGot.get(`func:${exportName}`); + if (localEntry) { + localEntry.value = wasmAddress( tableIdx, ptrWidth, - `${name}: GOT.func.${exportName}`, + `${name}: local GOT.func.${exportName}`, ); } - if (!alreadyDefined) { + const gotEntry = options.got.get(exportName); + if (globalVisibility && gotEntry) { + const gotKind = gotKinds.get(exportName); + if (gotKind !== undefined && gotKind !== "func") { + throw new Error(`${name}: GOT symbol ${exportName} changes kind`); + } + gotKinds.set(exportName, "func"); + if (!alreadyDefined) { + gotEntry.value = wasmAddress( + tableIdx, + ptrWidth, + `${name}: GOT.func.${exportName}`, + ); + } + } + if (globalVisibility && !alreadyDefined) { options.globalSymbols.set(exportName, exportValue as Function); + owners.set(exportName, name); } } else if (exportValue instanceof WebAssembly.Global) { const addr = (exportValue as WebAssembly.Global).value; - const gotEntry = options.got.get(exportName); - if (gotEntry && !alreadyDefined) { - gotEntry.value = requireWasmAddress( + const localEntry = localGot.get(`mem:${exportName}`); + if (localEntry) { + localEntry.value = requireWasmAddress( addr as WasmAddress, ptrWidth, - `${name}: GOT.mem.${exportName}`, + `${name}: local GOT.mem.${exportName}`, ); } - if (!alreadyDefined) { + const gotEntry = options.got.get(exportName); + if (globalVisibility && gotEntry) { + const gotKind = gotKinds.get(exportName); + if (gotKind !== undefined && gotKind !== "mem") { + throw new Error(`${name}: GOT symbol ${exportName} changes kind`); + } + gotKinds.set(exportName, "mem"); + if (!alreadyDefined) { + gotEntry.value = requireWasmAddress( + addr as WasmAddress, + ptrWidth, + `${name}: GOT.mem.${exportName}`, + ); + } + } + if (globalVisibility && !alreadyDefined) { options.globalSymbols.set(exportName, exportValue); + owners.set(exportName, name); } } } // Run data relocations const applyRelocs = instance.exports.__wasm_apply_data_relocs as Function | undefined; - if (applyRelocs) { - applyRelocs(); + if (applyRelocs && (!replay || replay.initializationStage !== undefined)) { + // A complete fork replay receives already-relocated live bytes from the + // parent. Re-running this entry would relocate pointers a second time. + // An in-flight replay still yields the full stage sequence so the + // archived selector can stop at the exact guest call being resumed. + yield { + libraryName: name, + stage: "relocations", + forkState: initializationForkState(tlsBase), + invoke: applyRelocs as () => void, + }; } - if (!replay) { + if (!replay || replay.initializationStage !== undefined) { // Skip ctors in replay: parent already ran them and post-startup state // (e.g. opcache accel_globals, registered INI entries) is in the // memcpy'd data; re-running would clobber it. const ctors = instance.exports.__wasm_call_ctors as Function | undefined; if (ctors) { - ctors(); + yield { + libraryName: name, + stage: "constructors", + forkState: initializationForkState(tlsBase), + invoke: ctors as () => void, + }; } } - const loaded: LoadedSharedLibrary = { - instance, - memoryBase, - tableBase, - exports: relocatedExports, - metadata, - name, - forkBufAddr: sideForkBufAddr || undefined, - forkContinuation: sideForkContinuation, - tlsBase, - forkCapable: importsFork, - functionImports, - functionExports, - importsDynamicLookup, - }; - - options.loadedLibraries.set(name, loaded); - return loaded; + provisionalLibrary.ownedTableEntries = + [...ownedTableEntries].sort((left, right) => left - right); + provisionalLibrary.gotImports = [...gotImports] + .sort(([left], [right]) => left.localeCompare(right)) + .map(([symbol, kind]) => Object.freeze({ name: symbol, kind })); + provisionalLibrary.providerDependencies = new Set(providerDependencies); + provisionalLibrary.allocations = allocations.map((allocation) => + copyForkMemoryAllocation(allocation, `${name}: live allocation`) + ); + provisionalLibrary.heapReservationEnd = options.heapPointer?.value; + provisionalLibrary.loadState = "loaded"; + return provisionalLibrary; } catch (error) { + if ( + provisionalLibrary + && options.loadedLibraries.get(name) === provisionalLibrary + ) { + options.loadedLibraries.delete(name); + } + let activationReleaseError: unknown; + try { + unregisterForkActivation(); + } catch (releaseError) { + activationReleaseError = releaseError; + } // Restore every mutable host-side linker structure we can. Table length and // Wasm memory cannot shrink, so clear newly-addressable table slots and let // the next successful archive entry record the resulting exact table base. @@ -1355,23 +2060,84 @@ function instantiateSharedLibrary( } options.globalSymbols.clear(); for (const [symbol, value] of symbolRollback) options.globalSymbols.set(symbol, value); + owners.clear(); + for (const [symbol, owner] of ownerRollback) owners.set(symbol, owner); options.got.clear(); for (const [symbol, snapshot] of gotRollback) { try { snapshot.global.value = snapshot.value; } catch { /* immutable should not occur */ } options.got.set(symbol, snapshot.global); } + gotKinds.clear(); + for (const [symbol, kind] of gotKindsRollback) { + gotKinds.set(symbol, kind); + } if (options.heapPointer && heapRollbackValue !== undefined) { options.heapPointer.value = heapRollbackValue; } if (options.deallocateMemory) { for (const allocation of allocations.reverse()) { - try { options.deallocateMemory(allocation.addr, allocation.size); } catch { /* preserve cause */ } + try { + options.deallocateMemory(allocation.address, allocation.size); + } catch { /* preserve cause */ } } } + if (activationReleaseError !== undefined) { + throw new AggregateError( + [error, activationReleaseError], + `${name}: side-module load failed and activation rollback was incomplete`, + ); + } throw error; } } +function driveDylinkInitialization( + steps: Generator< + DylinkInitializationStep, + LoadedSharedLibrary, + DylinkForkLibraryState | undefined + >, +): LoadedSharedLibrary { + let cursor = steps.next(); + while (!cursor.done) { + try { + cursor.value.invoke(); + } catch (error) { + // Re-enter the generator at its guarded yield so its ordinary rollback + // path releases allocations, table entries, and activation ownership. + // A process/libc staged driver deliberately does not do this for the + // private fork unwind: the same generator remains live until replay + // returns to the next state transition. + return steps.throw(error).value as never; + } + cursor = steps.next(); + } + return cursor.value; +} + +function instantiateSharedLibrary( + name: string, + wasmBytes: Uint8Array, + metadata: DylinkMetadata, + options: LoadSharedLibraryOptions, + replay?: DylinkReplayOptions, + globalVisibility = true, + dependencyScope: readonly LoadedSharedLibrary[] = [], +): LoadedSharedLibrary { + return driveDylinkInitialization( + instantiateSharedLibrarySteps( + name, + wasmBytes, + metadata, + options, + replay, + undefined, + globalVisibility, + dependencyScope, + ), + ); +} + /** * Load a shared library (.so / side module) into a process's address space. * Async version — uses async WebAssembly compilation for large modules and @@ -1384,10 +2150,17 @@ export async function loadSharedLibrary( name: string, wasmBytes: Uint8Array, options: LoadSharedLibraryOptions, + globalVisibility = true, ): Promise { validateLongjmpConfiguration(options); const existing = options.loadedLibraries.get(name); - if (existing) return existing; + if (existing) { + if (globalVisibility && !existing.globalVisibility) { + promoteLibraryGlobal(existing, options); + refreshGlobalGotEntries(options); + } + return existing; + } const metadata = parseDylinkSection(wasmBytes); if (!metadata) { @@ -1400,61 +2173,189 @@ export async function loadSharedLibrary( if (!options.resolveLibrary) { throw new Error(`${name}: depends on ${dep} but no resolveLibrary callback provided`); } - const depBytes = await options.resolveLibrary(dep); + const depBytes = await options.resolveLibrary(dep, name); if (!depBytes) { throw new Error(`${name}: dependency ${dep} not found`); } - await loadSharedLibrary(dep, depBytes, options); + await loadSharedLibrary(dep, depBytes, options, globalVisibility); } - return instantiateSharedLibrary(name, wasmBytes, metadata, options); + const dependencyScope: LoadedSharedLibrary[] = []; + appendDependencyScope( + dependencyScope, + metadata.neededDynlibs.map((dependencyName) => { + const dependency = options.loadedLibraries.get(dependencyName); + if (!dependency) { + throw new Error(`${name}: loaded dependency ${dependencyName} is missing`); + } + return dependency; + }), + options, + ); + return instantiateSharedLibrary( + name, + wasmBytes, + metadata, + options, + undefined, + globalVisibility, + dependencyScope, + ); } /** * Load a shared library synchronously. Required for dlopen() which must * return synchronously to C code. Uses synchronous WebAssembly compilation. */ -export function loadSharedLibrarySync( +function* loadSharedLibrarySyncSteps( name: string, wasmBytes: Uint8Array, options: LoadSharedLibraryOptions, replay?: DylinkReplayOptions, -): LoadedSharedLibrary { + replayModules?: ReadonlyMap, + loadContext?: DylinkLoadContext, + globalVisibility = true, +): Generator< + DylinkInitializationStep, + LoadedSharedLibrary, + DylinkForkLibraryState | undefined +> { validateLongjmpConfiguration(options); const existing = options.loadedLibraries.get(name); - if (existing) return existing; + if (existing) { + if (replay) { + throw new Error( + `${name}: fork replay cannot reuse an already-loaded library; ` + + "archive entries must be unique", + ); + } + if (globalVisibility && !existing.globalVisibility) { + promoteLibraryGlobal(existing, options); + refreshGlobalGotEntries(options); + } + return existing; + } const metadata = parseDylinkSection(wasmBytes); if (!metadata) { throw new Error(`${name}: not a shared library (no dylink.0 section)`); } - // Replay-with-deps would re-allocate the dep at the child's *current* - // mmap cursor (not the parent's address) and corrupt the replayed - // library's data-relocs, which encode the parent's dep memoryBase. - // Fail loudly instead of silently producing wrong addresses. - if (replay && metadata.neededDynlibs.length > 0) { - throw new Error( - `${name}: replay does not yet support NEEDED deps; ` + - `each dep would need its own DylinkReplayOptions in a future API extension`, - ); - } - - // Load dependencies first (sync). Replay is not forwarded: dep replay is - // out-of-scope (guarded above); the recursive call instantiates deps freshly. + // Parent archive entries are emitted in dependency-first load order. A + // child must replay each dependency with its own exact layout/activation + // record before replaying this consumer; silently allocating a missing dep + // here would choose fresh addresses and corrupt copied relocations. for (const dep of metadata.neededDynlibs) { if (options.loadedLibraries.has(dep)) continue; - if (!options.resolveLibrarySync) { + const archivedDependency = replayModules?.get(dep); + if (replay && !archivedDependency) { + throw new Error( + `${name}: fork replay is missing dependency ${dep}; ` + + "archive entries must be replayed in dependency order", + ); + } + if (!archivedDependency && !options.resolveLibrarySync) { throw new Error(`${name}: depends on ${dep} but no resolveLibrarySync callback provided`); } - const depBytes = options.resolveLibrarySync(dep); + const depBytes = archivedDependency + ? new Uint8Array(archivedDependency.moduleBytes) + : options.resolveLibrarySync!(dep, name); if (!depBytes) { throw new Error(`${name}: dependency ${dep} not found`); } - loadSharedLibrarySync(dep, depBytes, options); + const loadedDependency = yield* loadSharedLibrarySyncSteps( + dep, + depBytes, + options, + archivedDependency + ? { + memoryBase: archivedDependency.memoryBase, + tableBase: archivedDependency.tableBase, + activationId: archivedDependency.activationId, + tlsBase: archivedDependency.tlsBase, + globalVisibility: archivedDependency.globalVisibility, + committedGlobalRoot: archivedDependency.committedGlobalRoot, + providerDependencies: archivedDependency.providerDependencies, + allocations: archivedDependency.allocations, + initializationStage: + archivedDependency.initialization?.stage, + } + : undefined, + replayModules, + loadContext, + archivedDependency?.globalVisibility ?? globalVisibility, + ); + if (archivedDependency) { + loadedDependency.committedGlobalRoot = + archivedDependency.committedGlobalRoot; + loadedDependency.providerDependencies = new Set( + archivedDependency.providerDependencies + ?? loadedDependency.providerDependencies + ?? [], + ); + } } - return instantiateSharedLibrary(name, wasmBytes, metadata, options, replay); + const archivedSelf = replayModules?.get(name); + const effectiveGlobalVisibility = + archivedSelf?.globalVisibility + ?? replay?.globalVisibility + ?? globalVisibility; + const effectiveReplay = replay ?? (archivedSelf + ? { + memoryBase: archivedSelf.memoryBase, + tableBase: archivedSelf.tableBase, + activationId: archivedSelf.activationId, + tlsBase: archivedSelf.tlsBase, + globalVisibility: archivedSelf.globalVisibility, + committedGlobalRoot: archivedSelf.committedGlobalRoot, + providerDependencies: archivedSelf.providerDependencies, + allocations: archivedSelf.allocations, + initializationStage: archivedSelf.initialization?.stage, + } + : undefined); + const dependencyScope: LoadedSharedLibrary[] = []; + appendDependencyScope( + dependencyScope, + metadata.neededDynlibs.map((dependencyName) => { + const dependency = options.loadedLibraries.get(dependencyName); + if (!dependency) { + throw new Error(`${name}: loaded dependency ${dependencyName} is missing`); + } + return dependency; + }), + options, + ); + return yield* instantiateSharedLibrarySteps( + name, + wasmBytes, + metadata, + options, + effectiveReplay, + loadContext, + effectiveGlobalVisibility, + dependencyScope, + ); +} + +export function loadSharedLibrarySync( + name: string, + wasmBytes: Uint8Array, + options: LoadSharedLibraryOptions, + replay?: DylinkReplayOptions, + globalVisibility = true, +): LoadedSharedLibrary { + return driveDylinkInitialization( + loadSharedLibrarySyncSteps( + name, + wasmBytes, + options, + replay, + undefined, + undefined, + globalVisibility, + ), + ); } /** @@ -1466,105 +2367,1568 @@ export class DynamicLinker { private options: LoadSharedLibraryOptions; private handleCounter = DynamicLinker.MAIN_PROGRAM_HANDLE + 1; private handleMap = new Map(); + private libraryHandles = new Map(); + private handleRefCounts = new Map(); + /** One retain per live consumer -> immutable or runtime provider edge. */ + private dependencyRetainCounts = new Map(); + /** Consumers whose immutable NEEDED edges have been accounted exactly once. */ + private dependencyOwners = new Set(); + private pendingTokenCounter = 1; + private pendingDlopens = new Map(); private lastError: string | null = null; + private readonly baseGlobalSymbols: Map< + string, + Function | WebAssembly.Global + >; + private readonly baseGlobalSymbolOwners: Map; + private readonly baseGot: Map; constructor(options: LoadSharedLibraryOptions) { validateLongjmpConfiguration(options); this.options = options; + this.options.gotKinds ??= new Map(); + const owners = symbolOwners(this.options); + this.baseGlobalSymbols = new Map(options.globalSymbols); + this.baseGlobalSymbolOwners = new Map(owners); + this.baseGot = new Map(options.got); } - /** Return the stable opaque handle used by dlopen(NULL, ...). */ - dlopenMain(): number { - this.lastError = null; - return DynamicLinker.MAIN_PROGRAM_HANDLE; + private stateForLibrary(lib: LoadedSharedLibrary): DylinkForkLibraryState { + const providerDependencies = [...(lib.providerDependencies ?? [])] + .filter((dependency) => dependency !== lib.name) + .sort(); + const providerState = providerDependencies.length === 0 + ? {} + : { providerDependencies }; + const allocations = (lib.allocations ?? []).map((allocation, index) => + copyForkMemoryAllocation( + allocation, + `${lib.name}: live allocation ${index}`, + ) + ); + const allocationState = allocations.length === 0 + ? {} + : { allocations }; + const handle = this.libraryHandles.get(lib.name); + if (handle === undefined) { + return { + name: lib.name, + moduleBytes: lib.moduleBytes, + memoryBase: lib.memoryBase, + tableBase: lib.tableBase, + activationId: lib.activationId, + tlsBase: lib.tlsBase, + globalVisibility: lib.globalVisibility, + ...(lib.committedGlobalRoot + ? { committedGlobalRoot: true } + : {}), + ...providerState, + ...allocationState, + initialization: lib.initialization, + }; + } + if (this.handleMap.get(handle) !== lib) { + throw new Error( + `${lib.name}: dynamic-linker fork state points at a different instance`, + ); + } + const refCount = this.handleRefCounts.get(handle); + if (!Number.isInteger(refCount) || refCount! <= 0) { + throw new Error( + `${lib.name}: dynamic-linker fork state has invalid handle refcount`, + ); + } + return { + name: lib.name, + moduleBytes: lib.moduleBytes, + memoryBase: lib.memoryBase, + tableBase: lib.tableBase, + activationId: lib.activationId, + tlsBase: lib.tlsBase, + globalVisibility: lib.globalVisibility, + ...(lib.committedGlobalRoot + ? { committedGlobalRoot: true } + : {}), + ...providerState, + ...allocationState, + handle, + refCount, + initialization: lib.initialization, + }; } - /** Open a shared library. Returns a handle (>0) or 0 on error. - * When `replay` is provided, behaves as fork-replay: uses the parent's - * saved memoryBase and skips __wasm_call_ctors. See `DylinkReplayOptions` - * for preconditions. */ - dlopenSync(name: string, wasmBytes: Uint8Array, replay?: DylinkReplayOptions): number { - try { - const lib = loadSharedLibrarySync(name, wasmBytes, this.options, replay); - // Check if already mapped to a handle - for (const [h, l] of this.handleMap) { - if (l === lib) return h; - } - const handle = this.handleCounter++; - this.handleMap.set(handle, lib); - this.lastError = null; - return handle; - } catch (e) { - this.lastError = e instanceof Error ? e.message : String(e); - return 0; - } + /** O(1) lookup for incrementally updating one live-module archive record. */ + forkLibraryState(name: string): DylinkForkLibraryState | undefined { + const lib = this.options.loadedLibraries.get(name); + return lib ? this.stateForLibrary(lib) : undefined; } - private symbolAddress( - symbolName: string, - exp: Function | WebAssembly.Global | undefined, - ): number | null { - if (typeof exp === "function") { - // Return the table index for this function (C function pointers are table indices) - const table = this.options.table; - const length = tableLength(table); - for (let i = 0; i < length; i++) { - if (getTableEntry(table, i) === exp) { - this.lastError = null; - return i; + /** + * Read the compact live process state; closed modules and historical events + * are deliberately absent so archive size is bounded by the live closure. + */ + forkState(): DylinkForkState { + const transactions = Array.from( + this.pendingDlopens.values(), + (transaction): DylinkForkTransactionState => ({ + token: transaction.token, + name: transaction.name, + moduleBytes: transaction.moduleBytes, + globalVisibility: transaction.globalVisibility, + }), + ); + return { + nextHandle: this.handleCounter, + libraries: Array.from( + this.options.loadedLibraries.values(), + (lib) => this.stateForLibrary(lib), + ), + ...(transactions.length === 0 ? {} : { transactions }), + }; + } + + /** + * Materialize every archived module recipe into this Worker's table graph. + * + * This does not recreate user-visible dlopen handles: pthread replicas need + * callable functions and activation catalogs, while the process-wide handle + * snapshot remains owned by the archive/main API. Existing instances are + * verified rather than re-instantiated, so a generation check makes the + * steady-state path O(1). + */ + reconcileForkModules(state: DylinkForkState): void { + const archivedNames = new Set(); + let visibilityChanged = false; + let dependencyStateChanged = false; + for (const archived of state.libraries) { + if (archivedNames.has(archived.name)) { + throw new Error( + `${archived.name}: duplicate module in dynamic-linker table recipe state`, + ); + } + archivedNames.add(archived.name); + const live = this.options.loadedLibraries.get(archived.name); + if (live) { + // The immutable instance recipe must never drift, but a peer + // publication can legitimately advance TLS discovery and the issued + // initializer while this Worker's generator is suspended. + this.requireForkLibraryIdentity(live, archived, true); + if ( + live.globalVisibility !== archived.globalVisibility + || live.committedGlobalRoot !== archived.committedGlobalRoot + ) { + live.globalVisibility = archived.globalVisibility; + live.committedGlobalRoot = archived.committedGlobalRoot; + visibilityChanged = true; + } + const liveProviders = [...(live.providerDependencies ?? [])].sort(); + const archivedProviders = [...(archived.providerDependencies ?? [])] + .sort(); + if ( + liveProviders.length !== archivedProviders.length + || liveProviders.some( + (dependency, index) => dependency !== archivedProviders[index], + ) + ) { + live.providerDependencies = new Set(archivedProviders); + dependencyStateChanged = true; } } - // Not in table yet — add it - const idx = length; - growTable(table, 1); - setTableEntry(table, idx, exp as unknown as Function); - this.lastError = null; - return idx; } - - if (exp instanceof WebAssembly.Global) { - this.lastError = null; - return Number(exp.value); + if (visibilityChanged) this.rebuildRuntimeIndexes(); + + this.restorePendingDlopenTransactions(state); + + // A pthread Worker can observe dlclose after it materialized an earlier + // generation. Remove consumers before providers, clear their exact table + // slots, and release activation-owned reference catalogs before loading + // anything from the new authoritative closure. + const stale = [...this.options.loadedLibraries.values()] + .filter((lib) => !archivedNames.has(lib.name)) + .reverse(); + for (const lib of stale) { + const handle = this.libraryHandles.get(lib.name); + if (handle !== undefined) { + this.handleMap.delete(handle); + this.handleRefCounts.delete(handle); + this.libraryHandles.delete(lib.name); + } + this.clearLibraryTableEntries(lib); + this.options.loadedLibraries.delete(lib.name); + lib.unregisterForkActivation?.(); + for (const allocation of lib.allocations ?? []) { + // The publishing pthread already performed munmap. This replica only + // drops the copied Worker-local allocator index. + this.options.forgetMemoryAllocation?.(allocation); + } + lib.allocations = []; } - - this.lastError = `symbol not found: ${symbolName}`; - return null; - } - - /** Look up a symbol by name. Returns its function-table index or data address. */ - dlsym(handle: number, symbolName: string): number | null { - if (handle === DynamicLinker.MAIN_PROGRAM_HANDLE || handle === 0) { - return this.symbolAddress(symbolName, this.options.globalSymbols.get(symbolName)); + if (stale.length > 0) { + this.rebuildRuntimeIndexes(); } - const lib = this.handleMap.get(handle); - if (!lib) { - this.lastError = "invalid handle"; - return null; + for (const archived of state.libraries) { + if (this.options.loadedLibraries.has(archived.name)) continue; + if (archived.initialization !== undefined) continue; + this.loadModuleSync( + archived.name, + new Uint8Array(archived.moduleBytes), + { + memoryBase: archived.memoryBase, + tableBase: archived.tableBase, + activationId: archived.activationId, + tlsBase: archived.tlsBase, + globalVisibility: archived.globalVisibility, + committedGlobalRoot: archived.committedGlobalRoot, + providerDependencies: archived.providerDependencies, + allocations: archived.allocations, + }, + archived.globalVisibility, + false, + ); + } + // Replay can restore a constructor-created provider edge whose provider + // appears later in insertion order. Account lifetimes only after the exact + // module closure has been materialized. + this.rebuildDependencyBookkeeping(); + if (dependencyStateChanged) this.rebuildRuntimeIndexes(); + for (const archived of state.libraries) { + const live = this.options.loadedLibraries.get(archived.name); + if (!live) { + throw new Error( + `${archived.name}: dynamic-linker reconciliation lost its live instance`, + ); + } + this.requireForkLibraryIdentity(live, archived); } - - const exp = lib.exports[symbolName]; - return this.symbolAddress( - symbolName, - typeof exp === "function" || exp instanceof WebAssembly.Global - ? exp - : this.options.globalSymbols.get(symbolName), - ); } - /** Close a library handle. Returns 0 on success. */ - dlclose(handle: number): number { - if (handle === DynamicLinker.MAIN_PROGRAM_HANDLE) { - this.lastError = null; - return 0; + private restorePendingDlopenTransactions(state: DylinkForkState): void { + const transactions = state.transactions ?? []; + const archivedModules = new Map( + state.libraries.map((library) => [library.name, library]), + ); + const transactionStates = new Map(); + const activeStates = new Map(); + for (const transaction of transactions) { + if (transactionStates.has(transaction.token)) { + throw new Error( + `duplicate staged dlopen transaction ${transaction.token}`, + ); + } + transactionStates.set(transaction.token, transaction); } - if (!this.handleMap.has(handle)) { - this.lastError = "invalid handle"; - return -1; + for (const library of state.libraries) { + const initialization = library.initialization; + if (!initialization) continue; + if (!transactionStates.has(initialization.transactionToken)) { + throw new Error( + `${library.name}: issued initializer names missing staged dlopen ` + + `transaction ${initialization.transactionToken}`, + ); + } + if (activeStates.has(initialization.transactionToken)) { + throw new Error( + `staged dlopen transaction ${initialization.transactionToken} has ` + + "multiple issued entries", + ); + } + activeStates.set(initialization.transactionToken, library); } - this.handleMap.delete(handle); - this.lastError = null; - return 0; + for (const transaction of transactions) { + if (!activeStates.has(transaction.token)) { + throw new Error( + `staged dlopen transaction ${transaction.token} has no issued entry`, + ); + } + } + + const bytesEqual = ( + left: Readonly, + right: Readonly, + ): boolean => + left.length === right.length + && left.every((byte, index) => byte === right[index]); + const clearIssuedMarker = ( + transaction: PendingDlopenTransaction, + ): void => { + const current = transaction.currentStep; + if (!current) return; + const live = this.options.loadedLibraries.get(current.libraryName); + if ( + live?.initialization?.transactionToken === transaction.token + ) { + delete live.initialization; + } + transaction.currentStep = undefined; + transaction.awaitingCompletion = false; + }; + const clearTransactionTableEntry = ( + transaction: PendingDlopenTransaction, + ): void => { + if (transaction.tableIndex === undefined) return; + setTableEntry(this.options.table, transaction.tableIndex, null); + this.options.onTableMutation?.( + this.options.table, + transaction.tableIndex, + 1, + ); + }; + const installIssuedStep = ( + transaction: PendingDlopenTransaction, + archived: DylinkForkLibraryState, + ): void => { + const initialization = archived.initialization; + const current = transaction.currentStep; + if ( + !initialization + || !current + || current.libraryName !== archived.name + || current.stage !== initialization.stage + || transaction.tableIndex !== initialization.tableIndex + ) { + throw new Error( + `staged dlopen transaction ${transaction.token} could not ` + + `reconstruct ${archived.name}:${initialization?.stage ?? "none"}`, + ); + } + const tableLengthBefore = tableLength(this.options.table); + if (tableLengthBefore <= initialization.tableIndex) { + growTable( + this.options.table, + initialization.tableIndex + 1 - tableLengthBefore, + ); + } + setTableEntry( + this.options.table, + initialization.tableIndex, + current.invoke as unknown as Function, + ); + this.options.onTableMutation?.( + this.options.table, + initialization.tableIndex, + 1, + ); + const provisional = this.options.loadedLibraries.get(archived.name); + if (!provisional || provisional.loadState !== "initializing") { + throw new Error( + `${archived.name}: staged dlopen replay lost its provisional module`, + ); + } + provisional.providerDependencies = new Set( + archived.providerDependencies ?? [], + ); + provisional.initialization = Object.freeze({ ...initialization }); + transaction.awaitingCompletion = true; + }; + const refreshReplayModules = ( + transaction: PendingDlopenTransaction, + ): void => { + if (!transaction.replayModules) return; + transaction.replayModules.clear(); + for (const [name, library] of archivedModules) { + transaction.replayModules.set(name, library); + } + }; + const advanceWithoutGuestCalls = ( + transaction: PendingDlopenTransaction, + target?: DylinkForkLibraryState, + ): void => { + refreshReplayModules(transaction); + for (;;) { + if ( + target + && transaction.currentStep?.libraryName === target.name + && transaction.currentStep.stage === target.initialization?.stage + ) { + installIssuedStep(transaction, target); + return; + } + + const completedStep = transaction.currentStep; + const resumeState = completedStep + ? archivedModules.get(completedStep.libraryName) + : undefined; + clearIssuedMarker(transaction); + const cursor = transaction.steps.next(resumeState); + if (cursor.done) { + transaction.loaded = cursor.value; + clearTransactionTableEntry(transaction); + if (target) { + throw new Error( + `${target.name}: staged dlopen replay could not reach ` + + `${target.initialization?.stage ?? "an issued step"}`, + ); + } + return; + } + transaction.currentStep = cursor.value; + transaction.awaitingCompletion = true; + } + }; + const discardRolledBackTransaction = ( + transaction: PendingDlopenTransaction, + ): void => { + clearIssuedMarker(transaction); + clearTransactionTableEntry(transaction); + try { + transaction.steps.throw( + new Error( + `${transaction.name}: peer publication rolled back staged dlopen`, + ), + ); + } catch { + // The generator reports the synthetic rollback cause after releasing + // its activation/table/symbol ownership. The authoritative archive + // state, not that local exception, determines reconciliation. + } + }; + + for (const transaction of [...this.pendingDlopens.values()]) { + const archivedTransaction = transactionStates.get(transaction.token); + if (!archivedTransaction) { + const committed = archivedModules.get(transaction.name); + if (committed?.handle !== undefined) { + advanceWithoutGuestCalls(transaction); + } else { + discardRolledBackTransaction(transaction); + } + this.pendingDlopens.delete(transaction.token); + continue; + } + if ( + transaction.name !== archivedTransaction.name + || transaction.globalVisibility !== archivedTransaction.globalVisibility + || !bytesEqual(transaction.moduleBytes, archivedTransaction.moduleBytes) + ) { + throw new Error( + `staged dlopen transaction ${transaction.token} changed identity`, + ); + } + const target = activeStates.get(transaction.token)!; + if ( + transaction.tableIndex !== undefined + && transaction.tableIndex !== target.initialization!.tableIndex + ) { + throw new Error( + `staged dlopen transaction ${transaction.token} changed its ` + + "initialization table slot", + ); + } + transaction.tableIndex ??= target.initialization!.tableIndex; + advanceWithoutGuestCalls(transaction, target); + } + + for (const transactionState of transactions) { + if (this.pendingDlopens.has(transactionState.token)) continue; + const archived = activeStates.get(transactionState.token)!; + const initialization = archived.initialization!; + for (const prior of state.libraries) { + if (prior.name === archived.name) break; + if ( + this.options.loadedLibraries.has(prior.name) + || prior.initialization !== undefined + ) { + continue; + } + this.loadModuleSync( + prior.name, + new Uint8Array(prior.moduleBytes), + { + memoryBase: prior.memoryBase, + tableBase: prior.tableBase, + activationId: prior.activationId, + tlsBase: prior.tlsBase, + globalVisibility: prior.globalVisibility, + committedGlobalRoot: prior.committedGlobalRoot, + providerDependencies: prior.providerDependencies, + allocations: prior.allocations, + }, + prior.globalVisibility, + false, + ); + } + const replayModules = new Map(archivedModules); + const initialLibraries = new Set(this.options.loadedLibraries.values()); + const ownedLibraries = new Set(); + const steps = loadSharedLibrarySyncSteps( + transactionState.name, + new Uint8Array(transactionState.moduleBytes), + this.options, + undefined, + replayModules, + { ownedLibraries }, + transactionState.globalVisibility, + ); + const cursor = steps.next(); + if (cursor.done) { + throw new Error( + `${archived.name}: staged dlopen replay completed before ` + + `${initialization.stage}`, + ); + } + const tableLengthBefore = tableLength(this.options.table); + if (tableLengthBefore <= initialization.tableIndex) { + growTable( + this.options.table, + initialization.tableIndex + 1 - tableLengthBefore, + ); + } + const pending: PendingDlopenTransaction = { + token: transactionState.token, + name: transactionState.name, + moduleBytes: new Uint8Array(transactionState.moduleBytes), + globalVisibility: transactionState.globalVisibility, + steps, + replayModules, + initialLibraries, + initialVisibility: new Map( + Array.from( + initialLibraries, + (library) => [library, library.globalVisibility], + ), + ), + ownedLibraries, + initialHeapPointer: this.options.heapPointer?.value, + tableIndex: initialization.tableIndex, + awaitingCompletion: true, + currentStep: cursor.value, + }; + advanceWithoutGuestCalls(pending, archived); + this.pendingDlopens.set(transactionState.token, pending); + this.pendingTokenCounter = Math.max( + this.pendingTokenCounter, + transactionState.token + 1, + ); + } + } + + /** + * Restore the compact user-visible handle index after every archived module + * has been instantiated dependency-first. + * + * WHY: module-load order and dlopen-handle order are different domains. + * Dependencies are instantiated before their consumers, while handles are + * allocated only for explicit dlopen calls; final dlclose also leaves + * permanent gaps in the monotonic handle sequence. Replaying synthetic + * open/close events would either invent history or allocate the wrong next + * handle. The copied snapshot is the reconstruction owner instead. + */ + restoreForkHandleState(state: DylinkForkState): void { + this.applyForkHandleState(state, true); + } + + /** + * Replace this Worker's local handle index with the process publication. + * + * Pthread Workers can observe many generations, so unlike one-shot child + * replay this operation deliberately accepts an already populated index. + */ + reconcileForkHandleState(state: DylinkForkState): void { + this.applyForkHandleState(state, false); + } + + private applyForkHandleState( + state: DylinkForkState, + requirePristine: boolean, + ): void { + if ( + requirePristine + && ( + this.handleCounter !== DynamicLinker.MAIN_PROGRAM_HANDLE + 1 + || this.handleMap.size !== 0 + || this.libraryHandles.size !== 0 + || this.handleRefCounts.size !== 0 + ) + ) { + throw new Error( + "dynamic-linker fork handle state requires a pristine child handle index", + ); + } + if ( + !Number.isSafeInteger(state.nextHandle) + || state.nextHandle < DynamicLinker.MAIN_PROGRAM_HANDLE + 1 + || state.nextHandle > 0x1_0000_0000 + ) { + throw new RangeError( + `dynamic-linker fork next handle ${String(state.nextHandle)} is invalid`, + ); + } + + const liveByName = this.options.loadedLibraries; + if (state.libraries.length !== liveByName.size) { + throw new Error( + "dynamic-linker fork state does not describe the exact live module closure", + ); + } + + const restoredHandles = new Map(); + const restoredLibraryHandles = new Map(); + const restoredRefCounts = new Map(); + const seenNames = new Set(); + for (const archived of state.libraries) { + if (seenNames.has(archived.name)) { + throw new Error( + `${archived.name}: duplicate module in dynamic-linker fork state`, + ); + } + seenNames.add(archived.name); + const live = liveByName.get(archived.name); + if (!live) { + throw new Error( + `${archived.name}: dynamic-linker fork state has no live replay instance`, + ); + } + this.requireForkLibraryIdentity(live, archived); + + const hasHandle = archived.handle !== undefined; + if (hasHandle !== (archived.refCount !== undefined)) { + throw new Error( + `${archived.name}: dynamic-linker fork handle/refcount presence is inconsistent`, + ); + } + if (!hasHandle) continue; + const handle = archived.handle!; + const refCount = archived.refCount!; + if ( + !Number.isInteger(handle) + || handle <= DynamicLinker.MAIN_PROGRAM_HANDLE + || handle >= state.nextHandle + || handle > 0xffff_ffff + ) { + throw new RangeError( + `${archived.name}: dynamic-linker fork handle ${String(handle)} is invalid`, + ); + } + if (!Number.isInteger(refCount) || refCount <= 0 || refCount > 0xffff_ffff) { + throw new RangeError( + `${archived.name}: dynamic-linker fork refcount ${String(refCount)} is invalid`, + ); + } + if (restoredHandles.has(handle)) { + throw new Error( + `${archived.name}: duplicate dynamic-linker fork handle ${handle}`, + ); + } + restoredHandles.set(handle, live); + restoredLibraryHandles.set(archived.name, handle); + restoredRefCounts.set(handle, refCount); + } + for (const name of liveByName.keys()) { + if (!seenNames.has(name)) { + throw new Error( + `${name}: live replay module is missing from dynamic-linker fork state`, + ); + } + } + + this.handleMap = restoredHandles; + this.libraryHandles = restoredLibraryHandles; + this.handleRefCounts = restoredRefCounts; + this.handleCounter = state.nextHandle; + this.lastError = null; + } + + private requireForkLibraryIdentity( + live: LoadedSharedLibrary, + archived: DylinkForkLibraryState, + allowInitializationTransition = false, + ): void { + const liveProviders = [...(live.providerDependencies ?? [])].sort(); + const archivedProviders = [...(archived.providerDependencies ?? [])].sort(); + const liveAllocations = [...(live.allocations ?? [])]; + const archivedAllocations = [...(archived.allocations ?? [])]; + if ( + live.memoryBase !== archived.memoryBase + || live.tableBase !== archived.tableBase + || live.activationId !== archived.activationId + || liveAllocations.length !== archivedAllocations.length + || liveAllocations.some((allocation, index) => { + const expected = archivedAllocations[index]; + return ( + expected === undefined + || allocation.address !== expected.address + || allocation.size !== expected.size + || allocation.mappingAddress !== expected.mappingAddress + || allocation.mappingSize !== expected.mappingSize + ); + }) + || ( + !allowInitializationTransition + && ( + live.globalVisibility !== archived.globalVisibility + || live.committedGlobalRoot !== archived.committedGlobalRoot + || liveProviders.length !== archivedProviders.length + || liveProviders.some( + (dependency, index) => dependency !== archivedProviders[index], + ) + || live.tlsBase !== archived.tlsBase + || live.initialization?.transactionToken + !== archived.initialization?.transactionToken + || live.initialization?.stage !== archived.initialization?.stage + || live.initialization?.tableIndex + !== archived.initialization?.tableIndex + || (live.loadState === "initializing") + !== (archived.initialization !== undefined) + ) + ) + || live.moduleBytes.length !== archived.moduleBytes.length + || !live.moduleBytes.every( + (byte, index) => byte === archived.moduleBytes[index], + ) + ) { + throw new Error( + `${archived.name}: dynamic-linker replay instance does not match its fork state`, + ); + } + } + + /** Return the stable opaque handle used by dlopen(NULL, ...). */ + dlopenMain(): number { + this.lastError = null; + return DynamicLinker.MAIN_PROGRAM_HANDLE; + } + + /** + * Begin one process-driven dlopen transaction without entering guest code. + * + * The returned token is private to libc's prepare/next/commit loop and is + * never exposed as a user-visible dlopen handle. + */ + beginDlopenSync( + name: string, + wasmBytes: Uint8Array, + globalVisibility = true, + ): number { + try { + const token = requireNonzeroU32( + this.pendingTokenCounter, + `${name}: next staged dlopen token`, + ); + if (token === 0xffff_ffff) { + throw new RangeError(`${name}: staged dlopen token space is exhausted`); + } + this.pendingTokenCounter = token + 1; + const ownedBytes = wasmBytes.slice(); + const initialLibraries = new Set(this.options.loadedLibraries.values()); + const initialVisibility = new Map( + Array.from( + initialLibraries, + (library) => [library, library.globalVisibility], + ), + ); + const ownedLibraries = new Set(); + this.pendingDlopens.set(token, { + token, + name, + moduleBytes: ownedBytes, + globalVisibility, + steps: loadSharedLibrarySyncSteps( + name, + ownedBytes, + this.options, + undefined, + undefined, + { ownedLibraries }, + globalVisibility, + ), + initialLibraries, + initialVisibility, + ownedLibraries, + initialHeapPointer: this.options.heapPointer?.value, + awaitingCompletion: false, + }); + this.lastError = null; + return token; + } catch (error) { + this.lastError = error instanceof Error ? error.message : String(error); + return 0; + } + } + + /** + * Acknowledge the previously returned `() -> ()` entry and select the next. + * + * Zero means initialization is complete. The selected function remains + * rooted in one transaction-owned table slot until the following call. + */ + nextDlopenInitialization(token: number): number { + const transaction = this.pendingDlopens.get(token); + if (!transaction) { + this.lastError = `invalid staged dlopen token ${String(token)}`; + return -1; + } + try { + if (transaction.currentStep) { + const previous = this.options.loadedLibraries.get( + transaction.currentStep.libraryName, + ); + if ( + previous?.initialization?.transactionToken === transaction.token + ) { + delete previous.initialization; + } + transaction.currentStep = undefined; + } + transaction.awaitingCompletion = false; + const cursor = transaction.steps.next(); + if (cursor.done) { + transaction.loaded = cursor.value; + if (transaction.tableIndex !== undefined) { + setTableEntry(this.options.table, transaction.tableIndex, null); + this.options.onTableMutation?.( + this.options.table, + transaction.tableIndex, + 1, + ); + } + this.lastError = null; + return 0; + } + + let index = transaction.tableIndex; + if (index === undefined) { + index = tableLength(this.options.table); + if (index === 0) { + growTable(this.options.table, 1); + index = 1; + } + growTable(this.options.table, 1); + transaction.tableIndex = index; + } + if (!Number.isSafeInteger(index) || index <= 0 || index > 0x7fff_ffff) { + throw new RangeError( + `${transaction.name}: initialization table index ${String(index)} is invalid`, + ); + } + setTableEntry( + this.options.table, + index, + cursor.value.invoke as unknown as Function, + ); + this.options.onTableMutation?.(this.options.table, index, 1); + transaction.awaitingCompletion = true; + transaction.currentStep = cursor.value; + const provisional = this.options.loadedLibraries.get( + cursor.value.libraryName, + ); + if (!provisional || provisional.loadState !== "initializing") { + throw new Error( + `${cursor.value.libraryName}: initialization step has no provisional module`, + ); + } + provisional.initialization = Object.freeze({ + transactionToken: token, + stage: cursor.value.stage, + tableIndex: index, + }); + this.lastError = null; + return index; + } catch (error) { + this.abortDlopenTransaction(token, error); + return -1; + } + } + + /** + * Advance one staged load and atomically publish its public handle on finish. + * + * The process import uses this combined transition so no guest instruction + * can observe a completed generator whose transaction is still archived as + * an issued initializer. The separate next/commit methods remain useful to + * standalone embedders and focused state-machine tests. + */ + advanceDlopenSync(token: number): Readonly<{ + entry: number; + handle: number; + }> { + const entry = this.nextDlopenInitialization(token); + if (entry !== 0) return { entry, handle: 0 }; + const handle = this.commitDlopenSync(token); + return handle > 0 + ? { entry: 0, handle } + : { entry: -1, handle: 0 }; + } + + hasPendingDlopen(token: number): boolean { + return this.pendingDlopens.has(token); + } + + /** Commit the fully initialized module closure and return its stable handle. */ + commitDlopenSync(token: number): number { + const transaction = this.pendingDlopens.get(token); + if (!transaction) { + this.lastError = `invalid staged dlopen token ${String(token)}`; + return 0; + } + if (transaction.awaitingCompletion || !transaction.loaded) { + this.lastError = + `${transaction.name}: staged dlopen committed before initialization completed`; + return 0; + } + try { + this.registerDependencyEdges(); + if (transaction.globalVisibility) { + promoteLibraryGlobal(transaction.loaded, this.options); + transaction.loaded.committedGlobalRoot = true; + refreshGlobalGotEntries(this.options); + } + const handle = this.openLoadedLibrary(transaction.loaded); + this.pendingDlopens.delete(token); + this.lastError = null; + return handle; + } catch (error) { + this.abortDlopenTransaction(token, error); + return 0; + } + } + + private rollbackDlopenLibraries( + transaction: PendingDlopenTransaction, + ): unknown[] { + const failures: unknown[] = []; + const rolledBack = new Set(transaction.ownedLibraries); + const invalidNames = new Set( + Array.from(rolledBack, (library) => library.name), + ); + const loadedNow = [...this.options.loadedLibraries.values()]; + + // A constructor can complete a nested, independent dlopen before its outer + // initializer fails. Keep that nested transaction unless it captured an + // outer symbol or has a NEEDED edge into the failed closure. + let changed = true; + while (changed) { + changed = false; + for (const library of loadedNow) { + if ( + transaction.initialLibraries.has(library) + || rolledBack.has(library) + ) { + continue; + } + const dependsOnInvalid = [ + ...library.metadata.neededDynlibs, + ...(library.providerDependencies ?? []), + ].some((dependency) => invalidNames.has(dependency)); + if (!dependsOnInvalid) continue; + rolledBack.add(library); + invalidNames.add(library.name); + changed = true; + } + } + + for (const library of loadedNow.reverse()) { + if ( + !rolledBack.has(library) + || this.options.loadedLibraries.get(library.name) !== library + ) { + continue; + } + const handle = this.libraryHandles.get(library.name); + if (handle !== undefined) { + this.handleMap.delete(handle); + this.handleRefCounts.delete(handle); + this.libraryHandles.delete(library.name); + } + try { + this.clearLibraryTableEntries(library); + } catch (error) { + failures.push(error); + } + this.options.loadedLibraries.delete(library.name); + try { + library.unregisterForkActivation?.(); + } catch (error) { + failures.push(error); + } + if (this.options.deallocateMemory) { + for (const allocation of [...(library.allocations ?? [])].reverse()) { + try { + this.options.deallocateMemory( + allocation.address, + allocation.size, + ); + } catch (error) { + failures.push(error); + } + } + } + } + transaction.ownedLibraries.clear(); + + if ( + this.options.heapPointer + && transaction.initialHeapPointer !== undefined + ) { + let retainedEnd = transaction.initialHeapPointer; + for (const library of this.options.loadedLibraries.values()) { + if (transaction.initialLibraries.has(library)) continue; + retainedEnd = Math.max( + retainedEnd, + library.heapReservationEnd ?? retainedEnd, + ); + } + this.options.heapPointer.value = retainedEnd; + } + // Undo visibility changes made by this transaction, then reapply the + // closure of independently committed GLOBAL roots that survived it. + for (const [library, visibility] of transaction.initialVisibility) { + if (this.options.loadedLibraries.get(library.name) === library) { + library.globalVisibility = visibility; + } + } + for (const library of this.options.loadedLibraries.values()) { + if (library.committedGlobalRoot) { + promoteLibraryGlobal(library, this.options); + } + } + try { + this.rebuildDependencyBookkeeping(); + this.rebuildRuntimeIndexes(); + } catch (error) { + failures.push(error); + } + return failures; + } + + abortDlopenTransaction(token: number, cause?: unknown): void { + const transaction = this.pendingDlopens.get(token); + if (!transaction) return; + this.pendingDlopens.delete(token); + if (transaction.currentStep) { + const provisional = this.options.loadedLibraries.get( + transaction.currentStep.libraryName, + ); + if ( + provisional?.initialization?.transactionToken === transaction.token + ) { + delete provisional.initialization; + } + } + let failure = cause; + if (transaction.tableIndex !== undefined) { + try { + setTableEntry(this.options.table, transaction.tableIndex, null); + this.options.onTableMutation?.( + this.options.table, + transaction.tableIndex, + 1, + ); + } catch (error) { + failure ??= error; + } + } + try { + transaction.steps.throw( + cause ?? new Error(`${transaction.name}: staged dlopen aborted`), + ); + } catch (error) { + failure ??= error; + } + const rollbackFailures = this.rollbackDlopenLibraries(transaction); + if (rollbackFailures.length > 0) { + failure = failure === undefined + ? new AggregateError( + rollbackFailures, + `${transaction.name}: staged dlopen rollback was incomplete`, + ) + : new AggregateError( + [failure, ...rollbackFailures], + `${transaction.name}: staged dlopen rollback was incomplete`, + ); + } + this.lastError = failure instanceof Error + ? failure.message + : String(failure ?? "staged dlopen aborted"); + } + + /** + * Instantiate one module without manufacturing a dlopen handle. + * + * Fork replay records module loads separately from user-visible open/close + * events. This lets NEEDED dependencies be restored in exact parent order + * without accidentally incrementing their handle counts. + */ + loadModuleSync( + name: string, + wasmBytes: Uint8Array, + replay?: DylinkReplayOptions, + globalVisibility = replay?.globalVisibility ?? true, + registerDependencies = true, + ): LoadedSharedLibrary { + const loaded = loadSharedLibrarySync( + name, + wasmBytes, + this.options, + replay, + globalVisibility, + ); + if (replay) { + loaded.committedGlobalRoot = replay.committedGlobalRoot; + loaded.providerDependencies = new Set( + replay.providerDependencies ?? loaded.providerDependencies ?? [], + ); + } + if (registerDependencies) this.registerDependencyEdges(); + this.lastError = null; + return loaded; + } + + private registerDependencyEdges(): void { + // loadedLibraries insertion order is dependency-first. Scanning only the + // unaccounted suffix makes recursive dependency loads cheap while keeping + // the relationship derivable from immutable dylink metadata. + for (const lib of this.options.loadedLibraries.values()) { + if (this.dependencyOwners.has(lib.name)) continue; + for (const dependency of runtimeDependencyNames(lib)) { + if (!this.options.loadedLibraries.has(dependency)) { + throw new Error( + `${lib.name}: loaded dependency ${dependency} has no live provider`, + ); + } + const retains = this.dependencyRetainCounts.get(dependency) ?? 0; + if (retains >= 0xffff_ffff) { + throw new RangeError( + `${dependency}: dynamic-linker dependency retain count overflow`, + ); + } + this.dependencyRetainCounts.set(dependency, retains + 1); + } + this.dependencyOwners.add(lib.name); + } + } + + private rebuildDependencyBookkeeping(): void { + this.dependencyRetainCounts.clear(); + this.dependencyOwners.clear(); + this.registerDependencyEdges(); + } + + private clearLibraryTableEntries(lib: LoadedSharedLibrary): void { + const entries = [...new Set(lib.ownedTableEntries)] + .sort((left, right) => left - right); + const length = tableLength(this.options.table); + for (const index of entries) { + if ( + !Number.isSafeInteger(index) + || index < 0 + || index >= length + ) { + throw new Error( + `${lib.name}: owned table entry ${String(index)} is out of bounds`, + ); + } + } + for (const index of entries) { + setTableEntry(this.options.table, index, null); + } + for (let first = 0; first < entries.length;) { + let end = first + 1; + while (end < entries.length && entries[end] === entries[end - 1]! + 1) { + end++; + } + this.options.onTableMutation?.( + this.options.table, + entries[first]!, + entries[end - 1]! - entries[first]! + 1, + ); + first = end; + } + } + + /** + * Rebuild the loader-owned indexes from the exact live module closure. + * + * WHY: retaining an unloaded function in `globalSymbols`, a GOT cell, or a + * process-table slot would keep a stale callable GC root even though the + * archive no longer contains its activation recipe. + */ + private rebuildRuntimeIndexes(): void { + this.options.globalSymbols.clear(); + const owners = symbolOwners(this.options); + owners.clear(); + for (const [name, value] of this.baseGlobalSymbols) { + this.options.globalSymbols.set(name, value); + owners.set(name, this.baseGlobalSymbolOwners.get(name)); + } + for (const lib of this.options.loadedLibraries.values()) { + if (!lib.globalVisibility) continue; + for (const [name, value] of Object.entries(lib.exports)) { + if ( + !isPublicDylinkExport(name, value) + || this.options.globalSymbols.has(name) + ) { + continue; + } + this.options.globalSymbols.set(name, value); + owners.set(name, lib.name); + } + } + + const previousGot = new Map(this.options.got); + const liveGotKinds = new Map(); + for (const lib of this.options.loadedLibraries.values()) { + for (const { name, kind } of lib.gotImports) { + const previous = liveGotKinds.get(name); + if (previous !== undefined && previous !== kind) { + throw new Error( + `live GOT symbol ${name} is both ${previous} and ${kind}`, + ); + } + liveGotKinds.set(name, kind); + } + } + for (const name of this.baseGot.keys()) { + const kind = this.options.gotKinds!.get(name); + if (kind !== undefined) liveGotKinds.set(name, kind); + } + + this.options.got.clear(); + for (const [name, global] of this.baseGot) { + this.options.got.set(name, global); + } + for (const name of liveGotKinds.keys()) { + const global = previousGot.get(name); + if (!global) { + throw new Error(`live GOT symbol ${name} lost its Global cell`); + } + this.options.got.set(name, global); + } + for (const name of [...this.options.gotKinds!.keys()]) { + if (!this.options.got.has(name)) this.options.gotKinds!.delete(name); + } + for (const [name, kind] of liveGotKinds) { + this.options.gotKinds!.set(name, kind); + const global = this.options.got.get(name)!; + const symbol = this.options.globalSymbols.get(name); + if (kind === "mem" && symbol instanceof WebAssembly.Global) { + global.value = symbol.value; + continue; + } + if (kind === "func" && typeof symbol === "function") { + let index = -1; + const length = tableLength(this.options.table); + for (let candidate = 0; candidate < length; candidate++) { + if (getTableEntry(this.options.table, candidate) === symbol) { + index = candidate; + break; + } + } + if (index < 0) { + index = length; + growTable(this.options.table, 1); + setTableEntry(this.options.table, index, symbol); + this.options.onTableMutation?.(this.options.table, index, 1); + } + global.value = wasmAddress( + index, + this.options.ptrWidth ?? 4, + `GOT.func.${name}`, + ); + continue; + } + global.value = wasmAddress( + 0, + this.options.ptrWidth ?? 4, + `unresolved GOT.${kind}.${name}`, + ); + } + } + + private releaseUnretainedLibrary(lib: LoadedSharedLibrary): void { + if (this.libraryHandles.has(lib.name)) return; + if ((this.dependencyRetainCounts.get(lib.name) ?? 0) !== 0) return; + if (this.options.loadedLibraries.get(lib.name) !== lib) return; + + for (const dependency of runtimeDependencyNames(lib)) { + const retains = this.dependencyRetainCounts.get(dependency); + if (!Number.isInteger(retains) || retains! <= 0) { + throw new Error( + `${lib.name}: dependency ${dependency} has no matching retain`, + ); + } + } + // Remove the consumer before releasing its providers so recursive NEEDED + // chains observe the exact remaining live closure. + this.clearLibraryTableEntries(lib); + this.options.loadedLibraries.delete(lib.name); + this.dependencyOwners.delete(lib.name); + lib.unregisterForkActivation?.(); + const releaseFailures: unknown[] = []; + if (this.options.deallocateMemory) { + for (const allocation of [...(lib.allocations ?? [])].reverse()) { + try { + this.options.deallocateMemory(allocation.address, allocation.size); + } catch (error) { + releaseFailures.push(error); + } + } + } + lib.allocations = []; + for (const dependency of runtimeDependencyNames(lib)) { + const retains = this.dependencyRetainCounts.get(dependency)!; + if (retains === 1) this.dependencyRetainCounts.delete(dependency); + else this.dependencyRetainCounts.set(dependency, retains! - 1); + const provider = this.options.loadedLibraries.get(dependency); + if (provider) { + try { + this.releaseUnretainedLibrary(provider); + } catch (error) { + releaseFailures.push(error); + } + } + } + if (releaseFailures.length !== 0) { + throw new AggregateError( + releaseFailures, + `${lib.name}: final unload could not release every process mapping`, + ); + } + } + + private openLoadedLibrary( + lib: LoadedSharedLibrary, + replayHandle?: number, + ): number { + const existingHandle = this.libraryHandles.get(lib.name); + if (existingHandle !== undefined) { + if (this.handleMap.get(existingHandle) !== lib) { + throw new Error( + `${lib.name}: dynamic-linker handle index points at a different instance`, + ); + } + if (replayHandle !== undefined && replayHandle !== existingHandle) { + throw new Error( + `${lib.name}: replay open returned handle ${replayHandle}, ` + + `but the live handle is ${existingHandle}`, + ); + } + const references = this.handleRefCounts.get(existingHandle); + if ( + !Number.isInteger(references) + || references! <= 0 + || references! >= 0xffff_ffff + ) { + throw new Error( + `${lib.name}: dynamic-linker handle ${existingHandle} has invalid refcount`, + ); + } + this.handleRefCounts.set(existingHandle, references! + 1); + this.lastError = null; + return existingHandle; + } + + const handle = requireNonzeroU32( + replayHandle ?? this.handleCounter, + `${lib.name}: ${replayHandle === undefined ? "next" : "replay"} dlopen handle`, + ); + if (handle !== this.handleCounter) { + throw new Error( + `${lib.name}: replay dlopen handle ${handle} does not match ` + + `next handle ${this.handleCounter}`, + ); + } + if (this.handleMap.has(handle)) { + throw new Error(`${lib.name}: replay dlopen handle ${handle} is already in use`); + } + this.handleCounter = handle + 1; + this.handleMap.set(handle, lib); + this.libraryHandles.set(lib.name, handle); + this.handleRefCounts.set(handle, 1); + this.lastError = null; + return handle; + } + + /** Open a shared library. Returns a handle (>0) or 0 on error. + * + * New replay code must use `loadModuleSync` followed by `replayOpen`, because + * copied guest state requires the parent's exact handle rather than a newly + * allocated child handle. The replay option remains here only as a + * layout-preserving convenience for non-archived embedders. + */ + dlopenSync( + name: string, + wasmBytes: Uint8Array, + replay?: DylinkReplayOptions, + globalVisibility = true, + ): number { + try { + const loaded = this.loadModuleSync( + name, + wasmBytes, + replay, + globalVisibility, + ); + if (globalVisibility) { + promoteLibraryGlobal(loaded, this.options); + loaded.committedGlobalRoot = true; + refreshGlobalGotEntries(this.options); + } + return this.openLoadedLibrary(loaded); + } catch (e) { + this.lastError = e instanceof Error ? e.message : String(e); + return 0; + } + } + + /** + * Replay one successful parent dlopen event after its module-load event. + * + * The exact returned handle is part of process state: guest code may retain + * it in copied memory. A mismatch therefore rejects replay instead of + * silently allocating a child-local replacement. + */ + replayOpen(name: string, exactHandle: number): number { + const lib = this.options.loadedLibraries.get(name); + if (!lib) { + throw new Error(`${name}: replay dlopen requires a prior module-load event`); + } + return this.openLoadedLibrary(lib, exactHandle); + } + + private symbolAddress( + symbolName: string, + exp: Function | WebAssembly.Global | undefined, + ): number | null { + if (typeof exp === "function") { + // Return the table index for this function (C function pointers are table indices) + const table = this.options.table; + const length = tableLength(table); + for (let i = 0; i < length; i++) { + if (getTableEntry(table, i) === exp) { + this.lastError = null; + return i; + } + } + // Not in table yet — add it + const idx = length; + growTable(table, 1); + setTableEntry(table, idx, exp as unknown as Function); + this.options.onTableMutation?.(table, idx, 1); + this.lastError = null; + return idx; + } + + if (exp instanceof WebAssembly.Global) { + this.lastError = null; + return Number(exp.value); + } + + this.lastError = `symbol not found: ${symbolName}`; + return null; + } + + private recordConstructorProvider(owner: string | undefined): void { + if (owner === undefined) return; + const active = [...this.pendingDlopens.values()] + .reverse() + .find((transaction) => + transaction.currentStep?.stage === "constructors" + ); + const consumerName = active?.currentStep?.libraryName; + if (!consumerName || consumerName === owner) return; + const consumer = this.options.loadedLibraries.get(consumerName); + if (!consumer) return; + const dependencies = consumer.providerDependencies instanceof Set + ? consumer.providerDependencies + : new Set(consumer.providerDependencies ?? []); + dependencies.add(owner); + consumer.providerDependencies = dependencies; + } + + /** Look up a symbol by name. Returns its function-table index or data address. */ + dlsym(handle: number, symbolName: string): number | null { + if (isForkRuntimeExport(symbolName)) { + this.lastError = `symbol not found: ${symbolName}`; + return null; + } + if (handle === DynamicLinker.MAIN_PROGRAM_HANDLE || handle === 0) { + this.recordConstructorProvider( + symbolOwners(this.options).get(symbolName), + ); + return this.symbolAddress( + symbolName, + this.options.globalSymbols.get(symbolName), + ); + } + + const lib = this.handleMap.get(handle); + if (!lib) { + this.lastError = "invalid handle"; + return null; + } + + const scope: LoadedSharedLibrary[] = []; + const seen = new Set(); + const queue = [lib]; + for (let index = 0; index < queue.length; index++) { + const candidate = queue[index]!; + if (seen.has(candidate.name)) continue; + seen.add(candidate.name); + scope.push(candidate); + for (const dependencyName of candidate.metadata.neededDynlibs) { + const dependency = this.options.loadedLibraries.get(dependencyName); + if (dependency && !seen.has(dependency.name)) queue.push(dependency); + } + } + for (const candidate of scope) { + const exp = candidate.exports[symbolName]; + if (typeof exp !== "function" && !(exp instanceof WebAssembly.Global)) { + continue; + } + this.recordConstructorProvider(candidate.name); + return this.symbolAddress(symbolName, exp); + } + this.lastError = `symbol not found: ${symbolName}`; + return null; + } + + private closeHandle(handle: number): void { + if (handle === DynamicLinker.MAIN_PROGRAM_HANDLE) { + this.lastError = null; + return; + } + const lib = this.handleMap.get(handle); + if (!lib) { + throw new Error(`invalid dlopen handle ${handle}`); + } + if (this.libraryHandles.get(lib.name) !== handle) { + throw new Error( + `${lib.name}: dynamic-linker library index does not match handle ${handle}`, + ); + } + const references = this.handleRefCounts.get(handle); + if (!Number.isInteger(references) || references! <= 0) { + throw new Error(`${lib.name}: handle ${handle} has invalid refcount`); + } + if (references! > 1) { + this.handleRefCounts.set(handle, references! - 1); + this.lastError = null; + return; + } + this.handleMap.delete(handle); + this.libraryHandles.delete(lib.name); + this.handleRefCounts.delete(handle); + this.releaseUnretainedLibrary(lib); + this.rebuildRuntimeIndexes(); + this.lastError = null; + } + + /** Close a library handle. Returns 0 on success. */ + dlclose(handle: number): number { + try { + this.closeHandle(handle); + } catch (error) { + this.lastError = error instanceof Error ? error.message : String(error); + return -1; + } + return 0; + } + + /** Replay one successful parent dlclose event exactly. */ + replayClose(exactHandle: number): void { + requireNonzeroU32(exactHandle, "replay dlclose handle"); + this.closeHandle(exactHandle); } /** Get the last error message, or null if no error. */ diff --git a/host/src/fork-activation-registry.ts b/host/src/fork-activation-registry.ts new file mode 100644 index 0000000000..6707a10de7 --- /dev/null +++ b/host/src/fork-activation-registry.ts @@ -0,0 +1,1790 @@ +import { + WPK_FORK_EXPORT_MODULE_STATE_FINISH_RESTORE, + WPK_FORK_MODULE_STATE_IMPORT_RECORD_COMMIT, + WPK_FORK_MODULE_STATE_IMPORT_RECORD_FIND, + WPK_FORK_MODULE_STATE_IMPORT_RECORD_RESERVE, + WPK_FORK_MODULE_STATE_IMPORT_TABLE_DIRTY_COUNT, + WPK_FORK_MODULE_STATE_IMPORT_TABLE_DIRTY_MARK, + WPK_FORK_MODULE_STATE_IMPORT_TABLE_DIRTY_PAGE, + WPK_FORK_MODULE_STATE_IMPORT_TABLE_STATE_OWNED, + WPK_FORK_MODULE_STATE_TABLE_PAGE_SHIFT, + WPK_FORK_REFERENCE_IMPORT_DECODE_EXTERNREF, + WPK_FORK_REFERENCE_IMPORT_DECODE_FUNCREF, + WPK_FORK_REFERENCE_IMPORT_ENCODE_EXTERNREF, + WPK_FORK_REFERENCE_IMPORT_ENCODE_FUNCREF, + WPK_FORK_REFERENCE_IMPORT_GC_BROKER_ENCODE, + WPK_FORK_REFERENCE_IMPORT_GC_CAPTURE_LAYOUT, + WPK_FORK_REFERENCE_IMPORT_GC_CLAIM, + WPK_FORK_REFERENCE_IMPORT_GC_DEFINE, + WPK_FORK_REFERENCE_IMPORT_GC_I31, + WPK_FORK_REFERENCE_IMPORT_GC_LOAD, + WPK_FORK_REFERENCE_IMPORT_GC_LOOKUP, + WPK_FORK_REFERENCE_IMPORT_GC_PAYLOAD_LEN, + WPK_FORK_REFERENCE_IMPORT_GC_PROVENANCE_BEGIN, + WPK_FORK_REFERENCE_IMPORT_GC_PROVENANCE_END, + WPK_FORK_REFERENCE_IMPORT_GC_PROVENANCE_REF, + WPK_FORK_REFERENCE_IMPORT_GC_ROUTE, + WPK_FORK_REFERENCE_IMPORT_VECTOR_APPEND, + WPK_FORK_REFERENCE_IMPORT_VECTOR_BEGIN, + WPK_FORK_REFERENCE_IMPORT_VECTOR_FINISH, + WPK_FORK_REFERENCE_IMPORT_VECTOR_GET, + WPK_FORK_TABLE_CATALOG_EXPORT_PREFIX, +} from "./generated/abi"; +import { + FORK_ANYREF_TRANSIT_IMPORT, + ForkAnyrefTransitTable, +} from "./fork-anyref-transit"; +import { + ForkModuleStateArena, + ForkModuleStateRecordKind, + ForkTableDirtyTracker, + requireForkModuleTemplate, +} from "./fork-module-state"; +import { + FORK_FUNCTION_CATALOG_EXPORT, + ForkFunctionCatalog, +} from "./fork-function-catalog"; +import type { + DylinkForkTablePatch, + DylinkForkTablePatchRun, +} from "./dylink-fork-archive"; +import { + FORK_GC_LAYOUT_REQUIRES_PROVENANCE, + ForkGcProvenanceRegistry, + forkGcCodecProviderFromInstance, + type ForkGcCodecProvider, +} from "./fork-gc-codec"; +import { + FORK_HOST_EXCEPTION_ACTIVATION_ID, + ForkReferenceTransaction, + type ForkGcDefinitionProvenance, + type ForkExternrefRecipeProvider, + type ForkReferenceScratchAllocate, + type ForkReferenceScratchDeallocate, +} from "./fork-reference-transaction"; +import type { + DecodedSegmentedForkReferenceTransaction, +} from "./fork-reference-segments"; +import { + clearForkStaticRootTable, + FORK_STATIC_ROOT_CATALOG_EXPORT, + FORK_STATIC_ROOT_HARVEST_EXPORT, + ForkStaticRootCatalog, +} from "./fork-static-root-catalog"; + +export const FORK_MODULE_BOOTSTRAP_EXPORT = "wpk_fork_module_bootstrap"; +export const FORK_MODULE_STATE_SAVE_EXPORT = "wpk_fork_module_state_save"; +export const FORK_MODULE_STATE_RESTORE_EXPORT = "wpk_fork_module_state_restore"; +export const FORK_MODULE_STATE_FINISH_RESTORE_EXPORT = + WPK_FORK_EXPORT_MODULE_STATE_FINISH_RESTORE; +export const FORK_MODULE_TABLE_STATE_SAVE_EXPORT = + "wpk_fork_module_table_state_save"; +export const FORK_MODULE_TABLE_STATE_RESTORE_EXPORT = + "wpk_fork_module_table_state_restore"; +export const FORK_MODULE_TABLE_GENERATION_ADDR_IMPORT = + "__wpk_fork_module_state_table_generation_addr"; +export const FORK_MODULE_TABLE_MUTATION_BEGIN_IMPORT = + "__wpk_fork_module_state_table_mutation_begin"; +export const FORK_MODULE_TABLE_MUTATION_COMMIT_IMPORT = + "__wpk_fork_module_state_table_mutation_commit"; +export const FORK_MODULE_TABLE_MUTATION_ABORT_IMPORT = + "__wpk_fork_module_state_table_mutation_abort"; +export const FORK_MODULE_TABLE_RECONCILE_IMPORT = + "__wpk_fork_module_state_table_reconcile"; + +export interface ForkActivationTableReplication { + /** Immutable pointer-width address of the shared generation fence. */ + readonly generationAddress: WebAssembly.Global; + /** + * Acquire the process writer, apply the latest snapshot, and return its + * exact generation. Ownership remains live until commit() or abort(). + */ + beginMutation(): bigint; + /** Apply the latest process snapshot and return its exact generation. */ + reconcile(): bigint; + /** Publish a successful guest mutation and release writer ownership. */ + commit( + activationId: number, + ownerId: number, + firstIndex: number | bigint, + length: number | bigint, + ): void; + /** Release mutation writer ownership after a non-mutating failure/no-op. */ + abort(): void; +} + +export interface ForkActivationExceptionProvider { + /** Throw the exact exception currently rooted in an activation-local slot. */ + throwSlot(slot: number): never; + /** Throw an exception reconstructed from the process recipe graph. */ + throwRecipe(recipeId: number): never; + /** Route a host/JSTag ingress token into the process recipe graph. */ + encodeIngress(token: number): number; + /** Decode/cache a recipe without returning an `exnref` through JavaScript. */ + materialize?(recipeId: number): void; + /** Release transient roots after the outermost replay frame is restored. */ + clear(): void; + /** Release the same roots when capture or replay aborts. */ + abort(): void; +} + +export interface ForkActivationTypedReferenceProvider { + readonly activationId?: number; + readonly descriptor?: ForkGcCodecProvider["descriptor"]; + probe?(slot: number): bigint; + encodeSlot?(slot: number): number; + allocate?(recipeId: number): void; + fill?(recipeId: number): void; + publishExternref?(recipeId: number, value: unknown): void; + /** Release transient GC/reference codec roots after successful replay. */ + clear?(): void; + /** Release the same roots when capture or replay aborts. */ + abort?(): void; +} + +export interface ForkActivationModuleState { + /** Parent-only initialization: active segments followed by the original start. */ + bootstrap(): void; + /** Append this activation's globals, sparse tables, and segment lifetimes. */ + save(activationId: number): void; + /** Restore this activation before any continuation frame executes. */ + restore(activationId: number): void; + /** Drop passive segments after typed constructor replay has completed. */ + finishRestore(activationId: number): void; + /** Append only cumulative sparse table state to a peer-replication arena. */ + saveTables(activationId: number): void; + /** Restore only cumulative sparse table state in another Worker instance. */ + restoreTables(activationId: number): void; +} + +export interface ForkActivationRegistration { + readonly activationId: number; + readonly instance: WebAssembly.Instance; + /** SHA-256 of the exact instrumented module bytes. */ + readonly templateId: Uint8Array; + readonly functionCatalog: WebAssembly.Table; + /** + * Immutable GC/reference roots recreated by this exact instantiation. + * + * Recipes name these by activation and ordinal so `ref.eq` aliases resolve + * to the child's own canonical root instead of a structural clone. + */ + readonly staticRootCatalog: WebAssembly.Table; + /** Populate the one-shot static-root observation table before registration. */ + readonly staticRootHarvest: () => void; + readonly moduleState: ForkActivationModuleState; + readonly exceptionProvider?: ForkActivationExceptionProvider; + readonly typedReferenceProvider?: ForkActivationTypedReferenceProvider; + /** + * One journal per activation because generated owner ordinals are local to + * an artifact. Imported/shared table identity is deduplicated by the loader + * before it binds an activation to a journal. + */ + readonly tableDirty: ForkTableDirtyTracker; +} + +/** + * Replay-only scalar callbacks needed while a fresh child is still + * instantiating its activation graph. + * + * Capture callbacks deliberately remain registry-owned. Before `attachChild` + * there is no capture transaction, so invoking one is a phase error rather + * than an invitation to mutate the copied recipe graph. + */ +export interface ForkActivationReferenceReplayImports { + decodeFuncref(recipeId: number): CallableFunction | null; + decodeExternref(recipeId: number): unknown; + getReferenceVector(ordinal: number, index: number): number; + routeGc(recipeId: number, expectedActivation: number): number; + gcPayloadLength( + recipeId: number, + expectedActivation: number, + expectedLayoutId: number, + ): number; + loadGc( + recipeId: number, + moduleActivation: number, + typeOrdinal: number, + layoutId: number, + kind: number, + scalarDestination: number | bigint, + scalarByteLength: number, + ): number; +} + +type RegistryPhase = + | "idle" + | "capture" + | "table-capture" + | "sealed-parent" + | "parent-replay" + | "child-replay" + | "table-replay"; + +function assertActivationId(value: number): void { + if (!Number.isInteger(value) || value < 0 || value > 0xffff_ffff) { + throw new RangeError(`invalid fork module activation id ${value}`); + } +} + +function checkedTableMutationIndex( + value: number | bigint, + context: string, +): bigint { + if ( + typeof value === "number" + && (!Number.isSafeInteger(value) || value < 0) + ) { + throw new RangeError(`${context} must be an exact non-negative integer`); + } + const result = typeof value === "bigint" ? value : BigInt(value); + if (result < 0n || result >= (1n << 64n)) { + throw new RangeError(`${context} exceeds the WebAssembly table64 index space`); + } + return result; +} + +interface ForkRegisteredTableCoordinate { + readonly activationId: number; + readonly ownerId: number; + readonly tracker: ForkTableDirtyTracker; +} + +interface ForkActivationTableCatalogEntry { + readonly ownerId: number; + readonly table: WebAssembly.Table; +} + +function activationTableCatalog( + registration: ForkActivationRegistration, + label: string, +): ForkActivationTableCatalogEntry[] { + const entries: ForkActivationTableCatalogEntry[] = []; + for (const [name, value] of Object.entries(registration.instance.exports)) { + if (!name.startsWith(WPK_FORK_TABLE_CATALOG_EXPORT_PREFIX)) continue; + const suffix = name.slice(WPK_FORK_TABLE_CATALOG_EXPORT_PREFIX.length); + const ownerId = Number(suffix); + if ( + !/^[1-9][0-9]*$/.test(suffix) + || !Number.isSafeInteger(ownerId) + || ownerId > 0xffff_ffff + ) { + throw new Error(`${label}: malformed private table catalog export ${name}`); + } + if (!(value instanceof WebAssembly.Table)) { + throw new Error(`${label}: private table catalog ${name} is not a Table`); + } + entries.push({ ownerId, table: value }); + } + entries.sort((left, right) => left.ownerId - right.ownerId); + return entries; +} + +function copyTemplateId(value: Uint8Array): Uint8Array { + if (!(value instanceof Uint8Array) || value.byteLength !== 32) { + throw new TypeError("fork module template id must contain exactly 32 bytes"); + } + return value.slice(); +} + +function requireExportFunction( + instance: WebAssembly.Instance, + name: string, +): CallableFunction { + const value = instance.exports[name]; + if (typeof value !== "function") { + throw new Error(`fork module activation is missing function export ${name}`); + } + return value as CallableFunction; +} + +function requireExportTable( + instance: WebAssembly.Instance, + name: string, +): WebAssembly.Table { + const value = instance.exports[name]; + if (!(value instanceof WebAssembly.Table)) { + throw new Error(`fork module activation is missing table export ${name}`); + } + return value; +} + +/** + * Resolve the uniform ABI 43 activation exports after instantiation. + * + * Keeping this reflection in one place makes main modules, pthread instances, + * and dlopen activations obey the same state-ownership contract. + */ +export function forkActivationRegistrationFromInstance(options: { + activationId: number; + module?: WebAssembly.Module; + instance: WebAssembly.Instance; + templateId: Uint8Array; + tableDirty?: ForkTableDirtyTracker; + exceptionProvider?: ForkActivationExceptionProvider; + typedReferenceProvider?: ForkActivationTypedReferenceProvider; +}): ForkActivationRegistration { + const { + activationId, + instance, + exceptionProvider, + typedReferenceProvider, + } = options; + assertActivationId(activationId); + const bootstrap = requireExportFunction(instance, FORK_MODULE_BOOTSTRAP_EXPORT); + const save = requireExportFunction(instance, FORK_MODULE_STATE_SAVE_EXPORT); + const restore = requireExportFunction(instance, FORK_MODULE_STATE_RESTORE_EXPORT); + const finishRestore = requireExportFunction( + instance, + FORK_MODULE_STATE_FINISH_RESTORE_EXPORT, + ); + const saveTables = requireExportFunction( + instance, + FORK_MODULE_TABLE_STATE_SAVE_EXPORT, + ); + const restoreTables = requireExportFunction( + instance, + FORK_MODULE_TABLE_STATE_RESTORE_EXPORT, + ); + const harvestStaticRoots = requireExportFunction( + instance, + FORK_STATIC_ROOT_HARVEST_EXPORT, + ); + return { + activationId, + instance, + templateId: copyTemplateId(options.templateId), + functionCatalog: requireExportTable(instance, FORK_FUNCTION_CATALOG_EXPORT), + staticRootCatalog: requireExportTable( + instance, + FORK_STATIC_ROOT_CATALOG_EXPORT, + ), + staticRootHarvest: () => { harvestStaticRoots(); }, + moduleState: { + bootstrap: () => { bootstrap(); }, + save: (id) => { save(id); }, + restore: (id) => { restore(id); }, + finishRestore: (id) => { finishRestore(id); }, + saveTables: (id) => { saveTables(id); }, + restoreTables: (id) => { restoreTables(id); }, + }, + exceptionProvider, + typedReferenceProvider: typedReferenceProvider ?? ( + options.module + ? forkGcCodecProviderFromInstance(activationId, options.module, instance) + : undefined + ), + tableDirty: options.tableDirty ?? new ForkTableDirtyTracker(), + }; +} + +/** + * Scalar/JS-callable imports shared by every activation. + * + * Typed GC and exception codecs are activation-local Wasm functions and are + * bound separately by their providers. Keeping this helper to callbacks that + * JavaScript can represent prevents an accidental exnref/anyref round-trip + * through the embedding API. + */ +export function buildForkActivationStateImports( + activationId: number, + registry: ForkActivationRegistry, + referenceReplay: () => ForkActivationReferenceReplayImports = + () => registry.currentReferences(), + tableReplication?: ForkActivationTableReplication, +): Record { + assertActivationId(activationId); + const arena = () => registry.currentArena(); + const references = () => registry.currentReferences(); + const tableDirty = () => registry.tableDirty(activationId); + return { + [FORK_ANYREF_TRANSIT_IMPORT]: registry.gcTransitTable(), + [WPK_FORK_MODULE_STATE_IMPORT_RECORD_RESERVE]: ( + kind: number, + recordActivationId: number, + ownerId: number, + payloadSize: number | bigint, + ) => { + if (recordActivationId !== activationId) { + throw new Error( + `activation ${activationId} cannot reserve module state for ` + + `activation ${recordActivationId}`, + ); + } + return arena().reserveRecord( + kind, + recordActivationId, + ownerId, + payloadSize, + ); + }, + [WPK_FORK_MODULE_STATE_IMPORT_RECORD_COMMIT]: ( + payload: number | bigint, + ): void => arena().commitRecord(payload), + [WPK_FORK_MODULE_STATE_IMPORT_RECORD_FIND]: ( + kind: number, + recordActivationId: number, + ownerId: number, + ordinal: number, + ) => { + if (recordActivationId !== activationId) { + throw new Error( + `activation ${activationId} cannot restore module state for ` + + `activation ${recordActivationId}`, + ); + } + return arena().findRecord( + kind, + recordActivationId, + ownerId, + ordinal, + ); + }, + [WPK_FORK_MODULE_STATE_IMPORT_TABLE_DIRTY_MARK]: ( + ownerId: number, + firstPage: number | bigint, + pageCount: number | bigint, + ): void => tableDirty().markPages(ownerId, firstPage, pageCount), + [WPK_FORK_MODULE_STATE_IMPORT_TABLE_DIRTY_COUNT]: ( + ownerId: number, + ): number => tableDirty().pageCount(ownerId), + [WPK_FORK_MODULE_STATE_IMPORT_TABLE_DIRTY_PAGE]: ( + ownerId: number, + ordinal: number, + ): bigint => tableDirty().pageAt(ownerId, ordinal), + [WPK_FORK_MODULE_STATE_IMPORT_TABLE_STATE_OWNED]: ( + ownerId: number, + ): number => Number(tableDirty().ownsState(ownerId)), + [FORK_MODULE_TABLE_GENERATION_ADDR_IMPORT]: + tableReplication?.generationAddress + ?? new WebAssembly.Global({ value: "i64", mutable: false }, 0n), + [FORK_MODULE_TABLE_RECONCILE_IMPORT]: (): bigint => + tableReplication?.reconcile() ?? 0n, + [FORK_MODULE_TABLE_MUTATION_BEGIN_IMPORT]: (): bigint => + tableReplication?.beginMutation() ?? 0n, + [FORK_MODULE_TABLE_MUTATION_COMMIT_IMPORT]: ( + ownerId: number, + firstIndex: number | bigint, + length: number | bigint, + ): void => { + tableReplication?.commit( + activationId, + ownerId, + firstIndex, + length, + ); + }, + [FORK_MODULE_TABLE_MUTATION_ABORT_IMPORT]: (): void => { + tableReplication?.abort(); + }, + [WPK_FORK_REFERENCE_IMPORT_ENCODE_FUNCREF]: ( + value: unknown, + ): number => references().encodeFuncref(value), + [WPK_FORK_REFERENCE_IMPORT_DECODE_FUNCREF]: ( + recipeId: number, + ): CallableFunction | null => referenceReplay().decodeFuncref(recipeId >>> 0), + [WPK_FORK_REFERENCE_IMPORT_ENCODE_EXTERNREF]: ( + value: unknown, + ): number => references().encodeExternref(value), + [WPK_FORK_REFERENCE_IMPORT_DECODE_EXTERNREF]: ( + recipeId: number, + ): unknown => referenceReplay().decodeExternref(recipeId >>> 0), + [WPK_FORK_REFERENCE_IMPORT_VECTOR_BEGIN]: ( + expectedLength: number, + ): number => references().beginReferenceVector(expectedLength >>> 0), + [WPK_FORK_REFERENCE_IMPORT_VECTOR_APPEND]: ( + handle: number, + recipeId: number, + ): void => references().appendReferenceVector(handle >>> 0, recipeId >>> 0), + [WPK_FORK_REFERENCE_IMPORT_VECTOR_FINISH]: ( + handle: number, + ): number => references().finishReferenceVector(handle >>> 0), + [WPK_FORK_REFERENCE_IMPORT_VECTOR_GET]: ( + ordinal: number, + index: number, + ): number => referenceReplay().getReferenceVector( + ordinal >>> 0, + index >>> 0, + ), + [WPK_FORK_REFERENCE_IMPORT_GC_LOOKUP]: ( + slot: number, + ): number => registry.lookupGcSlot(activationId, slot), + [WPK_FORK_REFERENCE_IMPORT_GC_CLAIM]: ( + slot: number, + ): number => registry.claimGcSlot(slot), + [WPK_FORK_REFERENCE_IMPORT_GC_I31]: ( + value: number, + ): number => registry.encodeI31(value), + [WPK_FORK_REFERENCE_IMPORT_GC_DEFINE]: ( + recipeId: number, + recordActivationId: number, + typeOrdinal: number, + layoutId: number, + kind: number, + scalarPointer: number | bigint, + scalarByteLength: number, + referenceVectorOrdinal: number, + ): void => registry.defineGc( + activationId, + recipeId >>> 0, + recordActivationId >>> 0, + typeOrdinal, + layoutId, + kind, + scalarPointer, + scalarByteLength, + referenceVectorOrdinal >>> 0, + ), + [WPK_FORK_REFERENCE_IMPORT_GC_ROUTE]: ( + recipeId: number, + expectedActivation: number, + ): number => referenceReplay().routeGc( + recipeId >>> 0, + expectedActivation >>> 0, + ), + [WPK_FORK_REFERENCE_IMPORT_GC_PAYLOAD_LEN]: ( + recipeId: number, + expectedActivation: number, + expectedLayoutId: number, + ): number => referenceReplay().gcPayloadLength( + recipeId >>> 0, + expectedActivation >>> 0, + expectedLayoutId, + ), + [WPK_FORK_REFERENCE_IMPORT_GC_LOAD]: ( + recipeId: number, + moduleActivation: number, + typeOrdinal: number, + layoutId: number, + kind: number, + scalarDestination: number | bigint, + scalarByteLength: number, + ): number => referenceReplay().loadGc( + recipeId >>> 0, + moduleActivation >>> 0, + typeOrdinal, + layoutId, + kind, + scalarDestination, + scalarByteLength, + ), + [WPK_FORK_REFERENCE_IMPORT_GC_BROKER_ENCODE]: ( + slot: number, + ): number => registry.encodeGcFromSlot(activationId, slot), + [WPK_FORK_REFERENCE_IMPORT_GC_CAPTURE_LAYOUT]: ( + slot: number, + recordActivationId: number, + baseLayoutId: number, + ): number => { + if (recordActivationId !== activationId) { + throw new Error( + `activation ${activationId} cannot select GC layout for ` + + `activation ${recordActivationId}`, + ); + } + return registry.captureGcLayout( + activationId, + slot, + baseLayoutId, + ); + }, + [WPK_FORK_REFERENCE_IMPORT_GC_PROVENANCE_BEGIN]: ( + slot: number, + recordActivationId: number, + baseLayoutId: number, + specializedLayoutId: number, + scalarLo: bigint, + scalarHi: bigint, + referenceCount: number, + ): number => registry.beginGcProvenance( + activationId, + slot, + recordActivationId, + baseLayoutId, + specializedLayoutId, + scalarLo, + scalarHi, + referenceCount, + ), + [WPK_FORK_REFERENCE_IMPORT_GC_PROVENANCE_REF]: ( + token: number, + index: number, + slot: number, + ): void => registry.appendGcProvenanceReference(token, index, slot), + [WPK_FORK_REFERENCE_IMPORT_GC_PROVENANCE_END]: ( + token: number, + ): void => registry.endGcProvenance(token), + }; +} + +/** + * Process-worker owner for every Wasm module activation participating in fork. + * + * The registry owns no copied Wasm references. It rebuilds a transient + * function-identity catalog for each fork transaction, while durable state is + * represented only by bytes in the linked continuation/KFMS arena. Main, + * side-module, and pthread paths all register here before state restore. + */ +export class ForkActivationRegistry { + private readonly registrations = new Map(); + private readonly bootstrapped = new Set(); + private tableCoordinates = new WeakMap< + WebAssembly.Table, + ForkRegisteredTableCoordinate[] + >(); + private readonly activationTables = + new Map(); + /** + * Live process catalog used only for short table-delta recipes. + * + * Unlike a fork transaction catalog, this catalog follows dlopen/dlclose so + * a successful guest mutation can be encoded in O(changed range) without + * rebuilding every activation's function index on each table.set/fill. + */ + private readonly tablePatchFunctions = new ForkFunctionCatalog(); + private phase: RegistryPhase = "idle"; + private arena: ForkModuleStateArena | null = null; + private references: ForkReferenceTransaction | null = null; + private functions: ForkFunctionCatalog | null = null; + private readonly staticRoots = new ForkStaticRootCatalog(); + private readonly gcTransit = new ForkAnyrefTransitTable(); + private readonly gcProvenance = new ForkGcProvenanceRegistry(); + + constructor( + private readonly memory: WebAssembly.Memory, + private readonly externrefs: ForkExternrefRecipeProvider, + private readonly label: string, + private readonly allocateScratch?: ForkReferenceScratchAllocate, + private readonly deallocateScratch?: ForkReferenceScratchDeallocate, + ) {} + + registerActivation(registration: ForkActivationRegistration): void { + this.requireIdle("register a module activation"); + assertActivationId(registration.activationId); + if (this.registrations.has(registration.activationId)) { + throw new Error( + `${this.label}: module activation ${registration.activationId} is already registered`, + ); + } + const ownedRegistration = { + ...registration, + templateId: copyTemplateId(registration.templateId), + }; + const tableCatalog = activationTableCatalog(ownedRegistration, this.label); + try { + ownedRegistration.staticRootHarvest(); + this.staticRoots.register( + ownedRegistration.activationId, + ownedRegistration.staticRootCatalog, + ); + } catch (error) { + // A trapping harvest can have populated a strict prefix. Never let a + // failed dlopen/activation registration retain those temporary roots. + clearForkStaticRootTable(ownedRegistration.staticRootCatalog); + throw error; + } + try { + this.tablePatchFunctions.register( + ownedRegistration.activationId, + ownedRegistration.functionCatalog, + ); + } catch (error) { + this.staticRoots.unregister(ownedRegistration.activationId); + throw error; + } + this.registrations.set(registration.activationId, ownedRegistration); + this.activationTables.set(registration.activationId, tableCatalog); + const affectedTables = new Set(); + for (const { ownerId, table } of tableCatalog) { + const coordinates = this.tableCoordinates.get(table) ?? []; + coordinates.push({ + activationId: registration.activationId, + ownerId, + tracker: ownedRegistration.tableDirty, + }); + coordinates.sort( + (left, right) => + left.activationId - right.activationId + || left.ownerId - right.ownerId, + ); + this.tableCoordinates.set(table, coordinates); + affectedTables.add(table); + } + for (const table of affectedTables) { + this.bindTableCoordinates(table); + } + } + + getActivation(activationId: number): ForkActivationRegistration { + assertActivationId(activationId); + const registration = this.registrations.get(activationId); + if (!registration) { + throw new Error(`${this.label}: module activation ${activationId} is not registered`); + } + return registration; + } + + activations(): readonly ForkActivationRegistration[] { + return [...this.registrations.values()].sort( + (left, right) => left.activationId - right.activationId, + ); + } + + unregisterActivation(activationId: number): void { + this.requireIdle("unregister a module activation"); + const registration = this.getActivation(activationId); + // A provider may retain scratch roots even when no fork is active (for + // example, a caught exception awaiting an ingress callback). Abort is the + // stronger teardown operation and is therefore correct for dlclose/exec. + registration.exceptionProvider?.abort(); + registration.typedReferenceProvider?.abort?.(); + const affectedTables = new Set(); + for (const { table } of this.activationTables.get(activationId) ?? []) { + const remaining = (this.tableCoordinates.get(table) ?? []) + .filter((coordinate) => coordinate.activationId !== activationId); + if (remaining.length === 0) this.tableCoordinates.delete(table); + else { + this.tableCoordinates.set(table, remaining); + affectedTables.add(table); + } + } + this.activationTables.delete(activationId); + for (const table of affectedTables) { + this.bindTableCoordinates(table); + } + this.registrations.delete(activationId); + this.tablePatchFunctions.unregister(activationId); + this.bootstrapped.delete(activationId); + this.staticRoots.unregister(activationId); + } + + bootstrapActivation(activationId: number): void { + this.requireIdle("bootstrap a module activation"); + if (this.bootstrapped.has(activationId)) { + throw new Error(`${this.label}: module activation ${activationId} was bootstrapped twice`); + } + const registration = this.getActivation(activationId); + registration.moduleState.bootstrap(); + this.bootstrapped.add(activationId); + } + + tableDirty(activationId: number): ForkTableDirtyTracker { + return this.getActivation(activationId).tableDirty; + } + + /** + * Record a successful host-side mutation of an activation-owned Table. + * + * Dynamic-linker helpers can call `Table.grow`/`Table.set` without executing + * an instrumented Wasm opcode. Resolve the actual Table identity back to all + * live catalog coordinates and mark the same sparse pages the guest hook + * would have marked. Marking aliases is idempotent after journal union and + * also preserves mutations made before a newly loaded alias is bound. + */ + markTableMutation( + table: WebAssembly.Table, + firstIndexValue: number | bigint, + lengthValue: number | bigint, + ): void { + this.requireIdle("record a host table mutation"); + if (!(table instanceof WebAssembly.Table)) { + throw new TypeError(`${this.label}: host table mutation target is not a Table`); + } + const coordinates = this.tableCoordinates.get(table); + if (!coordinates || coordinates.length === 0) { + throw new Error( + `${this.label}: host mutated a Table outside the registered fork catalogs`, + ); + } + const firstIndex = checkedTableMutationIndex( + firstIndexValue, + "fork table mutation first index", + ); + const length = checkedTableMutationIndex( + lengthValue, + "fork table mutation length", + ); + if (length === 0n) return; + const end = firstIndex + length; + if (end > (1n << 64n)) { + throw new RangeError("fork table mutation range exceeds table64"); + } + const shift = BigInt(WPK_FORK_MODULE_STATE_TABLE_PAGE_SHIFT); + const firstPage = firstIndex >> shift; + const finalPage = (end - 1n) >> shift; + const pageCount = finalPage - firstPage + 1n; + const canonical = coordinates[0]!; + canonical.tracker.markPages(canonical.ownerId, firstPage, pageCount); + } + + /** + * Encode one successful null/funcref mutation using stable activation + * coordinates. + * + * `null` means that this exact range needs the full typed KFMS checkpoint: + * externref, exnref, GC values, and engine-hidden table kinds deliberately + * stay on the Wasm-owned codec path instead of crossing JavaScript. + */ + captureFuncrefTablePatch( + activationId: number, + ownerId: number, + firstIndexValue: number | bigint, + lengthValue: number | bigint, + ): DylinkForkTablePatch | null { + this.requireIdle("capture a table mutation patch"); + const table = this.requireActivationTable(activationId, ownerId); + const firstIndex = checkedTableMutationIndex( + firstIndexValue, + "fork table patch first index", + ); + const length = checkedTableMutationIndex( + lengthValue, + "fork table patch length", + ); + if (length === 0n) { + throw new Error(`${this.label}: cannot publish an empty table mutation`); + } + const end = firstIndex + length; + if ( + end > BigInt(table.length) + || firstIndex > BigInt(Number.MAX_SAFE_INTEGER) + || end > BigInt(Number.MAX_SAFE_INTEGER) + ) { + throw new RangeError( + `${this.label}: table patch range does not match its final Table`, + ); + } + + const start = Number(firstIndex); + const count = Number(length); + const runs: DylinkForkTablePatchRun[] = []; + for (let offset = 0; offset < count; offset++) { + let value: unknown; + try { + value = table.get(start + offset); + } catch { + return null; + } + let recipe: DylinkForkTablePatchRun["function"]; + if (value === null) { + recipe = null; + } else if (typeof value === "function") { + try { + const encoded = this.tablePatchFunctions.encode(value); + if (!encoded) return null; + recipe = { + activationId: encoded.moduleActivation, + ordinal: encoded.ordinal, + }; + } catch { + return null; + } + } else { + return null; + } + const previous = runs.at(-1); + if ( + previous + && ( + previous.function === null + ? recipe === null + : recipe !== null + && previous.function.activationId === recipe.activationId + && previous.function.ordinal === recipe.ordinal + ) + ) { + runs[runs.length - 1] = { + length: previous.length + 1, + function: previous.function, + }; + } else { + runs.push({ length: 1, function: recipe }); + } + } + return { + activationId, + ownerId, + start, + tableLength: table.length, + runs, + }; + } + + /** + * Apply one stable null/funcref patch with this Worker's own function + * objects. The process writer lock is held by the caller. + */ + applyFuncrefTablePatch(patch: DylinkForkTablePatch): void { + this.requireIdle("apply a table mutation patch"); + const table = this.requireActivationTable( + patch.activationId, + patch.ownerId, + ); + if ( + patch.generation === undefined + || !Number.isSafeInteger(patch.start) + || patch.start < 0 + || !Number.isSafeInteger(patch.tableLength) + || patch.tableLength < 0 + ) { + throw new Error(`${this.label}: table patch is not a published recipe`); + } + const decodedRuns: Array<{ + readonly length: number; + readonly value: CallableFunction | null; + }> = []; + let changedLength = 0; + for (const run of patch.runs) { + if (!Number.isSafeInteger(run.length) || run.length <= 0) { + throw new Error(`${this.label}: table patch has an invalid run`); + } + const value = run.function === null + ? null + : this.tablePatchFunctions.decode({ + moduleActivation: run.function.activationId, + ordinal: run.function.ordinal, + }); + changedLength += run.length; + if (!Number.isSafeInteger(changedLength)) { + throw new Error(`${this.label}: table patch changed range is inexact`); + } + decodedRuns.push({ length: run.length, value }); + } + if (patch.start + changedLength > patch.tableLength) { + throw new Error(`${this.label}: table patch exceeds its final length`); + } + if (table.length > patch.tableLength) { + throw new Error(`${this.label}: local Table is longer than its patch`); + } + if (table.length < patch.tableLength) { + const growthOffset = table.length - patch.start; + if (growthOffset < 0 || growthOffset >= changedLength) { + throw new Error( + `${this.label}: table patch cannot reconstruct its growth gap`, + ); + } + let remaining = growthOffset; + const initializer = decodedRuns.find((run) => { + if (remaining < run.length) return true; + remaining -= run.length; + return false; + })?.value; + if (initializer === undefined) { + throw new Error(`${this.label}: table patch has no growth initializer`); + } + // WHY: nullable tables accept null, but a non-nullable typed function + // table requires a real instance-local initializer. The patch covers + // every new entry from the old length, so any value at that coordinate + // is a safe temporary initializer before the exact runs are applied. + table.grow( + patch.tableLength - table.length, + initializer, + ); + } + let index = patch.start; + for (const run of decodedRuns) { + for (let offset = 0; offset < run.length; offset++) { + table.set(index++, run.value); + } + } + this.markTableMutation(table, patch.start, changedLength); + } + + currentArena(): ForkModuleStateArena { + if (!this.arena) { + throw new Error(`${this.label}: no fork module-state transaction is active`); + } + return this.arena; + } + + currentReferences(): ForkReferenceTransaction { + if (!this.references) { + throw new Error(`${this.label}: no fork reference transaction is active`); + } + return this.references; + } + + /** Host-owned typed scratch table imported by every activation codec. */ + gcTransitTable(): WebAssembly.Table { + return this.gcTransit.table; + } + + /** + * Reserve/read the same transit slots used by the normal replay owner while + * imported references are reconstructed before every activation exists. + */ + prepareEarlyGcTransit(maxRecipeId: number): void { + if ( + !Number.isInteger(maxRecipeId) + || maxRecipeId < 0 + || maxRecipeId > 0x7fff_fffe + ) { + throw new RangeError(`invalid early GC recipe maximum ${maxRecipeId}`); + } + this.gcTransit.clear(); + if (maxRecipeId > 0) this.gcTransit.ensureRecipeSlot(maxRecipeId); + } + + readEarlyGcTransit(recipeId: number): unknown { + if ( + !Number.isInteger(recipeId) + || recipeId <= 0 + || recipeId > 0x7fff_fffe + ) { + throw new RangeError(`invalid early GC recipe id ${recipeId}`); + } + return this.gcTransit.get(recipeId + 1); + } + + publishEarlyGcTransit(recipeId: number, value: unknown): void { + if ( + !Number.isInteger(recipeId) + || recipeId <= 0 + || recipeId > 0x7fff_fffe + ) { + throw new RangeError(`invalid early GC recipe id ${recipeId}`); + } + this.gcTransit.ensureRecipeSlot(recipeId); + this.gcTransit.set(recipeId + 1, value); + } + + abortEarlyGcTransit(): void { + this.gcTransit.clear(); + } + + decodeStaticRoot(activationId: number, ordinal: number): unknown { + return this.staticRoots.decode({ + moduleActivation: activationId, + ordinal, + }); + } + + lookupGcSlot(requestingActivation: number, slot: number): number { + const provenance = this.gcProvenance.find(this.gcTransit.get(slot)); + if (provenance && provenance.activationId !== requestingActivation) { + // Canonically equivalent recursive types can test true in more than one + // instance. Constructor/segment provenance decides the reconstruction + // owner before the requesting codec claims graph identity. + return this.requireTypedProvider(provenance.activationId).encodeSlot(slot); + } + return this.currentReferences().lookupGcSlot(this.gcTransit.table, slot); + } + + claimGcSlot(slot: number): number { + const recipeId = this.currentReferences().claimGcSlot( + this.gcTransit.table, + slot, + ); + this.gcTransit.ensureRecipeSlot(recipeId); + return recipeId; + } + + encodeI31(value: number): number { + const recipeId = this.currentReferences().encodeI31(value); + this.gcTransit.ensureRecipeSlot(recipeId); + return recipeId; + } + + captureGcLayout( + activationId: number, + slot: number, + baseLayoutId: number, + ): number { + const provider = this.requireTypedProvider(activationId); + const base = provider.descriptor.require(baseLayoutId); + const object = this.gcTransit.get(slot); + const provenance = this.gcProvenance.lookup( + object, + activationId, + provider.descriptor, + baseLayoutId, + ); + if (provenance) return provenance.layoutId; + if ((base.flags & FORK_GC_LAYOUT_REQUIRES_PROVENANCE) !== 0) { + throw new Error( + `${this.label}: GC layout ${activationId}:${baseLayoutId} ` + + "requires constructor provenance", + ); + } + return base.id; + } + + defineGc( + activationId: number, + recipeId: number, + recordActivationId: number, + typeOrdinal: number, + layoutId: number, + kind: number, + scalarPointer: number | bigint, + scalarByteLength: number, + referenceVectorOrdinal: number, + ): void { + if (recordActivationId !== activationId) { + throw new Error( + `activation ${activationId} cannot define GC state for ` + + `activation ${recordActivationId}`, + ); + } + const provider = this.requireTypedProvider(activationId); + const layout = provider.descriptor.require(layoutId); + const source = this.currentReferences().capturedGcValue(recipeId); + const record = this.gcProvenance.lookup( + source, + activationId, + provider.descriptor, + layout.baseLayoutId, + ); + let provenance: ForkGcDefinitionProvenance | null = null; + if (record) { + const recipeIds = record.references.map((reference) => + reference === null ? 0 : this.encodeGcObject(reference) + ); + provenance = { record, recipeIds }; + } + this.currentReferences().defineGc( + recipeId, + recordActivationId, + typeOrdinal, + layoutId, + kind, + scalarPointer, + scalarByteLength, + referenceVectorOrdinal, + provider.descriptor, + provenance, + ); + } + + routeGc(recipeId: number, expectedActivation: number): number { + return this.currentReferences().routeGc(recipeId, expectedActivation); + } + + gcPayloadLength( + recipeId: number, + expectedActivation: number, + expectedLayoutId: number, + ): number { + return this.currentReferences().gcPayloadLength( + recipeId, + expectedActivation, + expectedLayoutId, + ); + } + + loadGc( + recipeId: number, + moduleActivation: number, + typeOrdinal: number, + layoutId: number, + kind: number, + scalarDestination: number | bigint, + scalarByteLength: number, + ): number { + return this.currentReferences().loadGc( + recipeId, + moduleActivation, + typeOrdinal, + layoutId, + kind, + scalarDestination, + scalarByteLength, + ); + } + + encodeGcFromSlot(sourceActivation: number, slot: number): number { + const provenance = this.gcProvenance.find(this.gcTransit.get(slot)); + if ( + provenance + && provenance.activationId !== sourceActivation + ) { + return this.requireTypedProvider(provenance.activationId).encodeSlot(slot); + } + const candidates = this.activations().filter( + ({ activationId, typedReferenceProvider }) => + activationId !== sourceActivation && typedReferenceProvider !== undefined, + ); + for (const activation of candidates) { + const provider = this.requireTypedProvider(activation.activationId); + const packed = provider.probe(slot); + if (packed === 0n) continue; + const baseLayoutId = Number(packed & 0xffff_ffffn); + const typeOrdinal = Number(packed >> 32n); + const base = provider.descriptor.require(baseLayoutId); + if ( + base.baseLayoutId !== base.id + || base.typeOrdinal !== typeOrdinal + ) { + throw new Error( + `${this.label}: activation ${activation.activationId} returned ` + + "an invalid GC probe coordinate", + ); + } + return provider.encodeSlot(slot); + } + // No module codec recognized the internal value, so it is a hostref made + // by `any.convert_extern`. Its worker-local token names a process-owned + // broker handle; retain that handle as an externref leaf in the same graph. + const recipeId = this.currentReferences().encodeExternref( + this.gcTransit.get(slot), + ); + this.gcTransit.ensureRecipeSlot(recipeId); + return recipeId; + } + + beginGcProvenance( + expectedActivationId: number, + slot: number, + activationId: number, + baseLayoutId: number, + specializedLayoutId: number, + scalarLo: bigint, + scalarHi: bigint, + referenceCount: number, + ): number { + return this.gcProvenance.begin( + this.gcTransit.table, + this.requireTypedProvider(expectedActivationId).descriptor, + expectedActivationId, + slot, + activationId, + baseLayoutId, + specializedLayoutId, + scalarLo, + scalarHi, + referenceCount, + ); + } + + appendGcProvenanceReference( + token: number, + index: number, + slot: number, + ): void { + this.gcProvenance.appendReference( + this.gcTransit.table, + token, + index, + slot, + ); + } + + endGcProvenance(token: number): void { + this.gcProvenance.end(token); + } + + /** + * Start capture and snapshot every registered module before stack unwind. + * + * Frame codecs continue appending reference nodes while unwind walks + * outward. `sealCapture` publishes the single process graph only after the + * last committed frame exists. + */ + beginCapture(arena: ForkModuleStateArena): void { + this.requirePhase("idle", "begin fork activation capture"); + if (!arena.hasActiveArena() || arena.isSealed()) { + throw new Error(`${this.label}: capture requires a writable module-state arena`); + } + // A prior trap must never make a stale object appear as a recipe hit. + this.gcProvenance.abortPending(); + this.gcTransit.clear(); + const functions = this.buildFunctionCatalog(); + const references = new ForkReferenceTransaction( + functions, + this.externrefs, + this.memory, + this.allocateScratch, + this.deallocateScratch, + `${this.label}: references`, + this.staticRoots, + this.typedReplayOwner(), + ); + references.beginCapture(); + this.functions = functions; + this.references = references; + this.arena = arena; + this.phase = "capture"; + try { + for (const activation of this.activations()) { + arena.appendModule({ + activationId: activation.activationId, + templateId: activation.templateId, + }); + } + for (const activation of this.activations()) { + activation.moduleState.save(activation.activationId); + } + } catch (error) { + this.abort(); + throw error; + } + } + + /** + * Seal one process-wide, table-only snapshot for peer Workers. + * + * The generated helpers reuse the same typed reference codecs as fork, so + * exnref and Wasm-GC entries never cross the JavaScript Table API. Every + * cumulative dirty page is captured in one reference transaction; aliases + * therefore remain aliases even when they span tables, pages, or module + * activations. + */ + captureTableState(arena: ForkModuleStateArena): number { + this.requirePhase("idle", "capture peer table state"); + if (!arena.hasActiveArena() || arena.isSealed()) { + throw new Error( + `${this.label}: peer table capture requires a writable module-state arena`, + ); + } + this.gcProvenance.abortPending(); + this.gcTransit.clear(); + const functions = this.buildFunctionCatalog(); + const references = new ForkReferenceTransaction( + functions, + this.externrefs, + this.memory, + this.allocateScratch, + this.deallocateScratch, + `${this.label}: peer table references`, + this.staticRoots, + this.typedReplayOwner(), + ); + references.beginCapture(); + this.functions = functions; + this.references = references; + this.arena = arena; + this.phase = "table-capture"; + try { + for (const activation of this.activations()) { + arena.appendModule({ + activationId: activation.activationId, + templateId: activation.templateId, + }); + } + for (const activation of this.activations()) { + activation.moduleState.saveTables(activation.activationId); + } + references.sealInto(arena); + const root = arena.seal(); + // No live activation consumes capture-side recipe objects. Drop every + // transient codec/catalog root after the scalar arena is sealed. + this.abort(); + return root; + } catch (error) { + this.abort(); + throw error; + } + } + + /** + * Apply one validated table-only snapshot to this Worker's instance graph. + */ + restoreTableState(arena: ForkModuleStateArena): void { + this.requirePhase("idle", "restore peer table state"); + if (!arena.hasActiveArena() || !arena.isSealed()) { + throw new Error( + `${this.label}: peer table replay requires a validated sealed arena`, + ); + } + this.gcTransit.clear(); + this.gcProvenance.abortPending(); + const records = arena.recordViews(); + const declared = records + .filter((record) => record.kind === ForkModuleStateRecordKind.Module) + .map((record) => record.activationId) + .sort((left, right) => left - right); + const registered = this.activations().map(({ activationId }) => activationId); + if ( + declared.length !== registered.length + || declared.some((id, index) => id !== registered[index]) + ) { + throw new Error( + `${this.label}: peer table snapshot activations do not match the local registry`, + ); + } + for (const activation of this.activations()) { + requireForkModuleTemplate( + records, + activation.activationId, + activation.templateId, + ); + } + const functions = this.buildFunctionCatalog(); + const references = new ForkReferenceTransaction( + functions, + this.externrefs, + this.memory, + this.allocateScratch, + this.deallocateScratch, + `${this.label}: peer table references`, + this.staticRoots, + this.typedReplayOwner(), + ); + references.attachChild(records); + this.functions = functions; + this.references = references; + this.arena = arena; + this.phase = "table-replay"; + try { + references.materializeAllTyped(); + for (const activation of this.activations()) { + activation.moduleState.restoreTables(activation.activationId); + } + references.finishReplay(); + for (const activation of this.activations()) { + activation.exceptionProvider?.clear(); + activation.typedReferenceProvider?.clear?.(); + } + this.gcTransit.clear(); + this.resetTransaction(); + } catch (error) { + this.abort(); + throw error; + } + } + + sealCapture(): void { + this.requirePhase("capture", "seal fork activation capture"); + const references = this.currentReferences(); + const arena = this.currentArena(); + references.sealInto(arena); + arena.seal(); + this.phase = "sealed-parent"; + } + + beginParentReplay(): void { + this.requirePhase("sealed-parent", "begin parent activation replay"); + this.currentReferences().beginParentReplay(); + this.phase = "parent-replay"; + } + + /** + * Attach copied recipes only after every child activation and codec exists. + */ + attachChild( + arena: ForkModuleStateArena, + decodedReferences?: DecodedSegmentedForkReferenceTransaction, + ): void { + this.requirePhase("idle", "attach child activation state"); + if (!arena.hasActiveArena() || !arena.isSealed()) { + throw new Error(`${this.label}: child replay requires a validated sealed arena`); + } + this.gcTransit.clear(); + this.gcProvenance.abortPending(); + const records = arena.recordViews(); + const declared = records + .filter((record) => record.kind === ForkModuleStateRecordKind.Module) + .map((record) => record.activationId) + .sort((left, right) => left - right); + const registered = this.activations().map(({ activationId }) => activationId); + if ( + declared.length !== registered.length + || declared.some((id, index) => id !== registered[index]) + ) { + throw new Error( + `${this.label}: copied module activations do not match the fresh child registry`, + ); + } + for (const activation of this.activations()) { + requireForkModuleTemplate( + records, + activation.activationId, + activation.templateId, + ); + } + const functions = this.buildFunctionCatalog(); + const references = new ForkReferenceTransaction( + functions, + this.externrefs, + this.memory, + this.allocateScratch, + this.deallocateScratch, + `${this.label}: references`, + this.staticRoots, + this.typedReplayOwner(), + ); + references.attachChild(decodedReferences ?? records); + this.functions = functions; + this.references = references; + this.arena = arena; + this.phase = "child-replay"; + } + + restoreModuleState(): void { + if (this.phase !== "parent-replay" && this.phase !== "child-replay") { + throw new Error( + `${this.label}: cannot restore module state while registry is ${this.phase}`, + ); + } + if (this.phase === "child-replay") { + // WHY: generated global/table restore helpers decode recipe ids through + // the fresh instance's transit table. Publish every reconstructed typed + // identity first, while passive data/element segments are still intact + // for array.new_data/array.new_elem constructors. + this.currentReferences().materializeAllTyped(); + } + for (const activation of this.activations()) { + activation.moduleState.restore(activation.activationId); + } + for (const activation of this.activations()) { + activation.moduleState.finishRestore(activation.activationId); + } + } + + finishReplay(): void { + if (this.phase !== "parent-replay" && this.phase !== "child-replay") { + throw new Error( + `${this.label}: cannot finish activation replay while registry is ${this.phase}`, + ); + } + let failure: unknown; + try { + this.references?.finishReplay(); + } catch (error) { + failure = error; + } + for (const activation of this.activations()) { + for (const provider of [ + activation.exceptionProvider, + activation.typedReferenceProvider, + ]) { + try { + provider?.clear?.(); + } catch (error) { + failure ??= error; + } + } + } + try { + this.gcTransit.clear(); + } catch (error) { + failure ??= error; + } + this.resetTransaction(); + if (failure !== undefined) throw failure; + } + + abort(): void { + let failure: unknown; + try { + this.references?.abort(); + } catch (error) { + failure = error; + } + for (const activation of this.activations()) { + for (const provider of [ + activation.exceptionProvider, + activation.typedReferenceProvider, + ]) { + try { + provider?.abort?.(); + } catch (error) { + failure ??= error; + } + } + } + try { + this.gcTransit.clear(); + } catch (error) { + failure ??= error; + } + this.resetTransaction(); + if (failure !== undefined) throw failure; + } + + clear(): void { + this.abort(); + this.registrations.clear(); + this.bootstrapped.clear(); + this.activationTables.clear(); + this.tablePatchFunctions.clear(); + this.tableCoordinates = new WeakMap(); + this.staticRoots.clear(); + this.gcProvenance.clear(); + } + + phaseName(): RegistryPhase { + return this.phase; + } + + private buildFunctionCatalog(): ForkFunctionCatalog { + const functions = new ForkFunctionCatalog(); + for (const activation of this.activations()) { + functions.register(activation.activationId, activation.functionCatalog); + } + return functions; + } + + private requireActivationTable( + activationId: number, + ownerId: number, + ): WebAssembly.Table { + assertActivationId(activationId); + if ( + !Number.isInteger(ownerId) + || ownerId <= 0 + || ownerId > 0xffff_ffff + ) { + throw new RangeError(`invalid fork table owner id ${ownerId}`); + } + const entry = this.activationTables + .get(activationId) + ?.find((candidate) => candidate.ownerId === ownerId); + if (!entry) { + throw new Error( + `${this.label}: table coordinate ${activationId}:${ownerId} is not registered`, + ); + } + return entry.table; + } + + private bindTableCoordinates(table: WebAssembly.Table): void { + const coordinates = this.tableCoordinates.get(table); + if (!coordinates || coordinates.length === 0) return; + const canonical = coordinates[0]!; + canonical.tracker.setStateOwner(canonical.ownerId, true); + for (const coordinate of coordinates.slice(1)) { + coordinate.tracker.aliasOwner( + coordinate.ownerId, + canonical.tracker, + canonical.ownerId, + ); + coordinate.tracker.setStateOwner(coordinate.ownerId, false); + } + } + + private requireTypedProvider( + activationId: number, + ): ForkGcCodecProvider & ForkActivationTypedReferenceProvider { + const provider = this.getActivation(activationId).typedReferenceProvider; + if ( + !provider + || provider.activationId !== activationId + || !provider.descriptor + || typeof provider.probe !== "function" + || typeof provider.encodeSlot !== "function" + || typeof provider.allocate !== "function" + || typeof provider.fill !== "function" + || typeof provider.publishExternref !== "function" + ) { + throw new Error( + `${this.label}: module activation ${activationId} has no GC codec`, + ); + } + return provider as ForkGcCodecProvider & ForkActivationTypedReferenceProvider; + } + + private encodeGcObject(value: object): number { + this.gcTransit.set(0, value); + try { + return this.encodeGcFromSlot(-1, 0); + } finally { + this.gcTransit.clearSlot(0); + } + } + + private typedReplayOwner() { + return { + prepareTransit: (maxRecipeId: number): void => { + if (maxRecipeId > 0) this.gcTransit.ensureRecipeSlot(maxRecipeId); + }, + publishTransit: (recipeId: number, value: unknown): void => { + this.gcTransit.ensureRecipeSlot(recipeId); + this.gcTransit.set(recipeId + 1, value); + }, + publishExternref: (recipeId: number, value: unknown): void => { + const provider = this.activations() + .map(({ activationId, typedReferenceProvider }) => + typedReferenceProvider ? this.requireTypedProvider(activationId) : null + ) + .find((candidate) => candidate !== null); + if (!provider) { + throw new Error( + `${this.label}: externref replay has no generated GC codec`, + ); + } + this.gcTransit.ensureRecipeSlot(recipeId); + provider.publishExternref(recipeId, value); + if (!Object.is(this.gcTransit.get(recipeId + 1), value)) { + throw new Error( + `${this.label}: externref recipe ${recipeId} lost token identity ` + + "during anyref publication", + ); + } + }, + provider: (activationId: number): ForkGcCodecProvider => + this.requireTypedProvider(activationId), + providers: (): readonly ForkGcCodecProvider[] => + this.activations().flatMap(({ activationId, typedReferenceProvider }) => { + if ( + !typedReferenceProvider?.descriptor + || typeof typedReferenceProvider.probe !== "function" + || typeof typedReferenceProvider.encodeSlot !== "function" + || typeof typedReferenceProvider.allocate !== "function" + || typeof typedReferenceProvider.fill !== "function" + || typeof typedReferenceProvider.publishExternref !== "function" + ) { + return []; + } + return [this.requireTypedProvider(activationId)]; + }), + validateExceptionOwner: (activationId: number): void => { + if (activationId === FORK_HOST_EXCEPTION_ACTIVATION_ID) { + if (!this.activations().some(({ exceptionProvider }) => + exceptionProvider?.materialize + )) { + throw new Error( + `${this.label}: host exception replay has no local codec`, + ); + } + return; + } + const provider = this.getActivation(activationId).exceptionProvider; + if (!provider?.materialize) { + throw new Error( + `${this.label}: activation ${activationId} cannot materialize ` + + "exception recipes", + ); + } + }, + materializeException: ( + recipeId: number, + activationId: number, + ): void => { + const provider = activationId === FORK_HOST_EXCEPTION_ACTIVATION_ID + ? this.activations() + .map(({ exceptionProvider }) => exceptionProvider) + .find((candidate) => candidate?.materialize) + : this.getActivation(activationId).exceptionProvider; + if (!provider?.materialize) { + throw new Error( + `${this.label}: no exception materializer for activation ` + + `${activationId}`, + ); + } + provider.materialize(recipeId); + }, + }; + } + + private resetTransaction(): void { + this.functions?.clear(); + this.functions = null; + this.references = null; + this.arena = null; + this.phase = "idle"; + } + + private requireIdle(operation: string): void { + this.requirePhase("idle", operation); + } + + private requirePhase(expected: RegistryPhase, operation: string): void { + if (this.phase !== expected) { + throw new Error( + `${this.label}: cannot ${operation} while activation registry is ${this.phase}; ` + + `expected ${expected}`, + ); + } + } +} diff --git a/host/src/fork-anyref-transit.ts b/host/src/fork-anyref-transit.ts new file mode 100644 index 0000000000..0c44b0c4cb --- /dev/null +++ b/host/src/fork-anyref-transit.ts @@ -0,0 +1,135 @@ +/** + * The ABI 43 transaction-local Wasm-GC routing table. + * + * WebKit can import and export `(ref null any)` tables, but its JavaScript + * `WebAssembly.Table` constructor does not accept `element: "anyref"`. + * Creating the table in this fixed Wasm provider therefore gives Node and all + * browser engines the same host-owned object without weakening its type. + */ +export const FORK_ANYREF_TRANSIT_IMPORT = "__wpk_fork_ref_gc_transit"; +const FORK_ANYREF_TRANSIT_CLEAR_EXPORT = + "__wpk_fork_ref_gc_transit_clear"; + +/* + * Deterministic encoding of: + * + * (module + * (table (export "__wpk_fork_ref_gc_transit") 1 (ref null any)) + * (func (export "__wpk_fork_ref_gc_transit_clear") + * i32.const 0 + * ref.null any + * table.size 0 + * table.fill 0)) + * + * Keep this provider deliberately closed: no imports, memory, globals, start + * function, or mutable state other than the exported scratch table. + */ +const FORK_ANYREF_TRANSIT_PROVIDER_BYTES = Uint8Array.of( + 0x00, 0x61, 0x73, 0x6d, 0x01, 0x00, 0x00, 0x00, 0x01, 0x04, 0x01, 0x60, + 0x00, 0x00, 0x03, 0x02, 0x01, 0x00, 0x04, 0x04, 0x01, 0x6e, 0x00, 0x01, + 0x07, 0x3f, 0x02, 0x19, 0x5f, 0x5f, 0x77, 0x70, 0x6b, 0x5f, 0x66, 0x6f, + 0x72, 0x6b, 0x5f, 0x72, 0x65, 0x66, 0x5f, 0x67, 0x63, 0x5f, 0x74, 0x72, + 0x61, 0x6e, 0x73, 0x69, 0x74, 0x01, 0x00, 0x1f, 0x5f, 0x5f, 0x77, 0x70, + 0x6b, 0x5f, 0x66, 0x6f, 0x72, 0x6b, 0x5f, 0x72, 0x65, 0x66, 0x5f, 0x67, + 0x63, 0x5f, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x69, 0x74, 0x5f, 0x63, 0x6c, + 0x65, 0x61, 0x72, 0x00, 0x00, 0x0a, 0x0e, 0x01, 0x0c, 0x00, 0x41, 0x00, + 0xd0, 0x6e, 0xfc, 0x10, 0x00, 0xfc, 0x11, 0x00, 0x0b, +); + +let providerModule: WebAssembly.Module | undefined; + +function compileProviderModule(): WebAssembly.Module { + if (providerModule) return providerModule; + try { + providerModule = new WebAssembly.Module( + FORK_ANYREF_TRANSIT_PROVIDER_BYTES as BufferSource, + ); + } catch (cause) { + throw new Error( + "this host cannot construct the ABI 43 Wasm-GC transit table", + { cause }, + ); + } + return providerModule; +} + +/** Copy the audited provider binary for cross-engine contract tests. */ +export function forkAnyrefTransitProviderBytes(): Uint8Array { + return FORK_ANYREF_TRANSIT_PROVIDER_BYTES.slice(); +} + +/** + * One process-worker owner for the scratch table shared by all activations. + * + * The generated codecs may grow the table, but every entry is null-filled by + * Wasm at transaction boundaries. Using `table.fill` avoids one JS call per + * recipe while guaranteeing that no stale GC object remains a strong root. + */ +export class ForkAnyrefTransitTable { + readonly table: WebAssembly.Table; + private readonly clearTable: () => void; + + constructor() { + const instance = new WebAssembly.Instance(compileProviderModule()); + const table = instance.exports[FORK_ANYREF_TRANSIT_IMPORT]; + const clearTable = instance.exports[FORK_ANYREF_TRANSIT_CLEAR_EXPORT]; + if (!(table instanceof WebAssembly.Table) || typeof clearTable !== "function") { + throw new Error("invalid ABI 43 Wasm-GC transit provider exports"); + } + this.table = table; + this.clearTable = clearTable as () => void; + this.clear(); + } + + clear(): void { + this.clearTable(); + } + + /** + * Reserve the canonical `recipe + 1` slot before generated Wasm publishes + * an identity there. The table has no maximum, but keeping growth here lets + * the host reject integer overflow before it becomes an engine-dependent + * `table.grow` trap. + */ + ensureRecipeSlot(recipeId: number): void { + if ( + !Number.isInteger(recipeId) + || recipeId <= 0 + || recipeId > 0x7fff_fffe + ) { + throw new RangeError(`invalid Wasm-GC recipe id ${recipeId}`); + } + const requiredLength = recipeId + 2; + if (this.table.length >= requiredLength) return; + const delta = requiredLength - this.table.length; + const previous = this.table.grow(delta, null); + if (previous + delta !== requiredLength) { + throw new Error("Wasm-GC transit table grew to an unexpected length"); + } + } + + get(slot: number): unknown { + this.assertSlot(slot); + return this.table.get(slot); + } + + set(slot: number, value: unknown): void { + this.assertSlot(slot); + this.table.set(slot, value); + } + + clearSlot(slot: number): void { + this.assertSlot(slot); + this.table.set(slot, null); + } + + private assertSlot(slot: number): void { + if ( + !Number.isInteger(slot) + || slot < 0 + || slot >= this.table.length + ) { + throw new RangeError(`Wasm-GC transit slot ${slot} is out of bounds`); + } + } +} diff --git a/host/src/fork-continuation.ts b/host/src/fork-continuation.ts index 4fc4839253..f0c0d4da7f 100644 --- a/host/src/fork-continuation.ts +++ b/host/src/fork-continuation.ts @@ -211,6 +211,16 @@ interface ContinuationChunk { used: number; } +interface ValidatedReplayNode { + node: number; + payload: number; + previous: number; + nextReplay: { + chunkIndex: number; + expectedEnd: number; + }; +} + /** * Host-side owner and validator for one module instance's linked fork frames. * Allocations are ordinary anonymous process mappings, so kernel brk/mmap @@ -224,8 +234,10 @@ export class LinkedForkContinuation { private replayExpectedEnd = 0; private pending: PendingNode | null = null; private chunks: ContinuationChunk[] = []; - private committedFrames = 0; - private committedBytes = 0; + // Diagnostics must not become the first precision ceiling in a wasm64 + // continuation. The linked list is allocator-bounded, not Number-bounded. + private committedFrames = 0n; + private committedBytes = 0n; private abortFailure: AbortFailure | null = null; constructor( @@ -245,8 +257,8 @@ export class LinkedForkContinuation { this.format.alignment, ); const capacity = alignUp(Math.max(initialUsed, WASM_PAGE_SIZE), WASM_PAGE_SIZE); - this.committedFrames = 0; - this.committedBytes = 0; + this.committedFrames = 0n; + this.committedBytes = 0n; this.abortFailure = null; let root: number; try { @@ -421,12 +433,35 @@ export class LinkedForkContinuation { this.writePtr(this.root + 8 + 5 * this.format.ptrWidth, pending.node); const payloadSize = this.readPtr(pending.node + 8 + this.format.ptrWidth); this.committedFrames++; - this.committedBytes += payloadSize; + this.committedBytes += BigInt(payloadSize); this.pending = null; } + /** + * Validate and expose the next frame without advancing the replay cursor. + * + * Tail-call replay selects an activation-specific resume thunk from the + * common frame header before entering the original function. That function's + * ordinary preamble remains the sole consumer through `nextFrame`. + */ + peekFrame(expectedSize: number | bigint): number | bigint { + const expected = this.fromGuestPtr(expectedSize); + const validated = this.validateNextFrame(expected); + return this.asGuestPtr(validated.payload); + } + nextFrame(expectedSize: number | bigint): number | bigint { const expected = this.fromGuestPtr(expectedSize); + const validated = this.validateNextFrame(expected); + const { node, payload, previous, nextReplay } = validated; + this.replayNode = previous; + this.replayChunkIndex = nextReplay.chunkIndex; + this.replayExpectedEnd = nextReplay.expectedEnd; + this.view().setUint16(node + 6, NODE_CONSUMED, true); + return this.asGuestPtr(payload); + } + + private validateNextFrame(expected: number): ValidatedReplayNode { const node = this.replayNode; if (this.root === 0 || node === 0) { throw new Error(`${this.label}: linked continuation replay exhausted early`); @@ -456,11 +491,12 @@ export class LinkedForkContinuation { } const previous = this.readPtr(node + 8); const nextReplay = this.previousReplayPosition(previous, node); - this.replayNode = previous; - this.replayChunkIndex = nextReplay.chunkIndex; - this.replayExpectedEnd = nextReplay.expectedEnd; - view.setUint16(node + 6, NODE_CONSUMED, true); - return this.asGuestPtr(node + this.format.nodeHeaderSize); + return { + node, + payload: node + this.format.nodeHeaderSize, + previous, + nextReplay, + }; } finishUnwind(): void { diff --git a/host/src/fork-early-reference-provider.ts b/host/src/fork-early-reference-provider.ts new file mode 100644 index 0000000000..0fba95700c --- /dev/null +++ b/host/src/fork-early-reference-provider.ts @@ -0,0 +1,1604 @@ +import { + WPK_FORK_MODULE_STATE_GLOBAL_TYPE_ANYREF, + WPK_FORK_MODULE_STATE_GLOBAL_TYPE_EXNREF, + WPK_FORK_MODULE_STATE_GLOBAL_TYPE_EXTERNREF, + WPK_FORK_MODULE_STATE_GLOBAL_TYPE_FUNCREF, +} from "./generated/abi"; +import type { + ForkActivationExceptionProvider, +} from "./fork-activation-registry"; +import type { + ForkImportedReferenceProvider, +} from "./fork-imported-globals"; +import { + FORK_GC_FIELD_ALLOCATION_DEPENDENCY, + FORK_GC_FIELD_REFERENCE, + FORK_GC_LAYOUT_DEFAULTABLE_SHELL, + ForkGcConstructorKind, + type ForkGcCodecDescriptor, + type ForkGcCodecProvider, + type ForkGcLayoutDescriptor, +} from "./fork-gc-codec"; +import type { + ForkExceptionCodecDescriptor, +} from "./fork-exception-provider"; +import { + ForkImportedGlobalBindingKind, + ForkModuleStateRecordKind, + importedGlobalBindingsForChild, + type ForkModuleStateRecordView, +} from "./fork-module-state"; +import { + type ForkReferenceRecipeEntry, + type ForkReferenceRecipeNode, +} from "./fork-reference-recipes"; +import { + FORK_HOST_EXCEPTION_ACTIVATION_ID, + type ForkExternrefRecipeProvider, + type ForkReferenceChildReplayAdoption, + type ForkReferenceScratchAllocate, + type ForkReferenceScratchDeallocate, + ForkReferenceTransaction, +} from "./fork-reference-transaction"; +import { + findForkReferenceVectorOrdinal, + forkReferenceVectorFrom, + ForkReferenceDirectoryOverlay, + indexForkReferenceVector, + PagedForkReferenceDirectory, + type DecodedSegmentedForkReferenceTransaction, + type ForkReferenceDirectory, + type ForkReferenceVector, + type MutableForkReferenceVectorInternIndex, +} from "./fork-reference-segments"; + +const MAX_REFERENCE_VECTOR_ORDINAL = 0xffff_ffff; + +type ReferenceTypeCode = + | typeof WPK_FORK_MODULE_STATE_GLOBAL_TYPE_FUNCREF + | typeof WPK_FORK_MODULE_STATE_GLOBAL_TYPE_EXTERNREF + | typeof WPK_FORK_MODULE_STATE_GLOBAL_TYPE_EXNREF + | typeof WPK_FORK_MODULE_STATE_GLOBAL_TYPE_ANYREF; + +type ProviderPhase = "active" | "adopted" | "aborted"; + +export interface ForkEarlyReferenceActivationDeclaration { + readonly activationId: number; + /** + * Descriptor-only type evidence is available from the module before its + * instance exists. It is what makes owner planning and graph validation an + * actual pre-instantiation operation. + */ + readonly gcDescriptor?: ForkGcCodecDescriptor; + readonly exceptionDescriptor?: ForkExceptionCodecDescriptor; +} + +export interface ForkEarlyFunctionProvider { + decode(ordinal: number): CallableFunction; +} + +export interface ForkEarlyStaticRootProvider { + decode(ordinal: number): unknown; +} + +/** + * View of the same anyref transit table imported by generated activation + * codecs. Implementations must map recipe N to the canonical slot N + 1. + */ +export interface ForkEarlyReferenceTransit { + prepare(maxRecipeId: number): void; + /** + * Route an already-instantiated GC root at canonical slot `recipeId + 1`. + * Only instrumenter-proven anyref-compatible static roots use this path. + */ + publish(recipeId: number, value: unknown): void; + read(recipeId: number): unknown; + /** Release every early typed root if launch fails before adoption. */ + abort(): void; +} + +export interface ForkEarlyReferenceActivationProviders { + readonly activationId: number; + readonly functions?: ForkEarlyFunctionProvider; + readonly staticRoots?: ForkEarlyStaticRootProvider; + readonly typed?: ForkGcCodecProvider; + readonly exceptions?: ForkActivationExceptionProvider; + /** + * Optional activation-owned rollback for roots created before the registry + * takes over. It is not called after successful adoption. + */ + readonly abort?: () => void; +} + +export interface ForkEarlyChildReferenceProviderOptions { + readonly records: readonly ForkModuleStateRecordView[]; + /** One decoder result shared verbatim with ordinary child replay adoption. */ + readonly transaction: DecodedSegmentedForkReferenceTransaction; + readonly declarations: readonly ForkEarlyReferenceActivationDeclaration[]; + readonly externrefs: ForkExternrefRecipeProvider; + readonly transit: ForkEarlyReferenceTransit; + readonly memory: WebAssembly.Memory; + readonly allocateScratch: ForkReferenceScratchAllocate; + readonly deallocateScratch: ForkReferenceScratchDeallocate; + readonly label?: string; +} + +interface RegisteredActivation extends ForkEarlyReferenceActivationProviders {} + +interface ScratchChunk { + readonly addr: number; + readonly size: number; + used: number; +} + +interface ScratchReservation { + readonly addr: number; + readonly requestedSize: number; + readonly alignedSize: number; + readonly previousUsed: number; + readonly chunk: ScratchChunk; +} + +function assertU32(value: number, context: string): number { + if (!Number.isInteger(value) || value < 0 || value > 0xffff_ffff) { + throw new RangeError(`${context} is not a u32`); + } + return value; +} + +function assertRecipeId(value: number, nodeCount: number): number { + if ( + !Number.isInteger(value) + || value < 0 + || value > 0xffff_ffff + || value >= nodeCount + ) { + throw new RangeError(`invalid fork reference recipe id ${value}`); + } + return value; +} + +function requireReferenceTypeCode(value: number): ReferenceTypeCode { + switch (value) { + case WPK_FORK_MODULE_STATE_GLOBAL_TYPE_FUNCREF: + case WPK_FORK_MODULE_STATE_GLOBAL_TYPE_EXTERNREF: + case WPK_FORK_MODULE_STATE_GLOBAL_TYPE_EXNREF: + case WPK_FORK_MODULE_STATE_GLOBAL_TYPE_ANYREF: + return value; + default: + throw new Error(`invalid imported reference ABI type code ${value}`); + } +} + +function nodeEdges(node: ForkReferenceRecipeNode): readonly number[] { + switch (node.kind) { + case "exnref": + return node.payloads; + case "struct": + return node.fields; + case "array": + return node.elements; + case "null": + case "funcref": + case "externref": + case "i31": + case "static-root": + return []; + } +} + +function sameGcDescriptor( + left: ForkGcCodecDescriptor, + right: ForkGcCodecDescriptor, +): boolean { + if (left.layouts.length !== right.layouts.length) return false; + return left.layouts.every((layout, index) => { + const other = right.layouts[index]!; + return ( + layout.id === other.id + && layout.typeOrdinal === other.typeOrdinal + && layout.kind === other.kind + && layout.constructor === other.constructor + && layout.flags === other.flags + && layout.scalarLengthOrStride === other.scalarLengthOrStride + && layout.superTypeOrdinal === other.superTypeOrdinal + && layout.baseLayoutId === other.baseLayoutId + && layout.auxiliary === other.auxiliary + && layout.provenanceScalarLength === other.provenanceScalarLength + && layout.provenanceReferenceCount === other.provenanceReferenceCount + && layout.fields.length === other.fields.length + && layout.fields.every((field, fieldIndex) => { + const otherField = other.fields[fieldIndex]!; + return ( + field.storage === otherField.storage + && field.flags === otherField.flags + && field.scalarOffset === otherField.scalarOffset + && field.referenceOrdinal === otherField.referenceOrdinal + ); + }) + ); + }); +} + +function validateGcSnapshot( + layout: ForkGcLayoutDescriptor, + scalars: Uint8Array, + references: readonly number[], + context: string, +): void { + const referenceFieldCount = layout.fields.filter( + ({ flags }) => (flags & FORK_GC_FIELD_REFERENCE) !== 0, + ).length; + if (layout.kind === 1) { + if ( + scalars.byteLength !== layout.scalarLengthOrStride + || references.length !== referenceFieldCount + ) { + throw new Error(`${context} does not match struct layout ${layout.id}`); + } + return; + } + if (scalars.byteLength < 4) { + throw new Error(`${context} array length is truncated`); + } + const length = new DataView( + scalars.buffer, + scalars.byteOffset, + scalars.byteLength, + ).getUint32(0, true); + const referenceElements = + (layout.fields[0]!.flags & FORK_GC_FIELD_REFERENCE) !== 0; + const expectedScalarLength = referenceElements + ? 4 + : 4 + length * layout.scalarLengthOrStride; + if ( + !Number.isSafeInteger(expectedScalarLength) + || expectedScalarLength > 0xffff_ffff + || scalars.byteLength !== expectedScalarLength + || references.length !== (referenceElements ? length : 0) + || ( + layout.constructor === ForkGcConstructorKind.ArrayFixed + && layout.auxiliary !== length + ) + ) { + throw new Error(`${context} does not match array layout ${layout.id}`); + } +} + +function validateGcRecipe( + entry: ForkReferenceRecipeEntry, + descriptor: ForkGcCodecDescriptor, +): ForkGcLayoutDescriptor { + const node = entry.node; + if (node.kind !== "struct" && node.kind !== "array") { + throw new Error(`fork recipe ${entry.id} is not a GC aggregate`); + } + const layout = descriptor.require(node.layoutId ?? 0); + if ( + layout.typeOrdinal !== node.typeOrdinal + || (node.kind === "struct" ? 1 : 2) !== layout.kind + ) { + throw new Error( + `fork GC recipe ${entry.id} has an invalid type/layout coordinate`, + ); + } + const scalars = node.scalars ?? new Uint8Array(); + const references = node.kind === "struct" ? node.fields : node.elements; + if ( + scalars.byteLength < layout.provenanceScalarLength + || references.length < layout.provenanceReferenceCount + ) { + throw new Error( + `fork GC recipe ${entry.id} has truncated constructor provenance`, + ); + } + validateGcSnapshot( + layout, + scalars.subarray(layout.provenanceScalarLength), + references.slice(layout.provenanceReferenceCount), + `fork GC recipe ${entry.id}`, + ); + return layout; +} + +function gcAllocationDependencies( + node: Extract, + layout: ForkGcLayoutDescriptor, +): readonly number[] { + const edges = node.kind === "struct" ? node.fields : node.elements; + const dependencies = edges.slice(0, layout.provenanceReferenceCount); + const snapshotStart = layout.provenanceReferenceCount; + if (node.kind === "struct") { + for (const field of layout.fields) { + if ( + (field.flags & FORK_GC_FIELD_ALLOCATION_DEPENDENCY) !== 0 + && field.referenceOrdinal !== null + ) { + dependencies.push(edges[snapshotStart + field.referenceOrdinal]!); + } + } + return dependencies; + } + if ((layout.fields[0]!.flags & FORK_GC_FIELD_REFERENCE) === 0) { + return dependencies; + } + const snapshot = edges.slice(snapshotStart); + if (layout.constructor === ForkGcConstructorKind.ArrayFixed) { + if (layout.provenanceReferenceCount === 0) { + dependencies.push(...snapshot); + } + } else if ( + layout.constructor === ForkGcConstructorKind.ArrayNew + && layout.provenanceReferenceCount === 0 + && snapshot.length !== 0 + ) { + dependencies.push(snapshot[0]!); + } + return dependencies; +} + +/** + * Pre-instantiation child owner for raw imported reference globals. + * + * It never treats a parent Worker object as reconstruction evidence. Every + * non-null value comes from a deterministic recipe owner registered from a + * fresh activation, or from the process externref provider. + */ +export class ForkEarlyChildReferenceProvider + implements ForkImportedReferenceProvider +{ + private transaction: DecodedSegmentedForkReferenceTransaction | null; + private nodes: ForkReferenceDirectory; + private readonly referenceVectors = + new ForkReferenceDirectoryOverlay(); + private readonly referenceVectorIntern: + MutableForkReferenceVectorInternIndex = new Map(); + private readonly declarations = + new Map(); + private readonly registrations = new Map(); + private readonly materializedValues = new Map(); + private readonly publishedExternrefRecipes = new Set(); + private readonly allocatedTypedRecipes = new Set(); + private readonly filledTypedRecipes = new Set(); + private readonly materializedExceptionRecipes = new Set(); + private readonly exceptionCacheIndexes = new Map(); + private readonly gcLayouts = new Map(); + private readonly replayGcVectors = new Map(); + private readonly scratchChunks: ScratchChunk[] = []; + private readonly scratchReservations: ScratchReservation[] = []; + private readonly i31Owner: number | null; + private readonly hostExceptionOwner: number | null; + private transitPrepared = false; + private phase: ProviderPhase = "active"; + private readonly label: string; + private readonly externrefs: ForkExternrefRecipeProvider; + private readonly transit: ForkEarlyReferenceTransit; + private readonly memory: WebAssembly.Memory; + private readonly allocateScratch: ForkReferenceScratchAllocate; + private readonly deallocateScratch: ForkReferenceScratchDeallocate; + + constructor(options: ForkEarlyChildReferenceProviderOptions) { + this.label = options.label ?? "early child references"; + this.externrefs = options.externrefs; + this.transit = options.transit; + this.memory = options.memory; + this.allocateScratch = options.allocateScratch; + this.deallocateScratch = options.deallocateScratch; + this.transaction = options.transaction; + this.nodes = options.transaction.graph.nodes; + // WHY: keep the exact decoded KFRV vector directory as the immutable base. + // Early codec vectors append to a small overlay instead of copying every + // transaction vector into a second page tree. + this.referenceVectors.reset(options.transaction.vectors); + + const moduleIds = new Set(); + for (const record of options.records) { + if (record.kind !== ForkModuleStateRecordKind.Module) continue; + assertU32(record.activationId, "fork module activation"); + if (moduleIds.has(record.activationId)) { + throw new Error( + `${this.label}: duplicate module activation ${record.activationId}`, + ); + } + moduleIds.add(record.activationId); + } + if (moduleIds.size === 0) { + throw new Error(`${this.label}: reference graph has no module activations`); + } + + for (const declaration of options.declarations) { + const activationId = assertU32( + declaration.activationId, + "early reference activation", + ); + if (!moduleIds.has(activationId)) { + throw new Error( + `${this.label}: declaration names unknown activation ${activationId}`, + ); + } + if (this.declarations.has(activationId)) { + throw new Error( + `${this.label}: activation ${activationId} was declared twice`, + ); + } + this.declarations.set(activationId, declaration); + } + for (const activationId of moduleIds) { + if (!this.declarations.has(activationId)) { + throw new Error( + `${this.label}: module activation ${activationId} has no declaration`, + ); + } + } + + this.i31Owner = [...this.declarations.values()] + .filter(({ gcDescriptor }) => gcDescriptor !== undefined) + .map(({ activationId }) => activationId) + .sort((left, right) => left - right)[0] ?? null; + this.hostExceptionOwner = [...this.declarations.values()] + .filter(({ exceptionDescriptor }) => exceptionDescriptor !== undefined) + .map(({ activationId }) => activationId) + .sort((left, right) => left - right)[0] ?? null; + + for (const binding of importedGlobalBindingsForChild(options.records)) { + if ( + binding.kind === ForkImportedGlobalBindingKind.RawReference + && binding.typeCode === WPK_FORK_MODULE_STATE_GLOBAL_TYPE_EXNREF + && binding.recipeId !== 0 + ) { + // WHY: JavaScript cannot read a non-null exnref out of a Global or + // carry one as a raw import value. Parent capture therefore represents + // every real non-null exnref import as ActivationGlobal/BaseImport; a + // nonzero raw recipe can only be a malformed provenance manifest. + throw new Error( + `${this.label}: imported exnref ${binding.consumerActivation}:` + + `${binding.consumerOwner} has a non-null raw recipe instead of ` + + "an activation-owned Global carrier", + ); + } + } + + for (const entry of this.nodes) { + this.validateRecipeOwnership(entry, moduleIds); + if (entry.node.kind === "exnref") { + this.exceptionCacheIndexes.set( + entry.id, + this.exceptionCacheIndexes.size + 1, + ); + } + } + } + + registerActivation(providers: ForkEarlyReferenceActivationProviders): void { + this.requireActive("register an activation"); + const activationId = assertU32( + providers.activationId, + "early reference activation", + ); + const declaration = this.declarations.get(activationId); + if (!declaration) { + throw new Error( + `${this.label}: activation ${activationId} was not declared`, + ); + } + if (this.registrations.has(activationId)) { + throw new Error( + `${this.label}: activation ${activationId} was registered twice`, + ); + } + + let ownsFuncref = false; + let ownsStaticRoot = false; + let ownsTyped = false; + let ownsException = false; + for (const { node } of this.nodes) { + if (this.directOwner(node) !== activationId) continue; + ownsFuncref ||= node.kind === "funcref"; + ownsStaticRoot ||= node.kind === "static-root"; + ownsTyped ||= ( + node.kind === "struct" || node.kind === "array" || node.kind === "i31" + ); + ownsException ||= node.kind === "exnref"; + } + if (ownsFuncref && !providers.functions) { + throw new Error( + `${this.label}: activation ${activationId} has no function provider`, + ); + } + if ( + ownsStaticRoot && !providers.staticRoots + ) { + throw new Error( + `${this.label}: activation ${activationId} has no static-root provider`, + ); + } + if ( + ( + ownsTyped + || this.i31Owner === activationId + ) + ) { + if ( + !providers.typed + || providers.typed.activationId !== activationId + || !declaration.gcDescriptor + || !sameGcDescriptor( + providers.typed.descriptor, + declaration.gcDescriptor, + ) + ) { + throw new Error( + `${this.label}: activation ${activationId} has no matching GC provider`, + ); + } + } + if ( + ( + ownsException + || this.hostExceptionOwner === activationId + ) + && typeof providers.exceptions?.materialize !== "function" + ) { + throw new Error( + `${this.label}: activation ${activationId} has no exception materializer`, + ); + } + + this.registrations.set(activationId, { ...providers }); + } + + ownerActivation(recipeId: number, typeCode: number): number | null { + this.requireActive("plan a reference owner"); + const entry = this.requireCompatibleRecipe(recipeId, typeCode); + return this.directOwner(entry.node); + } + + /** + * Every activation needed to reconstruct the complete reachable identity. + * + * `ForkImportedReferenceProvider.ownerActivation` predates typed graphs and + * can name only the direct owner. Loaders should add this full set to their + * topological dependency graph before resolving a raw reference import. + */ + activationDependencies(recipeId: number, typeCode: number): number[] { + this.requireActive("plan reference dependencies"); + const entry = this.requireCompatibleRecipe(recipeId, typeCode); + const dependencies = new Set(); + const visited = new Set(); + const visit = (id: number): void => { + if (visited.has(id)) return; + visited.add(id); + const node = this.nodes.get(id)!.node; + const owner = this.directOwner(node); + if (owner !== null) dependencies.add(owner); + nodeEdges(node).forEach(visit); + }; + visit(entry.id); + return [...dependencies].sort((left, right) => left - right); + } + + materialize(recipeId: number, typeCode: number): unknown { + this.requireActive("materialize an imported reference"); + const entry = this.requireCompatibleRecipe(recipeId, typeCode); + if ( + entry.node.kind === "exnref" + && entry.id !== 0 + ) { + throw new Error( + `${this.label}: non-null exnref recipe ${entry.id} cannot cross ` + + "JavaScript; import its activation-owned WebAssembly.Global instead", + ); + } + this.requireRegisteredDependencies(entry.id, typeCode); + try { + const value = this.materializeRecipe(entry.id); + this.validateMaterializedValue(entry.id, typeCode, value); + return value; + } catch (error) { + // A provider that returned a malformed value may already have retained + // it in a catalog or transit slot. Poison the one-shot owner and release + // all early roots instead of permitting a retry over ambiguous state. + this.abortAfterFailure(error); + } + } + + decodeFuncref(recipeId: number): CallableFunction | null { + const value = this.materialize( + recipeId, + WPK_FORK_MODULE_STATE_GLOBAL_TYPE_FUNCREF, + ); + if (value !== null && typeof value !== "function") { + throw new TypeError( + `${this.label}: recipe ${recipeId} did not reconstruct a funcref`, + ); + } + return value as CallableFunction | null; + } + + decodeExternref(recipeId: number): unknown { + return this.materialize( + recipeId, + WPK_FORK_MODULE_STATE_GLOBAL_TYPE_EXTERNREF, + ); + } + + getReferenceVector(ordinal: number, index: number): number { + this.requireActive("read a reference vector"); + if ( + !Number.isInteger(ordinal) + || ordinal < 0 + || ordinal > 0xffff_ffff + ) { + throw new RangeError( + `${this.label}: reference vector ordinal is not a u32`, + ); + } + assertU32(index, "reference vector index"); + const vector = this.referenceVectors.get(ordinal); + if (!vector) { + throw new Error( + `${this.label}: reference vector ${ordinal} is not available`, + ); + } + const recipeId = vector.get(index); + if (recipeId === undefined) { + throw new Error( + `${this.label}: reference vector ${ordinal} index ${index} ` + + "is out of bounds", + ); + } + return recipeId; + } + + routeGc(recipeId: number, expectedActivation: number): number { + this.requireActive("route a GC recipe"); + const entry = this.requireRecipe(recipeId); + assertU32(expectedActivation, "GC route activation"); + if (entry.node.kind === "i31") return 0; + if ( + (entry.node.kind !== "struct" && entry.node.kind !== "array") + || entry.node.moduleActivation !== expectedActivation + ) { + return -1; + } + return entry.node.layoutId ?? 0; + } + + gcPayloadLength( + recipeId: number, + expectedActivation: number, + expectedLayoutId: number, + ): number { + this.requireActive("read a GC payload length"); + const entry = this.requireRecipe(recipeId); + assertU32(expectedActivation, "GC payload activation"); + assertU32(expectedLayoutId, "GC payload layout"); + if (entry.node.kind === "i31") { + if (expectedLayoutId !== 0) { + throw new Error( + `${this.label}: i31 recipe ${recipeId} has a nonzero layout`, + ); + } + return 4; + } + if ( + (entry.node.kind !== "struct" && entry.node.kind !== "array") + || entry.node.moduleActivation !== expectedActivation + || (entry.node.layoutId ?? 0) !== expectedLayoutId + ) { + throw new Error( + `${this.label}: GC recipe ${recipeId} does not match payload route ` + + `${expectedActivation}:${expectedLayoutId}`, + ); + } + return (entry.node.scalars ?? new Uint8Array()).byteLength; + } + + loadGc( + recipeId: number, + moduleActivation: number, + typeOrdinal: number, + layoutId: number, + kind: number, + scalarDestination: number | bigint, + scalarByteLength: number, + ): number { + this.requireActive("load a GC recipe"); + const entry = this.requireRecipe(recipeId); + assertU32(moduleActivation, "GC load activation"); + assertU32(typeOrdinal, "GC load type ordinal"); + assertU32(layoutId, "GC load layout"); + assertU32(kind, "GC load kind"); + assertU32(scalarByteLength, "GC scalar byte length"); + if (entry.node.kind === "i31") { + if ( + layoutId !== 0 + || typeOrdinal !== 0xffff_ffff + || kind !== 0 + || scalarByteLength !== 4 + ) { + throw new Error( + `${this.label}: i31 recipe ${recipeId} has an invalid load coordinate`, + ); + } + const bytes = new Uint8Array(4); + new DataView(bytes.buffer).setInt32(0, entry.node.value, true); + this.writeBytes( + scalarDestination, + bytes, + "early GC i31 destination", + ); + return 0; + } + if (entry.node.kind !== "struct" && entry.node.kind !== "array") { + throw new Error( + `${this.label}: recipe ${recipeId} is not a GC aggregate`, + ); + } + const nodeKind = entry.node.kind === "struct" ? 1 : 2; + const scalars = entry.node.scalars ?? new Uint8Array(); + if ( + entry.node.moduleActivation !== moduleActivation + || entry.node.typeOrdinal !== typeOrdinal + || (entry.node.layoutId ?? 0) !== layoutId + || nodeKind !== kind + || scalars.byteLength !== scalarByteLength + ) { + throw new Error( + `${this.label}: GC recipe ${recipeId} payload does not match ` + + "the generated codec", + ); + } + this.writeBytes( + scalarDestination, + scalars, + "early GC scalar destination", + ); + const edges = + entry.node.kind === "struct" ? entry.node.fields : entry.node.elements; + if (edges.length === 0) return 0; + const known = this.replayGcVectors.get(recipeId); + if (known !== undefined) return known; + const existing = findForkReferenceVectorOrdinal( + [ + this.transaction!.vectorIntern, + this.referenceVectorIntern, + ], + this.referenceVectors, + forkReferenceVectorFrom(edges, edges.length), + ); + if (existing !== undefined) { + this.replayGcVectors.set(recipeId, existing); + return existing; + } + const ordinal = this.referenceVectors.length; + if (ordinal > MAX_REFERENCE_VECTOR_ORDINAL) { + throw new RangeError( + `${this.label}: reference vector ordinal space exhausted`, + ); + } + const canonical = forkReferenceVectorFrom(edges, edges.length); + this.referenceVectors.push(canonical); + indexForkReferenceVector(this.referenceVectorIntern, canonical, ordinal); + this.replayGcVectors.set(recipeId, ordinal); + return ordinal; + } + + routeException(recipeId: number, expectedActivation: number): number { + this.requireActive("route an exception recipe"); + const entry = this.requireRecipe(recipeId); + assertU32(expectedActivation, "exception route activation"); + if ( + entry.node.kind !== "exnref" + || entry.node.moduleActivation !== expectedActivation + ) { + return -1; + } + return entry.node.layoutId ?? 0; + } + + exceptionOwner(recipeId: number): number { + this.requireActive("read an exception owner"); + const entry = this.requireRecipe(recipeId); + if (entry.node.kind !== "exnref") { + throw new Error( + `${this.label}: recipe ${recipeId} is not an exception`, + ); + } + // WHY: retain the process-graph owner here instead of the activation used + // to instantiate a host-exception codec. ForkExceptionBroker needs the + // sentinel to distinguish a host/JSTag value from an activation tag. + return entry.node.moduleActivation; + } + + materializeHostException(recipeId: number): unknown { + const owner = this.exceptionOwner(recipeId); + if (owner !== FORK_HOST_EXCEPTION_ACTIVATION_ID) { + throw new Error( + `${this.label}: exception recipe ${recipeId} is not host-owned`, + ); + } + const entry = this.requireRecipe(recipeId); + if ( + entry.node.kind !== "exnref" + || entry.node.payloads.length !== 1 + || this.nodes.get(entry.node.payloads[0]!)?.node.kind !== "externref" + ) { + throw new Error( + `${this.label}: host exception recipe ${recipeId} is malformed`, + ); + } + // The payload is an opaque process-owned handle; decodeExternref provides + // the same canonical child token to every early broker invocation. + return this.decodeExternref(entry.node.payloads[0]!); + } + + exceptionCacheIndex(recipeId: number): number { + this.requireActive("read an exception cache index"); + this.requireRecipe(recipeId); + const index = this.exceptionCacheIndexes.get(recipeId); + if (index === undefined) { + throw new Error( + `${this.label}: recipe ${recipeId} has no exception cache index`, + ); + } + return index; + } + + loadException( + recipeId: number, + moduleActivation: number, + tagOrdinal: number, + layoutId: number, + scalarDestination: number | bigint, + scalarByteLength: number, + referenceIdsDestination: number | bigint, + referenceCount: number, + ): number { + this.requireActive("load an exception recipe"); + const entry = this.requireRecipe(recipeId); + assertU32(moduleActivation, "exception load activation"); + assertU32(tagOrdinal, "exception load tag ordinal"); + assertU32(layoutId, "exception load layout"); + assertU32(scalarByteLength, "exception scalar byte length"); + assertU32(referenceCount, "exception reference count"); + if (entry.node.kind !== "exnref") { + throw new Error( + `${this.label}: recipe ${recipeId} is not an exception`, + ); + } + const scalars = entry.node.scalars ?? new Uint8Array(); + if ( + entry.node.moduleActivation !== moduleActivation + || entry.node.tagOrdinal !== tagOrdinal + || (entry.node.layoutId ?? 0) !== layoutId + || scalars.byteLength !== scalarByteLength + || entry.node.payloads.length !== referenceCount + ) { + throw new Error( + `${this.label}: exception recipe ${recipeId} payload does not match ` + + "the generated codec", + ); + } + this.writeBytes( + scalarDestination, + scalars, + "early exception scalar destination", + ); + this.writeRecipeIds( + referenceIdsDestination, + entry.node.payloads, + "early exception reference destination", + ); + return 1; + } + + reserveScratch(size: number | bigint): number { + this.requireActive("reserve reference scratch"); + const requestedSize = this.checkedScratchSize(size); + const alignedSize = this.alignScratch(requestedSize); + let chunk = this.scratchChunks[this.scratchChunks.length - 1]; + if (!chunk || alignedSize > chunk.size - chunk.used) { + const chunkSize = this.alignScratch(Math.max(65_536, alignedSize), 65_536); + const addr = this.allocateScratch(chunkSize); + if ( + !Number.isSafeInteger(addr) + || addr <= 0 + || addr % 16 !== 0 + || addr > this.memory.buffer.byteLength - chunkSize + ) { + if (Number.isSafeInteger(addr) && addr > 0) { + try { + this.deallocateScratch(addr, chunkSize); + } catch { + // Preserve the allocator contract violation. + } + } + throw new RangeError( + `${this.label}: scratch allocator returned an invalid mapping`, + ); + } + chunk = { addr, size: chunkSize, used: 0 }; + this.scratchChunks.push(chunk); + } + const previousUsed = chunk.used; + const addr = chunk.addr + previousUsed; + chunk.used += alignedSize; + new Uint8Array(this.memory.buffer, addr, alignedSize).fill(0); + this.scratchReservations.push({ + addr, + requestedSize, + alignedSize, + previousUsed, + chunk, + }); + return addr; + } + + releaseScratch(pointer: number | bigint, size: number | bigint): void { + this.requireActive("release reference scratch"); + const addr = this.checkedScratchPointer(pointer); + const requestedSize = this.checkedScratchSize(size); + const reservation = this.scratchReservations.pop(); + if ( + !reservation + || reservation.addr !== addr + || reservation.requestedSize !== requestedSize + ) { + if (reservation) this.scratchReservations.push(reservation); + throw new Error( + `${this.label}: scratch release is not the most recent reservation`, + ); + } + new Uint8Array( + this.memory.buffer, + reservation.addr, + reservation.alignedSize, + ).fill(0); + reservation.chunk.used = reservation.previousUsed; + const tail = this.scratchChunks[this.scratchChunks.length - 1]; + if ( + tail === reservation.chunk + && tail.used === 0 + && this.scratchChunks.length > 1 + ) { + this.scratchChunks.pop(); + this.deallocateScratch(tail.addr, tail.size); + } + } + + /** + * Adapter target for encode/claim/define imports while the child is still + * constructing activations. Those callbacks are capture-only by contract. + */ + captureUnavailable(operation: string): never { + this.requireActive(`run capture callback ${operation}`); + throw new Error( + `${this.label}: capture callback ${operation} is unavailable during ` + + "pre-instantiation child replay", + ); + } + + adoptInto(transaction: ForkReferenceTransaction): void { + this.requireActive("adopt reference replay"); + if (this.scratchReservations.length !== 0) { + throw new Error( + `${this.label}: cannot adopt with ` + + `${this.scratchReservations.length} live scratch reservation(s)`, + ); + } + this.releaseScratchChunks(); + const adoption: ForkReferenceChildReplayAdoption = { + transaction: this.transaction!, + materializedValues: this.materializedValues, + allocatedTypedRecipes: this.allocatedTypedRecipes, + filledTypedRecipes: this.filledTypedRecipes, + materializedExceptionRecipes: this.materializedExceptionRecipes, + }; + transaction.adoptChildReplay(adoption); + // WHY: the transaction copied every sparse value and milestone. Clear only + // this owner's JS collections; the registry now owns codec/transit cleanup. + this.releaseCollections(); + this.phase = "adopted"; + } + + abort(): void { + if (this.phase === "aborted") return; + if (this.phase === "adopted") { + throw new Error(`${this.label}: adopted reference replay cannot be aborted`); + } + const callbacks = [...this.registrations.values()] + .sort((left, right) => right.activationId - left.activationId) + .flatMap(({ abort }) => abort ? [abort] : []); + this.releaseCollections(); + this.phase = "aborted"; + + const failures: unknown[] = []; + try { + if (this.scratchReservations.length !== 0) { + // A trapping codec can bypass its generated release. All reservations + // still belong to this one-shot owner, so zero and release the complete + // retained chunk set during abort. + this.scratchReservations.length = 0; + } + this.releaseScratchChunks(); + } catch (error) { + failures.push(error); + } + for (const callback of callbacks) { + try { + callback(); + } catch (error) { + failures.push(error); + } + } + try { + this.transit.abort(); + } catch (error) { + failures.push(error); + } + if (failures.length === 1) throw failures[0]; + if (failures.length > 1) { + throw new AggregateError( + failures, + `${this.label}: early reference cleanup was incomplete`, + ); + } + } + + private validateRecipeOwnership( + entry: ForkReferenceRecipeEntry, + moduleIds: ReadonlySet, + ): void { + const node = entry.node; + const requireModule = (activationId: number, kind: string): void => { + if (!moduleIds.has(activationId)) { + throw new Error( + `${this.label}: ${kind} recipe ${entry.id} names missing ` + + `activation ${activationId}`, + ); + } + }; + switch (node.kind) { + case "funcref": + requireModule(node.moduleActivation, "funcref"); + break; + case "static-root": + requireModule(node.moduleActivation, "static-root"); + break; + case "struct": + case "array": { + requireModule(node.moduleActivation, node.kind); + const descriptor = + this.declarations.get(node.moduleActivation)?.gcDescriptor; + if (!descriptor) { + throw new Error( + `${this.label}: ${node.kind} recipe ${entry.id} owner ` + + `${node.moduleActivation} has no GC descriptor`, + ); + } + this.gcLayouts.set(entry.id, validateGcRecipe(entry, descriptor)); + break; + } + case "exnref": { + if (node.moduleActivation === FORK_HOST_EXCEPTION_ACTIVATION_ID) { + if ( + this.hostExceptionOwner === null + || node.tagOrdinal !== 0 + || (node.layoutId ?? 0) !== 0 + || (node.scalars?.byteLength ?? 0) !== 0 + || node.payloads.length !== 1 + || this.nodes.get(node.payloads[0]!)?.node.kind !== "externref" + ) { + throw new Error( + `${this.label}: host exception recipe ${entry.id} is malformed ` + + "or has no fresh-child codec", + ); + } + break; + } + requireModule(node.moduleActivation, "exnref"); + const descriptor = + this.declarations.get(node.moduleActivation)?.exceptionDescriptor; + const layout = descriptor?.tags[node.tagOrdinal]; + if ( + !layout + || layout.tagOrdinal !== node.tagOrdinal + || layout.layoutId !== (node.layoutId ?? 0) + || layout.scalarByteLength !== (node.scalars?.byteLength ?? 0) + || layout.referenceCount !== node.payloads.length + ) { + throw new Error( + `${this.label}: exnref recipe ${entry.id} does not match ` + + `activation ${node.moduleActivation}'s exception descriptor`, + ); + } + break; + } + case "i31": + if (this.i31Owner === null) { + throw new Error( + `${this.label}: i31 recipe ${entry.id} has no fresh-child GC codec`, + ); + } + break; + case "null": + case "externref": + break; + } + } + + private directOwner(node: ForkReferenceRecipeNode): number | null { + switch (node.kind) { + case "funcref": + case "struct": + case "array": + case "static-root": + return node.moduleActivation; + case "exnref": + return node.moduleActivation === FORK_HOST_EXCEPTION_ACTIVATION_ID + ? this.hostExceptionOwner + : node.moduleActivation; + case "i31": + return this.i31Owner; + case "null": + case "externref": + return null; + } + } + + private requireCompatibleRecipe( + recipeId: number, + typeCode: number, + ): ForkReferenceRecipeEntry { + const id = assertRecipeId(recipeId, this.nodes.length); + const code = requireReferenceTypeCode(typeCode); + const entry = this.nodes.get(id)!; + const kind = entry.node.kind; + const compatible = kind === "null" || ( + code === WPK_FORK_MODULE_STATE_GLOBAL_TYPE_FUNCREF + ? kind === "funcref" || kind === "static-root" + : code === WPK_FORK_MODULE_STATE_GLOBAL_TYPE_EXTERNREF + ? ( + kind === "funcref" + || kind === "externref" + || kind === "static-root" + ) + : code === WPK_FORK_MODULE_STATE_GLOBAL_TYPE_EXNREF + ? kind === "exnref" + : ( + kind === "i31" + || kind === "struct" + || kind === "array" + || kind === "static-root" + ) + ); + if (!compatible) { + throw new Error( + `${this.label}: ${kind} recipe ${id} cannot initialize ` + + `reference type code ${code}`, + ); + } + return entry; + } + + private requireRecipe(recipeId: number): ForkReferenceRecipeEntry { + return this.nodes.get(assertRecipeId(recipeId, this.nodes.length))!; + } + + private requireRegisteredDependencies( + recipeId: number, + typeCode: number, + ): void { + for (const activationId of this.activationDependencies(recipeId, typeCode)) { + if (!this.registrations.has(activationId)) { + throw new Error( + `${this.label}: recipe ${recipeId} needs unregistered activation ` + + `${activationId}`, + ); + } + } + } + + private materializeRecipe(recipeId: number): unknown { + if (this.materializedValues.has(recipeId)) { + return this.materializedValues.get(recipeId); + } + const entry = this.nodes.get(recipeId)!; + const node = entry.node; + let value: unknown; + switch (node.kind) { + case "null": + value = null; + break; + case "funcref": { + value = this.requireRegistration(node.moduleActivation).functions! + .decode(node.functionOrdinal); + if (typeof value !== "function") { + throw new TypeError( + `${this.label}: funcref recipe ${recipeId} did not produce a function`, + ); + } + break; + } + case "externref": + value = this.externrefs.materialize(node.handle); + break; + case "static-root": + this.prepareTransit(); + value = this.requireRegistration(node.moduleActivation).staticRoots! + .decode(node.staticRootOrdinal); + // Static-root catalogs contain only GC-domain references accepted by + // the instrumenter's `(ref null any)` harvest table. Unlike dynamic + // recipes, no generated allocator exists to publish this identity. + this.transit.publish(recipeId, value); + break; + case "i31": + case "struct": + case "array": + this.materializeTypedGraph(recipeId); + return this.materializedValues.get(recipeId); + case "exnref": + this.materializeException(recipeId, new Set()); + return undefined; + } + this.materializedValues.set(recipeId, value); + return value; + } + + private materializeTypedGraph(rootRecipeId: number): void { + this.prepareTransit(); + const reachable = this.reachableRecipes(rootRecipeId); + for (const entry of this.nodes) { + if (!reachable.has(entry.id) || entry.node.kind !== "externref") continue; + this.publishExternref(entry.id); + } + for (const entry of this.nodes) { + if (!reachable.has(entry.id)) continue; + if (entry.node.kind === "static-root") { + // WHY: immutable constructors can consume static roots while + // allocating, before the later identity walk. Publish every reachable + // instantiation-owned root first so both constructor dependencies and + // mutable field fills observe the activation's canonical identity. + this.materializeRecipe(entry.id); + } + } + for (const entry of this.nodes) { + if (!reachable.has(entry.id)) continue; + const { node } = entry; + if (node.kind !== "struct" && node.kind !== "array") continue; + const layout = this.gcLayouts.get(entry.id)!; + if ( + (layout.flags & FORK_GC_LAYOUT_DEFAULTABLE_SHELL) !== 0 + && !this.allocatedTypedRecipes.has(entry.id) + ) { + this.allocateTyped(entry.id, new Set(), true); + } + } + + const visiting = new Set(); + for (const entry of this.nodes) { + if (reachable.has(entry.id)) this.ensureIdentity(entry.id, visiting); + } + for (const entry of this.nodes) { + if (!reachable.has(entry.id)) continue; + const { node } = entry; + if ( + (node.kind !== "struct" && node.kind !== "array") + || this.filledTypedRecipes.has(entry.id) + ) { + continue; + } + nodeEdges(node).forEach((edge) => this.ensureIdentity(edge, visiting)); + this.requireRegistration(node.moduleActivation).typed!.fill(entry.id); + this.filledTypedRecipes.add(entry.id); + } + } + + private ensureIdentity(recipeId: number, visiting: Set): void { + const node = this.nodes.get(recipeId)!.node; + switch (node.kind) { + case "null": + case "funcref": + case "static-root": + this.materializeRecipe(recipeId); + return; + case "externref": + this.publishExternref(recipeId); + return; + case "exnref": + this.materializeException(recipeId, visiting); + return; + case "i31": + case "struct": + case "array": + this.allocateTyped(recipeId, visiting); + return; + } + } + + private prepareTransit(): void { + if (this.transitPrepared) return; + this.transit.prepare(Math.max(0, this.nodes.length - 1)); + this.transitPrepared = true; + } + + private publishExternref(recipeId: number): void { + if (this.publishedExternrefRecipes.has(recipeId)) return; + const entry = this.nodes.get(recipeId); + if (entry?.node.kind !== "externref") { + throw new Error(`${this.label}: recipe ${recipeId} is not an externref`); + } + const value = this.materializeRecipe(recipeId); + const publisher = [...this.registrations.values()] + .filter((registration) => registration.typed !== undefined) + .sort((left, right) => left.activationId - right.activationId)[0]?.typed; + if (!publisher) { + throw new Error( + `${this.label}: externref recipe ${recipeId} has no generated GC codec`, + ); + } + // WHY: the transit table stores anyref. Only generated Wasm can perform + // the required any.convert_extern for this Worker's canonical token. + publisher.publishExternref(recipeId, value); + if (!Object.is(this.transit.read(recipeId), value)) { + throw new Error( + `${this.label}: externref recipe ${recipeId} lost token identity ` + + "during anyref publication", + ); + } + this.publishedExternrefRecipes.add(recipeId); + } + + private allocateTyped( + recipeId: number, + visiting: Set, + defaultableShell = false, + ): void { + if (this.allocatedTypedRecipes.has(recipeId)) return; + if (visiting.has(recipeId)) { + throw new Error( + `${this.label}: typed replay has an unallocatable constructor cycle ` + + `at recipe ${recipeId}`, + ); + } + const node = this.nodes.get(recipeId)!.node; + if (node.kind !== "i31" && node.kind !== "struct" && node.kind !== "array") { + throw new Error(`${this.label}: recipe ${recipeId} is not a typed reference`); + } + visiting.add(recipeId); + try { + let activationId: number; + if (node.kind === "i31") { + activationId = this.i31Owner!; + } else { + activationId = node.moduleActivation; + if (!defaultableShell) { + const layout = this.gcLayouts.get(recipeId)!; + gcAllocationDependencies(node, layout) + .forEach((dependency) => this.ensureIdentity(dependency, visiting)); + } + } + this.requireRegistration(activationId).typed!.allocate(recipeId); + const value = this.transit.read(recipeId); + if (value === null || value === undefined) { + throw new Error( + `${this.label}: typed provider did not publish recipe ${recipeId}`, + ); + } + this.materializedValues.set(recipeId, value); + this.allocatedTypedRecipes.add(recipeId); + } finally { + visiting.delete(recipeId); + } + } + + private materializeException( + recipeId: number, + visiting: Set, + ): void { + if (this.materializedExceptionRecipes.has(recipeId)) return; + if (visiting.has(recipeId)) { + throw new Error( + `${this.label}: exception replay has an unallocatable cycle at ` + + `recipe ${recipeId}`, + ); + } + const node = this.nodes.get(recipeId)!.node; + if (node.kind !== "exnref") { + throw new Error(`${this.label}: recipe ${recipeId} is not an exception`); + } + visiting.add(recipeId); + try { + node.payloads.forEach((payload) => this.ensureIdentity(payload, visiting)); + const owner = this.directOwner(node)!; + this.requireRegistration(owner).exceptions!.materialize!(recipeId); + this.materializedExceptionRecipes.add(recipeId); + } finally { + visiting.delete(recipeId); + } + } + + private reachableRecipes(rootRecipeId: number): ReadonlySet { + const visited = new Set(); + const visit = (recipeId: number): void => { + if (visited.has(recipeId)) return; + visited.add(recipeId); + nodeEdges(this.nodes.get(recipeId)!.node).forEach(visit); + }; + visit(rootRecipeId); + return visited; + } + + private validateMaterializedValue( + recipeId: number, + typeCode: number, + value: unknown, + ): void { + if ( + typeCode === WPK_FORK_MODULE_STATE_GLOBAL_TYPE_FUNCREF + && value !== null + && typeof value !== "function" + ) { + throw new TypeError( + `${this.label}: recipe ${recipeId} did not materialize a funcref`, + ); + } + if ( + typeCode === WPK_FORK_MODULE_STATE_GLOBAL_TYPE_EXNREF + && value !== null + ) { + throw new TypeError( + `${this.label}: recipe ${recipeId} did not materialize a nullable exnref`, + ); + } + } + + private writeBytes( + pointer: number | bigint, + bytes: Uint8Array, + context: string, + ): void { + const { offset } = this.memoryRange(pointer, bytes.byteLength, context); + new Uint8Array(this.memory.buffer, offset, bytes.byteLength).set(bytes); + } + + private writeRecipeIds( + pointer: number | bigint, + ids: readonly number[], + context: string, + ): void { + const { offset } = this.memoryRange(pointer, ids.length * 4, context); + const view = new DataView(this.memory.buffer); + ids.forEach((id, index) => { + assertRecipeId(id, this.nodes.length); + view.setUint32(offset + index * 4, id, true); + }); + } + + private memoryRange( + pointer: number | bigint, + byteLength: number, + context: string, + ): { readonly offset: number; readonly length: number } { + assertU32(byteLength, `${context} byte length`); + const offset = typeof pointer === "bigint" ? Number(pointer) : pointer; + if ( + !Number.isSafeInteger(offset) + || offset < 0 + || (typeof pointer === "bigint" && BigInt(offset) !== pointer) + ) { + throw new RangeError(`${this.label}: ${context} has an invalid guest pointer`); + } + const memoryLength = this.memory.buffer.byteLength; + if (offset > memoryLength || byteLength > memoryLength - offset) { + throw new RangeError(`${this.label}: ${context} exceeds WebAssembly memory`); + } + return { offset, length: byteLength }; + } + + private checkedScratchPointer(value: number | bigint): number { + const result = typeof value === "bigint" ? Number(value) : value; + if ( + !Number.isSafeInteger(result) + || result <= 0 + || (typeof value === "bigint" && BigInt(result) !== value) + ) { + throw new RangeError(`${this.label}: scratch pointer is invalid`); + } + return result; + } + + private checkedScratchSize(value: number | bigint): number { + const result = typeof value === "bigint" ? Number(value) : value; + if ( + !Number.isSafeInteger(result) + || result <= 0 + || result > 0xffff_ffff + || (typeof value === "bigint" && BigInt(result) !== value) + ) { + throw new RangeError(`${this.label}: scratch size is not a nonzero u32`); + } + return result; + } + + private alignScratch(value: number, alignment = 16): number { + const result = Math.ceil(value / alignment) * alignment; + if (!Number.isSafeInteger(result) || result < value) { + throw new RangeError(`${this.label}: scratch alignment overflow`); + } + return result; + } + + private releaseScratchChunks(): void { + if (this.scratchReservations.length !== 0) { + throw new Error( + `${this.label}: cannot release scratch with live reservations`, + ); + } + const chunks = this.scratchChunks.splice(0).reverse(); + const failures: unknown[] = []; + for (const chunk of chunks) { + try { + new Uint8Array(this.memory.buffer, chunk.addr, chunk.size).fill(0); + this.deallocateScratch(chunk.addr, chunk.size); + } catch (error) { + failures.push(error); + } + } + if (failures.length === 1) throw failures[0]; + if (failures.length > 1) { + throw new AggregateError( + failures, + `${this.label}: scratch cleanup was incomplete`, + ); + } + } + + private requireRegistration(activationId: number): RegisteredActivation { + const registration = this.registrations.get(activationId); + if (!registration) { + throw new Error( + `${this.label}: activation ${activationId} is not registered`, + ); + } + return registration; + } + + private abortAfterFailure(cause: unknown): never { + try { + this.abort(); + } catch (cleanupError) { + throw new AggregateError( + [cause, cleanupError], + `${this.label}: reference materialization and cleanup both failed`, + ); + } + throw cause; + } + + private releaseCollections(): void { + this.materializedValues.clear(); + this.publishedExternrefRecipes.clear(); + this.allocatedTypedRecipes.clear(); + this.filledTypedRecipes.clear(); + this.materializedExceptionRecipes.clear(); + this.exceptionCacheIndexes.clear(); + this.gcLayouts.clear(); + this.replayGcVectors.clear(); + this.referenceVectorIntern.clear(); + this.registrations.clear(); + this.declarations.clear(); + this.nodes = new PagedForkReferenceDirectory(); + this.referenceVectors.clear(); + this.transaction = null; + } + + private requireActive(operation: string): void { + if (this.phase !== "active") { + throw new Error( + `${this.label}: cannot ${operation} after provider was ${this.phase}`, + ); + } + } +} diff --git a/host/src/fork-exception-provider.ts b/host/src/fork-exception-provider.ts new file mode 100644 index 0000000000..981efe8095 --- /dev/null +++ b/host/src/fork-exception-provider.ts @@ -0,0 +1,509 @@ +import { + type ForkActivationExceptionProvider, + ForkActivationRegistry, +} from "./fork-activation-registry"; +import { + FORK_HOST_EXCEPTION_ACTIVATION_ID, + type ForkExceptionSlotProvider, +} from "./fork-reference-transaction"; +import { + WPK_FORK_EXCEPTION_CODEC_HEADER_SIZE, + WPK_FORK_EXCEPTION_CODEC_SECTION, + WPK_FORK_EXCEPTION_CODEC_TAG_RECORD_SIZE, + WPK_FORK_EXCEPTION_CODEC_VERSION, + WPK_FORK_EXCEPTION_EXPORT_ABORT, + WPK_FORK_EXCEPTION_EXPORT_CLEAR, + WPK_FORK_EXCEPTION_EXPORT_DECODE, + WPK_FORK_EXCEPTION_EXPORT_ENCODE, + WPK_FORK_EXCEPTION_EXPORT_ENCODE_INGRESS, + WPK_FORK_EXCEPTION_EXPORT_MATERIALIZE, + WPK_FORK_EXCEPTION_EXPORT_THROW_RECIPE, + WPK_FORK_EXCEPTION_EXPORT_THROW_SLOT, + WPK_FORK_EXCEPTION_IMPORT_ACTIVATION, + WPK_FORK_EXCEPTION_IMPORT_BROKER_ENCODE, + WPK_FORK_EXCEPTION_IMPORT_BROKER_THROW_RECIPE, + WPK_FORK_EXCEPTION_IMPORT_CACHE_INDEX, + WPK_FORK_EXCEPTION_IMPORT_CLAIM, + WPK_FORK_EXCEPTION_IMPORT_DEFINE, + WPK_FORK_EXCEPTION_IMPORT_INGRESS_THROW, + WPK_FORK_EXCEPTION_IMPORT_LOAD, + WPK_FORK_EXCEPTION_IMPORT_LOOKUP, + WPK_FORK_EXCEPTION_IMPORT_ROUTE, + WPK_FORK_REFERENCE_IMPORT_SCRATCH_RELEASE, + WPK_FORK_REFERENCE_IMPORT_SCRATCH_RESERVE, +} from "./generated/abi"; + +export const FORK_EXCEPTION_CODEC_SECTION = + WPK_FORK_EXCEPTION_CODEC_SECTION; +export const FORK_EXCEPTION_CODEC_VERSION = WPK_FORK_EXCEPTION_CODEC_VERSION; +export const FORK_EXCEPTION_CODEC_HEADER_SIZE = + WPK_FORK_EXCEPTION_CODEC_HEADER_SIZE; +export const FORK_EXCEPTION_CODEC_TAG_RECORD_SIZE = + WPK_FORK_EXCEPTION_CODEC_TAG_RECORD_SIZE; + +export const FORK_EXCEPTION_ACTIVATION_IMPORT = + WPK_FORK_EXCEPTION_IMPORT_ACTIVATION; +export const FORK_EXCEPTION_LOOKUP_IMPORT = WPK_FORK_EXCEPTION_IMPORT_LOOKUP; +export const FORK_EXCEPTION_CLAIM_IMPORT = WPK_FORK_EXCEPTION_IMPORT_CLAIM; +export const FORK_EXCEPTION_DEFINE_IMPORT = WPK_FORK_EXCEPTION_IMPORT_DEFINE; +export const FORK_EXCEPTION_LOAD_IMPORT = WPK_FORK_EXCEPTION_IMPORT_LOAD; +export const FORK_EXCEPTION_ROUTE_IMPORT = WPK_FORK_EXCEPTION_IMPORT_ROUTE; +export const FORK_EXCEPTION_CACHE_INDEX_IMPORT = + WPK_FORK_EXCEPTION_IMPORT_CACHE_INDEX; +export const FORK_EXCEPTION_BROKER_ENCODE_IMPORT = + WPK_FORK_EXCEPTION_IMPORT_BROKER_ENCODE; +export const FORK_EXCEPTION_BROKER_THROW_RECIPE_IMPORT = + WPK_FORK_EXCEPTION_IMPORT_BROKER_THROW_RECIPE; +export const FORK_EXCEPTION_INGRESS_THROW_IMPORT = + WPK_FORK_EXCEPTION_IMPORT_INGRESS_THROW; +export const FORK_REFERENCE_SCRATCH_RESERVE_IMPORT = + WPK_FORK_REFERENCE_IMPORT_SCRATCH_RESERVE; +export const FORK_REFERENCE_SCRATCH_RELEASE_IMPORT = + WPK_FORK_REFERENCE_IMPORT_SCRATCH_RELEASE; + +export const FORK_EXCEPTION_ENCODE_EXPORT = WPK_FORK_EXCEPTION_EXPORT_ENCODE; +export const FORK_EXCEPTION_DECODE_EXPORT = WPK_FORK_EXCEPTION_EXPORT_DECODE; +export const FORK_EXCEPTION_THROW_SLOT_EXPORT = + WPK_FORK_EXCEPTION_EXPORT_THROW_SLOT; +export const FORK_EXCEPTION_THROW_RECIPE_EXPORT = + WPK_FORK_EXCEPTION_EXPORT_THROW_RECIPE; +export const FORK_EXCEPTION_ENCODE_INGRESS_EXPORT = + WPK_FORK_EXCEPTION_EXPORT_ENCODE_INGRESS; +export const FORK_EXCEPTION_MATERIALIZE_EXPORT = + WPK_FORK_EXCEPTION_EXPORT_MATERIALIZE; +export const FORK_EXCEPTION_CLEAR_EXPORT = WPK_FORK_EXCEPTION_EXPORT_CLEAR; +export const FORK_EXCEPTION_ABORT_EXPORT = WPK_FORK_EXCEPTION_EXPORT_ABORT; + +const MAX_RECIPE_ID = 0x7fff_fffe; +const MAX_ACTIVATION_ID = 0x7fff_ffff; + +export interface ForkExceptionTagLayout { + readonly tagOrdinal: number; + readonly layoutId: number; + readonly scalarByteLength: number; + readonly referenceCount: number; +} + +export interface ForkExceptionCodecDescriptor { + readonly version: number; + readonly tags: readonly ForkExceptionTagLayout[]; +} + +export interface ForkExceptionProvider + extends ForkActivationExceptionProvider, ForkExceptionSlotProvider +{ + readonly activationId: number; + readonly encode: CallableFunction; + readonly decode: CallableFunction; +} + +function assertI32(value: number, context: string): void { + if (!Number.isInteger(value) || value < -0x8000_0000 || value > 0x7fff_ffff) { + throw new RangeError(`${context} is not an i32`); + } +} + +function assertU31(value: number, context: string, allowZero = true): void { + if ( + !Number.isInteger(value) + || value < (allowZero ? 0 : 1) + || value > 0x7fff_ffff + ) { + throw new RangeError(`${context} is not ${allowZero ? "a" : "a nonzero"} u31`); + } +} + +function assertRecipeId(value: number, allowZero: boolean): void { + assertU31(value, "fork exception recipe id", allowZero); + if (value > MAX_RECIPE_ID) { + throw new RangeError(`fork exception recipe id ${value} is reserved`); + } +} + +function requireFunction( + exports: WebAssembly.Exports, + name: string, +): CallableFunction { + const value = exports[name]; + if (typeof value !== "function") { + throw new Error(`fork exception provider is missing function export ${name}`); + } + return value as CallableFunction; +} + +function checkedPointerResult( + value: number, + ptrWidth: 4 | 8, +): number | bigint { + return ptrWidth === 8 ? BigInt(value) : value; +} + +/** + * Parse and validate the exact-tag catalog emitted by the instrumenter. + * + * The module template hash binds tag identities and concrete payload types; + * this descriptor binds their deterministic codec ordinals and byte layout. + */ +export function readForkExceptionCodecDescriptor( + module: WebAssembly.Module, +): ForkExceptionCodecDescriptor { + const sections = WebAssembly.Module.customSections( + module, + FORK_EXCEPTION_CODEC_SECTION, + ); + if (sections.length !== 1) { + throw new Error( + `expected one ${FORK_EXCEPTION_CODEC_SECTION} section, found ${sections.length}`, + ); + } + const bytes = new Uint8Array(sections[0]!); + if (bytes.byteLength < FORK_EXCEPTION_CODEC_HEADER_SIZE) { + throw new Error("fork exception codec descriptor is truncated"); + } + const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); + const version = view.getUint8(0); + if (version !== FORK_EXCEPTION_CODEC_VERSION) { + throw new Error(`unsupported fork exception codec version ${version}`); + } + if (view.getUint8(1) !== 0 || view.getUint16(2, true) !== 0) { + throw new Error("fork exception codec descriptor reserved fields are nonzero"); + } + const count = view.getUint32(4, true); + const expected = FORK_EXCEPTION_CODEC_HEADER_SIZE + + count * FORK_EXCEPTION_CODEC_TAG_RECORD_SIZE; + if (!Number.isSafeInteger(expected) || bytes.byteLength !== expected) { + throw new Error("fork exception codec descriptor has an invalid size"); + } + const tags: ForkExceptionTagLayout[] = []; + const layouts = new Set(); + for (let index = 0; index < count; index++) { + const offset = FORK_EXCEPTION_CODEC_HEADER_SIZE + + index * FORK_EXCEPTION_CODEC_TAG_RECORD_SIZE; + const tagOrdinal = view.getUint32(offset, true); + const layoutId = view.getUint32(offset + 4, true); + const scalarByteLength = view.getUint32(offset + 8, true); + const referenceCount = view.getUint32(offset + 12, true); + if (tagOrdinal !== index) { + throw new Error( + `fork exception tag ordinal ${tagOrdinal} is noncanonical at ${index}`, + ); + } + if (layoutId > MAX_ACTIVATION_ID || layouts.has(layoutId)) { + throw new Error(`fork exception layout id ${layoutId} is invalid or duplicated`); + } + layouts.add(layoutId); + tags.push({ tagOrdinal, layoutId, scalarByteLength, referenceCount }); + } + return { version, tags }; +} + +/** Resolve one activation's local, exact-tag codec after instantiation. */ +export function forkExceptionProviderFromInstance( + activationId: number, + instance: WebAssembly.Instance, +): ForkExceptionProvider { + assertU31(activationId, "fork exception activation id"); + const throwSlot = requireFunction( + instance.exports, + FORK_EXCEPTION_THROW_SLOT_EXPORT, + ); + const throwRecipe = requireFunction( + instance.exports, + FORK_EXCEPTION_THROW_RECIPE_EXPORT, + ); + const encodeIngress = requireFunction( + instance.exports, + FORK_EXCEPTION_ENCODE_INGRESS_EXPORT, + ); + const materialize = requireFunction( + instance.exports, + FORK_EXCEPTION_MATERIALIZE_EXPORT, + ); + const clear = requireFunction(instance.exports, FORK_EXCEPTION_CLEAR_EXPORT); + const abort = requireFunction(instance.exports, FORK_EXCEPTION_ABORT_EXPORT); + return { + activationId, + encode: requireFunction(instance.exports, FORK_EXCEPTION_ENCODE_EXPORT), + decode: requireFunction(instance.exports, FORK_EXCEPTION_DECODE_EXPORT), + throwSlot(slot): never { + assertI32(slot, "fork exception scratch slot"); + throwSlot(slot); + throw new Error(`activation ${activationId} exception slot returned without throwing`); + }, + throwRecipe(recipeId): never { + assertRecipeId(recipeId, false); + throwRecipe(recipeId); + throw new Error(`activation ${activationId} exception recipe returned without throwing`); + }, + encodeIngress(token): number { + assertU31(token, "fork exception ingress token", false); + const recipeId = Number(encodeIngress(token)); + assertRecipeId(recipeId, true); + return recipeId; + }, + materialize(recipeId): void { + assertRecipeId(recipeId, false); + materialize(recipeId); + }, + clear(): void { + clear(); + }, + clearSlots(): void { + clear(); + }, + abort(): void { + abort(); + }, + }; +} + +function catchProviderThrow( + provider: ForkActivationExceptionProvider, + slot: number, +): unknown { + try { + provider.throwSlot(slot); + } catch (value) { + return value; + } + throw new Error(`fork exception slot ${slot} returned without throwing`); +} + +/** + * Scalar-only bridge for exceptions whose tag is owned by another activation. + * + * Providers are probed in activation order. During a probe, the candidate's + * own unknown-tag callback returns zero for the same thrown identity instead + * of recursively restarting discovery. A raw JavaScript/JSTag exception that + * no Wasm activation owns is represented by a host-owned recipe. + */ +export class ForkExceptionBroker { + private nextIngressToken = 1; + private readonly ingress = new Map(); + private readonly probes: unknown[] = []; + + constructor( + private readonly registry: ForkActivationRegistry, + private readonly label: string, + private readonly replayReferences: () => { + exceptionOwner(recipeId: number): number; + materializeHostException(recipeId: number): unknown; + } = () => registry.currentReferences(), + /** + * Called only after every exact activation codec declines an exception. + * It returns the owner-backed externref payload a fresh child can decode; + * the parent transaction still retains the original exception identity. + */ + private readonly normalizeUnclaimedHostException?: ( + value: unknown, + ) => unknown, + ) {} + + encodeFromSlot(sourceActivation: number, slot: number): number { + const source = this.requireProvider(sourceActivation); + const value = catchProviderThrow(source, slot); + if ( + this.probes.length !== 0 + && Object.is(this.probes[this.probes.length - 1], value) + ) { + return 0; + } + + const token = this.allocateIngress(value); + try { + for (const activation of this.registry.activations()) { + if (activation.activationId === sourceActivation) continue; + const provider = activation.exceptionProvider; + if (!provider) continue; + this.probes.push(value); + let recipeId: number; + try { + recipeId = provider.encodeIngress(token); + } finally { + this.probes.pop(); + } + assertRecipeId(recipeId, true); + if (recipeId !== 0) return recipeId; + } + const childPayload = this.normalizeUnclaimedHostException + ? this.normalizeUnclaimedHostException(value) + : value; + return this.registry.currentReferences().captureHostException( + value, + childPayload, + ); + } finally { + this.ingress.delete(token); + } + } + + throwIngress(token: number): never { + assertU31(token, "fork exception ingress token", false); + if (!this.ingress.has(token)) { + throw new Error(`${this.label}: unknown exception ingress token ${token}`); + } + throw this.ingress.get(token); + } + + throwRecipe(recipeId: number): never { + assertRecipeId(recipeId, false); + const references = this.replayReferences(); + const owner = references.exceptionOwner(recipeId); + if (owner === FORK_HOST_EXCEPTION_ACTIVATION_ID) { + throw references.materializeHostException(recipeId); + } + return this.requireProvider(owner).throwRecipe(recipeId); + } + + clear(): void { + this.ingress.clear(); + this.probes.length = 0; + } + + private requireProvider(activationId: number): ForkActivationExceptionProvider { + const provider = this.registry.getActivation(activationId).exceptionProvider; + if (!provider) { + throw new Error( + `${this.label}: activation ${activationId} has no exception provider`, + ); + } + return provider; + } + + private allocateIngress(value: unknown): number { + if (this.nextIngressToken > MAX_RECIPE_ID) { + this.nextIngressToken = 1; + } + const start = this.nextIngressToken; + do { + const token = this.nextIngressToken++; + if (!this.ingress.has(token)) { + this.ingress.set(token, value); + return token; + } + if (this.nextIngressToken > MAX_RECIPE_ID) this.nextIngressToken = 1; + } while (this.nextIngressToken !== start); + throw new RangeError(`${this.label}: exception ingress token space exhausted`); + } +} + +export interface ForkExceptionImportOptions { + readonly activationId: number; + readonly ptrWidth: 4 | 8; + readonly registry: ForkActivationRegistry; + readonly broker: ForkExceptionBroker; + /** Late-bound because imports must exist before the instance exports do. */ + readonly provider: () => ForkExceptionProvider; + /** + * Replay owner used before the complete fresh-child registry can attach. + * Capture-only callbacks remain bound to the registry transaction. + */ + readonly referenceReplay?: () => ForkExceptionReferenceReplayImports; +} + +export interface ForkExceptionReferenceReplayImports { + loadException( + recipeId: number, + moduleActivation: number, + tagOrdinal: number, + layoutId: number, + scalarPointer: number | bigint, + scalarByteLength: number, + referenceIdsPointer: number | bigint, + referenceCount: number, + ): number; + routeException(recipeId: number, expectedActivation: number): number; + exceptionCacheIndex(recipeId: number): number; + reserveScratch(size: number | bigint): number; + releaseScratch(pointer: number | bigint, size: number | bigint): void; +} + +/** + * Bind one in-module codec to the active process transaction. + * + * Every callback has only scalar parameters. The sole reference transfer is a + * thrown exception caught by JavaScript and immediately re-thrown into another + * provider; no reference enters the continuation or module-state arena. + */ +export function buildForkExceptionImports( + options: ForkExceptionImportOptions, +): Record { + const { activationId, ptrWidth, registry, broker } = options; + assertU31(activationId, "fork exception activation id"); + if (ptrWidth !== 4 && ptrWidth !== 8) { + throw new TypeError(`invalid fork exception pointer width ${ptrWidth}`); + } + const references = () => registry.currentReferences(); + const replayReferences = options.referenceReplay ?? references; + return { + [FORK_EXCEPTION_ACTIVATION_IMPORT]: new WebAssembly.Global( + { value: "i32", mutable: false }, + activationId, + ), + [FORK_EXCEPTION_LOOKUP_IMPORT]: (slot: number): number => + references().lookupExceptionSlot(slot, options.provider()), + [FORK_EXCEPTION_CLAIM_IMPORT]: (slot: number): number => + references().claimExceptionSlot(slot, options.provider()), + [FORK_EXCEPTION_DEFINE_IMPORT]: ( + recipeId: number, + moduleActivation: number, + tagOrdinal: number, + layoutId: number, + scalarPointer: number | bigint, + scalarByteLength: number, + referenceIdsPointer: number | bigint, + referenceCount: number, + ): void => references().defineException( + recipeId, + moduleActivation, + tagOrdinal, + layoutId, + scalarPointer, + scalarByteLength, + referenceIdsPointer, + referenceCount, + ), + [FORK_EXCEPTION_LOAD_IMPORT]: ( + recipeId: number, + moduleActivation: number, + tagOrdinal: number, + layoutId: number, + scalarPointer: number | bigint, + scalarByteLength: number, + referenceIdsPointer: number | bigint, + referenceCount: number, + ): number => replayReferences().loadException( + recipeId, + moduleActivation, + tagOrdinal, + layoutId, + scalarPointer, + scalarByteLength, + referenceIdsPointer, + referenceCount, + ), + [FORK_EXCEPTION_ROUTE_IMPORT]: ( + recipeId: number, + expectedActivation: number, + ): number => replayReferences().routeException( + recipeId, + expectedActivation, + ), + [FORK_EXCEPTION_CACHE_INDEX_IMPORT]: (recipeId: number): number => + replayReferences().exceptionCacheIndex(recipeId), + [FORK_EXCEPTION_BROKER_ENCODE_IMPORT]: (slot: number): number => + broker.encodeFromSlot(activationId, slot), + [FORK_EXCEPTION_BROKER_THROW_RECIPE_IMPORT]: (recipeId: number): never => + broker.throwRecipe(recipeId), + [FORK_EXCEPTION_INGRESS_THROW_IMPORT]: (token: number): never => + broker.throwIngress(token), + [FORK_REFERENCE_SCRATCH_RESERVE_IMPORT]: ( + size: number | bigint, + ): number | bigint => + checkedPointerResult(replayReferences().reserveScratch(size), ptrWidth), + [FORK_REFERENCE_SCRATCH_RELEASE_IMPORT]: ( + pointer: number | bigint, + size: number | bigint, + ): void => replayReferences().releaseScratch(pointer, size), + }; +} diff --git a/host/src/fork-externref-import-mailbox.ts b/host/src/fork-externref-import-mailbox.ts new file mode 100644 index 0000000000..cc86e79bad --- /dev/null +++ b/host/src/fork-externref-import-mailbox.ts @@ -0,0 +1,1314 @@ +import type { + ForkExternrefToken, + ForkExternrefTokenCache, +} from "./fork-reference-broker"; + +/** + * Version 2 is a catalog-sized, one-request-at-a-time mailbox. It is private + * to the host runtime, but versioning it prevents a mixed Worker/owner build + * from interpreting the same scalar words differently. + */ +export const FORK_EXTERNREF_IMPORT_MAILBOX_VERSION = 2; +export const FORK_EXTERNREF_IMPORT_DESCRIPTOR_VERSION = 1; + +const MAILBOX_MAGIC = 0x4b465849; // "KFXI" +const MAX_U32 = 0xffff_ffff; +const MAX_I32 = 0x7fff_ffff; +const MIN_I32 = -0x8000_0000; +const MAX_I64 = (1n << 63n) - 1n; +const MIN_I64 = -(1n << 63n); + +const enum HeaderWord { + Status = 0, + Magic = 1, + MailboxVersion = 2, + Pid = 3, + Generation = 4, + Sender = 5, + SequenceLow = 6, + SequenceHigh = 7, + DescriptorVersion = 8, + Ordinal = 9, + ParamCount = 10, + ResultCount = 11, + ParamCapacity = 12, + ResultCapacity = 13, + FailureCode = 14, + ExceptionHandle = 15, + CloseReason = 16, + Reserved1 = 17, +} + +const HEADER_WORDS = HeaderWord.Reserved1 + 1; +const HEADER_BYTES = HEADER_WORDS * Int32Array.BYTES_PER_ELEMENT; +const SLOT_BYTES = BigInt64Array.BYTES_PER_ELEMENT; +const TYPE_CODES_PER_BYTE = 2; + +export interface ForkExternrefImportMailboxCapacity { + readonly params: number; + readonly results: number; +} + +interface ForkExternrefImportMailboxLayout + extends ForkExternrefImportMailboxCapacity { + readonly paramTypesOffset: number; + readonly resultTypesOffset: number; + readonly paramOffset: number; + readonly resultOffset: number; + readonly byteLength: number; +} + +const enum MailboxStatus { + Idle = 0, + Writing = 1, + RequestReady = 2, + Dispatching = 3, + ResultReady = 4, + ExceptionReady = 5, + Failed = 6, + Closed = 7, +} + +export enum ForkExternrefImportFailureCode { + Protocol = 1, + Unauthorized = 2, + ArgumentAuthorization = 3, + HandlerContract = 4, + OwnerFailure = 5, + NotificationFailure = 6, + Teardown = 7, +} + +export type ForkExternrefImportValueType = + | "i32" + | "i64" + | "f32" + | "f64" + | "externref"; + +export type ForkExternrefImportValue = + | number + | bigint + | null + | ForkExternrefToken; + +export interface ForkExternrefImportDescriptor { + readonly version: typeof FORK_EXTERNREF_IMPORT_DESCRIPTOR_VERSION; + readonly ordinal: number; + readonly params: readonly ForkExternrefImportValueType[]; + readonly results: readonly ForkExternrefImportValueType[]; +} + +export interface ForkExternrefImportBinding { + readonly pid: number; + readonly generationId: number; + /** + * One nonzero u32 assigned to one process or pthread Worker. Side modules + * execute on that same Worker and deliberately reuse the same sender. + */ + readonly senderId: number; +} + +/** + * The only per-call message that needs to cross postMessage. All fields are + * scalar; the mailbox itself is transferred once in the Worker init message. + */ +export interface ForkExternrefImportWake { + readonly mailboxVersion: number; + readonly pid: number; + readonly generationId: number; + readonly senderId: number; + readonly sequenceLow: number; + readonly sequenceHigh: number; +} + +export interface ForkExternrefImportAuthority { + authorizeForWire( + pid: number, + generationId: number, + handle: number, + ): unknown; + registerForWire( + pid: number, + generationId: number, + value: unknown, + ): number; +} + +export interface ForkExternrefImportHandlerContext + extends ForkExternrefImportBinding { + readonly descriptor: ForkExternrefImportDescriptor; +} + +export type ForkExternrefImportHandler = ( + context: ForkExternrefImportHandlerContext, + ...args: unknown[] +) => unknown; + +export interface ForkExternrefImportOwnerEndpointOptions { + /** + * Revalidate the exact live Worker and process image for every request. + * Implementations should compare object identity owned by the entrypoint, + * not trust PID/generation numbers copied from the wake message. + */ + readonly authorizeSender: ( + binding: ForkExternrefImportBinding, + ) => void; + /** Owner-realm diagnostics; no Error or host value crosses the mailbox. */ + readonly onDiagnostic?: ( + error: unknown, + failure: ForkExternrefImportFailureCode, + ) => void; +} + +interface RegisteredHandler { + readonly descriptor: ForkExternrefImportDescriptor; + readonly handler: ForkExternrefImportHandler; +} + +function assertSafeByteCount(value: number, label: string): void { + if (!Number.isSafeInteger(value) || value < 0) { + throw new RangeError(`${label} exceeds JavaScript's safe byte range`); + } +} + +function checkedAdd(left: number, right: number, label: string): number { + const result = left + right; + assertSafeByteCount(result, label); + return result; +} + +function checkedMultiply( + left: number, + right: number, + label: string, +): number { + const result = left * right; + assertSafeByteCount(result, label); + return result; +} + +function alignToSlot(value: number): number { + const remainder = value % SLOT_BYTES; + return remainder === 0 + ? value + : checkedAdd(value, SLOT_BYTES - remainder, "mailbox alignment"); +} + +function validateCapacity( + capacity: ForkExternrefImportMailboxCapacity, +): void { + if ( + typeof capacity !== "object" + || capacity === null + ) { + throw new TypeError("fork externref import mailbox capacity is required"); + } + assertU32(capacity.params, "mailbox parameter capacity", true); + assertU32(capacity.results, "mailbox result capacity", true); +} + +function typeSignatureBytes(count: number): number { + return Math.ceil(count / TYPE_CODES_PER_BYTE); +} + +function mailboxLayout( + capacity: ForkExternrefImportMailboxCapacity, +): ForkExternrefImportMailboxLayout { + validateCapacity(capacity); + const paramTypesOffset = HEADER_BYTES; + const resultTypesOffset = checkedAdd( + paramTypesOffset, + typeSignatureBytes(capacity.params), + "mailbox parameter type signature", + ); + const typeEnd = checkedAdd( + resultTypesOffset, + typeSignatureBytes(capacity.results), + "mailbox result type signature", + ); + const paramOffset = alignToSlot(typeEnd); + const resultOffset = checkedAdd( + paramOffset, + checkedMultiply( + capacity.params, + SLOT_BYTES, + "mailbox parameter slots", + ), + "mailbox result offset", + ); + const byteLength = checkedAdd( + resultOffset, + checkedMultiply( + capacity.results, + SLOT_BYTES, + "mailbox result slots", + ), + "mailbox byte length", + ); + return Object.freeze({ + params: capacity.params, + results: capacity.results, + paramTypesOffset, + resultTypesOffset, + paramOffset, + resultOffset, + byteLength, + }); +} + +function assertU32(value: number, label: string, allowZero = false): void { + if ( + !Number.isInteger(value) + || value < (allowZero ? 0 : 1) + || value > MAX_U32 + ) { + throw new RangeError( + `${label} must be ${allowZero ? "an" : "a positive"} unsigned 32-bit integer`, + ); + } +} + +function validateBinding(binding: ForkExternrefImportBinding): void { + assertU32(binding.pid, "fork externref import pid"); + assertU32( + binding.generationId, + "fork externref import generation", + ); + assertU32(binding.senderId, "fork externref import sender"); +} + +function valueTypeCode(type: ForkExternrefImportValueType): number { + switch (type) { + case "i32": + return 1; + case "i64": + return 2; + case "f32": + return 3; + case "f64": + return 4; + case "externref": + return 5; + default: + throw new TypeError( + `unsupported fork externref import value type ${String(type)}`, + ); + } +} + +function validateTypes( + types: readonly ForkExternrefImportValueType[], + label: string, +): void { + if (!Array.isArray(types)) { + throw new TypeError(`${label} types must be an array`); + } + assertU32(types.length, `${label} count`, true); + for (const type of types) valueTypeCode(type); +} + +function writeTypeSequence( + view: DataView, + byteOffset: number, + types: readonly ForkExternrefImportValueType[], +): void { + for ( + let typeIndex = 0; + typeIndex < types.length; + typeIndex += TYPE_CODES_PER_BYTE + ) { + const low = valueTypeCode(types[typeIndex]!); + const high = typeIndex + 1 < types.length + ? valueTypeCode(types[typeIndex + 1]!) + : 0; + view.setUint8(byteOffset + typeIndex / TYPE_CODES_PER_BYTE, low | (high << 4)); + } +} + +function typeSequenceMatches( + view: DataView, + byteOffset: number, + types: readonly ForkExternrefImportValueType[], +): boolean { + for ( + let typeIndex = 0; + typeIndex < types.length; + typeIndex += TYPE_CODES_PER_BYTE + ) { + const low = valueTypeCode(types[typeIndex]!); + const high = typeIndex + 1 < types.length + ? valueTypeCode(types[typeIndex + 1]!) + : 0; + if ( + view.getUint8(byteOffset + typeIndex / TYPE_CODES_PER_BYTE) + !== (low | (high << 4)) + ) { + return false; + } + } + return true; +} + +export function defineForkExternrefImport( + ordinal: number, + params: readonly ForkExternrefImportValueType[], + results: readonly ForkExternrefImportValueType[], +): ForkExternrefImportDescriptor { + assertU32(ordinal, "fork externref import ordinal", true); + validateTypes(params, "parameter"); + validateTypes(results, "result"); + return Object.freeze({ + version: FORK_EXTERNREF_IMPORT_DESCRIPTOR_VERSION, + ordinal, + params: Object.freeze([...params]), + results: Object.freeze([...results]), + }); +} + +export function forkExternrefImportMailboxBytes( + capacity: ForkExternrefImportMailboxCapacity, +): number { + return mailboxLayout(capacity).byteLength; +} + +export function createForkExternrefImportMailbox( + capacity: ForkExternrefImportMailboxCapacity, +): SharedArrayBuffer { + const layout = mailboxLayout(capacity); + const buffer = new SharedArrayBuffer(layout.byteLength); + const words = new Int32Array(buffer); + words[HeaderWord.Magic] = MAILBOX_MAGIC; + words[HeaderWord.MailboxVersion] = + FORK_EXTERNREF_IMPORT_MAILBOX_VERSION; + words[HeaderWord.ParamCapacity] = capacity.params; + words[HeaderWord.ResultCapacity] = capacity.results; + Atomics.store(words, HeaderWord.Status, MailboxStatus.Idle); + return buffer; +} + +function readMailboxLayout( + buffer: SharedArrayBuffer, +): ForkExternrefImportMailboxLayout { + if ( + !(buffer instanceof SharedArrayBuffer) + || buffer.byteLength < HEADER_BYTES + || buffer.byteLength % Int32Array.BYTES_PER_ELEMENT !== 0 + ) { + throw new TypeError( + "fork externref import mailbox is not a complete shared header", + ); + } + const words = new Int32Array(buffer); + if ((words[HeaderWord.Magic]! >>> 0) !== MAILBOX_MAGIC) { + throw new Error("invalid fork externref import mailbox magic"); + } + if ( + (words[HeaderWord.MailboxVersion]! >>> 0) + !== FORK_EXTERNREF_IMPORT_MAILBOX_VERSION + ) { + throw new Error( + `unsupported fork externref import mailbox version ` + + `${words[HeaderWord.MailboxVersion]! >>> 0}`, + ); + } + const layout = mailboxLayout({ + params: words[HeaderWord.ParamCapacity]! >>> 0, + results: words[HeaderWord.ResultCapacity]! >>> 0, + }); + if (buffer.byteLength !== layout.byteLength) { + throw new TypeError( + `fork externref import mailbox has ${buffer.byteLength} bytes; ` + + `declared capacity requires exactly ${layout.byteLength}`, + ); + } + return layout; +} + +function validateMailboxHeader( + words: Int32Array, + layout: ForkExternrefImportMailboxLayout, +): void { + if ((words[HeaderWord.Magic]! >>> 0) !== MAILBOX_MAGIC) { + throw new Error("invalid fork externref import mailbox magic"); + } + if ( + (words[HeaderWord.MailboxVersion]! >>> 0) + !== FORK_EXTERNREF_IMPORT_MAILBOX_VERSION + ) { + throw new Error( + `unsupported fork externref import mailbox version ` + + `${words[HeaderWord.MailboxVersion]! >>> 0}`, + ); + } + if ( + (words[HeaderWord.ParamCapacity]! >>> 0) !== layout.params + || (words[HeaderWord.ResultCapacity]! >>> 0) !== layout.results + ) { + throw new Error("fork externref import mailbox capacity changed"); + } +} + +function writeU32(view: DataView, byteOffset: number, value: number): void { + view.setUint32(byteOffset, value >>> 0, true); +} + +function readU32(view: DataView, byteOffset: number): number { + return view.getUint32(byteOffset, true); +} + +function wordOffset(word: HeaderWord): number { + return word * Int32Array.BYTES_PER_ELEMENT; +} + +function writeHeaderU32( + view: DataView, + word: HeaderWord, + value: number, +): void { + writeU32(view, wordOffset(word), value); +} + +function readHeaderU32(view: DataView, word: HeaderWord): number { + return readU32(view, wordOffset(word)); +} + +function slotOffset(base: number, index: number): number { + return base + index * SLOT_BYTES; +} + +function writeScalarSlot( + view: DataView, + base: number, + index: number, + type: Exclude, + value: unknown, +): void { + const offset = slotOffset(base, index); + view.setBigUint64(offset, 0n, true); + switch (type) { + case "i32": + if ( + typeof value !== "number" + || !Number.isInteger(value) + || value < MIN_I32 + || value > MAX_I32 + ) { + throw new TypeError(`i32 value at slot ${index} is not signed i32`); + } + view.setInt32(offset, value, true); + return; + case "i64": + if ( + typeof value !== "bigint" + || value < MIN_I64 + || value > MAX_I64 + ) { + throw new TypeError(`i64 value at slot ${index} is not signed i64`); + } + view.setBigInt64(offset, value, true); + return; + case "f32": + if (typeof value !== "number") { + throw new TypeError(`f32 value at slot ${index} is not a number`); + } + view.setFloat32(offset, value, true); + return; + case "f64": + if (typeof value !== "number") { + throw new TypeError(`f64 value at slot ${index} is not a number`); + } + view.setFloat64(offset, value, true); + return; + } +} + +function readScalarSlot( + view: DataView, + base: number, + index: number, + type: Exclude, +): number | bigint { + const offset = slotOffset(base, index); + switch (type) { + case "i32": + return view.getInt32(offset, true); + case "i64": + return view.getBigInt64(offset, true); + case "f32": + return view.getFloat32(offset, true); + case "f64": + return view.getFloat64(offset, true); + } +} + +function nextSequence( + low: number, + high: number, +): { low: number; high: number } { + if (low === MAX_U32) { + if (high === MAX_U32) { + throw new RangeError( + "fork externref import mailbox sequence space exhausted", + ); + } + return { low: 0, high: high + 1 }; + } + return { low: low + 1, high }; +} + +function bindingEquals( + first: ForkExternrefImportBinding, + second: ForkExternrefImportBinding, +): boolean { + return first.pid === second.pid + && first.generationId === second.generationId + && first.senderId === second.senderId; +} + +function failureDescription(code: number): string { + const known = ForkExternrefImportFailureCode[ + code as ForkExternrefImportFailureCode + ]; + return known ?? `Unknown(${code})`; +} + +export class ForkExternrefImportRemoteFailure extends Error { + constructor( + readonly failureCode: number, + options?: ErrorOptions, + ) { + super( + `fork externref host import failed: ${failureDescription(failureCode)}`, + options, + ); + this.name = "ForkExternrefImportRemoteFailure"; + } +} + +export class ForkExternrefImportClosedError extends Error { + constructor(readonly reasonCode: number) { + super( + `fork externref host import mailbox is closed: ` + + `${failureDescription(reasonCode)}`, + ); + this.name = "ForkExternrefImportClosedError"; + } +} + +/** + * One synchronous caller per process or pthread Worker. + * + * The caller can bind imports from the main module and any side module to this + * same object. Its single atomic state rejects reentrancy instead of allowing + * two Wasm activations to overwrite one mailbox and deadlock each other. + */ +export class ForkExternrefImportWorkerCaller { + private readonly words: Int32Array; + private readonly view: DataView; + private readonly layout: ForkExternrefImportMailboxLayout; + private sequenceLow = 0; + private sequenceHigh = 0; + + constructor( + readonly mailbox: SharedArrayBuffer, + readonly binding: ForkExternrefImportBinding, + private readonly tokens: ForkExternrefTokenCache, + private readonly notifyOwner: (wake: ForkExternrefImportWake) => void, + ) { + this.layout = readMailboxLayout(mailbox); + validateBinding(binding); + if (tokens.generationId !== binding.generationId) { + throw new Error( + `fork externref token generation ${tokens.generationId} does not ` + + `match mailbox generation ${binding.generationId}`, + ); + } + this.words = new Int32Array(mailbox); + this.view = new DataView(mailbox); + validateMailboxHeader(this.words, this.layout); + } + + bind( + descriptor: ForkExternrefImportDescriptor, + ): (...args: ForkExternrefImportValue[]) => unknown { + this.validateDescriptor(descriptor); + return (...args: ForkExternrefImportValue[]) => + this.call(descriptor, args); + } + + call( + descriptor: ForkExternrefImportDescriptor, + args: readonly ForkExternrefImportValue[], + ): unknown { + this.validateDescriptor(descriptor); + if (args.length !== descriptor.params.length) { + throw new TypeError( + `fork externref import ${descriptor.ordinal} expects ` + + `${descriptor.params.length} arguments, received ${args.length}`, + ); + } + + const prior = Atomics.compareExchange( + this.words, + HeaderWord.Status, + MailboxStatus.Idle, + MailboxStatus.Writing, + ); + if (prior === MailboxStatus.Closed) throw this.closedError(); + if (prior !== MailboxStatus.Idle) { + throw new Error( + `reentrant fork externref host import while mailbox state=${prior}`, + ); + } + + try { + const sequence = nextSequence( + this.sequenceLow, + this.sequenceHigh, + ); + this.sequenceLow = sequence.low; + this.sequenceHigh = sequence.high; + this.writeRequest(descriptor, args, sequence); + + if ( + Atomics.compareExchange( + this.words, + HeaderWord.Status, + MailboxStatus.Writing, + MailboxStatus.RequestReady, + ) !== MailboxStatus.Writing + ) { + throw this.closedError(); + } + + const wake: ForkExternrefImportWake = Object.freeze({ + mailboxVersion: FORK_EXTERNREF_IMPORT_MAILBOX_VERSION, + pid: this.binding.pid, + generationId: this.binding.generationId, + senderId: this.binding.senderId, + sequenceLow: sequence.low, + sequenceHigh: sequence.high, + }); + try { + this.notifyOwner(wake); + } catch (error) { + const reset = Atomics.compareExchange( + this.words, + HeaderWord.Status, + MailboxStatus.RequestReady, + MailboxStatus.Idle, + ); + if (reset === MailboxStatus.RequestReady) { + throw new ForkExternrefImportRemoteFailure( + ForkExternrefImportFailureCode.NotificationFailure, + { cause: error }, + ); + } + // The owner already claimed the request. It owns completion now, so + // waiting is the only state-safe choice even if notification reported + // a local error after publishing the wake. + } + return this.waitForCompletion(descriptor); + } catch (error) { + Atomics.compareExchange( + this.words, + HeaderWord.Status, + MailboxStatus.Writing, + MailboxStatus.Idle, + ); + throw error; + } + } + + private writeRequest( + descriptor: ForkExternrefImportDescriptor, + args: readonly ForkExternrefImportValue[], + sequence: { low: number; high: number }, + ): void { + writeHeaderU32(this.view, HeaderWord.Pid, this.binding.pid); + writeHeaderU32( + this.view, + HeaderWord.Generation, + this.binding.generationId, + ); + writeHeaderU32(this.view, HeaderWord.Sender, this.binding.senderId); + writeHeaderU32(this.view, HeaderWord.SequenceLow, sequence.low); + writeHeaderU32(this.view, HeaderWord.SequenceHigh, sequence.high); + writeHeaderU32( + this.view, + HeaderWord.DescriptorVersion, + descriptor.version, + ); + writeHeaderU32(this.view, HeaderWord.Ordinal, descriptor.ordinal); + writeHeaderU32( + this.view, + HeaderWord.ParamCount, + descriptor.params.length, + ); + writeHeaderU32( + this.view, + HeaderWord.ResultCount, + descriptor.results.length, + ); + writeTypeSequence( + this.view, + this.layout.paramTypesOffset, + descriptor.params, + ); + writeTypeSequence( + this.view, + this.layout.resultTypesOffset, + descriptor.results, + ); + writeHeaderU32(this.view, HeaderWord.FailureCode, 0); + writeHeaderU32(this.view, HeaderWord.ExceptionHandle, 0); + + for (let index = 0; index < descriptor.params.length; index++) { + const type = descriptor.params[index]!; + const value = args[index]; + if (type === "externref") { + const handle = value === null ? 0 : this.tokens.encode(value); + if (handle === null) { + throw new Error( + `externref argument ${index} for import ${descriptor.ordinal} ` + + `did not come from this process-image owner`, + ); + } + this.view.setBigUint64( + slotOffset(this.layout.paramOffset, index), + BigInt(handle), + true, + ); + } else { + writeScalarSlot( + this.view, + this.layout.paramOffset, + index, + type, + value, + ); + } + } + } + + private waitForCompletion( + descriptor: ForkExternrefImportDescriptor, + ): unknown { + for (;;) { + const status = Atomics.load(this.words, HeaderWord.Status); + if ( + status === MailboxStatus.RequestReady + || status === MailboxStatus.Dispatching + ) { + Atomics.wait(this.words, HeaderWord.Status, status); + continue; + } + if (status === MailboxStatus.ResultReady) { + if (!this.takeCompletion(MailboxStatus.ResultReady)) continue; + return this.readResults(descriptor); + } + if (status === MailboxStatus.ExceptionReady) { + if (!this.takeCompletion(MailboxStatus.ExceptionReady)) continue; + const handle = readHeaderU32( + this.view, + HeaderWord.ExceptionHandle, + ); + throw handle === 0 ? null : this.tokens.materialize(handle); + } + if (status === MailboxStatus.Failed) { + const code = readHeaderU32(this.view, HeaderWord.FailureCode); + if (!this.takeCompletion(MailboxStatus.Failed)) continue; + throw new ForkExternrefImportRemoteFailure(code); + } + if (status === MailboxStatus.Closed) throw this.closedError(); + throw new Error( + `invalid fork externref import completion state ${status}`, + ); + } + } + + private takeCompletion(expected: MailboxStatus): boolean { + // Once the Worker returns the status to IDLE, only its own synchronous JS + // stack can start another request. Result slots therefore remain stable + // while this call decodes them, without an extra per-call buffer. + return Atomics.compareExchange( + this.words, + HeaderWord.Status, + expected, + MailboxStatus.Idle, + ) === expected; + } + + private readResults( + descriptor: ForkExternrefImportDescriptor, + ): unknown { + const results = descriptor.results.map((type, index) => { + if (type === "externref") { + const bits = this.view.getBigUint64( + slotOffset(this.layout.resultOffset, index), + true, + ); + if (bits > BigInt(MAX_U32)) { + throw new Error( + `invalid externref result handle ${bits} at slot ${index}`, + ); + } + const handle = Number(bits); + return handle === 0 ? null : this.tokens.materialize(handle); + } + return readScalarSlot( + this.view, + this.layout.resultOffset, + index, + type, + ); + }); + if (results.length === 0) return undefined; + if (results.length === 1) return results[0]; + return results; + } + + private closedError(): ForkExternrefImportClosedError { + return new ForkExternrefImportClosedError( + readHeaderU32(this.view, HeaderWord.CloseReason), + ); + } + + private validateDescriptor( + descriptor: ForkExternrefImportDescriptor, + ): void { + if ( + descriptor.version !== FORK_EXTERNREF_IMPORT_DESCRIPTOR_VERSION + ) { + throw new Error( + `unsupported fork externref import descriptor version ` + + `${descriptor.version}`, + ); + } + assertU32(descriptor.ordinal, "fork externref import ordinal", true); + validateTypes(descriptor.params, "parameter"); + validateTypes(descriptor.results, "result"); + if ( + descriptor.params.length > this.layout.params + || descriptor.results.length > this.layout.results + ) { + throw new RangeError( + `fork externref import ${descriptor.ordinal} signature ` + + `(${descriptor.params.length}, ${descriptor.results.length}) exceeds ` + + `mailbox capacity (${this.layout.params}, ${this.layout.results})`, + ); + } + } +} + +/** + * Immutable owner-realm descriptor catalog. The ordinal alone never selects a + * handler: every request must also match its complete packed type sequence. + */ +export class ForkExternrefImportOwnerCatalog { + private readonly handlers = new Map(); + private maxParams = 0; + private maxResults = 0; + + register( + descriptor: ForkExternrefImportDescriptor, + handler: ForkExternrefImportHandler, + ): void { + if (this.handlers.has(descriptor.ordinal)) { + throw new Error( + `duplicate fork externref import ordinal ${descriptor.ordinal}`, + ); + } + const canonical = defineForkExternrefImport( + descriptor.ordinal, + descriptor.params, + descriptor.results, + ); + if (descriptor.version !== canonical.version) { + throw new Error( + `unsupported fork externref import descriptor version ` + + `${descriptor.version}`, + ); + } + this.handlers.set(descriptor.ordinal, { + descriptor: canonical, + handler, + }); + this.maxParams = Math.max(this.maxParams, canonical.params.length); + this.maxResults = Math.max(this.maxResults, canonical.results.length); + } + + lookup(ordinal: number): RegisteredHandler | undefined { + return this.handlers.get(ordinal); + } + + get mailboxCapacity(): ForkExternrefImportMailboxCapacity { + return Object.freeze({ + params: this.maxParams, + results: this.maxResults, + }); + } +} + +/** + * Owner-side endpoint bound to one exact Worker mailbox. + * + * Entry-point message handlers pass the independently observed sender binding + * into dispatch(). That prevents a numeric sender copied from an untrusted + * wake message from authorizing itself. + */ +export class ForkExternrefImportOwnerEndpoint { + private readonly words: Int32Array; + private readonly view: DataView; + private readonly layout: ForkExternrefImportMailboxLayout; + + constructor( + readonly mailbox: SharedArrayBuffer, + readonly binding: ForkExternrefImportBinding, + private readonly catalog: ForkExternrefImportOwnerCatalog, + private readonly authority: ForkExternrefImportAuthority, + private readonly options: ForkExternrefImportOwnerEndpointOptions, + ) { + this.layout = readMailboxLayout(mailbox); + validateBinding(binding); + const required = catalog.mailboxCapacity; + if ( + required.params > this.layout.params + || required.results > this.layout.results + ) { + throw new RangeError( + `fork externref import catalog requires capacity ` + + `(${required.params}, ${required.results}); mailbox provides ` + + `(${this.layout.params}, ${this.layout.results})`, + ); + } + this.words = new Int32Array(mailbox); + this.view = new DataView(mailbox); + validateMailboxHeader(this.words, this.layout); + } + + /** + * Dispatch one ready request. False means the wake was stale, duplicated, or + * routed from a different Worker; in those cases this endpoint does not + * disturb a possibly newer live request. + */ + dispatch( + wake: ForkExternrefImportWake, + observedSender: ForkExternrefImportBinding, + ): boolean { + validateBinding(observedSender); + if ( + !bindingEquals(observedSender, this.binding) + || wake.mailboxVersion !== FORK_EXTERNREF_IMPORT_MAILBOX_VERSION + || wake.pid !== this.binding.pid + || wake.generationId !== this.binding.generationId + || wake.senderId !== this.binding.senderId + || wake.sequenceLow !== readHeaderU32( + this.view, + HeaderWord.SequenceLow, + ) + || wake.sequenceHigh !== readHeaderU32( + this.view, + HeaderWord.SequenceHigh, + ) + ) { + return false; + } + if ( + Atomics.compareExchange( + this.words, + HeaderWord.Status, + MailboxStatus.RequestReady, + MailboxStatus.Dispatching, + ) !== MailboxStatus.RequestReady + ) { + return false; + } + + let registered: RegisteredHandler; + try { + this.validateClaimedRequest(wake); + registered = this.requireRegisteredHandler(); + } catch (error) { + this.completeFailure( + ForkExternrefImportFailureCode.Protocol, + error, + ); + return true; + } + try { + this.options.authorizeSender(this.binding); + } catch (error) { + this.completeFailure( + ForkExternrefImportFailureCode.Unauthorized, + error, + ); + return true; + } + + let args: unknown[]; + try { + args = this.readArguments(registered.descriptor); + } catch (error) { + this.completeFailure( + ForkExternrefImportFailureCode.ArgumentAuthorization, + error, + ); + return true; + } + + let returned: unknown; + try { + returned = registered.handler( + { + ...this.binding, + descriptor: registered.descriptor, + }, + ...args, + ); + } catch (thrown) { + try { + // WHY: exception completion never uses the externref-null sentinel. + // JavaScript may throw null, undefined, or any other primitive; each + // still needs a nonzero owner handle so CatchAllRef cannot retain a + // raw Worker-local value that the fork recipe provider cannot encode. + const handle = this.authority.registerForWire( + this.binding.pid, + this.binding.generationId, + thrown, + ); + assertU32( + handle, + "fork externref exception handle", + ); + writeHeaderU32( + this.view, + HeaderWord.ExceptionHandle, + handle, + ); + this.complete(MailboxStatus.ExceptionReady); + } catch (error) { + this.completeFailure( + ForkExternrefImportFailureCode.OwnerFailure, + error, + ); + } + return true; + } + + try { + this.writeResults(registered.descriptor, returned); + this.complete(MailboxStatus.ResultReady); + } catch (error) { + this.completeFailure( + ForkExternrefImportFailureCode.HandlerContract, + error, + ); + } + return true; + } + + /** + * Close on exec, exit, Worker crash, or host destruction. Any blocked call is + * woken even if teardown races request publication or owner dispatch. + */ + close( + reason: ForkExternrefImportFailureCode = + ForkExternrefImportFailureCode.Teardown, + ): void { + // WHY: failure completion and teardown can race. Keep the terminal close + // reason in its own word so a losing dispatch cannot rewrite what wakes a + // blocked Worker after the owner has retired this process image. + writeHeaderU32(this.view, HeaderWord.CloseReason, reason); + Atomics.exchange( + this.words, + HeaderWord.Status, + MailboxStatus.Closed, + ); + Atomics.notify(this.words, HeaderWord.Status); + } + + private validateClaimedRequest(wake: ForkExternrefImportWake): void { + validateMailboxHeader(this.words, this.layout); + const requestBinding: ForkExternrefImportBinding = { + pid: readHeaderU32(this.view, HeaderWord.Pid), + generationId: readHeaderU32( + this.view, + HeaderWord.Generation, + ), + senderId: readHeaderU32(this.view, HeaderWord.Sender), + }; + if (!bindingEquals(requestBinding, this.binding)) { + throw new Error("fork externref mailbox request binding mismatch"); + } + if ( + readHeaderU32(this.view, HeaderWord.SequenceLow) + !== wake.sequenceLow + || readHeaderU32(this.view, HeaderWord.SequenceHigh) + !== wake.sequenceHigh + ) { + throw new Error("fork externref mailbox sequence changed after claim"); + } + } + + private requireRegisteredHandler(): RegisteredHandler { + const descriptorVersion = readHeaderU32( + this.view, + HeaderWord.DescriptorVersion, + ); + if ( + descriptorVersion !== FORK_EXTERNREF_IMPORT_DESCRIPTOR_VERSION + ) { + throw new Error( + `unsupported fork externref descriptor version ` + + `${descriptorVersion}`, + ); + } + const ordinal = readHeaderU32(this.view, HeaderWord.Ordinal); + const registered = this.catalog.lookup(ordinal); + if (!registered) { + throw new Error(`unknown fork externref import ordinal ${ordinal}`); + } + const descriptor = registered.descriptor; + const matches = + readHeaderU32(this.view, HeaderWord.ParamCount) + === descriptor.params.length + && readHeaderU32(this.view, HeaderWord.ResultCount) + === descriptor.results.length + && typeSequenceMatches( + this.view, + this.layout.paramTypesOffset, + descriptor.params, + ) + && typeSequenceMatches( + this.view, + this.layout.resultTypesOffset, + descriptor.results, + ); + if (!matches) { + throw new Error( + `fork externref import ${ordinal} signature mismatch`, + ); + } + return registered; + } + + private readArguments( + descriptor: ForkExternrefImportDescriptor, + ): unknown[] { + return descriptor.params.map((type, index) => { + if (type === "externref") { + const bits = this.view.getBigUint64( + slotOffset(this.layout.paramOffset, index), + true, + ); + if (bits > BigInt(MAX_U32)) { + throw new RangeError( + `externref argument handle ${bits} exceeds u32`, + ); + } + const handle = Number(bits); + return handle === 0 + ? null + : this.authority.authorizeForWire( + this.binding.pid, + this.binding.generationId, + handle, + ); + } + return readScalarSlot( + this.view, + this.layout.paramOffset, + index, + type, + ); + }); + } + + private writeResults( + descriptor: ForkExternrefImportDescriptor, + returned: unknown, + ): void { + let values: readonly unknown[]; + if (descriptor.results.length === 0) { + // Match the WebAssembly JS embedding: a return value from a void import + // is ignored. This also lets every host import use the exception- + // normalization path without imposing a new result-value policy. + values = []; + } else if (descriptor.results.length === 1) { + values = [returned]; + } else { + if ( + !Array.isArray(returned) + || returned.length !== descriptor.results.length + ) { + throw new TypeError( + `fork externref import ${descriptor.ordinal} must return ` + + `${descriptor.results.length} values`, + ); + } + values = returned; + } + + for (let index = 0; index < descriptor.results.length; index++) { + const type = descriptor.results[index]!; + const value = values[index]; + if (type === "externref") { + const handle = value === null + ? 0 + : this.authority.registerForWire( + this.binding.pid, + this.binding.generationId, + value, + ); + assertU32( + handle, + "fork externref result handle", + value === null, + ); + this.view.setBigUint64( + slotOffset(this.layout.resultOffset, index), + BigInt(handle), + true, + ); + } else { + writeScalarSlot( + this.view, + this.layout.resultOffset, + index, + type, + value, + ); + } + } + } + + private complete(status: MailboxStatus): void { + const previous = Atomics.compareExchange( + this.words, + HeaderWord.Status, + MailboxStatus.Dispatching, + status, + ); + // Teardown wins a race with dispatch. Never resurrect a closed mailbox or + // publish a result from an image whose generation was already retired. + if (previous === MailboxStatus.Dispatching) { + Atomics.notify(this.words, HeaderWord.Status); + } + } + + private completeFailure( + failure: ForkExternrefImportFailureCode, + error: unknown, + ): void { + try { + this.options.onDiagnostic?.(error, failure); + } catch { + // WHY: diagnostics are observational. A throwing logger must not leave + // the Worker asleep forever with the mailbox stuck in DISPATCHING. + } + writeHeaderU32(this.view, HeaderWord.FailureCode, failure); + this.complete(MailboxStatus.Failed); + } +} diff --git a/host/src/fork-externref-process-owner.ts b/host/src/fork-externref-process-owner.ts new file mode 100644 index 0000000000..5c564d3b0b --- /dev/null +++ b/host/src/fork-externref-process-owner.ts @@ -0,0 +1,234 @@ +import { + ForkExternrefBroker, + type ForkExternrefGeneration, +} from "./fork-reference-broker"; +import { + ForkModuleStateArena, + ForkModuleStateRecordKind, + readForkModuleStateRoot, +} from "./fork-module-state"; +import { + FORK_REFERENCE_TRANSACTION_OWNER_ID, +} from "./fork-reference-transaction"; +import { + scanSegmentedForkReferenceExternrefHandles, +} from "./fork-reference-segments"; +import { + unwrapForkWorkerExceptionCapability, +} from "./fork-worker-exception-capability"; + +export interface ForkExternrefForkGrant { + readonly generation: ForkExternrefGeneration; + readonly handleCount: number; +} + +/** + * Kernel-Worker owner for opaque host references across process lifetimes. + * + * Process and pthread Workers receive only `generation.id` plus Worker-local + * handle tokens. Real JavaScript values stay in this owner and are reached by + * host-import adapters through `registerForWire` / `authorizeForWire`. + * + * This is intentionally independent of activation-frame layout: fork leases + * are acquired from the process-wide reference-recipe record already copied + * through linear memory, so supporting an externref adds no bytes to each + * activation frame. + */ +export class ForkExternrefProcessOwner { + private readonly current = new Map(); + + constructor( + private readonly broker = new ForkExternrefBroker(), + ) {} + + /** Start a PID that does not already have a live Wasm image. */ + startGeneration(pid: number): ForkExternrefGeneration { + if (this.current.has(pid)) { + throw new Error(`externref process pid ${pid} already has a live generation`); + } + const generation = this.broker.createGeneration(pid); + this.current.set(pid, generation); + return generation; + } + + /** + * Replace one exact process image at exec's irreversible commit point. + * + * The broker retires the old token before returning the replacement, so an + * async callback from the discarded Worker cannot authorize a post-exec + * operation merely because the PID stayed the same. + */ + replaceGeneration( + expected: ForkExternrefGeneration, + ): ForkExternrefGeneration { + this.requireCurrent(expected); + const replacement = this.broker.createGeneration(expected.pid); + this.current.set(expected.pid, replacement); + return replacement; + } + + /** + * Grant a fresh fork child the unique externref handles named by the exact + * sealed continuation it will replay. + */ + forkGenerationFromContinuation( + parent: ForkExternrefGeneration, + childPid: number, + memory: WebAssembly.Memory, + ptrWidth: 4 | 8, + moduleBufferAddress: number, + label = `fork child pid=${childPid}: externref owner`, + ): ForkExternrefForkGrant { + this.requireCurrent(parent); + if (this.current.has(childPid)) { + throw new Error( + `externref fork child pid ${childPid} already has a live generation`, + ); + } + + const root = readForkModuleStateRoot( + memory, + moduleBufferAddress, + ptrWidth, + ); + if (root === 0) { + throw new Error(`${label}: copied continuation has no module-state arena`); + } + + // Inspect only the KFRV payload. The scanner validates the sealed chunk + // chain and every record envelope without copying unrelated table pages; + // the fresh child performs the full semantic arena validation before + // execution. Its allocation callbacks remain deliberately impossible: + // the arena belongs to the blocked parent and cannot be mutated here. + const arena = new ForkModuleStateArena( + memory, + ptrWidth, + () => { + throw new Error(`${label}: read-only arena attempted allocation`); + }, + () => { + throw new Error(`${label}: read-only arena attempted release`); + }, + `${label}: copied module state`, + ); + const records = arena.inspectSealedRecordViews( + root, + [ + ForkModuleStateRecordKind.ReferenceRecipeSegment, + ForkModuleStateRecordKind.ReferenceRecipe, + ], + ); + const handles = scanSegmentedForkReferenceExternrefHandles( + records, + FORK_REFERENCE_TRANSACTION_OWNER_ID, + ); + + const child = this.broker.createGeneration(childPid); + try { + const lease = this.broker.acquireFork(parent, child, handles); + this.current.set(childPid, child); + return Object.freeze({ + generation: child, + handleCount: lease.handleCount, + }); + } catch (error) { + // `acquireFork` is transactional, and retiring the provisional + // generation also clears any lease bookkeeping if a future broker + // implementation adds a fallible step after publication. + this.broker.releaseGeneration(child); + throw error; + } + } + + /** + * Release an exact process image after its process and pthread Workers can no + * longer execute. Returns false for already-retired generations. + */ + releaseGeneration(generation: ForkExternrefGeneration): boolean { + const current = this.current.get(generation.pid); + if (current === generation) this.current.delete(generation.pid); + return this.broker.releaseGeneration(generation); + } + + generationId(generation: ForkExternrefGeneration): number { + this.requireCurrent(generation); + return generation.id; + } + + /** + * Owner-side endpoint for an externref-producing host import. + * + * The adapter executes in this Realm, registers the real value here, and + * returns only the u32 handle to the process Worker. + */ + registerForWire( + pid: number, + generationId: number, + value: unknown, + ): number { + return this.broker.register( + this.requireWireGeneration(pid, generationId), + value, + ); + } + + /** Resolve an externref-consuming host import under exact image authority. */ + authorizeForWire( + pid: number, + generationId: number, + handle: number, + ): unknown { + return unwrapForkWorkerExceptionCapability( + this.broker.authorize( + this.requireWireGeneration(pid, generationId), + handle, + ), + ); + } + + /** Permanently close a host resource and invalidate all fork aliases. */ + tombstoneForWire( + pid: number, + generationId: number, + handle: number, + ): void { + this.broker.tombstone( + this.requireWireGeneration(pid, generationId), + handle, + ); + } + + private requireCurrent( + generation: ForkExternrefGeneration, + ): ForkExternrefGeneration { + if (this.current.get(generation.pid) !== generation) { + throw new Error( + `stale externref process generation ${generation.id} ` + + `for pid ${generation.pid}`, + ); + } + return generation; + } + + private requireWireGeneration( + pid: number, + generationId: number, + ): ForkExternrefGeneration { + if ( + !Number.isInteger(generationId) + || generationId <= 0 + || generationId > 0xffff_ffff + ) { + throw new RangeError( + `invalid externref process generation id ${generationId}`, + ); + } + const generation = this.current.get(pid); + if (!generation || generation.id !== generationId) { + throw new Error( + `stale externref process generation ${generationId} for pid ${pid}`, + ); + } + return generation; + } +} diff --git a/host/src/fork-function-catalog.ts b/host/src/fork-function-catalog.ts new file mode 100644 index 0000000000..31efeea36e --- /dev/null +++ b/host/src/fork-function-catalog.ts @@ -0,0 +1,120 @@ +/** + * Deterministic fresh-instance recipes for Wasm function references. + * + * A WebAssembly function object belongs to one JS Agent and cannot be moved to + * a fork child's Worker. Instrumented modules therefore export an immutable + * catalog table containing every function that can become a reference. The + * parent records `(module activation, catalog ordinal)`; after main/side + * modules are instantiated in the child, the same pair resolves to that + * instance's fresh function object. + */ + +export const FORK_FUNCTION_CATALOG_EXPORT = "__wpk_fork_function_catalog"; + +export interface ForkFunctionRecipe { + readonly moduleActivation: number; + readonly ordinal: number; +} + +interface RegisteredCatalog { + table: WebAssembly.Table; + entries: Array; +} + +function assertU32(value: number, label: string): void { + if (!Number.isInteger(value) || value < 0 || value > 0xffff_ffff) { + throw new RangeError(`invalid ${label} ${value}`); + } +} + +function recipeKey(moduleActivation: number, ordinal: number): string { + return `${moduleActivation}:${ordinal}`; +} + +export class ForkFunctionCatalog { + private readonly catalogs = new Map(); + private recipesByFunction = + new WeakMap(); + + register(moduleActivation: number, table: WebAssembly.Table): void { + assertU32(moduleActivation, "module activation"); + if (this.catalogs.has(moduleActivation)) { + throw new Error(`function catalog ${moduleActivation} is already registered`); + } + const entries: Array = []; + for (let ordinal = 0; ordinal < table.length; ordinal++) { + const value = table.get(ordinal); + if (typeof value !== "function") { + throw new Error( + `function catalog ${moduleActivation} has non-function entry ${ordinal}`, + ); + } + entries.push(value); + const recipes = this.recipesByFunction.get(value) ?? []; + if (!recipes.some((recipe) => + recipe.moduleActivation === moduleActivation + && recipe.ordinal === ordinal + )) { + recipes.push({ moduleActivation, ordinal }); + recipes.sort( + (left, right) => + left.moduleActivation - right.moduleActivation + || left.ordinal - right.ordinal, + ); + this.recipesByFunction.set(value, recipes); + } + } + this.catalogs.set(moduleActivation, { table, entries }); + } + + unregister(moduleActivation: number): void { + assertU32(moduleActivation, "module activation"); + const catalog = this.catalogs.get(moduleActivation); + if (!catalog) { + throw new Error(`function catalog ${moduleActivation} is not registered`); + } + for (const value of new Set(catalog.entries)) { + const remaining = (this.recipesByFunction.get(value) ?? []) + .filter((recipe) => recipe.moduleActivation !== moduleActivation); + this.recipesByFunction.set(value, remaining); + } + this.catalogs.delete(moduleActivation); + } + + encode(value: unknown): ForkFunctionRecipe | null { + if (value === null) return null; + if (typeof value !== "function") { + throw new TypeError("funcref encoder received a non-function value"); + } + const recipe = this.recipesByFunction.get(value)?.[0]; + if (!recipe) { + throw new Error("funcref is absent from the process module catalogs"); + } + return recipe; + } + + decode(recipe: ForkFunctionRecipe | null): CallableFunction | null { + if (recipe === null) return null; + assertU32(recipe.moduleActivation, "module activation"); + assertU32(recipe.ordinal, "function ordinal"); + const catalog = this.catalogs.get(recipe.moduleActivation); + if (!catalog) { + throw new Error(`function catalog ${recipe.moduleActivation} is not registered`); + } + const value = catalog.entries[recipe.ordinal]; + if (!value) { + throw new Error( + `function recipe ${recipeKey(recipe.moduleActivation, recipe.ordinal)} is out of bounds`, + ); + } + return value; + } + + clear(): void { + // WeakMap entries disappear with their function objects. Catalog entries + // are the only strong roots owned here and must not outlive replay/abort. + this.catalogs.clear(); + this.recipesByFunction = + new WeakMap(); + } +} diff --git a/host/src/fork-gc-codec.ts b/host/src/fork-gc-codec.ts new file mode 100644 index 0000000000..7b6a4c7ae3 --- /dev/null +++ b/host/src/fork-gc-codec.ts @@ -0,0 +1,905 @@ +import { + WPK_FORK_GC_CODEC_FIELD_RECORD_SIZE, + WPK_FORK_GC_CODEC_HEADER_SIZE, + WPK_FORK_GC_CODEC_LAYOUT_RECORD_SIZE, + WPK_FORK_GC_CODEC_MAGIC, + WPK_FORK_GC_CODEC_SECTION, + WPK_FORK_GC_CODEC_VERSION, + WPK_FORK_REFERENCE_EXPORT_GC_ALLOCATE, + WPK_FORK_REFERENCE_EXPORT_GC_ENCODE_SLOT, + WPK_FORK_REFERENCE_EXPORT_GC_FILL, + WPK_FORK_REFERENCE_EXPORT_GC_PUBLISH_EXTERNREF, + WPK_FORK_REFERENCE_EXPORT_GC_PROBE, +} from "./generated/abi"; + +export const enum ForkGcLayoutKind { + Struct = 1, + Array = 2, +} + +export const enum ForkGcConstructorKind { + Struct = 0, + ArrayGeneric = 1, + ArrayNew = 2, + ArrayDefault = 3, + ArrayFixed = 4, + ArrayData = 5, + ArrayElement = 6, +} + +export const FORK_GC_LAYOUT_REQUIRES_PROVENANCE = 1 << 0; +export const FORK_GC_LAYOUT_DEFAULTABLE_SHELL = 1 << 1; +const FORK_GC_LAYOUT_KNOWN_FLAGS = + FORK_GC_LAYOUT_REQUIRES_PROVENANCE + | FORK_GC_LAYOUT_DEFAULTABLE_SHELL; + +export const FORK_GC_FIELD_MUTABLE = 1 << 0; +export const FORK_GC_FIELD_NULLABLE = 1 << 1; +export const FORK_GC_FIELD_REFERENCE = 1 << 2; +export const FORK_GC_FIELD_ALLOCATION_DEPENDENCY = 1 << 3; +const FORK_GC_FIELD_KNOWN_FLAGS = + FORK_GC_FIELD_MUTABLE + | FORK_GC_FIELD_NULLABLE + | FORK_GC_FIELD_REFERENCE + | FORK_GC_FIELD_ALLOCATION_DEPENDENCY; + +const NO_ORDINAL = 0xffff_ffff; +const MAX_RECIPE_ID = 0x7fff_fffe; + +export interface ForkGcFieldDescriptor { + readonly storage: number; + readonly flags: number; + readonly scalarOffset: number | null; + readonly referenceOrdinal: number | null; +} + +export interface ForkGcLayoutDescriptor { + readonly id: number; + readonly typeOrdinal: number; + readonly kind: ForkGcLayoutKind; + readonly constructor: ForkGcConstructorKind; + readonly flags: number; + readonly scalarLengthOrStride: number; + readonly fields: readonly ForkGcFieldDescriptor[]; + readonly superTypeOrdinal: number | null; + readonly baseLayoutId: number; + readonly auxiliary: number; + readonly provenanceScalarLength: number; + readonly provenanceReferenceCount: number; +} + +export interface ForkGcCodecProvider { + readonly activationId: number; + readonly descriptor: ForkGcCodecDescriptor; + /** Probe the object in a shared transit slot without crossing `anyref`. */ + probe(slot: number): bigint; + /** Encode the object in a shared transit slot into the active recipe graph. */ + encodeSlot(slot: number): number; + /** Allocate one routed aggregate/i31 recipe into `recipe + 1`. */ + allocate(recipeId: number): void; + /** Restore mutable aggregate fields after every shell has been allocated. */ + fill(recipeId: number): void; + /** + * Convert one process-owned token inside Wasm and publish it at recipe+1. + * + * JavaScript cannot directly create an anyref value for the shared transit + * table, so this remains an activation-local scalar/externref entry point. + */ + publishExternref(recipeId: number, value: unknown): void; +} + +function assertU31(value: number, context: string): void { + if (!Number.isInteger(value) || value <= 0 || value > 0x7fff_ffff) { + throw new Error(`${context} is not a nonzero u31`); + } +} + +function assertU32(value: number, context: string): void { + if (!Number.isInteger(value) || value < 0 || value > 0xffff_ffff) { + throw new Error(`${context} is not a u32`); + } +} + +function checkedProduct(left: number, right: number, context: string): number { + const result = left * right; + if (!Number.isSafeInteger(result) || result > 0xffff_ffff) { + throw new Error(`${context} exceeds the u32 format`); + } + return result; +} + +function storageByteLength(storage: number): number { + switch (storage) { + case 1: + return 1; + case 2: + return 2; + case 3: + case 5: + return 4; + case 4: + case 6: + return 8; + case 7: + return 16; + case 8: + return 4; + default: + throw new Error(`unsupported GC storage code ${storage}`); + } +} + +function requireFunction( + exports: WebAssembly.Exports, + name: string, +): CallableFunction { + const value = exports[name]; + if (typeof value !== "function") { + throw new Error(`fork GC codec is missing function export ${name}`); + } + return value as CallableFunction; +} + +function assertSlot(value: number, context: string): void { + assertU32(value, context); + if (value > 0x7fff_ffff) { + throw new RangeError(`${context} is not a routable table slot`); + } +} + +function assertRecipeId(value: number): void { + assertU31(value, "GC recipe id"); + if (value > MAX_RECIPE_ID) { + throw new RangeError(`GC recipe id ${value} is reserved`); + } +} + +/** + * Validated, activation-local structural type evidence. + * + * A host callback may select only a constructor record whose `baseLayoutId` + * points at the exact base record supplied by generated Wasm. This prevents a + * stale or malicious provenance map from changing the concrete replay helper. + */ +export class ForkGcCodecDescriptor { + private readonly byId = new Map(); + private readonly baseByType = new Map(); + + constructor(readonly layouts: readonly ForkGcLayoutDescriptor[]) { + layouts.forEach((layout, index) => { + if (layout.id !== index + 1 || this.byId.has(layout.id)) { + throw new Error( + `GC codec layout ${layout.id} is not in canonical id order`, + ); + } + this.byId.set(layout.id, layout); + if (layout.baseLayoutId === layout.id) { + if (this.baseByType.has(layout.typeOrdinal)) { + throw new Error( + `GC type ordinal ${layout.typeOrdinal} has multiple base layouts`, + ); + } + this.baseByType.set(layout.typeOrdinal, layout); + } + }); + for (const layout of layouts) { + const base = this.byId.get(layout.baseLayoutId); + if ( + !base + || base.baseLayoutId !== base.id + || base.typeOrdinal !== layout.typeOrdinal + || base.kind !== layout.kind + || ( + layout.id !== base.id + && layout.constructor === ForkGcConstructorKind.ArrayGeneric + ) + ) { + throw new Error( + `GC codec layout ${layout.id} has invalid base layout ` + + `${layout.baseLayoutId}`, + ); + } + if ( + layout.kind === ForkGcLayoutKind.Struct + ? layout.constructor !== ForkGcConstructorKind.Struct + : layout.fields.length !== 1 + || ( + layout.id === base.id + ? layout.constructor !== ForkGcConstructorKind.ArrayGeneric + : layout.constructor === ForkGcConstructorKind.ArrayGeneric + ) + ) { + throw new Error(`GC codec layout ${layout.id} has an invalid constructor`); + } + if ( + layout.id !== base.id + && ( + layout.scalarLengthOrStride !== base.scalarLengthOrStride + || layout.superTypeOrdinal !== base.superTypeOrdinal + || layout.flags + !== ( + base.flags + | FORK_GC_LAYOUT_REQUIRES_PROVENANCE + ) + || !sameFields(layout.fields, base.fields) + ) + ) { + throw new Error( + `GC constructor layout ${layout.id} does not match base ` + + `${base.id}`, + ); + } + if ( + layout.provenanceScalarLength > 16 + || ( + (layout.flags & FORK_GC_LAYOUT_REQUIRES_PROVENANCE) === 0 + && ( + layout.provenanceScalarLength !== 0 + || layout.provenanceReferenceCount !== 0 + ) + ) + ) { + throw new Error(`GC codec layout ${layout.id} has invalid provenance`); + } + validateLayoutPayload(layout); + } + } + + require(layoutId: number): ForkGcLayoutDescriptor { + assertU31(layoutId, "GC layout id"); + const layout = this.byId.get(layoutId); + if (!layout) throw new Error(`unknown GC layout ${layoutId}`); + return layout; + } + + requireCaptureLayout( + baseLayoutId: number, + specializedLayoutId: number, + ): ForkGcLayoutDescriptor { + const base = this.require(baseLayoutId); + const selected = this.require(specializedLayoutId); + if ( + base.baseLayoutId !== base.id + || selected.baseLayoutId !== base.id + || selected.typeOrdinal !== base.typeOrdinal + || selected.kind !== base.kind + || ( + (base.flags & FORK_GC_LAYOUT_REQUIRES_PROVENANCE) !== 0 + && selected.id === base.id + && selected.kind === ForkGcLayoutKind.Array + ) + ) { + throw new Error( + `GC constructor layout ${specializedLayoutId} does not belong to ` + + `base layout ${baseLayoutId}`, + ); + } + return selected; + } +} + +function sameFields( + left: readonly ForkGcFieldDescriptor[], + right: readonly ForkGcFieldDescriptor[], +): boolean { + return left.length === right.length && left.every((field, index) => { + const other = right[index]!; + return field.storage === other.storage + && field.flags === other.flags + && field.scalarOffset === other.scalarOffset + && field.referenceOrdinal === other.referenceOrdinal; + }); +} + +function validateLayoutPayload(layout: ForkGcLayoutDescriptor): void { + let expectedReferenceOrdinal = 0; + let minimumScalarLength = 0; + for (const [index, field] of layout.fields.entries()) { + const isReference = (field.flags & FORK_GC_FIELD_REFERENCE) !== 0; + const isMutable = (field.flags & FORK_GC_FIELD_MUTABLE) !== 0; + const isNullable = (field.flags & FORK_GC_FIELD_NULLABLE) !== 0; + const isDependency = + (field.flags & FORK_GC_FIELD_ALLOCATION_DEPENDENCY) !== 0; + if ( + isReference !== (field.storage === 8) + || (!isReference && (isNullable || isDependency)) + || (isDependency && isMutable) + ) { + throw new Error( + `GC layout ${layout.id} field ${index} has inconsistent flags`, + ); + } + if (isReference) { + if (field.referenceOrdinal !== expectedReferenceOrdinal) { + throw new Error( + `GC layout ${layout.id} field ${index} has noncanonical ` + + `reference ordinal`, + ); + } + expectedReferenceOrdinal++; + continue; + } + const scalarOffset = field.scalarOffset!; + const end = scalarOffset + storageByteLength(field.storage); + if ( + !Number.isSafeInteger(end) + || end > 0xffff_ffff + || scalarOffset < minimumScalarLength + ) { + throw new Error( + `GC layout ${layout.id} field ${index} has an invalid scalar offset`, + ); + } + minimumScalarLength = end; + } + if ( + layout.kind === ForkGcLayoutKind.Struct + && minimumScalarLength > layout.scalarLengthOrStride + ) { + throw new Error(`GC struct layout ${layout.id} scalar fields overflow`); + } + if ( + layout.kind === ForkGcLayoutKind.Array + && layout.fields[0]!.storage !== 8 + && layout.scalarLengthOrStride + !== storageByteLength(layout.fields[0]!.storage) + ) { + throw new Error(`GC array layout ${layout.id} has an invalid stride`); + } + + switch (layout.constructor) { + case ForkGcConstructorKind.Struct: + if (layout.provenanceScalarLength !== 0) { + throw new Error( + `GC struct layout ${layout.id} has unexpected scalar provenance`, + ); + } + break; + case ForkGcConstructorKind.ArrayGeneric: + case ForkGcConstructorKind.ArrayDefault: + if ( + layout.provenanceScalarLength !== 0 + || layout.provenanceReferenceCount !== 0 + ) { + throw new Error( + `GC layout ${layout.id} has unexpected constructor provenance`, + ); + } + break; + case ForkGcConstructorKind.ArrayFixed: + if ( + layout.provenanceScalarLength !== 0 + || ( + layout.provenanceReferenceCount !== 0 + && layout.provenanceReferenceCount !== layout.auxiliary + ) + ) { + throw new Error( + `GC array.new_fixed layout ${layout.id} is malformed`, + ); + } + break; + case ForkGcConstructorKind.ArrayNew: + if ( + layout.provenanceReferenceCount > 1 + || ( + layout.provenanceReferenceCount !== 0 + && layout.provenanceScalarLength !== 0 + ) + ) { + throw new Error(`GC array.new layout ${layout.id} is malformed`); + } + break; + case ForkGcConstructorKind.ArrayData: + if ( + layout.fields[0]!.storage === 8 + || layout.provenanceScalarLength !== 8 + || layout.provenanceReferenceCount !== 0 + ) { + throw new Error(`GC array.new_data layout ${layout.id} is malformed`); + } + break; + case ForkGcConstructorKind.ArrayElement: + if ( + layout.fields[0]!.storage !== 8 + || layout.provenanceScalarLength !== 8 + || layout.provenanceReferenceCount !== 0 + ) { + throw new Error(`GC array.new_elem layout ${layout.id} is malformed`); + } + break; + } +} + +export function decodeForkGcCodecDescriptor( + bytes: Uint8Array, +): ForkGcCodecDescriptor { + if (bytes.byteLength < WPK_FORK_GC_CODEC_HEADER_SIZE) { + throw new Error("GC codec descriptor is truncated"); + } + if ( + WPK_FORK_GC_CODEC_MAGIC.some((byte, index) => bytes[index] !== byte) + ) { + throw new Error("GC codec descriptor has an invalid magic"); + } + const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); + if ( + view.getUint16(4, true) !== WPK_FORK_GC_CODEC_VERSION + || view.getUint16(6, true) !== WPK_FORK_GC_CODEC_HEADER_SIZE + ) { + throw new Error("GC codec descriptor has an unsupported version/header"); + } + const layoutCount = view.getUint32(8, true); + const fieldCount = view.getUint32(12, true); + const layoutsLength = checkedProduct( + layoutCount, + WPK_FORK_GC_CODEC_LAYOUT_RECORD_SIZE, + "GC layout catalog", + ); + const fieldsLength = checkedProduct( + fieldCount, + WPK_FORK_GC_CODEC_FIELD_RECORD_SIZE, + "GC field catalog", + ); + const expectedLength = + WPK_FORK_GC_CODEC_HEADER_SIZE + layoutsLength + fieldsLength; + if (expectedLength !== bytes.byteLength) { + throw new Error( + `GC codec descriptor has ${bytes.byteLength} bytes; ` + + `expected ${expectedLength}`, + ); + } + + const rawLayouts: Array<{ + id: number; + typeOrdinal: number; + kind: ForkGcLayoutKind; + constructor: ForkGcConstructorKind; + flags: number; + scalarLengthOrStride: number; + fieldStart: number; + fieldCount: number; + superTypeOrdinal: number | null; + baseLayoutId: number; + auxiliary: number; + provenanceScalarLength: number; + provenanceReferenceCount: number; + }> = []; + let expectedFieldStart = 0; + for (let index = 0; index < layoutCount; index++) { + const offset = + WPK_FORK_GC_CODEC_HEADER_SIZE + + index * WPK_FORK_GC_CODEC_LAYOUT_RECORD_SIZE; + const id = view.getUint32(offset, true); + assertU31(id, `GC layout ${index} id`); + const kind = view.getUint8(offset + 8); + const constructor = view.getUint8(offset + 9); + const flags = view.getUint16(offset + 10, true); + const fieldStart = view.getUint32(offset + 16, true); + const layoutFieldCount = view.getUint32(offset + 20, true); + if ( + (kind !== ForkGcLayoutKind.Struct && kind !== ForkGcLayoutKind.Array) + || constructor > ForkGcConstructorKind.ArrayElement + || (flags & ~FORK_GC_LAYOUT_KNOWN_FLAGS) !== 0 + || fieldStart !== expectedFieldStart + || layoutFieldCount > fieldCount - Math.min(fieldStart, fieldCount) + ) { + throw new Error(`GC layout ${id} has unsupported kind/flags`); + } + expectedFieldStart += layoutFieldCount; + rawLayouts.push({ + id, + typeOrdinal: view.getUint32(offset + 4, true), + kind, + constructor, + flags, + scalarLengthOrStride: view.getUint32(offset + 12, true), + fieldStart, + fieldCount: layoutFieldCount, + superTypeOrdinal: + view.getUint32(offset + 24, true) === NO_ORDINAL + ? null + : view.getUint32(offset + 24, true), + baseLayoutId: view.getUint32(offset + 28, true), + auxiliary: view.getUint32(offset + 32, true), + provenanceScalarLength: view.getUint32(offset + 36, true), + provenanceReferenceCount: view.getUint32(offset + 40, true), + }); + } + if (expectedFieldStart !== fieldCount) { + throw new Error("GC codec descriptor has unowned field records"); + } + + const fields: ForkGcFieldDescriptor[] = []; + const fieldsOffset = WPK_FORK_GC_CODEC_HEADER_SIZE + layoutsLength; + for (let index = 0; index < fieldCount; index++) { + const offset = fieldsOffset + index * WPK_FORK_GC_CODEC_FIELD_RECORD_SIZE; + const storage = view.getUint8(offset); + const flags = view.getUint8(offset + 1); + const reserved = view.getUint16(offset + 2, true); + const scalarOffset = view.getUint32(offset + 4, true); + const referenceOrdinal = view.getUint32(offset + 8, true); + if ( + storage < 1 + || storage > 8 + || (flags & ~FORK_GC_FIELD_KNOWN_FLAGS) !== 0 + || reserved !== 0 + || ( + (flags & FORK_GC_FIELD_REFERENCE) !== 0 + && (scalarOffset !== NO_ORDINAL || referenceOrdinal === NO_ORDINAL) + ) + || ( + (flags & FORK_GC_FIELD_REFERENCE) === 0 + && (scalarOffset === NO_ORDINAL || referenceOrdinal !== NO_ORDINAL) + ) + ) { + throw new Error(`GC field ${index} is malformed`); + } + fields.push({ + storage, + flags, + scalarOffset: scalarOffset === NO_ORDINAL ? null : scalarOffset, + referenceOrdinal: + referenceOrdinal === NO_ORDINAL ? null : referenceOrdinal, + }); + } + + const layouts = rawLayouts.map((layout) => { + if ( + layout.fieldStart > fieldCount + || layout.fieldCount > fieldCount - layout.fieldStart + ) { + throw new Error(`GC layout ${layout.id} field range is out of bounds`); + } + const selectedFields = fields.slice( + layout.fieldStart, + layout.fieldStart + layout.fieldCount, + ); + if ( + (layout.kind === ForkGcLayoutKind.Array && selectedFields.length !== 1) + || (layout.kind === ForkGcLayoutKind.Struct + && layout.constructor !== ForkGcConstructorKind.Struct) + ) { + throw new Error(`GC layout ${layout.id} has inconsistent shape`); + } + return { + ...layout, + fields: selectedFields, + }; + }); + return new ForkGcCodecDescriptor(layouts); +} + +export function readForkGcCodecDescriptor( + module: WebAssembly.Module, +): ForkGcCodecDescriptor { + const sections = WebAssembly.Module.customSections( + module, + WPK_FORK_GC_CODEC_SECTION, + ); + if (sections.length !== 1) { + throw new Error( + `expected one ${WPK_FORK_GC_CODEC_SECTION} section, ` + + `found ${sections.length}`, + ); + } + return decodeForkGcCodecDescriptor(new Uint8Array(sections[0]!)); +} + +/** + * Bind the four scalar-callable entry points generated for one activation. + * + * No method accepts or returns `anyref`; values move only through the + * process-owned transit table imported by both the parent and fresh child. + */ +export function forkGcCodecProviderFromInstance( + activationId: number, + module: WebAssembly.Module, + instance: WebAssembly.Instance, +): ForkGcCodecProvider { + assertU32(activationId, "GC codec activation"); + const probe = requireFunction( + instance.exports, + WPK_FORK_REFERENCE_EXPORT_GC_PROBE, + ); + const encodeSlot = requireFunction( + instance.exports, + WPK_FORK_REFERENCE_EXPORT_GC_ENCODE_SLOT, + ); + const allocate = requireFunction( + instance.exports, + WPK_FORK_REFERENCE_EXPORT_GC_ALLOCATE, + ); + const fill = requireFunction( + instance.exports, + WPK_FORK_REFERENCE_EXPORT_GC_FILL, + ); + const publishExternref = requireFunction( + instance.exports, + WPK_FORK_REFERENCE_EXPORT_GC_PUBLISH_EXTERNREF, + ); + return { + activationId, + descriptor: readForkGcCodecDescriptor(module), + probe(slot): bigint { + assertSlot(slot, "GC probe slot"); + const packed = probe(slot); + if (typeof packed !== "bigint") { + throw new TypeError("GC probe did not return an i64"); + } + return BigInt.asUintN(64, packed); + }, + encodeSlot(slot): number { + assertSlot(slot, "GC encode slot"); + const recipeId = Number(encodeSlot(slot)); + assertRecipeId(recipeId); + return recipeId; + }, + allocate(recipeId): void { + assertRecipeId(recipeId); + if (recipeId === 0) { + throw new RangeError("the null recipe cannot be allocated"); + } + allocate(recipeId); + }, + fill(recipeId): void { + assertRecipeId(recipeId); + if (recipeId === 0) { + throw new RangeError("the null recipe cannot be filled"); + } + fill(recipeId); + }, + publishExternref(recipeId, value): void { + assertRecipeId(recipeId); + if (recipeId === 0) { + throw new RangeError("the null recipe cannot publish an externref"); + } + publishExternref(recipeId, value); + }, + }; +} + +export interface ForkGcConstructorProvenance { + readonly activationId: number; + readonly baseLayoutId: number; + readonly layoutId: number; + readonly scalars: Uint8Array; + readonly references: readonly (object | null)[]; +} + +interface PendingProvenance { + readonly token: number; + readonly object: object; + readonly activationId: number; + readonly baseLayoutId: number; + readonly layout: ForkGcLayoutDescriptor; + readonly scalars: Uint8Array; + readonly references: (object | null)[]; +} + +/** + * Weak-keyed constructor evidence for non-shell GC objects. + * + * The registry owns a key strongly only between `begin` and `end`; finalized + * records are ephemerons and disappear with their Wasm wrapper. `abortPending` + * is called at every transaction/activation teardown so a trapping wrapper + * cannot leave a hidden strong root. + */ +export class ForkGcProvenanceRegistry { + private finalized = new WeakMap(); + private pending: PendingProvenance | null = null; + private nextToken = 1; + + begin( + table: WebAssembly.Table, + descriptor: ForkGcCodecDescriptor, + expectedActivationId: number, + slot: number, + activationId: number, + baseLayoutId: number, + specializedLayoutId: number, + scalarLo: bigint, + scalarHi: bigint, + referenceCount: number, + ): number { + try { + if (this.pending) { + throw new Error( + `GC provenance registration ${this.pending.token} is still pending`, + ); + } + assertU32(expectedActivationId, "expected GC activation"); + assertU32(activationId, "GC provenance activation"); + assertU32(slot, "GC provenance slot"); + assertU32(referenceCount, "GC provenance reference count"); + if (activationId !== expectedActivationId) { + throw new Error( + `activation ${expectedActivationId} cannot register GC provenance ` + + `for activation ${activationId}`, + ); + } + if (slot >= table.length) { + throw new Error(`GC provenance slot ${slot} is out of bounds`); + } + const object = table.get(slot); + if ( + (typeof object !== "object" || object === null) + && typeof object !== "function" + ) { + throw new Error("GC provenance source is not a non-null Wasm object"); + } + const layout = descriptor.requireCaptureLayout( + baseLayoutId, + specializedLayoutId, + ); + if (layout.provenanceReferenceCount !== referenceCount) { + throw new Error( + `GC layout ${layout.id} expects ` + + `${layout.provenanceReferenceCount} provenance references, ` + + `found ${referenceCount}`, + ); + } + const scalarBytes = new Uint8Array(16); + const scalarView = new DataView(scalarBytes.buffer); + scalarView.setBigUint64(0, BigInt.asUintN(64, scalarLo), true); + scalarView.setBigUint64(8, BigInt.asUintN(64, scalarHi), true); + const token = this.nextToken++; + if (!Number.isSafeInteger(token) || token > 0x7fff_ffff) { + this.nextToken = 1; + throw new Error("GC provenance token space exhausted"); + } + this.pending = { + token, + object: object as object, + activationId, + baseLayoutId, + layout, + scalars: scalarBytes.slice(0, layout.provenanceScalarLength), + references: [], + }; + return token; + } catch (error) { + this.abortPending(); + try { + if (Number.isInteger(slot) && slot >= 0 && slot < table.length) { + table.set(slot, null); + } + } catch { + // Preserve the fail-closed provenance error. + } + throw error; + } + } + + appendReference( + table: WebAssembly.Table, + token: number, + index: number, + slot: number, + ): void { + try { + const pending = this.requirePending(token); + assertU32(index, "GC provenance reference index"); + assertU32(slot, "GC provenance reference slot"); + if ( + index !== pending.references.length + || index >= pending.layout.provenanceReferenceCount + ) { + throw new Error( + `GC provenance reference ${index} is out of canonical order`, + ); + } + if (slot >= table.length) { + throw new Error(`GC provenance reference slot ${slot} is out of bounds`); + } + const value = table.get(slot); + if ( + value !== null + && (typeof value !== "object") + && typeof value !== "function" + ) { + throw new Error( + `GC provenance reference ${index} is neither null nor an object`, + ); + } + // A nullable seed for a zero-length immutable array is unobservable but + // still a typed constructor operand. Preserve it as recipe zero so a + // replayed child can register the same constructor evidence. + pending.references.push(value as object | null); + } catch (error) { + this.abortPending(); + try { + if (Number.isInteger(slot) && slot >= 0 && slot < table.length) { + table.set(slot, null); + } + } catch { + // Preserve the fail-closed provenance error. + } + throw error; + } + } + + end(token: number): void { + try { + const pending = this.requirePending(token); + if ( + pending.references.length + !== pending.layout.provenanceReferenceCount + ) { + throw new Error( + `GC provenance registration ${token} has ` + + `${pending.references.length} references; expected ` + + `${pending.layout.provenanceReferenceCount}`, + ); + } + this.finalized.set(pending.object, { + activationId: pending.activationId, + baseLayoutId: pending.baseLayoutId, + layoutId: pending.layout.id, + scalars: pending.scalars, + references: [...pending.references], + }); + this.pending = null; + } catch (error) { + this.abortPending(); + throw error; + } + } + + lookup( + object: unknown, + expectedActivationId: number, + descriptor: ForkGcCodecDescriptor, + baseLayoutId: number, + ): ForkGcConstructorProvenance | null { + if ( + (typeof object !== "object" || object === null) + && typeof object !== "function" + ) { + return null; + } + const provenance = this.finalized.get(object as object); + if (!provenance) return null; + if (provenance.activationId !== expectedActivationId) { + throw new Error( + `GC provenance belongs to activation ${provenance.activationId}, ` + + `not ${expectedActivationId}`, + ); + } + descriptor.requireCaptureLayout(baseLayoutId, provenance.layoutId); + if (provenance.baseLayoutId !== baseLayoutId) { + throw new Error( + `GC provenance base ${provenance.baseLayoutId} does not match ` + + `${baseLayoutId}`, + ); + } + return provenance; + } + + find(object: unknown): ForkGcConstructorProvenance | null { + if ( + (typeof object !== "object" || object === null) + && typeof object !== "function" + ) { + return null; + } + return this.finalized.get(object as object) ?? null; + } + + abortPending(): void { + this.pending = null; + } + + clear(): void { + this.abortPending(); + this.finalized = new WeakMap(); + } + + private requirePending(token: number): PendingProvenance { + assertU31(token, "GC provenance token"); + if (!this.pending || this.pending.token !== token) { + throw new Error(`GC provenance token ${token} is not active`); + } + return this.pending; + } +} diff --git a/host/src/fork-host-import-runtime.ts b/host/src/fork-host-import-runtime.ts new file mode 100644 index 0000000000..e069013c0f --- /dev/null +++ b/host/src/fork-host-import-runtime.ts @@ -0,0 +1,497 @@ +import { + readWasmFunctionImports, + type WasmFunctionImportType, + type WasmFunctionSignature, + type WasmValueType, +} from "./constants"; +import { + createForkExternrefImportMailbox, + type ForkExternrefImportAuthority, + type ForkExternrefImportBinding, + type ForkExternrefImportDescriptor, + type ForkExternrefImportHandler, + ForkExternrefImportOwnerCatalog, + ForkExternrefImportOwnerEndpoint, + type ForkExternrefImportWake, + ForkExternrefImportWorkerCaller, +} from "./fork-externref-import-mailbox"; +import { + FORK_WORKER_EXCEPTION_RESERVED_ORDINAL_START, + ForkWorkerExceptionCapabilityOwner, + ForkWorkerLocalImportExceptionNormalizer, + type ForkWorkerLocalImportExceptionNormalizerOptions, +} from "./fork-worker-import-exceptions"; +import { + ForkExternrefTokenCache, +} from "./fork-reference-broker"; + +export interface ForkOwnerImportWireRegistration { + readonly module: string; + readonly name: string; + readonly descriptor: ForkExternrefImportDescriptor; +} + +export interface ForkHostImportWorkerInit { + readonly mailbox: SharedArrayBuffer; + readonly senderId: number; + readonly ownerImports: readonly ForkOwnerImportWireRegistration[]; +} + +export interface ForkHostImportOwnerWorkerOptions { + readonly pid: number; + readonly generationId: number; + /** + * Must compare the exact live Worker object/generation held by the host + * entrypoint. Numeric wake fields are never sufficient authorization. + */ + readonly authorizeSender: (binding: ForkExternrefImportBinding) => void; + readonly onDiagnostic?: (error: unknown) => void; +} + +function importKey(module: string, name: string): string { + return `${module.length}:${module}${name}`; +} + +function validateImportName(value: string, label: string): void { + if (typeof value !== "string" || value.length === 0) { + throw new TypeError(`${label} must be a nonempty string`); + } +} + +function freezeDescriptor( + descriptor: ForkExternrefImportDescriptor, +): ForkExternrefImportDescriptor { + return Object.freeze({ + version: descriptor.version, + ordinal: descriptor.ordinal, + params: Object.freeze([...descriptor.params]), + results: Object.freeze([...descriptor.results]), + }); +} + +function cloneWireRegistration( + registration: ForkOwnerImportWireRegistration, +): ForkOwnerImportWireRegistration { + return Object.freeze({ + module: registration.module, + name: registration.name, + descriptor: freezeDescriptor(registration.descriptor), + }); +} + +function wasmTypeForMailbox(type: string): number { + switch (type) { + case "i32": + return 0x7f; + case "i64": + return 0x7e; + case "f32": + return 0x7d; + case "f64": + return 0x7c; + case "externref": + return 0x6f; + default: + throw new Error(`unknown fork host-import mailbox type ${type}`); + } +} + +function isExternReferenceType(type: WasmValueType): boolean { + return ( + type.code === 0x6f + || type.code === 0x72 + || ( + (type.code === 0x62 || type.code === 0x63 || type.code === 0x64) + && (type.heapType === -17 || type.heapType === -14) + ) + ); +} + +/** + * Vectors and exception/continuation references deliberately cannot enter a + * JavaScript host function. They remain valid on a direct Wasm-to-Wasm import + * and are reconstructed by the scalar frame, exception codec, or activation + * codec respectively. + */ +function requiresDirectWasmBoundary(type: WasmValueType): boolean { + if ( + type.code === 0x7b // v128 + || type.code === 0x69 // exnref + || type.code === 0x74 // noexnref + || type.code === 0x68 // contref + || type.code === 0x75 // nocontref + ) { + return true; + } + return ( + (type.code === 0x62 || type.code === 0x63 || type.code === 0x64) + && ( + type.heapType === -23 // exn + || type.heapType === -12 // noexn + || type.heapType === -24 // cont + || type.heapType === -11 // nocont + ) + ); +} + +function signatureRequiresDirectWasmBoundary( + signature: WasmFunctionSignature, +): boolean { + for (const type of signature.paramTypes) { + if (requiresDirectWasmBoundary(type)) return true; + } + for (const type of signature.resultTypes) { + if (requiresDirectWasmBoundary(type)) return true; + } + return false; +} + +function signatureMatchesDescriptor( + signature: WasmFunctionSignature, + descriptor: ForkExternrefImportDescriptor, +): boolean { + return ( + signature.params.length === descriptor.params.length + && signature.results.length === descriptor.results.length + && signature.params.every( + (type, index) => { + const expected = descriptor.params[index]!; + return expected === "externref" + ? isExternReferenceType(signature.paramTypes[index]!) + : type === wasmTypeForMailbox(expected); + }, + ) + && signature.results.every( + (type, index) => { + const expected = descriptor.results[index]!; + return expected === "externref" + ? isExternReferenceType(signature.resultTypes[index]!) + : type === wasmTypeForMailbox(expected); + }, + ) + ); +} + +function signatureText(signature: WasmFunctionSignature): string { + const valueName = (value: WasmValueType): string => { + switch (value.code) { + case 0x7f: + return "i32"; + case 0x7e: + return "i64"; + case 0x7d: + return "f32"; + case 0x7c: + return "f64"; + case 0x6f: + return "externref"; + case 0x70: + return "funcref"; + case 0x6e: + return "anyref"; + case 0x6d: + return "eqref"; + case 0x6c: + return "i31ref"; + case 0x6b: + return "structref"; + case 0x6a: + return "arrayref"; + case 0x69: + return "exnref"; + case 0x68: + return "contref"; + case 0x7b: + return "v128"; + case 0x62: + case 0x63: + case 0x64: + return `${value.code === 0x62 ? "exact" : "ref"}` + + `${value.code === 0x63 ? " null" : ""}` + + `${value.shared ? " shared" : ""} ${String(value.heapType)}`; + default: + return `0x${value.code.toString(16)}`; + } + }; + return `(${signature.paramTypes.map(valueName).join(",")}) -> (` + + `${signature.resultTypes.map(valueName).join(",")})`; +} + +/** + * One owner-realm catalog shared by every process/pthread Worker. + * + * The catalog is sealed when the first Worker is created. This guarantees a + * side module and the process main module see the same immutable routing + * policy, while every Worker still gets a distinct mailbox/sender identity. + */ +export class ForkHostImportOwnerRuntime { + private readonly catalog = new ForkExternrefImportOwnerCatalog(); + private readonly exceptionCapabilities = + new ForkWorkerExceptionCapabilityOwner(); + private readonly registrations = + new Map(); + private nextSenderId = 1; + private sealed = false; + + constructor( + private readonly authority: ForkExternrefImportAuthority, + ) { + this.exceptionCapabilities.install(this.catalog); + } + + register( + module: string, + name: string, + descriptor: ForkExternrefImportDescriptor, + handler: ForkExternrefImportHandler, + ): void { + if (this.sealed) { + throw new Error( + "fork owner host-import catalog is sealed by a live Worker", + ); + } + validateImportName(module, "fork owner import module"); + validateImportName(name, "fork owner import name"); + if (descriptor.ordinal >= FORK_WORKER_EXCEPTION_RESERVED_ORDINAL_START) { + throw new RangeError( + `fork owner import ordinal ${descriptor.ordinal} is reserved`, + ); + } + const key = importKey(module, name); + if (this.registrations.has(key)) { + throw new Error(`duplicate fork owner import ${module}.${name}`); + } + const registration = cloneWireRegistration({ + module, + name, + descriptor, + }); + this.catalog.register(registration.descriptor, handler); + this.registrations.set(key, registration); + } + + createWorker( + options: ForkHostImportOwnerWorkerOptions, + ): ForkHostImportOwnerWorker { + this.sealed = true; + if (this.nextSenderId > 0xffff_ffff) { + throw new RangeError("fork host-import sender id space exhausted"); + } + const binding: ForkExternrefImportBinding = Object.freeze({ + pid: options.pid, + generationId: options.generationId, + senderId: this.nextSenderId++, + }); + // WHY: the sealed catalog is the reconstruction owner for these imports. + // Size one reusable mailbox from its widest exact signature so valid wide + // imports need neither a continuation-frame field nor per-call buffers. + const mailbox = createForkExternrefImportMailbox( + this.catalog.mailboxCapacity, + ); + const endpoint = new ForkExternrefImportOwnerEndpoint( + mailbox, + binding, + this.catalog, + this.authority, + { + authorizeSender: options.authorizeSender, + onDiagnostic: (error) => options.onDiagnostic?.(error), + }, + ); + return new ForkHostImportOwnerWorker( + endpoint, + this.exceptionCapabilities, + Object.freeze( + [...this.registrations.values()].map(cloneWireRegistration), + ), + ); + } +} + +export class ForkHostImportOwnerWorker { + readonly init: ForkHostImportWorkerInit; + private closed = false; + + constructor( + private readonly endpoint: ForkExternrefImportOwnerEndpoint, + private readonly exceptionCapabilities: + ForkWorkerExceptionCapabilityOwner, + ownerImports: readonly ForkOwnerImportWireRegistration[], + ) { + this.init = Object.freeze({ + mailbox: endpoint.mailbox, + senderId: endpoint.binding.senderId, + ownerImports, + }); + } + + get binding(): ForkExternrefImportBinding { + return this.endpoint.binding; + } + + dispatch(wake: ForkExternrefImportWake): boolean { + if (this.closed) return false; + // This method is called only by the listener attached to this exact Worker + // object; pass that independently observed binding into the core endpoint. + return this.endpoint.dispatch(wake, this.endpoint.binding); + } + + close(): void { + if (this.closed) return; + this.closed = true; + this.exceptionCapabilities.clearBinding(this.endpoint.binding); + this.endpoint.close(); + } +} + +/** + * One Worker-side caller shared by its main module and every side module. + */ +export class ForkHostImportWorkerRuntime { + readonly caller: ForkExternrefImportWorkerCaller; + readonly localExceptions: ForkWorkerLocalImportExceptionNormalizer; + private readonly ownerImports = + new Map(); + + constructor( + init: ForkHostImportWorkerInit, + pid: number, + generationId: number, + tokens: ForkExternrefTokenCache, + notifyOwner: (wake: ForkExternrefImportWake) => void, + normalizerOptions: + ForkWorkerLocalImportExceptionNormalizerOptions = {}, + ) { + const binding: ForkExternrefImportBinding = { + pid, + generationId, + senderId: init.senderId, + }; + this.caller = new ForkExternrefImportWorkerCaller( + init.mailbox, + binding, + tokens, + notifyOwner, + ); + this.localExceptions = new ForkWorkerLocalImportExceptionNormalizer( + this.caller, + tokens, + normalizerOptions, + ); + for (const raw of init.ownerImports) { + validateImportName(raw.module, "fork owner import module"); + validateImportName(raw.name, "fork owner import name"); + const registration = cloneWireRegistration(raw); + if ( + registration.descriptor.ordinal + >= FORK_WORKER_EXCEPTION_RESERVED_ORDINAL_START + ) { + throw new Error( + `fork owner import ${raw.module}.${raw.name} uses reserved ordinal ` + + `${registration.descriptor.ordinal}`, + ); + } + const key = importKey(raw.module, raw.name); + if (this.ownerImports.has(key)) { + throw new Error(`duplicate fork owner import ${raw.module}.${raw.name}`); + } + this.ownerImports.set(key, registration); + } + } + + /** + * Select the owner RPC for a registered opaque-value import, otherwise wrap + * the same-Worker intrinsic so nested Wasm traps retain trap semantics. + * Ordinary thrown values remain exact until fork capture. + */ + routeFunction( + imported: WasmFunctionImportType, + localImplementation: CallableFunction, + ): CallableFunction { + const owner = this.ownerImports.get( + importKey(imported.module, imported.name), + ); + if (owner) { + if (!signatureMatchesDescriptor(imported.signature, owner.descriptor)) { + throw new Error( + `owner import ${imported.module}.${imported.name} descriptor does ` + + `not match artifact signature ${signatureText(imported.signature)}`, + ); + } + return this.caller.bind(owner.descriptor); + } + if (signatureRequiresDirectWasmBoundary(imported.signature)) { + // WHY: wrapping creates a JavaScript host function. The JS embedding + // rejects v128/exnref (and has no continuation-reference conversion), + // while a direct imported Wasm function is valid and preserves its + // instance-local typed value. Those values are captured by the typed + // scalar/exception/module codec path, never by the externref mailbox. + return localImplementation; + } + return this.localExceptions.wrap( + imported.importOrdinal, + localImplementation, + ); + } + + /** + * Parse the artifact once and route all function imports in a conventional + * import object. Dynamic-linker Proxies can instead call routeFunction at + * their final property-resolution boundary. + */ + routeImportObject( + programBytes: ArrayBuffer, + imports: WebAssembly.Imports, + ): WebAssembly.Imports { + const grouped = new Map(); + for (const imported of readWasmFunctionImports(programBytes)) { + const key = importKey(imported.module, imported.name); + const entries = grouped.get(key) ?? []; + entries.push(imported); + grouped.set(key, entries); + } + + const routed: WebAssembly.Imports = { ...imports }; + const modules = new Map>(); + for (const entries of grouped.values()) { + const imported = entries[0]!; + const originalModule = imports[imported.module] as + | Record + | undefined; + if (!originalModule) continue; + let routedModule = modules.get(imported.module); + if (!routedModule) { + routedModule = { ...originalModule }; + modules.set(imported.module, routedModule); + routed[imported.module] = routedModule; + } + const implementation = originalModule[imported.name]; + if (typeof implementation !== "function") continue; + + const owner = this.ownerImports.get( + importKey(imported.module, imported.name), + ); + if ( + owner + && entries.some( + (entry) => + !signatureMatchesDescriptor(entry.signature, owner.descriptor), + ) + ) { + throw new Error( + `owner import ${imported.module}.${imported.name} has multiple ` + + "artifact signatures that do not share its exact descriptor", + ); + } + routedModule[imported.name] = this.routeFunction( + imported, + implementation, + ) as WebAssembly.ImportValue; + } + return routed; + } + + clear(): void { + this.localExceptions.clear(); + } +} diff --git a/host/src/fork-imported-globals.ts b/host/src/fork-imported-globals.ts new file mode 100644 index 0000000000..06991b54a5 --- /dev/null +++ b/host/src/fork-imported-globals.ts @@ -0,0 +1,1151 @@ +import { + WPK_FORK_GLOBAL_CATALOG_EXPORT_PREFIX, + WPK_FORK_MODULE_STATE_GLOBAL_TYPE_ANYREF, + WPK_FORK_MODULE_STATE_GLOBAL_TYPE_EXNREF, + WPK_FORK_MODULE_STATE_GLOBAL_TYPE_EXTERNREF, + WPK_FORK_MODULE_STATE_GLOBAL_TYPE_FUNCREF, + WPK_FORK_TABLE_CATALOG_EXPORT_PREFIX, +} from "./generated/abi"; +import { + findForkGlobalSnapshot, + ForkImportedGlobalBindingKind, + ForkImportedTableBindingKind, + type ForkGlobalSnapshot, + type ForkImportedGlobalBinding, + type ForkImportedGlobalState, + type ForkImportedTableBinding, + type ForkImportedTableState, + type ForkModuleStateArena, + type ForkModuleStateRecord, + type ForkTableDirtyTracker, + importedGlobalBindingsForChild, + importedTableBindingsForChild, + readForkImportedGlobals, + readForkImportedTables, +} from "./fork-module-state"; + +export type ForkWasmImports = Readonly< + Record>> +>; + +interface ParentActivation { + readonly activationId: number; + readonly module: WebAssembly.Module; + readonly globalDescriptors: readonly ForkImportedGlobalState[]; + readonly tableDescriptors: readonly ForkImportedTableState[]; + readonly globalBindings: ReadonlyMap; + readonly tableBindings: ReadonlyMap; + instance?: WebAssembly.Instance; +} + +interface GlobalCoordinate { + readonly activationId: number; + readonly ownerId: number; + readonly imported: boolean; +} + +interface TableCoordinate { + readonly activationId: number; + readonly ownerId: number; + readonly imported: boolean; +} + +/** + * Early child-side view of the process reference transaction. + * + * This interface is intentionally smaller than the replay transaction. The + * loader needs only raw immutable import values and their owning activation; + * ordinary global/table/frame restore still uses the full transaction after + * every activation is registered. + */ +export interface ForkImportedReferenceProvider { + ownerActivation(recipeId: number, typeCode: number): number | null; + /** + * Complete activation closure needed to materialize a typed recipe. + * + * A GC aggregate may be owned by one activation while its constructor or + * fields depend on codecs/catalogs from several earlier activations. The + * direct owner alone is therefore insufficient for child instantiation + * ordering. Scalar/funcref-only providers may omit this and retain the + * direct-owner behavior. + */ + activationDependencies?(recipeId: number, typeCode: number): number[]; + materialize(recipeId: number, typeCode: number): unknown; +} + +export interface PreparedForkParentActivation { + readonly imports: ForkWasmImports; + complete(instance: WebAssembly.Instance): void; + abort(): void; +} + +function assertU32(value: number, context: string, allowZero = true): number { + if ( + !Number.isInteger(value) + || value < (allowZero ? 0 : 1) + || value > 0xffff_ffff + ) { + throw new RangeError(`${context} is not ${allowZero ? "a" : "a nonzero"} u32`); + } + return value; +} + +function bindingKey(activationId: number, ownerId: number): string { + return `${activationId}:${ownerId}`; +} + +function importKey(module: string, name: string): string { + return `${module.length}:${module}${name}`; +} + +function catalogName(ownerId: number): string { + return `${WPK_FORK_GLOBAL_CATALOG_EXPORT_PREFIX}${ownerId}`; +} + +function tableCatalogName(ownerId: number): string { + return `${WPK_FORK_TABLE_CATALOG_EXPORT_PREFIX}${ownerId}`; +} + +function isReferenceType(typeCode: number): boolean { + return typeCode === WPK_FORK_MODULE_STATE_GLOBAL_TYPE_FUNCREF + || typeCode === WPK_FORK_MODULE_STATE_GLOBAL_TYPE_EXTERNREF + || typeCode === WPK_FORK_MODULE_STATE_GLOBAL_TYPE_EXNREF + || typeCode === WPK_FORK_MODULE_STATE_GLOBAL_TYPE_ANYREF; +} + +interface TableCatalogActivation { + readonly activationId: number; + readonly instance: WebAssembly.Instance; +} + +function aliasTableDirtyTrackers( + activations: readonly TableCatalogActivation[], + trackers: ReadonlyMap, + label: string, +): void { + const identities = new Map< + WebAssembly.Table, + Array<{ activationId: number; ownerId: number }> + >(); + for (const activation of activations) { + const tracker = trackers.get(activation.activationId); + if (!tracker) { + throw new Error( + `${label}: activation ${activation.activationId} has no table dirty tracker`, + ); + } + for (const [name, value] of Object.entries(activation.instance.exports)) { + if (!name.startsWith(WPK_FORK_TABLE_CATALOG_EXPORT_PREFIX)) continue; + const ownerText = name.slice(WPK_FORK_TABLE_CATALOG_EXPORT_PREFIX.length); + if (!/^[1-9][0-9]*$/.test(ownerText)) { + throw new Error(`${label}: malformed private table catalog export ${name}`); + } + if (!(value instanceof WebAssembly.Table)) { + throw new Error(`${label}: private table catalog ${name} is not a Table`); + } + const ownerId = assertU32( + Number(ownerText), + "fork table catalog owner", + false, + ); + const coordinates = identities.get(value) ?? []; + coordinates.push({ activationId: activation.activationId, ownerId }); + identities.set(value, coordinates); + } + } + for (const coordinates of identities.values()) { + coordinates.sort( + (left, right) => + left.activationId - right.activationId + || left.ownerId - right.ownerId, + ); + const source = coordinates[0]!; + const sourceTracker = trackers.get(source.activationId)!; + sourceTracker.setStateOwner(source.ownerId, true); + for (const coordinate of coordinates.slice(1)) { + const tracker = trackers.get(coordinate.activationId)!; + tracker.aliasOwner( + coordinate.ownerId, + sourceTracker, + source.ownerId, + ); + tracker.setStateOwner(coordinate.ownerId, false); + } + } +} + +function f64Bits(value: number): bigint { + const bytes = new ArrayBuffer(8); + const view = new DataView(bytes); + view.setFloat64(0, value, true); + return view.getBigUint64(0, true); +} + +function numberFromF64Bits(value: bigint): number { + const bytes = new ArrayBuffer(8); + const view = new DataView(bytes); + view.setBigUint64(0, value, true); + return view.getFloat64(0, true); +} + +function requireImportNamespace( + imports: ForkWasmImports, + moduleName: string, +): Readonly> { + const namespace = imports[moduleName]; + if (!namespace || (typeof namespace !== "object" && typeof namespace !== "function")) { + throw new Error(`fork import object is missing namespace ${JSON.stringify(moduleName)}`); + } + return namespace; +} + +function validateImportedDescriptors( + module: WebAssembly.Module, + globalDescriptors: readonly ForkImportedGlobalState[], + tableDescriptors: readonly ForkImportedTableState[], + context: string, +): readonly WebAssembly.ModuleImportDescriptor[] { + const imports = WebAssembly.Module.imports(module); + const ordinals = new Set(); + for (const descriptor of globalDescriptors) { + const declaration = imports[descriptor.importOrdinal]; + if ( + !declaration + || declaration.kind !== "global" + || declaration.module !== descriptor.module + || declaration.name !== descriptor.name + ) { + throw new Error( + `${context}: KFIG owner ${descriptor.ownerId} does not match ` + + `global import ordinal ${descriptor.importOrdinal}`, + ); + } + if (ordinals.has(descriptor.importOrdinal)) { + throw new Error( + `${context}: duplicate ownership for import ordinal ${descriptor.importOrdinal}`, + ); + } + ordinals.add(descriptor.importOrdinal); + } + for (const descriptor of tableDescriptors) { + const declaration = imports[descriptor.importOrdinal]; + if ( + !declaration + || declaration.kind !== "table" + || declaration.module !== descriptor.module + || declaration.name !== descriptor.name + ) { + throw new Error( + `${context}: KFIT owner ${descriptor.ownerId} does not match ` + + `table import ordinal ${descriptor.importOrdinal}`, + ); + } + if (ordinals.has(descriptor.importOrdinal)) { + throw new Error( + `${context}: duplicate ownership for import ordinal ${descriptor.importOrdinal}`, + ); + } + ordinals.add(descriptor.importOrdinal); + } + return imports; +} + +type OwnedImport = + | { readonly kind: "global"; readonly descriptor: ForkImportedGlobalState } + | { readonly kind: "table"; readonly descriptor: ForkImportedTableState }; + +interface ImportAccess { + readonly owned?: OwnedImport; +} + +function importAccesses( + module: WebAssembly.Module, + globalDescriptors: readonly ForkImportedGlobalState[], + tableDescriptors: readonly ForkImportedTableState[], + context: string, +): Map { + const imports = validateImportedDescriptors( + module, + globalDescriptors, + tableDescriptors, + context, + ); + const byOrdinal = new Map(); + for (const descriptor of globalDescriptors) { + byOrdinal.set(descriptor.importOrdinal, { kind: "global", descriptor }); + } + for (const descriptor of tableDescriptors) { + byOrdinal.set(descriptor.importOrdinal, { kind: "table", descriptor }); + } + const interesting = new Set( + [...globalDescriptors, ...tableDescriptors] + .map(({ module, name }) => importKey(module, name)), + ); + const accesses = new Map(); + imports.forEach((declaration, importOrdinal) => { + const key = importKey(declaration.module, declaration.name); + if (!interesting.has(key)) return; + const values = accesses.get(key) ?? []; + values.push({ owned: byOrdinal.get(importOrdinal) }); + accesses.set(key, values); + }); + return accesses; +} + +/** + * Wrap an import object so each global declaration records the exact raw + * JavaScript value observed by WebAssembly instantiation. + * + * Duplicate `(module,name)` declarations are not collapsed. A getter may + * legally return a different value for each declaration, and Wasm performs + * each declaration's own type conversion. The child planner installs the same + * ordered getter sequence. + */ +function recordingImports( + module: WebAssembly.Module, + imports: ForkWasmImports, + globalDescriptors: readonly ForkImportedGlobalState[], + tableDescriptors: readonly ForkImportedTableState[], + capturedGlobals: Map, + capturedTables: Map, +): ForkWasmImports { + const accessPlan = importAccesses( + module, + globalDescriptors, + tableDescriptors, + "fork parent import capture", + ); + const byModule = new Map>(); + for (const descriptor of [...globalDescriptors, ...tableDescriptors]) { + let names = byModule.get(descriptor.module); + if (!names) { + names = new Map(); + byModule.set(descriptor.module, names); + } + names.set( + descriptor.name, + accessPlan.get(importKey(descriptor.module, descriptor.name))!, + ); + } + + const topLevel = new Map(); + for (const [moduleName, names] of byModule) { + const source = requireImportNamespace(imports, moduleName); + const ordinals = new Map(); + topLevel.set(moduleName, new Proxy(source as object, { + get(target, property, receiver) { + if (typeof property !== "string") { + return Reflect.get(target, property, receiver); + } + const declarations = names.get(property); + if (!declarations) return Reflect.get(target, property, receiver); + const ordinal = ordinals.get(property) ?? 0; + const access = declarations[ordinal]; + if (!access) { + throw new Error( + `WebAssembly read imported global ${JSON.stringify(moduleName)}.` + + `${JSON.stringify(property)} more than ${declarations.length} time(s)`, + ); + } + ordinals.set(property, ordinal + 1); + const value = Reflect.get(target, property, receiver); + if (access.owned?.kind === "global") { + capturedGlobals.set(access.owned.descriptor.ownerId, value); + } else if (access.owned?.kind === "table") { + capturedTables.set(access.owned.descriptor.ownerId, value); + } + return value; + }, + })); + } + + return new Proxy(imports as object, { + get(target, property, receiver) { + if (typeof property === "string" && topLevel.has(property)) { + return topLevel.get(property); + } + return Reflect.get(target, property, receiver); + }, + }) as ForkWasmImports; +} + +/** + * Parent-side capture of imported-global identity and raw binding semantics. + * + * This owner is process-lifetime state, not per-fork reference state. It holds + * only already-live activation instances/import values and drops the temporary + * declaration map after each prepared instantiation completes or aborts. + */ +export class ForkImportedGlobalCapture { + private readonly activations = new Map(); + private readonly prepared = new Set(); + + constructor(private readonly label: string) {} + + prepareActivation( + activationId: number, + module: WebAssembly.Module, + imports: ForkWasmImports, + ): PreparedForkParentActivation { + assertU32(activationId, "fork imported-global activation"); + if (this.activations.has(activationId) || this.prepared.has(activationId)) { + throw new Error(`${this.label}: activation ${activationId} is already prepared`); + } + const globalDescriptors = readForkImportedGlobals(module); + const tableDescriptors = readForkImportedTables(module); + const capturedGlobals = new Map(); + const capturedTables = new Map(); + this.prepared.add(activationId); + let finished = false; + const finish = (): void => { + if (finished) { + throw new Error(`${this.label}: activation ${activationId} preparation is finished`); + } + finished = true; + this.prepared.delete(activationId); + }; + return { + imports: recordingImports( + module, + imports, + globalDescriptors, + tableDescriptors, + capturedGlobals, + capturedTables, + ), + complete: (instance) => { + finish(); + for (const descriptor of globalDescriptors) { + if (!capturedGlobals.has(descriptor.ownerId)) { + throw new Error( + `${this.label}: WebAssembly did not resolve imported global ` + + `${activationId}:${descriptor.ownerId}`, + ); + } + } + const tables = new Map(); + for (const descriptor of tableDescriptors) { + const value = capturedTables.get(descriptor.ownerId); + if (!(value instanceof WebAssembly.Table)) { + throw new Error( + `${this.label}: WebAssembly did not resolve imported table ` + + `${activationId}:${descriptor.ownerId}`, + ); + } + tables.set(descriptor.ownerId, value); + } + this.activations.set(activationId, { + activationId, + module, + globalDescriptors, + tableDescriptors, + globalBindings: capturedGlobals, + tableBindings: tables, + instance, + }); + }, + abort: finish, + }; + } + + unregisterActivation(activationId: number): void { + assertU32(activationId, "fork imported-global activation"); + if (!this.activations.delete(activationId)) { + throw new Error(`${this.label}: activation ${activationId} is not registered`); + } + } + + /** + * Join activation-local dirty journals that name the same live Table. + * + * Call this after every activation is registered and before bootstrap/start + * mutations when possible. Joining is still correct after mutations because + * `aliasOwner` merges both existing interval sets before sharing the journal. + */ + bindTableDirtyTrackers( + trackers: ReadonlyMap, + ): void { + if (this.prepared.size !== 0) { + throw new Error( + `${this.label}: cannot bind table journals with ` + + `${this.prepared.size} incomplete activation(s)`, + ); + } + aliasTableDirtyTrackers( + this.orderedActivations().map(({ activationId, instance }) => ({ + activationId, + instance: instance!, + })), + trackers, + this.label, + ); + } + + appendTo(arena: ForkModuleStateArena): readonly ForkImportedGlobalBinding[] { + if (this.prepared.size !== 0) { + throw new Error( + `${this.label}: cannot snapshot with ${this.prepared.size} incomplete activation(s)`, + ); + } + const records = arena.recordsForCapture(); + const globalCoordinates = this.globalCoordinates(); + const tableCoordinates = this.tableCoordinates(); + const bindings: ForkImportedGlobalBinding[] = []; + const tableBindings: ForkImportedTableBinding[] = []; + for (const activation of this.orderedActivations()) { + for (const descriptor of activation.globalDescriptors) { + const snapshot = findForkGlobalSnapshot( + records, + activation.activationId, + descriptor.ownerId, + ); + if (snapshot.typeCode !== descriptor.typeCode) { + throw new Error( + `${this.label}: imported global ${activation.activationId}:` + + `${descriptor.ownerId} snapshot type does not match KFIG`, + ); + } + const value = activation.globalBindings.get(descriptor.ownerId)!; + bindings.push(this.captureBinding( + activation.activationId, + descriptor, + snapshot, + value, + globalCoordinates, + )); + } + for (const descriptor of activation.tableDescriptors) { + tableBindings.push(this.captureTableBinding( + activation.activationId, + descriptor, + activation.tableBindings.get(descriptor.ownerId)!, + tableCoordinates, + )); + } + } + bindings.sort( + (left, right) => + left.consumerActivation - right.consumerActivation + || left.consumerOwner - right.consumerOwner, + ); + tableBindings.sort( + (left, right) => + left.consumerActivation - right.consumerActivation + || left.consumerOwner - right.consumerOwner, + ); + arena.appendImportedGlobalBindings(bindings); + arena.appendImportedTableBindings(tableBindings); + return bindings; + } + + clear(): void { + this.activations.clear(); + this.prepared.clear(); + } + + private orderedActivations(): ParentActivation[] { + return [...this.activations.values()].sort( + (left, right) => left.activationId - right.activationId, + ); + } + + private globalCoordinates(): WeakMap { + const coordinates = new WeakMap(); + for (const activation of this.orderedActivations()) { + const importedOwners = new Set( + activation.globalDescriptors.map(({ ownerId }) => ownerId), + ); + for (const [name, value] of Object.entries(activation.instance!.exports)) { + if (!name.startsWith(WPK_FORK_GLOBAL_CATALOG_EXPORT_PREFIX)) continue; + const ownerText = name.slice(WPK_FORK_GLOBAL_CATALOG_EXPORT_PREFIX.length); + if (!/^[1-9][0-9]*$/.test(ownerText)) { + throw new Error(`${this.label}: malformed private global catalog export ${name}`); + } + const ownerId = Number(ownerText); + assertU32(ownerId, "fork global catalog owner", false); + if (!(value instanceof WebAssembly.Global)) { + throw new Error(`${this.label}: private global catalog ${name} is not a Global`); + } + const entries = coordinates.get(value) ?? []; + entries.push({ + activationId: activation.activationId, + ownerId, + imported: importedOwners.has(ownerId), + }); + coordinates.set(value, entries); + } + } + return coordinates; + } + + private tableCoordinates(): WeakMap { + const coordinates = new WeakMap(); + for (const activation of this.orderedActivations()) { + const importedOwners = new Set( + activation.tableDescriptors.map(({ ownerId }) => ownerId), + ); + for (const [name, value] of Object.entries(activation.instance!.exports)) { + if (!name.startsWith(WPK_FORK_TABLE_CATALOG_EXPORT_PREFIX)) continue; + const ownerText = name.slice(WPK_FORK_TABLE_CATALOG_EXPORT_PREFIX.length); + if (!/^[1-9][0-9]*$/.test(ownerText)) { + throw new Error(`${this.label}: malformed private table catalog export ${name}`); + } + const ownerId = Number(ownerText); + assertU32(ownerId, "fork table catalog owner", false); + if (!(value instanceof WebAssembly.Table)) { + throw new Error(`${this.label}: private table catalog ${name} is not a Table`); + } + const entries = coordinates.get(value) ?? []; + entries.push({ + activationId: activation.activationId, + ownerId, + imported: importedOwners.has(ownerId), + }); + coordinates.set(value, entries); + } + } + return coordinates; + } + + private captureBinding( + activationId: number, + descriptor: ForkImportedGlobalState, + snapshot: ForkGlobalSnapshot, + value: unknown, + coordinates: WeakMap, + ): ForkImportedGlobalBinding { + const base = { + consumerActivation: activationId, + consumerOwner: descriptor.ownerId, + sourceActivation: 0, + sourceOwner: 0, + reserved: 0, + recipeId: 0, + rawBits: 0n, + mutable: descriptor.mutable, + shared: descriptor.shared, + typeCode: descriptor.typeCode, + }; + if (value instanceof WebAssembly.Global) { + const candidates = (coordinates.get(value) ?? []) + .filter((coordinate) => !coordinate.imported) + .sort( + (left, right) => + left.activationId - right.activationId + || left.ownerId - right.ownerId, + ); + const provider = candidates[0]; + if (provider) { + return { + ...base, + kind: ForkImportedGlobalBindingKind.ActivationGlobal, + sourceActivation: provider.activationId, + sourceOwner: provider.ownerId, + }; + } + return { + ...base, + // The final child import builder owns process cells such as GOT, + // stack-pointer, and dylink base globals. Re-resolving the exact + // declaration preserves its identity without copying a JS handle. + kind: ForkImportedGlobalBindingKind.BaseImport, + }; + } + + if (isReferenceType(descriptor.typeCode)) { + if (snapshot.recipeId === undefined) { + throw new Error( + `${this.label}: reference import ${activationId}:${descriptor.ownerId} ` + + "has no recipe id", + ); + } + if ( + descriptor.typeCode === WPK_FORK_MODULE_STATE_GLOBAL_TYPE_EXNREF + && snapshot.recipeId !== 0 + ) { + // WHY: JavaScript cannot read or carry a non-null exnref. A legitimate + // non-null import is necessarily a WebAssembly.Global carrier and was + // handled above as ActivationGlobal/BaseImport. Emitting RawReference + // here would manufacture a child transport that the embedding API + // cannot represent. + throw new Error( + `${this.label}: non-null exnref import ${activationId}:` + + `${descriptor.ownerId} has no WebAssembly.Global carrier`, + ); + } + return { + ...base, + kind: ForkImportedGlobalBindingKind.RawReference, + recipeId: snapshot.recipeId, + }; + } + if (typeof value === "number") { + return { + ...base, + kind: ForkImportedGlobalBindingKind.RawNumber, + rawBits: f64Bits(value), + }; + } + if (typeof value === "bigint") { + return { + ...base, + kind: ForkImportedGlobalBindingKind.RawBigInt, + rawBits: BigInt.asUintN(64, value), + }; + } + return { + ...base, + kind: ForkImportedGlobalBindingKind.BaseImport, + }; + } + + private captureTableBinding( + activationId: number, + descriptor: ForkImportedTableState, + value: WebAssembly.Table, + coordinates: WeakMap, + ): ForkImportedTableBinding { + const candidates = (coordinates.get(value) ?? []) + .filter((coordinate) => !coordinate.imported) + .sort( + (left, right) => + left.activationId - right.activationId + || left.ownerId - right.ownerId, + ); + const provider = candidates[0]; + if (provider) { + return { + consumerActivation: activationId, + consumerOwner: descriptor.ownerId, + sourceActivation: provider.activationId, + sourceOwner: provider.ownerId, + reserved: 0, + kind: ForkImportedTableBindingKind.ActivationTable, + }; + } + return { + consumerActivation: activationId, + consumerOwner: descriptor.ownerId, + sourceActivation: 0, + sourceOwner: 0, + reserved: 0, + // Process tables are reconstructed by the same main/dylink import + // builder that created them in the parent. The planner deliberately + // leaves this declaration's lazy getter in control. + kind: ForkImportedTableBindingKind.BaseImport, + }; + } +} + +interface ChildActivation { + readonly activationId: number; + readonly module: WebAssembly.Module; + readonly globalDescriptors: readonly ForkImportedGlobalState[]; + readonly tableDescriptors: readonly ForkImportedTableState[]; + readonly globalBindings: readonly ForkImportedGlobalBinding[]; + readonly tableBindings: readonly ForkImportedTableBinding[]; +} + +/** + * Fresh-child instantiation planner for imported globals. + * + * Provider activations are topologically ordered before consumers. Mutable + * Global cells may still contain their deterministic baseline while consumers + * bind them; KFMS restore updates that one shared cell after all activations + * exist. Immutable cells are already final by definition, so const + * initializers and direct re-exports observe the exact provider identity at + * instantiation time. + */ +export class ForkImportedGlobalPlanner { + private readonly activations = new Map(); + private readonly instances = new Map(); + private readonly globalBindingsByConsumer = + new Map(); + private readonly tableBindingsByConsumer = + new Map(); + + constructor( + records: readonly ForkModuleStateRecord[], + modules: ReadonlyMap, + private readonly references: ForkImportedReferenceProvider, + private readonly label: string, + ) { + const globalBindings = importedGlobalBindingsForChild(records); + const tableBindings = importedTableBindingsForChild(records); + for (const binding of globalBindings) { + this.globalBindingsByConsumer.set( + bindingKey(binding.consumerActivation, binding.consumerOwner), + binding, + ); + } + for (const binding of tableBindings) { + this.tableBindingsByConsumer.set( + bindingKey(binding.consumerActivation, binding.consumerOwner), + binding, + ); + } + for (const [activationId, module] of [...modules].sort( + ([left], [right]) => left - right, + )) { + assertU32(activationId, "fork imported-global activation"); + const globalDescriptors = readForkImportedGlobals(module); + const tableDescriptors = readForkImportedTables(module); + const activationGlobalBindings = globalDescriptors.map((descriptor) => { + const binding = this.globalBindingsByConsumer.get( + bindingKey(activationId, descriptor.ownerId), + ); + if (!binding) { + throw new Error( + `${this.label}: missing imported-global binding ` + + `${activationId}:${descriptor.ownerId}`, + ); + } + if ( + binding.typeCode !== descriptor.typeCode + || binding.mutable !== descriptor.mutable + || binding.shared !== descriptor.shared + ) { + throw new Error( + `${this.label}: imported-global binding ${activationId}:` + + `${descriptor.ownerId} does not match KFIG`, + ); + } + if ( + binding.kind === ForkImportedGlobalBindingKind.RawReference + && binding.typeCode === WPK_FORK_MODULE_STATE_GLOBAL_TYPE_EXNREF + && binding.recipeId !== 0 + ) { + throw new Error( + `${this.label}: imported exnref binding ${activationId}:` + + `${descriptor.ownerId} has impossible raw non-null provenance`, + ); + } + return binding; + }); + const activationTableBindings = tableDescriptors.map((descriptor) => { + const binding = this.tableBindingsByConsumer.get( + bindingKey(activationId, descriptor.ownerId), + ); + if (!binding) { + throw new Error( + `${this.label}: missing imported-table binding ` + + `${activationId}:${descriptor.ownerId}`, + ); + } + return binding; + }); + this.activations.set(activationId, { + activationId, + module, + globalDescriptors, + tableDescriptors, + globalBindings: activationGlobalBindings, + tableBindings: activationTableBindings, + }); + } + if ( + globalBindings.length + !== [...this.activations.values()] + .reduce((count, activation) => count + activation.globalBindings.length, 0) + ) { + throw new Error(`${this.label}: imported-global bindings name unknown declarations`); + } + if ( + tableBindings.length + !== [...this.activations.values()] + .reduce((count, activation) => count + activation.tableBindings.length, 0) + ) { + throw new Error(`${this.label}: imported-table bindings name unknown declarations`); + } + } + + instantiationOrder(): number[] { + const ids = [...this.activations.keys()].sort((left, right) => left - right); + const dependencies = new Map>( + ids.map((id) => [id, new Set(this.dependenciesFor(id))]), + ); + const order: number[] = []; + const remaining = new Set(ids); + while (remaining.size !== 0) { + const ready = [...remaining] + .filter((id) => + [...dependencies.get(id)!].every( + (dependency) => !remaining.has(dependency), + )) + .sort((left, right) => left - right); + if (ready.length === 0) { + const cycle = [...remaining].sort((left, right) => left - right); + throw new Error( + `${this.label}: imported-global provider cycle among activations ` + + cycle.join(", "), + ); + } + for (const id of ready) { + remaining.delete(id); + order.push(id); + } + } + return order; + } + + dependenciesFor(activationId: number): number[] { + const activation = this.requireActivation(activationId); + const dependencies = new Set(); + for (const binding of activation.globalBindings) { + let dependency: number | null = null; + switch (binding.kind) { + case ForkImportedGlobalBindingKind.ActivationGlobal: + dependency = binding.sourceActivation; + break; + case ForkImportedGlobalBindingKind.RawReference: { + const closure = this.references.activationDependencies?.( + binding.recipeId, + binding.typeCode, + ); + if (closure) { + for (const activationDependency of closure) { + if (activationDependency === activationId) continue; + if (!this.activations.has(activationDependency)) { + throw new Error( + `${this.label}: activation ${activationId} depends on missing ` + + `provider activation ${activationDependency}`, + ); + } + dependencies.add(activationDependency); + } + break; + } + dependency = this.references.ownerActivation( + binding.recipeId, + binding.typeCode, + ); + break; + } + case ForkImportedGlobalBindingKind.BaseImport: + case ForkImportedGlobalBindingKind.RawNumber: + case ForkImportedGlobalBindingKind.RawBigInt: + break; + } + if (dependency === null || dependency === activationId) continue; + if (!this.activations.has(dependency)) { + throw new Error( + `${this.label}: activation ${activationId} depends on missing ` + + `provider activation ${dependency}`, + ); + } + dependencies.add(dependency); + } + for (const binding of activation.tableBindings) { + let dependency: number | null = null; + switch (binding.kind) { + case ForkImportedTableBindingKind.ActivationTable: + dependency = binding.sourceActivation; + break; + case ForkImportedTableBindingKind.BaseImport: + break; + } + if (dependency === null || dependency === activationId) continue; + if (!this.activations.has(dependency)) { + throw new Error( + `${this.label}: activation ${activationId} depends on missing ` + + `provider activation ${dependency}`, + ); + } + dependencies.add(dependency); + } + return [...dependencies].sort((left, right) => left - right); + } + + importsForActivation( + activationId: number, + baseImports: ForkWasmImports, + ): ForkWasmImports { + const activation = this.requireActivation(activationId); + const resolvedByOrdinal = new Map< + number, + { override: boolean; value?: unknown } + >(); + activation.globalDescriptors.forEach((descriptor, index) => { + const binding = activation.globalBindings[index]!; + resolvedByOrdinal.set( + descriptor.importOrdinal, + binding.kind === ForkImportedGlobalBindingKind.BaseImport + ? { override: false } + : { override: true, value: this.resolveGlobal(binding) }, + ); + }); + activation.tableDescriptors.forEach((descriptor, index) => { + const binding = activation.tableBindings[index]!; + resolvedByOrdinal.set( + descriptor.importOrdinal, + binding.kind === ForkImportedTableBindingKind.BaseImport + ? { override: false } + : { override: true, value: this.resolveTable(binding) }, + ); + }); + const accessPlan = importAccesses( + activation.module, + activation.globalDescriptors, + activation.tableDescriptors, + `${this.label}: activation ${activationId}`, + ); + const byModule = new Map< + string, + Map> + >(); + for ( + const descriptor of [ + ...activation.globalDescriptors, + ...activation.tableDescriptors, + ] + ) { + let names = byModule.get(descriptor.module); + if (!names) { + names = new Map(); + byModule.set(descriptor.module, names); + } + names.set( + descriptor.name, + accessPlan.get(importKey(descriptor.module, descriptor.name))!.map( + (access) => access.owned + ? resolvedByOrdinal.get(access.owned.descriptor.importOrdinal)! + : { override: false }, + ), + ); + } + + const namespaces = new Map(); + for (const [moduleName, names] of byModule) { + const source = baseImports[moduleName] ?? {}; + const ordinals = new Map(); + namespaces.set(moduleName, new Proxy(source as object, { + get(target, property, receiver) { + if (typeof property !== "string") { + return Reflect.get(target, property, receiver); + } + const accesses = names.get(property); + if (!accesses) return Reflect.get(target, property, receiver); + const ordinal = ordinals.get(property) ?? 0; + const access = accesses[ordinal]; + if (!access) { + throw new Error( + `WebAssembly read reconstructed global ${JSON.stringify(moduleName)}.` + + `${JSON.stringify(property)} more than ${accesses.length} time(s)`, + ); + } + ordinals.set(property, ordinal + 1); + return access.override + ? access.value + : Reflect.get(target, property, receiver); + }, + })); + } + return new Proxy(baseImports as object, { + get(target, property, receiver) { + if (typeof property === "string" && namespaces.has(property)) { + return namespaces.get(property); + } + return Reflect.get(target, property, receiver); + }, + }) as ForkWasmImports; + } + + registerInstance(activationId: number, instance: WebAssembly.Instance): void { + this.requireActivation(activationId); + if (this.instances.has(activationId)) { + throw new Error(`${this.label}: activation ${activationId} was instantiated twice`); + } + this.instances.set(activationId, instance); + } + + /** + * Join child journals before KFMS restore marks replayed sparse pages. + * + * All instances are required so aliases are derived from actual provider + * identity instead of guessed from import names. + */ + bindTableDirtyTrackers( + trackers: ReadonlyMap, + ): void { + if (this.instances.size !== this.activations.size) { + throw new Error( + `${this.label}: cannot bind table journals before all ` + + `${this.activations.size} activation(s) are instantiated`, + ); + } + aliasTableDirtyTrackers( + [...this.instances] + .sort(([left], [right]) => left - right) + .map(([activationId, instance]) => ({ activationId, instance })), + trackers, + this.label, + ); + } + + clear(): void { + this.instances.clear(); + } + + private resolveGlobal(binding: ForkImportedGlobalBinding): unknown { + switch (binding.kind) { + case ForkImportedGlobalBindingKind.RawNumber: + return numberFromF64Bits(binding.rawBits); + case ForkImportedGlobalBindingKind.RawBigInt: + return BigInt.asIntN(64, binding.rawBits); + case ForkImportedGlobalBindingKind.RawReference: + return this.references.materialize(binding.recipeId, binding.typeCode); + case ForkImportedGlobalBindingKind.ActivationGlobal: { + const provider = this.instances.get(binding.sourceActivation); + if (!provider) { + throw new Error( + `${this.label}: provider activation ${binding.sourceActivation} ` + + "is not instantiated", + ); + } + const value = provider.exports[catalogName(binding.sourceOwner)]; + if (!(value instanceof WebAssembly.Global)) { + throw new Error( + `${this.label}: provider global ${binding.sourceActivation}:` + + `${binding.sourceOwner} is missing`, + ); + } + return value; + } + case ForkImportedGlobalBindingKind.BaseImport: + throw new Error(`${this.label}: base import cannot be eagerly resolved`); + } + } + + private resolveTable(binding: ForkImportedTableBinding): WebAssembly.Table { + switch (binding.kind) { + case ForkImportedTableBindingKind.ActivationTable: { + const provider = this.instances.get(binding.sourceActivation); + if (!provider) { + throw new Error( + `${this.label}: provider activation ${binding.sourceActivation} ` + + "is not instantiated", + ); + } + const value = provider.exports[tableCatalogName(binding.sourceOwner)]; + if (!(value instanceof WebAssembly.Table)) { + throw new Error( + `${this.label}: provider table ${binding.sourceActivation}:` + + `${binding.sourceOwner} is missing`, + ); + } + return value; + } + case ForkImportedTableBindingKind.BaseImport: + throw new Error(`${this.label}: base table import cannot be eagerly resolved`); + } + } + + private requireActivation(activationId: number): ChildActivation { + assertU32(activationId, "fork imported-global activation"); + const activation = this.activations.get(activationId); + if (!activation) { + throw new Error(`${this.label}: activation ${activationId} is not declared`); + } + return activation; + } +} diff --git a/host/src/fork-module-state.ts b/host/src/fork-module-state.ts new file mode 100644 index 0000000000..029c02ad81 --- /dev/null +++ b/host/src/fork-module-state.ts @@ -0,0 +1,3662 @@ +import { WASM_PAGE_SIZE } from "./constants"; +import { + ContinuationAllocationError, + type ContinuationAllocate, + type ContinuationDeallocate, +} from "./fork-continuation"; +import { + type ForkReplayEventCaptureSource, + type ForkReplayEventWire, + validateForkReplayEventWire, +} from "./fork-replay-events"; +/* + * Keep the allocation error as a runtime import: fork() must return its errno + * after an arena mmap failure, not turn an ordinary resource failure into a + * process trap. + */ +import { + WPK_FORK_ACTIVATION_CONTINUATIONS_ENTRY_KNOWN_FLAGS, + WPK_FORK_ACTIVATION_CONTINUATIONS_ENTRY_SIZE, + WPK_FORK_ACTIVATION_CONTINUATIONS_HEADER_SIZE, + WPK_FORK_ACTIVATION_CONTINUATIONS_KNOWN_FLAGS, + WPK_FORK_ACTIVATION_CONTINUATIONS_MAGIC, + WPK_FORK_ACTIVATION_CONTINUATIONS_OWNER, + WPK_FORK_ACTIVATION_CONTINUATIONS_VERSION, + WPK_FORK_MODULE_STATE_ARENA_VERSION, + WPK_FORK_MODULE_STATE_CHUNK_FLAG_ROOT, + WPK_FORK_MODULE_STATE_CHUNK_FLAG_SEALED, + WPK_FORK_MODULE_STATE_CHUNK_MAGIC, + WPK_FORK_MODULE_STATE_DATA_SEGMENT_HEADER_SIZE, + WPK_FORK_MODULE_STATE_DESCRIPTOR_SIZE, + WPK_FORK_MODULE_STATE_ELEMENT_SEGMENT_HEADER_SIZE, + WPK_FORK_MODULE_STATE_FLAG_EXPLICIT_OWNERS, + WPK_FORK_MODULE_STATE_FLAG_ROOT_PREFIX_POINTER, + WPK_FORK_MODULE_STATE_FLAG_SPARSE_TABLES, + WPK_FORK_MODULE_STATE_FORMAT_MAGIC, + WPK_FORK_MODULE_STATE_FORMAT_SECTION, + WPK_FORK_MODULE_STATE_FORMAT_VERSION, + WPK_FORK_MODULE_STATE_GLOBAL_HEADER_SIZE, + WPK_FORK_MODULE_STATE_GLOBAL_TYPE_ANYREF, + WPK_FORK_MODULE_STATE_GLOBAL_TYPE_EXNREF, + WPK_FORK_MODULE_STATE_GLOBAL_TYPE_EXTERNREF, + WPK_FORK_MODULE_STATE_GLOBAL_TYPE_F32, + WPK_FORK_MODULE_STATE_GLOBAL_TYPE_F64, + WPK_FORK_MODULE_STATE_GLOBAL_TYPE_FUNCREF, + WPK_FORK_MODULE_STATE_GLOBAL_TYPE_I32, + WPK_FORK_MODULE_STATE_GLOBAL_TYPE_I64, + WPK_FORK_MODULE_STATE_GLOBAL_TYPE_V128, + WPK_FORK_MODULE_STATE_KNOWN_FLAGS, + WPK_FORK_MODULE_STATE_MAX_TABLE_PAGE_SHIFT, + WPK_FORK_MODULE_STATE_MIN_TABLE_PAGE_SHIFT, + WPK_FORK_MODULE_STATE_MODULE_RECORD_KNOWN_FLAGS, + WPK_FORK_MODULE_STATE_MODULE_RECORD_PAYLOAD_SIZE, + WPK_FORK_MODULE_STATE_MODULE_TEMPLATE_ID_SIZE, + WPK_FORK_MODULE_STATE_POINTER_WIDTHS, + WPK_FORK_MODULE_STATE_RECORD_ALIGNMENT, + WPK_FORK_MODULE_STATE_RECORD_HEADER_SIZE, + WPK_FORK_MODULE_STATE_RECORD_KIND_DATA_SEGMENTS, + WPK_FORK_MODULE_STATE_RECORD_KIND_ACTIVATION_CONTINUATIONS, + WPK_FORK_MODULE_STATE_RECORD_KIND_ELEMENT_SEGMENTS, + WPK_FORK_MODULE_STATE_RECORD_KIND_IMPORTED_GLOBAL_BINDINGS, + WPK_FORK_MODULE_STATE_RECORD_KIND_IMPORTED_TABLE_BINDINGS, + WPK_FORK_MODULE_STATE_RECORD_KIND_MODULE, + WPK_FORK_MODULE_STATE_RECORD_KIND_MUTABLE_GLOBAL, + WPK_FORK_MODULE_STATE_RECORD_KIND_REFERENCE_RECIPE, + WPK_FORK_MODULE_STATE_RECORD_KIND_REFERENCE_RECIPE_SEGMENT, + WPK_FORK_MODULE_STATE_RECORD_KIND_REPLAY_EVENTS, + WPK_FORK_MODULE_STATE_RECORD_KIND_REPLAY_EVENT_SEGMENT, + WPK_FORK_MODULE_STATE_RECORD_KIND_TABLE, + WPK_FORK_MODULE_STATE_RECORD_KIND_TABLE_PAGE, + WPK_FORK_MODULE_STATE_RECORD_MAGIC, + WPK_FORK_MODULE_STATE_RECORD_VERSION, + WPK_FORK_MODULE_STATE_REQUIRED_FLAGS, + WPK_FORK_MODULE_STATE_ROOT_POINTER_WORD_OFFSET, + WPK_FORK_MODULE_STATE_TABLE_BASELINE_FINGERPRINT_SIZE, + WPK_FORK_MODULE_STATE_TABLE_DESCRIPTOR_PAYLOAD_SIZE, + WPK_FORK_MODULE_STATE_TABLE_FLAG_SPARSE_OVERRIDES, + WPK_FORK_MODULE_STATE_TABLE_KNOWN_FLAGS, + WPK_FORK_MODULE_STATE_TABLE_PAGE_HEADER_SIZE, + WPK_FORK_MODULE_STATE_TABLE_RUN_HEADER_SIZE, + WPK_FORK_MODULE_STATE_REPLAY_EVENTS_OWNER, + WPK_FORK_IMPORTED_GLOBALS_FLAG_MUTABLE, + WPK_FORK_IMPORTED_GLOBALS_FLAG_SHARED, + WPK_FORK_IMPORTED_GLOBALS_HEADER_SIZE, + WPK_FORK_IMPORTED_GLOBALS_KNOWN_FLAGS, + WPK_FORK_IMPORTED_GLOBALS_MAGIC, + WPK_FORK_IMPORTED_GLOBALS_RECORD_HEADER_SIZE, + WPK_FORK_IMPORTED_GLOBALS_SECTION, + WPK_FORK_IMPORTED_GLOBALS_VERSION, + WPK_FORK_IMPORTED_GLOBAL_BINDINGS_ENTRY_SIZE, + WPK_FORK_IMPORTED_GLOBAL_BINDINGS_HEADER_SIZE, + WPK_FORK_IMPORTED_GLOBAL_BINDINGS_KNOWN_FLAGS, + WPK_FORK_IMPORTED_GLOBAL_BINDINGS_MAGIC, + WPK_FORK_IMPORTED_GLOBAL_BINDINGS_OWNER, + WPK_FORK_IMPORTED_GLOBAL_BINDINGS_VERSION, + WPK_FORK_IMPORTED_GLOBAL_BINDING_ACTIVATION_GLOBAL, + WPK_FORK_IMPORTED_GLOBAL_BINDING_BASE_IMPORT, + WPK_FORK_IMPORTED_GLOBAL_BINDING_RAW_BIGINT, + WPK_FORK_IMPORTED_GLOBAL_BINDING_RAW_NUMBER, + WPK_FORK_IMPORTED_GLOBAL_BINDING_RAW_REFERENCE, + WPK_FORK_IMPORTED_TABLES_FLAG_TABLE64, + WPK_FORK_IMPORTED_TABLES_HEADER_SIZE, + WPK_FORK_IMPORTED_TABLES_KNOWN_FLAGS, + WPK_FORK_IMPORTED_TABLES_MAGIC, + WPK_FORK_IMPORTED_TABLES_RECORD_HEADER_SIZE, + WPK_FORK_IMPORTED_TABLES_SECTION, + WPK_FORK_IMPORTED_TABLES_VERSION, + WPK_FORK_IMPORTED_TABLE_BINDINGS_ENTRY_SIZE, + WPK_FORK_IMPORTED_TABLE_BINDINGS_HEADER_SIZE, + WPK_FORK_IMPORTED_TABLE_BINDINGS_KNOWN_FLAGS, + WPK_FORK_IMPORTED_TABLE_BINDINGS_MAGIC, + WPK_FORK_IMPORTED_TABLE_BINDINGS_OWNER, + WPK_FORK_IMPORTED_TABLE_BINDINGS_VERSION, + WPK_FORK_IMPORTED_TABLE_BINDING_ACTIVATION_TABLE, + WPK_FORK_IMPORTED_TABLE_BINDING_BASE_IMPORT, +} from "./generated/abi"; + +/** + * Versioned artifact metadata for activation-owned module-state recipes. + * + * Keep the short aliases as the arena's public API, but source every wire + * literal from the generated shared ABI contract. + */ +export const FORK_MODULE_STATE_SECTION = WPK_FORK_MODULE_STATE_FORMAT_SECTION; +export const FORK_MODULE_STATE_DESCRIPTOR_MAGIC = WPK_FORK_MODULE_STATE_FORMAT_MAGIC; +export const FORK_MODULE_STATE_DESCRIPTOR_VERSION = WPK_FORK_MODULE_STATE_FORMAT_VERSION; +export const FORK_MODULE_STATE_DESCRIPTOR_SIZE = WPK_FORK_MODULE_STATE_DESCRIPTOR_SIZE; +export const FORK_MODULE_STATE_RECORD_ALIGNMENT = WPK_FORK_MODULE_STATE_RECORD_ALIGNMENT; +export const FORK_MODULE_STATE_ARENA_VERSION = WPK_FORK_MODULE_STATE_ARENA_VERSION; +export const FORK_MODULE_STATE_RECORD_VERSION = WPK_FORK_MODULE_STATE_RECORD_VERSION; +export const FORK_MODULE_STATE_ROOT_POINTER_WORD_OFFSET = + WPK_FORK_MODULE_STATE_ROOT_POINTER_WORD_OFFSET; + +export const FORK_MODULE_STATE_FLAG_ROOT_PREFIX_POINTER = + WPK_FORK_MODULE_STATE_FLAG_ROOT_PREFIX_POINTER; +export const FORK_MODULE_STATE_FLAG_EXPLICIT_OWNERS = + WPK_FORK_MODULE_STATE_FLAG_EXPLICIT_OWNERS; +export const FORK_MODULE_STATE_FLAG_SPARSE_TABLES = + WPK_FORK_MODULE_STATE_FLAG_SPARSE_TABLES; +export const FORK_MODULE_STATE_REQUIRED_FLAGS = WPK_FORK_MODULE_STATE_REQUIRED_FLAGS; +export const FORK_MODULE_STATE_KNOWN_FLAGS = WPK_FORK_MODULE_STATE_KNOWN_FLAGS; + +const CHUNK_MAGIC = littleEndianMagic(WPK_FORK_MODULE_STATE_CHUNK_MAGIC); +const RECORD_MAGIC = littleEndianMagic(WPK_FORK_MODULE_STATE_RECORD_MAGIC); +const CHUNK_FLAG_ROOT = WPK_FORK_MODULE_STATE_CHUNK_FLAG_ROOT; +const CHUNK_FLAG_SEALED = WPK_FORK_MODULE_STATE_CHUNK_FLAG_SEALED; +const RECORD_HEADER_SIZE = WPK_FORK_MODULE_STATE_RECORD_HEADER_SIZE; +export const FORK_MODULE_STATE_TEMPLATE_ID_SIZE = + WPK_FORK_MODULE_STATE_MODULE_TEMPLATE_ID_SIZE; +export const FORK_MODULE_STATE_BASELINE_FINGERPRINT_SIZE = + WPK_FORK_MODULE_STATE_TABLE_BASELINE_FINGERPRINT_SIZE; +const MODULE_TEMPLATE_ID_SIZE = FORK_MODULE_STATE_TEMPLATE_ID_SIZE; +const MODULE_RECORD_PAYLOAD_SIZE = WPK_FORK_MODULE_STATE_MODULE_RECORD_PAYLOAD_SIZE; +const MODULE_RECORD_KNOWN_FLAGS = WPK_FORK_MODULE_STATE_MODULE_RECORD_KNOWN_FLAGS; +const TABLE_DESCRIPTOR_PAYLOAD_SIZE = WPK_FORK_MODULE_STATE_TABLE_DESCRIPTOR_PAYLOAD_SIZE; +const TABLE_FLAG_SPARSE_OVERRIDES = WPK_FORK_MODULE_STATE_TABLE_FLAG_SPARSE_OVERRIDES; +const TABLE_KNOWN_FLAGS = WPK_FORK_MODULE_STATE_TABLE_KNOWN_FLAGS; +const TABLE_PAGE_HEADER_SIZE = WPK_FORK_MODULE_STATE_TABLE_PAGE_HEADER_SIZE; +const TABLE_RUN_HEADER_SIZE = WPK_FORK_MODULE_STATE_TABLE_RUN_HEADER_SIZE; +const ELEMENT_SEGMENT_HEADER_SIZE = WPK_FORK_MODULE_STATE_ELEMENT_SEGMENT_HEADER_SIZE; +const DATA_SEGMENT_HEADER_SIZE = WPK_FORK_MODULE_STATE_DATA_SEGMENT_HEADER_SIZE; +const GLOBAL_HEADER_SIZE = WPK_FORK_MODULE_STATE_GLOBAL_HEADER_SIZE; +const MIN_TABLE_PAGE_SHIFT = WPK_FORK_MODULE_STATE_MIN_TABLE_PAGE_SHIFT; +const MAX_TABLE_PAGE_SHIFT = WPK_FORK_MODULE_STATE_MAX_TABLE_PAGE_SHIFT; + +export interface ForkModuleStateDescriptor { + version: number; + ptrWidth: 4 | 8; + alignment: number; + flags: number; + arenaVersion: number; + recordVersion: number; + rootPointerWordOffset: number; +} + +export const ForkModuleStateRecordKind = { + Module: WPK_FORK_MODULE_STATE_RECORD_KIND_MODULE, + ReferenceRecipe: WPK_FORK_MODULE_STATE_RECORD_KIND_REFERENCE_RECIPE, + MutableGlobal: WPK_FORK_MODULE_STATE_RECORD_KIND_MUTABLE_GLOBAL, + Table: WPK_FORK_MODULE_STATE_RECORD_KIND_TABLE, + TablePage: WPK_FORK_MODULE_STATE_RECORD_KIND_TABLE_PAGE, + ElementSegments: WPK_FORK_MODULE_STATE_RECORD_KIND_ELEMENT_SEGMENTS, + DataSegments: WPK_FORK_MODULE_STATE_RECORD_KIND_DATA_SEGMENTS, + ReplayEvents: WPK_FORK_MODULE_STATE_RECORD_KIND_REPLAY_EVENTS, + ImportedGlobalBindings: WPK_FORK_MODULE_STATE_RECORD_KIND_IMPORTED_GLOBAL_BINDINGS, + ActivationContinuations: WPK_FORK_MODULE_STATE_RECORD_KIND_ACTIVATION_CONTINUATIONS, + ImportedTableBindings: WPK_FORK_MODULE_STATE_RECORD_KIND_IMPORTED_TABLE_BINDINGS, + ReferenceRecipeSegment: + WPK_FORK_MODULE_STATE_RECORD_KIND_REFERENCE_RECIPE_SEGMENT, + ReplayEventSegment: WPK_FORK_MODULE_STATE_RECORD_KIND_REPLAY_EVENT_SEGMENT, +} as const; + +export type ForkModuleStateRecordKind = + typeof ForkModuleStateRecordKind[keyof typeof ForkModuleStateRecordKind]; + +const RECORD_KINDS = new Set(Object.values(ForkModuleStateRecordKind)); + +export interface ForkModuleStateRecord { + kind: ForkModuleStateRecordKind; + activationId: number; + ownerId: number; + payload: Uint8Array; +} + +/** + * A validated record envelope whose payload may alias the sealed arena. + * + * The view is valid only while its owning process memory and arena mapping are + * alive. Consumers that need a longer lifetime must copy the specific bytes + * they retain; streaming decoders should keep the view to avoid duplicating a + * whole segmented transaction. + */ +export interface ForkModuleStateRecordView { + readonly kind: ForkModuleStateRecordKind; + readonly activationId: number; + readonly ownerId: number; + readonly payload: Uint8Array; +} + +export interface ForkImportedGlobalState { + module: string; + name: string; + importOrdinal: number; + ownerId: number; + typeCode: number; + mutable: boolean; + shared: boolean; +} + +export interface ForkImportedTableState { + module: string; + name: string; + importOrdinal: number; + ownerId: number; + typeCode: number; + table64: boolean; +} + +export const ForkImportedGlobalBindingKind = { + RawNumber: WPK_FORK_IMPORTED_GLOBAL_BINDING_RAW_NUMBER, + RawBigInt: WPK_FORK_IMPORTED_GLOBAL_BINDING_RAW_BIGINT, + RawReference: WPK_FORK_IMPORTED_GLOBAL_BINDING_RAW_REFERENCE, + ActivationGlobal: WPK_FORK_IMPORTED_GLOBAL_BINDING_ACTIVATION_GLOBAL, + BaseImport: WPK_FORK_IMPORTED_GLOBAL_BINDING_BASE_IMPORT, +} as const; + +export type ForkImportedGlobalBindingKind = + typeof ForkImportedGlobalBindingKind[keyof typeof ForkImportedGlobalBindingKind]; + +export interface ForkImportedGlobalBinding { + consumerActivation: number; + consumerOwner: number; + sourceActivation: number; + sourceOwner: number; + reserved: number; + recipeId: number; + rawBits: bigint; + kind: ForkImportedGlobalBindingKind; + mutable: boolean; + shared: boolean; + typeCode: number; +} + +export const ForkImportedTableBindingKind = { + ActivationTable: WPK_FORK_IMPORTED_TABLE_BINDING_ACTIVATION_TABLE, + BaseImport: WPK_FORK_IMPORTED_TABLE_BINDING_BASE_IMPORT, +} as const; + +export type ForkImportedTableBindingKind = + typeof ForkImportedTableBindingKind[keyof typeof ForkImportedTableBindingKind]; + +export interface ForkImportedTableBinding { + consumerActivation: number; + consumerOwner: number; + sourceActivation: number; + sourceOwner: number; + reserved: number; + kind: ForkImportedTableBindingKind; +} + +export interface ForkGlobalSnapshot { + typeCode: number; + value: Uint8Array; + recipeId?: number; +} + +export interface ForkActivationContinuation { + activationId: number; + root: bigint; +} + +export interface ForkModuleDescriptorRecord { + activationId: number; + templateId: Uint8Array; + flags?: number; +} + +export interface ForkSparseTableRun { + start: number; + recipeIds: readonly number[] | Uint32Array; +} + +export interface ForkSparseTablePage { + pageIndex: number | bigint; + runs: readonly ForkSparseTableRun[]; +} + +export interface ForkSparseTableSnapshot { + activationId: number; + ownerId: number; + indexWidth: 4 | 8; + pageShift: number; + length: number | bigint; + baselineLength: number | bigint; + baselineFingerprint: Uint8Array; + pages: readonly ForkSparseTablePage[]; +} + +export interface DecodedForkSparseTableRun { + start: number; + recipeIds: Uint32Array; +} + +export interface DecodedForkSparseTablePage { + pageIndex: bigint; + runs: DecodedForkSparseTableRun[]; +} + +export interface DecodedForkSparseTableSnapshot { + activationId: number; + ownerId: number; + indexWidth: 4 | 8; + pageShift: number; + length: bigint; + baselineLength: bigint; + baselineFingerprint: Uint8Array; + pages: DecodedForkSparseTablePage[]; +} + +interface DirtyPageJournal { + intervals: Array<{ start: bigint; end: bigint }>; + cumulativeEnds: bigint[]; + count: bigint; +} + +interface DirtyPageJournalNode { + parent: DirtyPageJournalNode; + journal: DirtyPageJournal; + stateOwner: boolean; +} + +/** + * Process-lifetime journal of table pages changed from the deterministic + * instantiation baseline. + * + * Generated Wasm calls `markPages` after successful table mutations. The + * journal stores merged intervals rather than one object per page, while + * `pageAt` exposes a deterministic sorted enumeration to the KFMS save helper. + * It intentionally outlives one fork transaction: a replayed child seeds the + * same journal while applying overlays so a later child does not mistake the + * restored parent state for its static baseline. + */ +export class ForkTableDirtyTracker { + private readonly journals = new Map(); + + /** + * Make two activation-local owner ordinals describe one physical Table. + * + * Imported table aliases are discovered only after instantiation. Unioning + * journals (including journals that already contain start-function writes) + * ensures a mutation through any alias reaches one canonical KFMS sparse + * snapshot. Every activation still replays its static element baseline. + * State ownership is elected separately because the union root may belong + * to a provider activation that was later unloaded while an imported alias + * remains live. + */ + aliasOwner( + ownerId: number, + source: ForkTableDirtyTracker, + sourceOwnerId: number, + ): void { + checkedU32(ownerId, "table dirty owner", false); + checkedU32(sourceOwnerId, "table dirty source owner", false); + const targetNode = this.node(ownerId); + const sourceNode = source.node(sourceOwnerId); + targetNode.stateOwner = false; + sourceNode.stateOwner = true; + const targetRoot = this.root(targetNode); + const sourceRoot = source.root(sourceNode); + if (targetRoot === sourceRoot) return; + mergeDirtyPageJournals(sourceRoot.journal, targetRoot.journal); + targetRoot.parent = sourceRoot; + } + + /** Whether this activation-local coordinate owns the physical table state. */ + ownsState(ownerId: number): boolean { + checkedU32(ownerId, "table state owner", false); + return this.node(ownerId).stateOwner; + } + + /** + * Elect or retire this live activation coordinate as sparse-state owner. + * + * The journal's union topology deliberately remains intact so mutations + * accumulated through a now-unloaded provider are not lost when ownership + * moves to a surviving alias. + */ + setStateOwner(ownerId: number, owned: boolean): void { + checkedU32(ownerId, "table state owner", false); + this.node(ownerId).stateOwner = owned; + } + + markPages( + ownerId: number, + firstPageValue: number | bigint, + pageCountValue: number | bigint, + ): void { + checkedU32(ownerId, "table dirty owner", false); + const firstPage = checkedWasmU64(firstPageValue, "table dirty first page"); + const pageCount = checkedWasmU64(pageCountValue, "table dirty page count"); + if (pageCount === 0n) return; + const end = firstPage + pageCount; + if (end > (1n << 64n)) { + throw new RangeError("table dirty page range exceeds u64"); + } + const journal = this.root(this.node(ownerId)).journal; + let insertion = 0; + while ( + insertion < journal.intervals.length + && journal.intervals[insertion]!.end < firstPage + ) { + insertion++; + } + let mergedStart = firstPage; + let mergedEnd = end; + let removalEnd = insertion; + while ( + removalEnd < journal.intervals.length + && journal.intervals[removalEnd]!.start <= mergedEnd + ) { + const interval = journal.intervals[removalEnd]!; + if (interval.start < mergedStart) mergedStart = interval.start; + if (interval.end > mergedEnd) mergedEnd = interval.end; + removalEnd++; + } + journal.intervals.splice( + insertion, + removalEnd - insertion, + { start: mergedStart, end: mergedEnd }, + ); + rebuildDirtyPageJournal(journal); + } + + pageCount(ownerId: number): number { + checkedU32(ownerId, "table dirty owner", false); + const node = this.journals.get(ownerId); + const count = node ? this.root(node).journal.count : 0n; + if (count > 0xffff_ffffn) { + throw new RangeError("table dirty page count exceeds KFMS u32 record count"); + } + return Number(count); + } + + pageAt(ownerId: number, ordinal: number): bigint { + checkedU32(ownerId, "table dirty owner", false); + checkedU32(ordinal, "table dirty page ordinal"); + const node = this.journals.get(ownerId); + const journal = node ? this.root(node).journal : undefined; + if (!journal || BigInt(ordinal) >= journal.count) { + throw new RangeError( + `table dirty owner ${ownerId} has no page ordinal ${ordinal}`, + ); + } + const target = BigInt(ordinal); + let low = 0; + let high = journal.cumulativeEnds.length; + while (low < high) { + const mid = low + ((high - low) >> 1); + if (target < journal.cumulativeEnds[mid]!) high = mid; + else low = mid + 1; + } + const previous = low === 0 ? 0n : journal.cumulativeEnds[low - 1]!; + const page = journal.intervals[low]!.start + (target - previous); + // WebAssembly i64 crosses JavaScript as signed BigInt. Preserve the exact + // unsigned page bits for the generated helper's i64 shifts/stores. + return BigInt.asIntN(64, page); + } + + private node(ownerId: number): DirtyPageJournalNode { + const existing = this.journals.get(ownerId); + if (existing) return existing; + const journal: DirtyPageJournal = { + intervals: [], + cumulativeEnds: [], + count: 0n, + }; + const node = {} as DirtyPageJournalNode; + node.parent = node; + node.journal = journal; + node.stateOwner = true; + this.journals.set(ownerId, node); + return node; + } + + private root(node: DirtyPageJournalNode): DirtyPageJournalNode { + let root = node; + while (root.parent !== root) root = root.parent; + let cursor = node; + while (cursor.parent !== cursor) { + const parent = cursor.parent; + cursor.parent = root; + cursor = parent; + } + return root; + } +} + +function checkedWasmU64(value: number | bigint, context: string): bigint { + if (typeof value === "number" && (!Number.isSafeInteger(value) || value < 0)) { + throw new RangeError(`${context}: expected an exact non-negative integer`); + } + const signed = typeof value === "bigint" ? value : BigInt(value); + const exact = BigInt.asUintN(64, signed); + if (signed >= 0n && signed !== exact) { + throw new RangeError(`${context}: value exceeds u64`); + } + return exact; +} + +function rebuildDirtyPageJournal(journal: DirtyPageJournal): void { + let count = 0n; + journal.cumulativeEnds = journal.intervals.map((interval) => { + count += interval.end - interval.start; + return count; + }); + journal.count = count; +} + +function mergeDirtyPageJournals( + target: DirtyPageJournal, + source: DirtyPageJournal, +): void { + if (source.intervals.length === 0) return; + const intervals = [...target.intervals, ...source.intervals] + .sort((left, right) => + left.start < right.start ? -1 : left.start > right.start ? 1 : 0 + ); + target.intervals = []; + for (const interval of intervals) { + const previous = target.intervals[target.intervals.length - 1]; + if (!previous || previous.end < interval.start) { + target.intervals.push({ ...interval }); + } else if (interval.end > previous.end) { + previous.end = interval.end; + } + } + rebuildDirtyPageJournal(target); +} + +export interface ForkElementSegmentState { + activationId: number; + ownerId: number; + segmentCount: number; + dropped: Uint8Array; +} + +export interface ForkDataSegmentState { + activationId: number; + ownerId: number; + segmentCount: number; + dropped: Uint8Array; +} + +export async function computeForkModuleTemplateId( + bytes: ArrayBuffer | ArrayBufferView, +): Promise { + if (!globalThis.crypto?.subtle) { + throw new Error("SHA-256 is unavailable for fork module template identity"); + } + const source = bytes instanceof ArrayBuffer + ? new Uint8Array(bytes) + : new Uint8Array(bytes.buffer, bytes.byteOffset, bytes.byteLength); + // WebCrypto excludes SharedArrayBuffer-backed views. A module template is + // immutable input, so one exact owned copy also avoids hashing a concurrently + // changing shared view. + const owned = new Uint8Array(source); + return new Uint8Array(await globalThis.crypto.subtle.digest("SHA-256", owned)); +} + +const SHA256_INITIAL_STATE = new Uint32Array([ + 0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, + 0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19, +]); + +const SHA256_ROUND_CONSTANTS = new Uint32Array([ + 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, + 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, + 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, + 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, + 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, + 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, + 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, + 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, + 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, + 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, + 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, + 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, + 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, + 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3, + 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, + 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2, +]); + +function rotateRight32(value: number, amount: number): number { + return (value >>> amount) | (value << (32 - amount)); +} + +/** + * Synchronous, host-neutral SHA-256 for the synchronous `dlopen` import. + * + * The implementation streams one 64-byte block at a time, so exact module + * identity does not require a second module-sized padding allocation. Keep + * the WebCrypto implementation above for async main-program admission; tests + * require both paths to produce byte-identical digests. + */ +export function computeForkModuleTemplateIdSync( + bytes: ArrayBuffer | ArrayBufferView, +): Uint8Array { + const source = bytes instanceof ArrayBuffer + ? new Uint8Array(bytes) + : new Uint8Array(bytes.buffer, bytes.byteOffset, bytes.byteLength); + const totalLength = Math.ceil((source.byteLength + 9) / 64) * 64; + if (!Number.isSafeInteger(totalLength)) { + throw new RangeError("fork module template is too large to hash safely"); + } + + const state = new Uint32Array(SHA256_INITIAL_STATE); + const schedule = new Uint32Array(64); + const block = new Uint8Array(64); + const blockView = new DataView(block.buffer); + const bitLength = BigInt(source.byteLength) * 8n; + + for (let offset = 0; offset < totalLength; offset += 64) { + block.fill(0); + const sourceEnd = Math.min(offset + 64, source.byteLength); + if (offset < sourceEnd) { + block.set(source.subarray(offset, sourceEnd)); + } + if (source.byteLength >= offset && source.byteLength < offset + 64) { + block[source.byteLength - offset] = 0x80; + } + if (offset + 64 === totalLength) { + blockView.setBigUint64(56, bitLength, false); + } + + for (let word = 0; word < 16; word++) { + schedule[word] = blockView.getUint32(word * 4, false); + } + for (let word = 16; word < 64; word++) { + const x = schedule[word - 15]!; + const y = schedule[word - 2]!; + const sigma0 = rotateRight32(x, 7) ^ rotateRight32(x, 18) ^ (x >>> 3); + const sigma1 = rotateRight32(y, 17) ^ rotateRight32(y, 19) ^ (y >>> 10); + schedule[word] = ( + schedule[word - 16]! + + sigma0 + + schedule[word - 7]! + + sigma1 + ) >>> 0; + } + + let a = state[0]!; + let b = state[1]!; + let c = state[2]!; + let d = state[3]!; + let e = state[4]!; + let f = state[5]!; + let g = state[6]!; + let h = state[7]!; + for (let round = 0; round < 64; round++) { + const upper = rotateRight32(e, 6) + ^ rotateRight32(e, 11) + ^ rotateRight32(e, 25); + const choose = (e & f) ^ (~e & g); + const temporary1 = ( + h + + upper + + choose + + SHA256_ROUND_CONSTANTS[round]! + + schedule[round]! + ) >>> 0; + const lower = rotateRight32(a, 2) + ^ rotateRight32(a, 13) + ^ rotateRight32(a, 22); + const majority = (a & b) ^ (a & c) ^ (b & c); + const temporary2 = (lower + majority) >>> 0; + h = g; + g = f; + f = e; + e = (d + temporary1) >>> 0; + d = c; + c = b; + b = a; + a = (temporary1 + temporary2) >>> 0; + } + state[0] = (state[0]! + a) >>> 0; + state[1] = (state[1]! + b) >>> 0; + state[2] = (state[2]! + c) >>> 0; + state[3] = (state[3]! + d) >>> 0; + state[4] = (state[4]! + e) >>> 0; + state[5] = (state[5]! + f) >>> 0; + state[6] = (state[6]! + g) >>> 0; + state[7] = (state[7]! + h) >>> 0; + } + + const digest = new Uint8Array(32); + const digestView = new DataView(digest.buffer); + for (let word = 0; word < state.length; word++) { + digestView.setUint32(word * 4, state[word]!, false); + } + return digest; +} + +export function requireForkModuleTemplate( + records: readonly ForkModuleStateRecord[], + activationId: number, + expectedTemplateId: Uint8Array, +): void { + checkedU32(activationId, "module activation id"); + if (expectedTemplateId.byteLength !== MODULE_TEMPLATE_ID_SIZE) { + throw new RangeError( + `module template id has ${expectedTemplateId.byteLength} bytes, ` + + `expected ${MODULE_TEMPLATE_ID_SIZE}`, + ); + } + const record = records.find( + (candidate) => + candidate.kind === ForkModuleStateRecordKind.Module + && candidate.activationId === activationId, + ); + if (!record) { + throw new Error(`module-state arena is missing activation ${activationId}`); + } + if ( + !bytesEqual( + record.payload.subarray(0, MODULE_TEMPLATE_ID_SIZE), + expectedTemplateId, + ) + ) { + throw new Error(`module-state activation ${activationId} has the wrong template`); + } +} + +interface ArenaChunk { + addr: number; + size: number; + used: number; + recordCount: number; +} + +interface PendingRecord { + chunk: ArenaChunk; + kind: ForkModuleStateRecordKind; + activationId: number; + ownerId: number; + payloadAddr: number; + totalSize: number; + payloadSize: number; +} + +interface DecodedTableDescriptor { + activationId: number; + ownerId: number; + indexWidth: 4 | 8; + pageShift: number; + flags: number; + pageCount: number; + length: bigint; + baselineLength: bigint; + baselineFingerprint: Uint8Array; +} + +interface DecodedTablePage { + activationId: number; + ownerId: number; + pageIndex: bigint; + runs: DecodedForkSparseTableRun[]; + entryCount: number; +} + +interface ValidatedTablePage { + pageIndex: bigint; + runCount: number; + entryCount: number; +} + +interface ValidatedSparseTablePage { + pageIndex: bigint; + entryCount: number; + payloadSize: number; +} + +function littleEndianMagic(bytes: readonly number[]): number { + return ( + bytes[0]! + | (bytes[1]! << 8) + | (bytes[2]! << 16) + | (bytes[3]! << 24) + ) >>> 0; +} + +function alignUp(value: number, alignment: number): number { + const result = Math.ceil(value / alignment) * alignment; + if (!Number.isSafeInteger(result)) { + throw new RangeError(`module-state alignment overflow: ${value}`); + } + return result; +} + +function checkedEnd(addr: number, size: number, context: string): number { + const end = addr + size; + if ( + !Number.isSafeInteger(addr) + || !Number.isSafeInteger(size) + || addr < 0 + || size < 0 + || !Number.isSafeInteger(end) + ) { + throw new RangeError(`${context}: invalid range addr=${addr} size=${size}`); + } + return end; +} + +function checkedMemoryRange( + memory: WebAssembly.Memory, + addr: number, + size: number, + context: string, +): void { + if (checkedEnd(addr, size, context) > memory.buffer.byteLength) { + throw new RangeError(`${context}: range exceeds WebAssembly memory`); + } +} + +function checkedU32(value: number, context: string, allowZero = true): number { + if ( + !Number.isInteger(value) + || value < (allowZero ? 0 : 1) + || value > 0xffff_ffff + ) { + throw new RangeError(`${context}: expected ${allowZero ? "a" : "a nonzero"} u32`); + } + return value; +} + +function checkedU64(value: number | bigint, context: string): bigint { + if ( + typeof value === "number" + && (!Number.isSafeInteger(value) || value < 0) + ) { + throw new RangeError(`${context}: expected a u64`); + } + const result = typeof value === "bigint" ? value : BigInt(value); + if (result < 0n || result > 0xffff_ffff_ffff_ffffn) { + throw new RangeError(`${context}: expected a u64`); + } + return result; +} + +function checkedPointer( + value: number | bigint, + ptrWidth: 4 | 8, + context: string, + allowZero: boolean, +): number { + const result = typeof value === "bigint" ? Number(value) : value; + if ( + (typeof value === "bigint" && BigInt(result) !== value) + || !Number.isSafeInteger(result) + || result < (allowZero ? 0 : 1) + || (ptrWidth === 4 && result > 0xffff_ffff) + ) { + throw new RangeError(`${context}: invalid ${ptrWidth * 8}-bit guest pointer`); + } + return result; +} + +function writePointer( + memory: WebAssembly.Memory, + ptrWidth: 4 | 8, + addr: number, + value: number, +): void { + checkedMemoryRange(memory, addr, ptrWidth, "module-state pointer write"); + const checked = checkedPointer(value, ptrWidth, "module-state pointer write", true); + const view = new DataView(memory.buffer); + if (ptrWidth === 8) view.setBigUint64(addr, BigInt(checked), true); + else view.setUint32(addr, checked, true); +} + +function readPointer( + memory: WebAssembly.Memory, + ptrWidth: 4 | 8, + addr: number, + context: string, +): number { + checkedMemoryRange(memory, addr, ptrWidth, context); + const view = new DataView(memory.buffer); + const raw = ptrWidth === 8 + ? view.getBigUint64(addr, true) + : BigInt(view.getUint32(addr, true)); + return checkedPointer(raw, ptrWidth, context, true); +} + +function bytesEqual(actual: Uint8Array, expected: ArrayLike): boolean { + if (actual.byteLength < expected.length) return false; + for (let index = 0; index < expected.length; index++) { + if (actual[index] !== expected[index]) return false; + } + return true; +} + +function requireZeroBytes(bytes: Uint8Array, context: string): void { + if (bytes.some((value) => value !== 0)) { + throw new Error(`${context}: reserved or padding bytes must be zero`); + } +} + +function chunkHeaderSize(ptrWidth: 4 | 8): number { + const format = WPK_FORK_MODULE_STATE_POINTER_WIDTHS.find( + ({ bytes }) => bytes === ptrWidth, + ); + if (!format) { + throw new Error(`unsupported module-state pointer width ${ptrWidth}`); + } + return format.chunkHeaderSize; +} + +function chunkOffset(ptrWidth: 4 | 8, field: 0 | 1 | 2 | 3 | 4): number { + return 8 + field * ptrWidth; +} + +function chunkRecordCountOffset(ptrWidth: 4 | 8): number { + return 8 + 5 * ptrWidth; +} + +function chunkReservedOffset(ptrWidth: 4 | 8): number { + return 12 + 5 * ptrWidth; +} + +function tableKey(activationId: number, ownerId: number): string { + return `${activationId}:${ownerId}`; +} + +function ownerKey( + kind: ForkModuleStateRecordKind, + activationId: number, + ownerId: number, +): string { + return `${kind}:${activationId}:${ownerId}`; +} + +/** + * Encode the artifact descriptor that binds an instrumented module to this + * exact arena/record/root-prefix contract. + */ +export function encodeForkModuleStateDescriptor( + ptrWidth: 4 | 8, +): Uint8Array { + const bytes = new Uint8Array(FORK_MODULE_STATE_DESCRIPTOR_SIZE); + const view = new DataView(bytes.buffer); + bytes.set(FORK_MODULE_STATE_DESCRIPTOR_MAGIC, 0); + view.setUint16(4, FORK_MODULE_STATE_DESCRIPTOR_VERSION, true); + view.setUint16(6, FORK_MODULE_STATE_DESCRIPTOR_SIZE, true); + view.setUint8(8, ptrWidth); + view.setUint8(9, FORK_MODULE_STATE_RECORD_ALIGNMENT); + view.setUint16(10, FORK_MODULE_STATE_REQUIRED_FLAGS, true); + view.setUint16(12, FORK_MODULE_STATE_ARENA_VERSION, true); + view.setUint16(14, FORK_MODULE_STATE_RECORD_VERSION, true); + view.setUint32(16, FORK_MODULE_STATE_ROOT_POINTER_WORD_OFFSET, true); + view.setUint32(20, 0, true); + return bytes; +} + +export function decodeForkModuleStateDescriptor( + bytes: Uint8Array, +): ForkModuleStateDescriptor { + if (bytes.byteLength !== FORK_MODULE_STATE_DESCRIPTOR_SIZE) { + throw new Error( + `module-state descriptor has ${bytes.byteLength} bytes, ` + + `expected ${FORK_MODULE_STATE_DESCRIPTOR_SIZE}`, + ); + } + if (!bytesEqual(bytes, FORK_MODULE_STATE_DESCRIPTOR_MAGIC)) { + throw new Error("module-state descriptor has invalid magic"); + } + const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); + const version = view.getUint16(4, true); + if (version !== FORK_MODULE_STATE_DESCRIPTOR_VERSION) { + throw new Error(`unsupported module-state descriptor version ${version}`); + } + if (view.getUint16(6, true) !== FORK_MODULE_STATE_DESCRIPTOR_SIZE) { + throw new Error("module-state descriptor declares an invalid size"); + } + const ptrWidth = view.getUint8(8); + if (ptrWidth !== 4 && ptrWidth !== 8) { + throw new Error(`unsupported module-state pointer width ${ptrWidth}`); + } + const alignment = view.getUint8(9); + if (alignment !== FORK_MODULE_STATE_RECORD_ALIGNMENT) { + throw new Error(`unsupported module-state record alignment ${alignment}`); + } + const flags = view.getUint16(10, true); + if ((flags & ~FORK_MODULE_STATE_KNOWN_FLAGS) !== 0) { + throw new Error(`unknown module-state descriptor flags 0x${flags.toString(16)}`); + } + if ((flags & FORK_MODULE_STATE_REQUIRED_FLAGS) !== FORK_MODULE_STATE_REQUIRED_FLAGS) { + throw new Error("module-state descriptor omits required ownership features"); + } + const arenaVersion = view.getUint16(12, true); + if (arenaVersion !== FORK_MODULE_STATE_ARENA_VERSION) { + throw new Error(`unsupported module-state arena version ${arenaVersion}`); + } + const recordVersion = view.getUint16(14, true); + if (recordVersion !== FORK_MODULE_STATE_RECORD_VERSION) { + throw new Error(`unsupported module-state record version ${recordVersion}`); + } + const rootPointerWordOffset = view.getUint32(16, true); + if (rootPointerWordOffset !== FORK_MODULE_STATE_ROOT_POINTER_WORD_OFFSET) { + throw new Error( + `unsupported module-state root-pointer word offset ${rootPointerWordOffset}`, + ); + } + if (view.getUint32(20, true) !== 0) { + throw new Error("module-state descriptor reserved field is nonzero"); + } + return { + version, + ptrWidth, + alignment, + flags, + arenaVersion, + recordVersion, + rootPointerWordOffset, + }; +} + +export function readForkModuleStateDescriptor( + module: WebAssembly.Module, +): ForkModuleStateDescriptor { + const sections = WebAssembly.Module.customSections(module, FORK_MODULE_STATE_SECTION); + if (sections.length !== 1) { + throw new Error( + `expected one ${FORK_MODULE_STATE_SECTION} section, found ${sections.length}`, + ); + } + return decodeForkModuleStateDescriptor(new Uint8Array(sections[0]!)); +} + +const IMPORTED_GLOBAL_TYPE_CODES = new Set([ + WPK_FORK_MODULE_STATE_GLOBAL_TYPE_I32, + WPK_FORK_MODULE_STATE_GLOBAL_TYPE_I64, + WPK_FORK_MODULE_STATE_GLOBAL_TYPE_F32, + WPK_FORK_MODULE_STATE_GLOBAL_TYPE_F64, + WPK_FORK_MODULE_STATE_GLOBAL_TYPE_V128, + WPK_FORK_MODULE_STATE_GLOBAL_TYPE_FUNCREF, + WPK_FORK_MODULE_STATE_GLOBAL_TYPE_EXTERNREF, + WPK_FORK_MODULE_STATE_GLOBAL_TYPE_EXNREF, + WPK_FORK_MODULE_STATE_GLOBAL_TYPE_ANYREF, +]); + +/** + * Read the pre-instantiation ownership recipe for imported globals. + * + * Immutable imports must be supplied with their saved parent value before + * WebAssembly instantiation: exported imported Globals and const initializers + * observe that exact binding and cannot be repaired by a later `global.set`. + */ +export function readForkImportedGlobals( + module: WebAssembly.Module, +): readonly ForkImportedGlobalState[] { + const sections = WebAssembly.Module.customSections( + module, + WPK_FORK_IMPORTED_GLOBALS_SECTION, + ); + if (sections.length !== 1) { + throw new Error( + `expected one ${WPK_FORK_IMPORTED_GLOBALS_SECTION} section, found ${sections.length}`, + ); + } + const bytes = new Uint8Array(sections[0]!); + if (bytes.byteLength < WPK_FORK_IMPORTED_GLOBALS_HEADER_SIZE) { + throw new Error("imported-global descriptor is truncated"); + } + const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); + if ( + view.getUint32(0, true) + !== littleEndianMagic(WPK_FORK_IMPORTED_GLOBALS_MAGIC) + ) { + throw new Error("imported-global descriptor has the wrong magic"); + } + if (view.getUint16(4, true) !== WPK_FORK_IMPORTED_GLOBALS_VERSION) { + throw new Error( + `unsupported imported-global descriptor version ${view.getUint16(4, true)}`, + ); + } + if (view.getUint16(6, true) !== WPK_FORK_IMPORTED_GLOBALS_HEADER_SIZE) { + throw new Error("imported-global descriptor declares an invalid header size"); + } + const count = view.getUint32(8, true); + if (view.getUint32(12, true) !== 0) { + throw new Error("imported-global descriptor reserved field is nonzero"); + } + + const decoder = new TextDecoder("utf-8", { fatal: true }); + const owners = new Set(); + const importOrdinals = new Set(); + const globals: ForkImportedGlobalState[] = []; + let previousImportOrdinal = -1; + let offset = WPK_FORK_IMPORTED_GLOBALS_HEADER_SIZE; + for (let index = 0; index < count; index++) { + if (offset + WPK_FORK_IMPORTED_GLOBALS_RECORD_HEADER_SIZE > bytes.byteLength) { + throw new Error(`imported-global record ${index} header is truncated`); + } + const recordSize = view.getUint32(offset, true); + const ownerId = view.getUint32(offset + 4, true); + const typeCode = view.getUint8(offset + 8); + const flags = view.getUint8(offset + 9); + const moduleLength = view.getUint32(offset + 12, true); + const nameLength = view.getUint32(offset + 16, true); + const importOrdinal = view.getUint32(offset + 20, true); + const expectedSize = WPK_FORK_IMPORTED_GLOBALS_RECORD_HEADER_SIZE + + moduleLength + + nameLength; + if ( + recordSize !== expectedSize + || recordSize < WPK_FORK_IMPORTED_GLOBALS_RECORD_HEADER_SIZE + || offset + recordSize > bytes.byteLength + ) { + throw new Error(`imported-global record ${index} has invalid bounds`); + } + checkedU32(ownerId, `imported-global record ${index} owner`, false); + if (owners.has(ownerId)) { + throw new Error(`imported-global record ${index} duplicates owner ${ownerId}`); + } + owners.add(ownerId); + if (!IMPORTED_GLOBAL_TYPE_CODES.has(typeCode)) { + throw new Error( + `imported-global record ${index} has unknown value type ${typeCode}`, + ); + } + if ((flags & ~WPK_FORK_IMPORTED_GLOBALS_KNOWN_FLAGS) !== 0) { + throw new Error( + `imported-global record ${index} has unknown flags 0x${flags.toString(16)}`, + ); + } + if (view.getUint16(offset + 10, true) !== 0) { + throw new Error(`imported-global record ${index} reserved field is nonzero`); + } + const namesOffset = offset + WPK_FORK_IMPORTED_GLOBALS_RECORD_HEADER_SIZE; + let moduleName: string; + let fieldName: string; + try { + moduleName = decoder.decode( + bytes.subarray(namesOffset, namesOffset + moduleLength), + ); + fieldName = decoder.decode( + bytes.subarray( + namesOffset + moduleLength, + namesOffset + moduleLength + nameLength, + ), + ); + } catch { + throw new Error(`imported-global record ${index} contains invalid UTF-8`); + } + if ( + importOrdinals.has(importOrdinal) + || importOrdinal <= previousImportOrdinal + ) { + throw new Error( + `imported-global record ${index} has duplicated or unordered import ordinal`, + ); + } + importOrdinals.add(importOrdinal); + previousImportOrdinal = importOrdinal; + globals.push({ + module: moduleName, + name: fieldName, + importOrdinal, + ownerId, + typeCode, + mutable: (flags & WPK_FORK_IMPORTED_GLOBALS_FLAG_MUTABLE) !== 0, + shared: (flags & WPK_FORK_IMPORTED_GLOBALS_FLAG_SHARED) !== 0, + }); + offset += recordSize; + } + if (offset !== bytes.byteLength) { + throw new Error("imported-global descriptor has trailing bytes"); + } + return globals; +} + +const IMPORTED_TABLE_ELEMENT_TYPE_CODES = new Set([ + WPK_FORK_MODULE_STATE_GLOBAL_TYPE_FUNCREF, + WPK_FORK_MODULE_STATE_GLOBAL_TYPE_EXTERNREF, + WPK_FORK_MODULE_STATE_GLOBAL_TYPE_EXNREF, + WPK_FORK_MODULE_STATE_GLOBAL_TYPE_ANYREF, +]); + +/** + * Read exact import-section coordinates for every application-owned table. + * + * A table import is an identity edge in the module graph, not just an initial + * sequence of elements. The fresh child must wire that edge before + * instantiation so aliases, active element initializers, and exported imported + * tables all observe the same reconstructed Table object. + */ +export function readForkImportedTables( + module: WebAssembly.Module, +): readonly ForkImportedTableState[] { + const sections = WebAssembly.Module.customSections( + module, + WPK_FORK_IMPORTED_TABLES_SECTION, + ); + if (sections.length !== 1) { + throw new Error( + `expected one ${WPK_FORK_IMPORTED_TABLES_SECTION} section, found ${sections.length}`, + ); + } + const bytes = new Uint8Array(sections[0]!); + if (bytes.byteLength < WPK_FORK_IMPORTED_TABLES_HEADER_SIZE) { + throw new Error("imported-table descriptor is truncated"); + } + const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); + if ( + view.getUint32(0, true) + !== littleEndianMagic(WPK_FORK_IMPORTED_TABLES_MAGIC) + ) { + throw new Error("imported-table descriptor has the wrong magic"); + } + if (view.getUint16(4, true) !== WPK_FORK_IMPORTED_TABLES_VERSION) { + throw new Error( + `unsupported imported-table descriptor version ${view.getUint16(4, true)}`, + ); + } + if (view.getUint16(6, true) !== WPK_FORK_IMPORTED_TABLES_HEADER_SIZE) { + throw new Error("imported-table descriptor declares an invalid header size"); + } + const count = view.getUint32(8, true); + if (view.getUint32(12, true) !== 0) { + throw new Error("imported-table descriptor reserved field is nonzero"); + } + + const decoder = new TextDecoder("utf-8", { fatal: true }); + const owners = new Set(); + const importOrdinals = new Set(); + const tables: ForkImportedTableState[] = []; + let previousImportOrdinal = -1; + let offset = WPK_FORK_IMPORTED_TABLES_HEADER_SIZE; + for (let index = 0; index < count; index++) { + if (offset + WPK_FORK_IMPORTED_TABLES_RECORD_HEADER_SIZE > bytes.byteLength) { + throw new Error(`imported-table record ${index} header is truncated`); + } + const recordSize = view.getUint32(offset, true); + const ownerId = view.getUint32(offset + 4, true); + const typeCode = view.getUint8(offset + 8); + const flags = view.getUint8(offset + 9); + const moduleLength = view.getUint32(offset + 12, true); + const nameLength = view.getUint32(offset + 16, true); + const importOrdinal = view.getUint32(offset + 20, true); + const expectedSize = WPK_FORK_IMPORTED_TABLES_RECORD_HEADER_SIZE + + moduleLength + + nameLength; + if ( + recordSize !== expectedSize + || recordSize < WPK_FORK_IMPORTED_TABLES_RECORD_HEADER_SIZE + || offset + recordSize > bytes.byteLength + ) { + throw new Error(`imported-table record ${index} has invalid bounds`); + } + checkedU32(ownerId, `imported-table record ${index} owner`, false); + if (owners.has(ownerId)) { + throw new Error(`imported-table record ${index} duplicates owner ${ownerId}`); + } + owners.add(ownerId); + if (!IMPORTED_TABLE_ELEMENT_TYPE_CODES.has(typeCode)) { + throw new Error( + `imported-table record ${index} has unknown element type ${typeCode}`, + ); + } + if ((flags & ~WPK_FORK_IMPORTED_TABLES_KNOWN_FLAGS) !== 0) { + throw new Error( + `imported-table record ${index} has unknown flags 0x${flags.toString(16)}`, + ); + } + if (view.getUint16(offset + 10, true) !== 0) { + throw new Error(`imported-table record ${index} reserved field is nonzero`); + } + const namesOffset = offset + WPK_FORK_IMPORTED_TABLES_RECORD_HEADER_SIZE; + let moduleName: string; + let fieldName: string; + try { + moduleName = decoder.decode( + bytes.subarray(namesOffset, namesOffset + moduleLength), + ); + fieldName = decoder.decode( + bytes.subarray( + namesOffset + moduleLength, + namesOffset + moduleLength + nameLength, + ), + ); + } catch { + throw new Error(`imported-table record ${index} contains invalid UTF-8`); + } + if ( + importOrdinals.has(importOrdinal) + || importOrdinal <= previousImportOrdinal + ) { + throw new Error( + `imported-table record ${index} has duplicated or unordered import ordinal`, + ); + } + importOrdinals.add(importOrdinal); + previousImportOrdinal = importOrdinal; + tables.push({ + module: moduleName, + name: fieldName, + importOrdinal, + ownerId, + typeCode, + table64: (flags & WPK_FORK_IMPORTED_TABLES_FLAG_TABLE64) !== 0, + }); + offset += recordSize; + } + if (offset !== bytes.byteLength) { + throw new Error("imported-table descriptor has trailing bytes"); + } + return tables; +} + +/** + * Publish the arena root in the second pointer word of the module prefix. + * + * WHY: the first word is the activation-frame cursor. The linked runtime has + * reserved the `+P` word since ABI 42, so using it gives copied module state an + * activation-owned root without changing activation-frame replay ordering. + */ +export function writeForkModuleStateRoot( + memory: WebAssembly.Memory, + moduleBufferAddr: number, + ptrWidth: 4 | 8, + arenaRoot: number, +): void { + const moduleBuffer = checkedPointer( + moduleBufferAddr, + ptrWidth, + "module-state module buffer", + false, + ); + const root = checkedPointer(arenaRoot, ptrWidth, "module-state arena root", true); + if (root !== 0 && root % WASM_PAGE_SIZE !== 0) { + throw new RangeError("module-state arena root must be page-aligned"); + } + writePointer( + memory, + ptrWidth, + moduleBuffer + FORK_MODULE_STATE_ROOT_POINTER_WORD_OFFSET * ptrWidth, + root, + ); +} + +export function readForkModuleStateRoot( + memory: WebAssembly.Memory, + moduleBufferAddr: number, + ptrWidth: 4 | 8, +): number { + const moduleBuffer = checkedPointer( + moduleBufferAddr, + ptrWidth, + "module-state module buffer", + false, + ); + const root = readPointer( + memory, + ptrWidth, + moduleBuffer + FORK_MODULE_STATE_ROOT_POINTER_WORD_OFFSET * ptrWidth, + "module-state root-prefix pointer", + ); + if (root !== 0 && root % WASM_PAGE_SIZE !== 0) { + throw new Error("module-state root-prefix pointer is not page-aligned"); + } + return root; +} + +function encodeModulePayload(record: ForkModuleDescriptorRecord): Uint8Array { + checkedU32(record.activationId, "module activation id"); + if (record.templateId.byteLength !== MODULE_TEMPLATE_ID_SIZE) { + throw new RangeError( + `module template id has ${record.templateId.byteLength} bytes, ` + + `expected ${MODULE_TEMPLATE_ID_SIZE}`, + ); + } + const flags = checkedU32(record.flags ?? 0, "module flags"); + if ((flags & ~MODULE_RECORD_KNOWN_FLAGS) !== 0) { + throw new RangeError(`unknown module-state module flags 0x${flags.toString(16)}`); + } + const payload = new Uint8Array(MODULE_RECORD_PAYLOAD_SIZE); + payload.set(record.templateId, 0); + const view = new DataView(payload.buffer); + view.setUint32(MODULE_TEMPLATE_ID_SIZE, flags, true); + view.setUint32(MODULE_TEMPLATE_ID_SIZE + 4, 0, true); + return payload; +} + +function decodeModulePayload(payload: Uint8Array, context: string): void { + if (payload.byteLength !== MODULE_RECORD_PAYLOAD_SIZE) { + throw new Error( + `${context}: module payload has ${payload.byteLength} bytes, ` + + `expected ${MODULE_RECORD_PAYLOAD_SIZE}`, + ); + } + const view = new DataView(payload.buffer, payload.byteOffset, payload.byteLength); + const flags = view.getUint32(MODULE_TEMPLATE_ID_SIZE, true); + if ((flags & ~MODULE_RECORD_KNOWN_FLAGS) !== 0) { + throw new Error(`${context}: unknown module flags 0x${flags.toString(16)}`); + } + if (view.getUint32(MODULE_TEMPLATE_ID_SIZE + 4, true) !== 0) { + throw new Error(`${context}: module payload reserved field is nonzero`); + } +} + +function encodeTableDescriptor(snapshot: ForkSparseTableSnapshot): Uint8Array { + const indexWidth = snapshot.indexWidth; + if (indexWidth !== 4 && indexWidth !== 8) { + throw new RangeError(`table index width ${String(indexWidth)} is unsupported`); + } + if ( + !Number.isInteger(snapshot.pageShift) + || snapshot.pageShift < MIN_TABLE_PAGE_SHIFT + || snapshot.pageShift > MAX_TABLE_PAGE_SHIFT + ) { + throw new RangeError( + `table page shift must be ${MIN_TABLE_PAGE_SHIFT}..${MAX_TABLE_PAGE_SHIFT}`, + ); + } + const length = checkedU64(snapshot.length, "table length"); + const baselineLength = checkedU64(snapshot.baselineLength, "table baseline length"); + if (indexWidth === 4 && length > 0xffff_ffffn) { + throw new RangeError("table32 length exceeds u32"); + } + if (baselineLength > length) { + throw new RangeError("table baseline length exceeds final length"); + } + if ( + snapshot.baselineFingerprint.byteLength + !== FORK_MODULE_STATE_BASELINE_FINGERPRINT_SIZE + ) { + throw new RangeError( + `table baseline fingerprint must be ` + + `${FORK_MODULE_STATE_BASELINE_FINGERPRINT_SIZE} bytes`, + ); + } + checkedU32(snapshot.pages.length, "table page count"); + const payload = new Uint8Array(TABLE_DESCRIPTOR_PAYLOAD_SIZE); + const view = new DataView(payload.buffer); + view.setUint8(0, indexWidth); + view.setUint8(1, snapshot.pageShift); + view.setUint16(2, TABLE_FLAG_SPARSE_OVERRIDES, true); + view.setUint32(4, snapshot.pages.length, true); + view.setBigUint64(8, length, true); + view.setBigUint64(16, baselineLength, true); + payload.set(snapshot.baselineFingerprint, 24); + return payload; +} + +function decodeTableDescriptor( + record: ForkModuleStateRecord, + context: string, +): DecodedTableDescriptor { + if (record.payload.byteLength !== TABLE_DESCRIPTOR_PAYLOAD_SIZE) { + throw new Error( + `${context}: table descriptor has ${record.payload.byteLength} bytes, ` + + `expected ${TABLE_DESCRIPTOR_PAYLOAD_SIZE}`, + ); + } + const view = new DataView( + record.payload.buffer, + record.payload.byteOffset, + record.payload.byteLength, + ); + const indexWidth = view.getUint8(0); + if (indexWidth !== 4 && indexWidth !== 8) { + throw new Error(`${context}: unsupported table index width ${indexWidth}`); + } + const pageShift = view.getUint8(1); + if (pageShift < MIN_TABLE_PAGE_SHIFT || pageShift > MAX_TABLE_PAGE_SHIFT) { + throw new Error(`${context}: unsupported table page shift ${pageShift}`); + } + const flags = view.getUint16(2, true); + if ( + (flags & ~TABLE_KNOWN_FLAGS) !== 0 + || (flags & TABLE_FLAG_SPARSE_OVERRIDES) === 0 + ) { + throw new Error(`${context}: invalid table flags 0x${flags.toString(16)}`); + } + const length = view.getBigUint64(8, true); + const baselineLength = view.getBigUint64(16, true); + if (indexWidth === 4 && length > 0xffff_ffffn) { + throw new Error(`${context}: table32 length exceeds u32`); + } + if (baselineLength > length) { + throw new Error(`${context}: table baseline length exceeds final length`); + } + return { + activationId: record.activationId, + ownerId: record.ownerId, + indexWidth, + pageShift, + flags, + pageCount: view.getUint32(4, true), + length, + baselineLength, + baselineFingerprint: record.payload.slice(24, 56), + }; +} + +function validateSparseTablePage( + descriptor: DecodedTableDescriptor, + page: ForkSparseTablePage, + previousPageIndex: bigint | null, +): ValidatedSparseTablePage { + const pageIndex = checkedU64(page.pageIndex, "table page index"); + if (previousPageIndex !== null && pageIndex <= previousPageIndex) { + throw new RangeError("sparse table pages must be strictly increasing"); + } + checkedU32(page.runs.length, "table page run count"); + const pageSize = 1 << descriptor.pageShift; + let previousEnd = 0; + let entryCount = 0; + let payloadSize = TABLE_PAGE_HEADER_SIZE; + for (const [runIndex, run] of page.runs.entries()) { + if (!Number.isInteger(run.start) || run.start < previousEnd || run.start >= pageSize) { + throw new RangeError(`table page run ${runIndex} is unordered or out of bounds`); + } + if (run.recipeIds.length === 0) { + throw new RangeError(`table page run ${runIndex} is empty`); + } + checkedU32(run.recipeIds.length, `table page run ${runIndex} length`, false); + const end = run.start + run.recipeIds.length; + if (!Number.isSafeInteger(end) || end > pageSize) { + throw new RangeError(`table page run ${runIndex} exceeds its page`); + } + const absoluteEnd = pageIndex * BigInt(pageSize) + BigInt(end); + if (absoluteEnd > descriptor.length) { + throw new RangeError(`table page run ${runIndex} exceeds final table length`); + } + for (let entryIndex = 0; entryIndex < run.recipeIds.length; entryIndex++) { + checkedU32( + run.recipeIds[entryIndex]!, + `table page run ${runIndex} recipe ${entryIndex}`, + ); + } + previousEnd = end; + entryCount += run.recipeIds.length; + payloadSize += TABLE_RUN_HEADER_SIZE + run.recipeIds.length * 4; + } + if (entryCount === 0) { + throw new RangeError("sparse table page must contain at least one override"); + } + checkedU32(entryCount, "table page entry count"); + return { pageIndex, entryCount, payloadSize }; +} + +function encodeTablePage( + descriptor: DecodedTableDescriptor, + page: ForkSparseTablePage, + previousPageIndex: bigint | null, +): Uint8Array { + const validated = validateSparseTablePage( + descriptor, + page, + previousPageIndex, + ); + const payload = new Uint8Array(validated.payloadSize); + const view = new DataView(payload.buffer); + view.setBigUint64(0, validated.pageIndex, true); + view.setUint32(8, page.runs.length, true); + view.setUint32(12, validated.entryCount, true); + let offset = TABLE_PAGE_HEADER_SIZE; + for (const run of page.runs) { + view.setUint32(offset, run.start, true); + view.setUint32(offset + 4, run.recipeIds.length, true); + offset += TABLE_RUN_HEADER_SIZE; + for (const recipeId of run.recipeIds) { + view.setUint32(offset, recipeId, true); + offset += 4; + } + } + return payload; +} + +function validateTablePage( + record: ForkModuleStateRecord, + descriptor: DecodedTableDescriptor, + context: string, +): ValidatedTablePage { + if (record.payload.byteLength < TABLE_PAGE_HEADER_SIZE) { + throw new Error(`${context}: table page payload is truncated`); + } + const view = new DataView( + record.payload.buffer, + record.payload.byteOffset, + record.payload.byteLength, + ); + const pageIndex = view.getBigUint64(0, true); + const runCount = view.getUint32(8, true); + const declaredEntryCount = view.getUint32(12, true); + const pageSize = 1 << descriptor.pageShift; + let previousEnd = 0; + let entryCount = 0; + let offset = TABLE_PAGE_HEADER_SIZE; + for (let runIndex = 0; runIndex < runCount; runIndex++) { + if (offset + TABLE_RUN_HEADER_SIZE > record.payload.byteLength) { + throw new Error(`${context}: table page run ${runIndex} header is truncated`); + } + const start = view.getUint32(offset, true); + const count = view.getUint32(offset + 4, true); + offset += TABLE_RUN_HEADER_SIZE; + if (count === 0 || start < previousEnd || start >= pageSize || start + count > pageSize) { + throw new Error(`${context}: table page run ${runIndex} is unordered or out of bounds`); + } + if (offset + count * 4 > record.payload.byteLength) { + throw new Error(`${context}: table page run ${runIndex} recipes are truncated`); + } + const absoluteEnd = pageIndex * BigInt(pageSize) + BigInt(start + count); + if (absoluteEnd > descriptor.length) { + throw new Error(`${context}: table page run ${runIndex} exceeds final table length`); + } + // WHY: attachment only needs structural validity and page ordering. Read + // each recipe in place so validating an arbitrarily long segmented table + // does not allocate and immediately discard one Uint32Array per run. + for (let index = 0; index < count; index++) { + view.getUint32(offset, true); + offset += 4; + } + previousEnd = start + count; + entryCount += count; + } + if ( + runCount === 0 + || entryCount !== declaredEntryCount + || offset !== record.payload.byteLength + ) { + throw new Error(`${context}: table page counts or payload size are inconsistent`); + } + return { pageIndex, runCount, entryCount }; +} + +function decodeTablePage( + record: ForkModuleStateRecord, + descriptor: DecodedTableDescriptor, + context: string, +): DecodedTablePage { + const validated = validateTablePage(record, descriptor, context); + const view = new DataView( + record.payload.buffer, + record.payload.byteOffset, + record.payload.byteLength, + ); + let offset = TABLE_PAGE_HEADER_SIZE; + const runs: DecodedForkSparseTableRun[] = []; + for (let runIndex = 0; runIndex < validated.runCount; runIndex++) { + const start = view.getUint32(offset, true); + const count = view.getUint32(offset + 4, true); + offset += TABLE_RUN_HEADER_SIZE; + const recipeIds = new Uint32Array(count); + for (let index = 0; index < count; index++) { + recipeIds[index] = view.getUint32(offset, true); + offset += 4; + } + runs.push({ start, recipeIds }); + } + return { + activationId: record.activationId, + ownerId: record.ownerId, + pageIndex: validated.pageIndex, + runs, + entryCount: validated.entryCount, + }; +} + +function encodeSegmentBitmap( + state: ForkElementSegmentState | ForkDataSegmentState, + label: "element" | "data", + headerSize: number, +): Uint8Array { + checkedU32(state.segmentCount, `${label} segment count`); + const expectedBytes = Math.ceil(state.segmentCount / 8); + if (state.dropped.byteLength !== expectedBytes) { + throw new RangeError( + `${label} drop bitmap has ${state.dropped.byteLength} bytes, expected ${expectedBytes}`, + ); + } + if (expectedBytes > 0 && state.segmentCount % 8 !== 0) { + const liveBits = state.segmentCount % 8; + const invalidMask = 0xff << liveBits; + if ((state.dropped[expectedBytes - 1]! & invalidMask) !== 0) { + throw new RangeError(`${label} drop bitmap has nonzero bits beyond segment count`); + } + } + const payload = new Uint8Array(headerSize + expectedBytes); + const view = new DataView(payload.buffer); + view.setUint32(0, state.segmentCount, true); + view.setUint32(4, expectedBytes, true); + payload.set(state.dropped, headerSize); + return payload; +} + +function decodeSegmentBitmap( + payload: Uint8Array, + context: string, + label: "element" | "data", + headerSize: number, +): void { + if (payload.byteLength < headerSize) { + throw new Error(`${context}: ${label}-segment payload is truncated`); + } + const view = new DataView(payload.buffer, payload.byteOffset, payload.byteLength); + const segmentCount = view.getUint32(0, true); + const bitmapBytes = view.getUint32(4, true); + const expectedBytes = Math.ceil(segmentCount / 8); + if ( + bitmapBytes !== expectedBytes + || payload.byteLength !== headerSize + expectedBytes + ) { + throw new Error(`${context}: ${label}-segment bitmap size is inconsistent`); + } + if (expectedBytes > 0 && segmentCount % 8 !== 0) { + const invalidMask = 0xff << (segmentCount % 8); + if ((payload[payload.byteLength - 1]! & invalidMask) !== 0) { + throw new Error(`${context}: ${label}-segment bitmap has nonzero trailing bits`); + } + } +} + +function encodeElementSegments(state: ForkElementSegmentState): Uint8Array { + return encodeSegmentBitmap(state, "element", ELEMENT_SEGMENT_HEADER_SIZE); +} + +function encodeDataSegments(state: ForkDataSegmentState): Uint8Array { + return encodeSegmentBitmap(state, "data", DATA_SEGMENT_HEADER_SIZE); +} + +function decodeElementSegments(payload: Uint8Array, context: string): void { + decodeSegmentBitmap(payload, context, "element", ELEMENT_SEGMENT_HEADER_SIZE); +} + +function decodeDataSegments(payload: Uint8Array, context: string): void { + decodeSegmentBitmap(payload, context, "data", DATA_SEGMENT_HEADER_SIZE); +} + +function assertActivationContinuationSet( + continuations: readonly ForkActivationContinuation[], + activeActivationIds: ReadonlySet, + context: string, +): void { + const expected = [...activeActivationIds].sort((left, right) => left - right); + const actual = continuations.map(({ activationId }) => activationId); + if ( + actual.length !== expected.length + || actual.some((activationId, index) => activationId !== expected[index]) + ) { + throw new Error( + `${context}: activation set does not exactly match replay events ` + + `(continuations ${actual.join(",")}; replay ${expected.join(",")})`, + ); + } +} + +/** + * Encode the copied continuation root for every activation participating in + * this fork. Roots are fixed-width u64 values so one process record can name + * wasm32 and wasm64 activations without an archive-private side channel. + */ +export function encodeForkActivationContinuations( + continuations: readonly ForkActivationContinuation[], +): Uint8Array { + if (continuations.length === 0) { + throw new Error("activation-continuation manifest must not be empty"); + } + if (continuations.length > 0xffff_ffff) { + throw new RangeError("activation-continuation count exceeds u32"); + } + const payloadSize = WPK_FORK_ACTIVATION_CONTINUATIONS_HEADER_SIZE + + continuations.length * WPK_FORK_ACTIVATION_CONTINUATIONS_ENTRY_SIZE; + if (!Number.isSafeInteger(payloadSize)) { + throw new RangeError( + "activation-continuation payload size exceeds JavaScript safe integer", + ); + } + const payload = new Uint8Array(payloadSize); + const view = new DataView(payload.buffer); + view.setUint32( + 0, + littleEndianMagic(WPK_FORK_ACTIVATION_CONTINUATIONS_MAGIC), + true, + ); + view.setUint16(4, WPK_FORK_ACTIVATION_CONTINUATIONS_VERSION, true); + view.setUint16(6, WPK_FORK_ACTIVATION_CONTINUATIONS_HEADER_SIZE, true); + view.setUint16(8, WPK_FORK_ACTIVATION_CONTINUATIONS_ENTRY_SIZE, true); + view.setUint16(10, WPK_FORK_ACTIVATION_CONTINUATIONS_KNOWN_FLAGS, true); + view.setUint32(12, continuations.length, true); + view.setBigUint64(16, 0n, true); + + let previousActivation = -1; + for (const [index, continuation] of continuations.entries()) { + checkedU32( + continuation.activationId, + `activation continuation ${index} activation`, + ); + if (continuation.activationId <= previousActivation) { + throw new Error( + `activation continuation ${index}: activations must be unique and strictly ordered`, + ); + } + const root = checkedU64( + continuation.root, + `activation continuation ${index} root`, + ); + if (root === 0n) { + throw new RangeError(`activation continuation ${index} root is zero`); + } + const offset = WPK_FORK_ACTIVATION_CONTINUATIONS_HEADER_SIZE + + index * WPK_FORK_ACTIVATION_CONTINUATIONS_ENTRY_SIZE; + view.setUint32(offset, continuation.activationId, true); + view.setUint32( + offset + 4, + WPK_FORK_ACTIVATION_CONTINUATIONS_ENTRY_KNOWN_FLAGS, + true, + ); + view.setBigUint64(offset + 8, root, true); + previousActivation = continuation.activationId; + } + return payload; +} + +export function decodeForkActivationContinuations( + payload: Uint8Array, + context = "module-state activation continuations", +): ForkActivationContinuation[] { + if (payload.byteLength < WPK_FORK_ACTIVATION_CONTINUATIONS_HEADER_SIZE) { + throw new Error(`${context}: payload is truncated`); + } + const view = new DataView(payload.buffer, payload.byteOffset, payload.byteLength); + if ( + view.getUint32(0, true) + !== littleEndianMagic(WPK_FORK_ACTIVATION_CONTINUATIONS_MAGIC) + ) { + throw new Error(`${context}: wrong magic`); + } + const version = view.getUint16(4, true); + if (version !== WPK_FORK_ACTIVATION_CONTINUATIONS_VERSION) { + throw new Error(`${context}: unsupported version ${version}`); + } + if ( + view.getUint16(6, true) !== WPK_FORK_ACTIVATION_CONTINUATIONS_HEADER_SIZE + || view.getUint16(8, true) !== WPK_FORK_ACTIVATION_CONTINUATIONS_ENTRY_SIZE + ) { + throw new Error(`${context}: header or entry size is inconsistent`); + } + const flags = view.getUint16(10, true); + if ((flags & ~WPK_FORK_ACTIVATION_CONTINUATIONS_KNOWN_FLAGS) !== 0) { + throw new Error(`${context}: unknown flags 0x${flags.toString(16)}`); + } + const count = view.getUint32(12, true); + if (view.getBigUint64(16, true) !== 0n) { + throw new Error(`${context}: reserved header field is nonzero`); + } + const expectedSize = WPK_FORK_ACTIVATION_CONTINUATIONS_HEADER_SIZE + + count * WPK_FORK_ACTIVATION_CONTINUATIONS_ENTRY_SIZE; + if (payload.byteLength !== expectedSize) { + throw new Error(`${context}: entry count is inconsistent with payload size`); + } + if (count === 0) { + throw new Error(`${context}: manifest is empty`); + } + + const continuations: ForkActivationContinuation[] = []; + let previousActivation = -1; + for (let index = 0; index < count; index++) { + const offset = WPK_FORK_ACTIVATION_CONTINUATIONS_HEADER_SIZE + + index * WPK_FORK_ACTIVATION_CONTINUATIONS_ENTRY_SIZE; + const activationId = view.getUint32(offset, true); + const entryFlags = view.getUint32(offset + 4, true); + if ( + (entryFlags & ~WPK_FORK_ACTIVATION_CONTINUATIONS_ENTRY_KNOWN_FLAGS) !== 0 + ) { + throw new Error( + `${context} entry ${index}: unknown flags 0x${entryFlags.toString(16)}`, + ); + } + if (activationId <= previousActivation) { + throw new Error( + `${context} entry ${index}: activations are duplicated or unordered`, + ); + } + const root = view.getBigUint64(offset + 8, true); + if (root === 0n) { + throw new Error(`${context} entry ${index}: continuation root is zero`); + } + continuations.push({ activationId, root }); + previousActivation = activationId; + } + return continuations; +} + +const IMPORTED_GLOBAL_BINDING_KINDS = new Set( + Object.values(ForkImportedGlobalBindingKind), +); + +function importedGlobalBindingFlags(binding: ForkImportedGlobalBinding): number { + return (binding.mutable ? WPK_FORK_IMPORTED_GLOBALS_FLAG_MUTABLE : 0) + | (binding.shared ? WPK_FORK_IMPORTED_GLOBALS_FLAG_SHARED : 0); +} + +function validateImportedGlobalBinding( + binding: ForkImportedGlobalBinding, + context: string, +): void { + checkedU32(binding.consumerActivation, `${context} consumer activation`); + checkedU32(binding.consumerOwner, `${context} consumer owner`, false); + checkedU32(binding.sourceActivation, `${context} source activation`); + checkedU32(binding.sourceOwner, `${context} source owner`); + checkedU32(binding.reserved, `${context} reserved field`); + checkedU32(binding.recipeId, `${context} recipe id`); + checkedU64(binding.rawBits, `${context} raw bits`); + if (!IMPORTED_GLOBAL_BINDING_KINDS.has(binding.kind)) { + throw new Error(`${context}: unknown binding kind ${binding.kind}`); + } + if (!IMPORTED_GLOBAL_TYPE_CODES.has(binding.typeCode)) { + throw new Error(`${context}: unknown value type ${binding.typeCode}`); + } + + const zeroSource = (): boolean => + binding.sourceActivation === 0 && binding.sourceOwner === 0; + const zeroRaw = (): boolean => binding.rawBits === 0n; + switch (binding.kind) { + case ForkImportedGlobalBindingKind.RawNumber: + case ForkImportedGlobalBindingKind.RawBigInt: + if (!zeroSource() || binding.reserved !== 0 || binding.recipeId !== 0) { + throw new Error(`${context}: raw scalar binding has nonzero owner fields`); + } + break; + case ForkImportedGlobalBindingKind.RawReference: + if (!zeroSource() || binding.reserved !== 0 || !zeroRaw()) { + throw new Error(`${context}: raw reference binding has nonzero owner fields`); + } + break; + case ForkImportedGlobalBindingKind.ActivationGlobal: + if ( + binding.sourceOwner === 0 + || binding.reserved !== 0 + || binding.recipeId !== 0 + || !zeroRaw() + ) { + throw new Error(`${context}: activation-global binding fields are inconsistent`); + } + break; + case ForkImportedGlobalBindingKind.BaseImport: + if ( + binding.reserved !== 0 + || !zeroSource() + || binding.recipeId !== 0 + || !zeroRaw() + ) { + throw new Error(`${context}: base-import binding fields are inconsistent`); + } + break; + } +} + +/** + * Encode the process-wide provenance of every imported global declaration. + * + * Entries are declaration-owned, while repeated `(module,name)` properties + * intentionally carry the same raw recipe/provider coordinates. This keeps + * independent Wasm coercions valid without losing the JavaScript binding that + * produced them. + */ +export function encodeForkImportedGlobalBindings( + bindings: readonly ForkImportedGlobalBinding[], +): Uint8Array { + if (bindings.length > 0xffff_ffff) { + throw new RangeError("imported-global binding count exceeds u32"); + } + const payloadSize = WPK_FORK_IMPORTED_GLOBAL_BINDINGS_HEADER_SIZE + + bindings.length * WPK_FORK_IMPORTED_GLOBAL_BINDINGS_ENTRY_SIZE; + if (!Number.isSafeInteger(payloadSize)) { + throw new RangeError( + "imported-global binding payload size exceeds JavaScript safe integer", + ); + } + const payload = new Uint8Array(payloadSize); + const view = new DataView(payload.buffer); + view.setUint32( + 0, + littleEndianMagic(WPK_FORK_IMPORTED_GLOBAL_BINDINGS_MAGIC), + true, + ); + view.setUint16(4, WPK_FORK_IMPORTED_GLOBAL_BINDINGS_VERSION, true); + view.setUint16(6, WPK_FORK_IMPORTED_GLOBAL_BINDINGS_HEADER_SIZE, true); + view.setUint16(8, WPK_FORK_IMPORTED_GLOBAL_BINDINGS_ENTRY_SIZE, true); + view.setUint16(10, WPK_FORK_IMPORTED_GLOBAL_BINDINGS_KNOWN_FLAGS, true); + view.setUint32(12, bindings.length, true); + view.setBigUint64(16, 0n, true); + + let previousKey = ""; + for (const [index, binding] of bindings.entries()) { + const context = `imported-global binding ${index}`; + validateImportedGlobalBinding(binding, context); + const key = `${binding.consumerActivation.toString(16).padStart(8, "0")}:` + + binding.consumerOwner.toString(16).padStart(8, "0"); + if (key <= previousKey) { + throw new Error( + `${context}: consumer declarations must be unique and strictly ordered`, + ); + } + previousKey = key; + const offset = WPK_FORK_IMPORTED_GLOBAL_BINDINGS_HEADER_SIZE + + index * WPK_FORK_IMPORTED_GLOBAL_BINDINGS_ENTRY_SIZE; + view.setUint32(offset, binding.consumerActivation, true); + view.setUint32(offset + 4, binding.consumerOwner, true); + view.setUint32(offset + 8, binding.sourceActivation, true); + view.setUint32(offset + 12, binding.sourceOwner, true); + view.setUint32(offset + 16, binding.reserved, true); + view.setUint32(offset + 20, binding.recipeId, true); + view.setBigUint64(offset + 24, binding.rawBits, true); + view.setUint8(offset + 32, binding.kind); + view.setUint8(offset + 33, importedGlobalBindingFlags(binding)); + view.setUint8(offset + 34, binding.typeCode); + } + return payload; +} + +export function decodeForkImportedGlobalBindings( + payload: Uint8Array, + context = "module-state imported-global bindings", +): ForkImportedGlobalBinding[] { + if (payload.byteLength < WPK_FORK_IMPORTED_GLOBAL_BINDINGS_HEADER_SIZE) { + throw new Error(`${context}: payload is truncated`); + } + const view = new DataView(payload.buffer, payload.byteOffset, payload.byteLength); + if ( + view.getUint32(0, true) + !== littleEndianMagic(WPK_FORK_IMPORTED_GLOBAL_BINDINGS_MAGIC) + ) { + throw new Error(`${context}: wrong magic`); + } + if (view.getUint16(4, true) !== WPK_FORK_IMPORTED_GLOBAL_BINDINGS_VERSION) { + throw new Error(`${context}: unsupported version ${view.getUint16(4, true)}`); + } + if ( + view.getUint16(6, true) !== WPK_FORK_IMPORTED_GLOBAL_BINDINGS_HEADER_SIZE + || view.getUint16(8, true) !== WPK_FORK_IMPORTED_GLOBAL_BINDINGS_ENTRY_SIZE + ) { + throw new Error(`${context}: header or entry size is inconsistent`); + } + const flags = view.getUint16(10, true); + if ((flags & ~WPK_FORK_IMPORTED_GLOBAL_BINDINGS_KNOWN_FLAGS) !== 0) { + throw new Error(`${context}: unknown flags 0x${flags.toString(16)}`); + } + const count = view.getUint32(12, true); + if (view.getBigUint64(16, true) !== 0n) { + throw new Error(`${context}: reserved header field is nonzero`); + } + const expectedSize = WPK_FORK_IMPORTED_GLOBAL_BINDINGS_HEADER_SIZE + + count * WPK_FORK_IMPORTED_GLOBAL_BINDINGS_ENTRY_SIZE; + if (payload.byteLength !== expectedSize) { + throw new Error(`${context}: entry count is inconsistent with payload size`); + } + + const bindings: ForkImportedGlobalBinding[] = []; + let previousKey = ""; + for (let index = 0; index < count; index++) { + const offset = WPK_FORK_IMPORTED_GLOBAL_BINDINGS_HEADER_SIZE + + index * WPK_FORK_IMPORTED_GLOBAL_BINDINGS_ENTRY_SIZE; + if (view.getUint8(offset + 35) !== 0 || view.getUint32(offset + 36, true) !== 0) { + throw new Error(`${context} entry ${index}: reserved fields are nonzero`); + } + const entryFlags = view.getUint8(offset + 33); + if ((entryFlags & ~WPK_FORK_IMPORTED_GLOBALS_KNOWN_FLAGS) !== 0) { + throw new Error( + `${context} entry ${index}: unknown flags 0x${entryFlags.toString(16)}`, + ); + } + const binding: ForkImportedGlobalBinding = { + consumerActivation: view.getUint32(offset, true), + consumerOwner: view.getUint32(offset + 4, true), + sourceActivation: view.getUint32(offset + 8, true), + sourceOwner: view.getUint32(offset + 12, true), + reserved: view.getUint32(offset + 16, true), + recipeId: view.getUint32(offset + 20, true), + rawBits: view.getBigUint64(offset + 24, true), + kind: view.getUint8(offset + 32) as ForkImportedGlobalBindingKind, + mutable: (entryFlags & WPK_FORK_IMPORTED_GLOBALS_FLAG_MUTABLE) !== 0, + shared: (entryFlags & WPK_FORK_IMPORTED_GLOBALS_FLAG_SHARED) !== 0, + typeCode: view.getUint8(offset + 34), + }; + validateImportedGlobalBinding(binding, `${context} entry ${index}`); + const key = `${binding.consumerActivation.toString(16).padStart(8, "0")}:` + + binding.consumerOwner.toString(16).padStart(8, "0"); + if (key <= previousKey) { + throw new Error( + `${context} entry ${index}: consumer declarations are duplicated or unordered`, + ); + } + previousKey = key; + bindings.push(binding); + } + return bindings; +} + +const IMPORTED_TABLE_BINDING_KINDS = new Set( + Object.values(ForkImportedTableBindingKind), +); + +function validateImportedTableBinding( + binding: ForkImportedTableBinding, + context: string, +): void { + checkedU32(binding.consumerActivation, `${context} consumer activation`); + checkedU32(binding.consumerOwner, `${context} consumer owner`, false); + checkedU32(binding.sourceActivation, `${context} source activation`); + checkedU32(binding.sourceOwner, `${context} source owner`); + checkedU32(binding.reserved, `${context} reserved field`); + if (!IMPORTED_TABLE_BINDING_KINDS.has(binding.kind)) { + throw new Error(`${context}: unknown binding kind ${binding.kind}`); + } + switch (binding.kind) { + case ForkImportedTableBindingKind.ActivationTable: + if (binding.sourceOwner === 0 || binding.reserved !== 0) { + throw new Error(`${context}: activation-table binding fields are inconsistent`); + } + break; + case ForkImportedTableBindingKind.BaseImport: + if ( + binding.sourceActivation !== 0 + || binding.sourceOwner !== 0 + || binding.reserved !== 0 + ) { + throw new Error(`${context}: base-import binding fields are inconsistent`); + } + break; + } +} + +/** + * Encode the process-wide identity graph for imported Table declarations. + * + * KFMS table records own element contents. This manifest separately owns + * pre-instantiation identity: every consumer declaration is wired either to a + * table exported by another activation or re-resolved through the child's + * exact base import builder. The split avoids copying a full table per + * activation while preserving aliases. + */ +export function encodeForkImportedTableBindings( + bindings: readonly ForkImportedTableBinding[], +): Uint8Array { + if (bindings.length > 0xffff_ffff) { + throw new RangeError("imported-table binding count exceeds u32"); + } + const payloadSize = WPK_FORK_IMPORTED_TABLE_BINDINGS_HEADER_SIZE + + bindings.length * WPK_FORK_IMPORTED_TABLE_BINDINGS_ENTRY_SIZE; + if (!Number.isSafeInteger(payloadSize)) { + throw new RangeError( + "imported-table binding payload size exceeds JavaScript safe integer", + ); + } + const payload = new Uint8Array(payloadSize); + const view = new DataView(payload.buffer); + view.setUint32( + 0, + littleEndianMagic(WPK_FORK_IMPORTED_TABLE_BINDINGS_MAGIC), + true, + ); + view.setUint16(4, WPK_FORK_IMPORTED_TABLE_BINDINGS_VERSION, true); + view.setUint16(6, WPK_FORK_IMPORTED_TABLE_BINDINGS_HEADER_SIZE, true); + view.setUint16(8, WPK_FORK_IMPORTED_TABLE_BINDINGS_ENTRY_SIZE, true); + view.setUint16(10, WPK_FORK_IMPORTED_TABLE_BINDINGS_KNOWN_FLAGS, true); + view.setUint32(12, bindings.length, true); + view.setBigUint64(16, 0n, true); + + let previousKey = ""; + for (const [index, binding] of bindings.entries()) { + const context = `imported-table binding ${index}`; + validateImportedTableBinding(binding, context); + const key = `${binding.consumerActivation.toString(16).padStart(8, "0")}:` + + binding.consumerOwner.toString(16).padStart(8, "0"); + if (key <= previousKey) { + throw new Error( + `${context}: consumer declarations must be unique and strictly ordered`, + ); + } + previousKey = key; + const offset = WPK_FORK_IMPORTED_TABLE_BINDINGS_HEADER_SIZE + + index * WPK_FORK_IMPORTED_TABLE_BINDINGS_ENTRY_SIZE; + view.setUint32(offset, binding.consumerActivation, true); + view.setUint32(offset + 4, binding.consumerOwner, true); + view.setUint32(offset + 8, binding.sourceActivation, true); + view.setUint32(offset + 12, binding.sourceOwner, true); + view.setUint32(offset + 16, binding.reserved, true); + view.setUint8(offset + 20, binding.kind); + } + return payload; +} + +export function decodeForkImportedTableBindings( + payload: Uint8Array, + context = "module-state imported-table bindings", +): ForkImportedTableBinding[] { + if (payload.byteLength < WPK_FORK_IMPORTED_TABLE_BINDINGS_HEADER_SIZE) { + throw new Error(`${context}: payload is truncated`); + } + const view = new DataView(payload.buffer, payload.byteOffset, payload.byteLength); + if ( + view.getUint32(0, true) + !== littleEndianMagic(WPK_FORK_IMPORTED_TABLE_BINDINGS_MAGIC) + ) { + throw new Error(`${context}: wrong magic`); + } + if (view.getUint16(4, true) !== WPK_FORK_IMPORTED_TABLE_BINDINGS_VERSION) { + throw new Error(`${context}: unsupported version ${view.getUint16(4, true)}`); + } + if ( + view.getUint16(6, true) !== WPK_FORK_IMPORTED_TABLE_BINDINGS_HEADER_SIZE + || view.getUint16(8, true) !== WPK_FORK_IMPORTED_TABLE_BINDINGS_ENTRY_SIZE + ) { + throw new Error(`${context}: header or entry size is inconsistent`); + } + const flags = view.getUint16(10, true); + if ((flags & ~WPK_FORK_IMPORTED_TABLE_BINDINGS_KNOWN_FLAGS) !== 0) { + throw new Error(`${context}: unknown flags 0x${flags.toString(16)}`); + } + const count = view.getUint32(12, true); + if (view.getBigUint64(16, true) !== 0n) { + throw new Error(`${context}: reserved header field is nonzero`); + } + const expectedSize = WPK_FORK_IMPORTED_TABLE_BINDINGS_HEADER_SIZE + + count * WPK_FORK_IMPORTED_TABLE_BINDINGS_ENTRY_SIZE; + if (payload.byteLength !== expectedSize) { + throw new Error(`${context}: entry count is inconsistent with payload size`); + } + + const bindings: ForkImportedTableBinding[] = []; + let previousKey = ""; + for (let index = 0; index < count; index++) { + const offset = WPK_FORK_IMPORTED_TABLE_BINDINGS_HEADER_SIZE + + index * WPK_FORK_IMPORTED_TABLE_BINDINGS_ENTRY_SIZE; + if (view.getUint8(offset + 21) !== 0 || view.getUint16(offset + 22, true) !== 0) { + throw new Error(`${context} entry ${index}: reserved fields are nonzero`); + } + const binding: ForkImportedTableBinding = { + consumerActivation: view.getUint32(offset, true), + consumerOwner: view.getUint32(offset + 4, true), + sourceActivation: view.getUint32(offset + 8, true), + sourceOwner: view.getUint32(offset + 12, true), + reserved: view.getUint32(offset + 16, true), + kind: view.getUint8(offset + 20) as ForkImportedTableBindingKind, + }; + validateImportedTableBinding(binding, `${context} entry ${index}`); + const key = `${binding.consumerActivation.toString(16).padStart(8, "0")}:` + + binding.consumerOwner.toString(16).padStart(8, "0"); + if (key <= previousKey) { + throw new Error( + `${context} entry ${index}: consumer declarations are duplicated or unordered`, + ); + } + previousKey = key; + bindings.push(binding); + } + return bindings; +} + +export function importedGlobalBindingsForChild( + records: readonly ForkModuleStateRecord[], +): ForkImportedGlobalBinding[] { + const matches = records.filter( + (record) => record.kind === ForkModuleStateRecordKind.ImportedGlobalBindings, + ); + if (matches.length !== 1) { + throw new Error( + `module-state arena has ${matches.length} imported-global binding records; expected one`, + ); + } + const record = matches[0]!; + if ( + record.activationId !== 0 + || record.ownerId !== WPK_FORK_IMPORTED_GLOBAL_BINDINGS_OWNER + ) { + throw new Error("module-state imported-global bindings have invalid ownership"); + } + return decodeForkImportedGlobalBindings(record.payload); +} + +export function importedTableBindingsForChild( + records: readonly ForkModuleStateRecord[], +): ForkImportedTableBinding[] { + const matches = records.filter( + (record) => record.kind === ForkModuleStateRecordKind.ImportedTableBindings, + ); + if (matches.length !== 1) { + throw new Error( + `module-state arena has ${matches.length} imported-table binding records; expected one`, + ); + } + const record = matches[0]!; + if ( + record.activationId !== 0 + || record.ownerId !== WPK_FORK_IMPORTED_TABLE_BINDINGS_OWNER + ) { + throw new Error("module-state imported-table bindings have invalid ownership"); + } + return decodeForkImportedTableBindings(record.payload); +} + +export function activationContinuationsForChild( + records: readonly ForkModuleStateRecordView[], + ptrWidth: 4 | 8, +): ForkActivationContinuation[] { + const matches = records.filter( + (record) => record.kind === ForkModuleStateRecordKind.ActivationContinuations, + ); + if (matches.length !== 1) { + throw new Error( + `module-state arena has ${matches.length} activation-continuation records; expected one`, + ); + } + const record = matches[0]!; + if ( + record.activationId !== 0 + || record.ownerId !== WPK_FORK_ACTIVATION_CONTINUATIONS_OWNER + ) { + throw new Error("module-state activation continuations have invalid ownership"); + } + const continuations = decodeForkActivationContinuations(record.payload); + const replay = validateForkReplayEventWire(replayEventsForChild(records)); + assertActivationContinuationSet( + continuations, + replay.activationIds, + "module-state activation continuations", + ); + const maxRoot = ptrWidth === 4 ? 0xffff_ffffn : 0xffff_ffff_ffff_ffffn; + for (const continuation of continuations) { + if (continuation.root > maxRoot) { + throw new RangeError( + `module-state activation ${continuation.activationId} continuation root ` + + `does not fit wasm${ptrWidth * 8}`, + ); + } + } + return continuations; +} + +/** Select and validate the ordered process replay-event segment stream. */ +export function replayEventsForChild( + records: readonly ForkModuleStateRecordView[], +): ForkReplayEventWire { + let manifest: Uint8Array | null = null; + for (const record of records) { + if ( + record.kind !== ForkModuleStateRecordKind.ReplayEventSegment + && record.kind !== ForkModuleStateRecordKind.ReplayEvents + ) { + continue; + } + if ( + record.activationId !== 0 + || record.ownerId !== WPK_FORK_MODULE_STATE_REPLAY_EVENTS_OWNER + ) { + throw new Error("module-state replay events have invalid process ownership"); + } + if (record.kind === ForkModuleStateRecordKind.ReplayEventSegment) { + if (manifest) { + throw new Error( + "module-state replay-event segment follows its final manifest", + ); + } + continue; + } + if (manifest) { + throw new Error("module-state arena has duplicate process replay-event records"); + } + manifest = record.payload; + } + if (!manifest) { + throw new Error("module-state arena has no process replay-event manifest"); + } + const segments: Iterable = { + *[Symbol.iterator]() { + for (const record of records) { + if (record.kind === ForkModuleStateRecordKind.ReplayEventSegment) { + yield record.payload; + } + } + }, + }; + const wire = { manifest, segments }; + validateForkReplayEventWire(wire); + return wire; +} + +export function decodeForkGlobalSnapshot( + payload: Uint8Array, + context = "module-state global", +): ForkGlobalSnapshot { + if (payload.byteLength < GLOBAL_HEADER_SIZE) { + throw new Error(`${context}: mutable-global payload is truncated`); + } + const valueSizes = new Map([ + [WPK_FORK_MODULE_STATE_GLOBAL_TYPE_I32, 4], + [WPK_FORK_MODULE_STATE_GLOBAL_TYPE_I64, 8], + [WPK_FORK_MODULE_STATE_GLOBAL_TYPE_F32, 4], + [WPK_FORK_MODULE_STATE_GLOBAL_TYPE_F64, 8], + [WPK_FORK_MODULE_STATE_GLOBAL_TYPE_V128, 16], + [WPK_FORK_MODULE_STATE_GLOBAL_TYPE_FUNCREF, 4], + [WPK_FORK_MODULE_STATE_GLOBAL_TYPE_EXTERNREF, 4], + [WPK_FORK_MODULE_STATE_GLOBAL_TYPE_EXNREF, 4], + [WPK_FORK_MODULE_STATE_GLOBAL_TYPE_ANYREF, 4], + ]); + const view = new DataView(payload.buffer, payload.byteOffset, payload.byteLength); + const type = view.getUint8(0); + const expectedValueSize = valueSizes.get(type); + if (expectedValueSize === undefined) { + throw new Error(`${context}: unknown mutable-global value type ${type}`); + } + const valueSize = view.getUint8(1); + if ( + valueSize !== expectedValueSize + || payload.byteLength !== GLOBAL_HEADER_SIZE + expectedValueSize + ) { + throw new Error(`${context}: mutable-global value size is inconsistent`); + } + if (view.getUint16(2, true) !== 0 || view.getUint32(4, true) !== 0) { + throw new Error(`${context}: mutable-global reserved fields are nonzero`); + } + const value = payload.slice(GLOBAL_HEADER_SIZE); + const snapshot: ForkGlobalSnapshot = { typeCode: type, value }; + if ( + type === WPK_FORK_MODULE_STATE_GLOBAL_TYPE_FUNCREF + || type === WPK_FORK_MODULE_STATE_GLOBAL_TYPE_EXTERNREF + || type === WPK_FORK_MODULE_STATE_GLOBAL_TYPE_EXNREF + || type === WPK_FORK_MODULE_STATE_GLOBAL_TYPE_ANYREF + ) { + snapshot.recipeId = new DataView( + value.buffer, + value.byteOffset, + value.byteLength, + ).getUint32(0, true); + } + return snapshot; +} + +export function findForkGlobalSnapshot( + records: readonly ForkModuleStateRecord[], + activationId: number, + ownerId: number, +): ForkGlobalSnapshot { + checkedU32(activationId, "global snapshot activation"); + checkedU32(ownerId, "global snapshot owner", false); + const matches = records.filter( + (record) => + record.kind === ForkModuleStateRecordKind.MutableGlobal + && record.activationId === activationId + && record.ownerId === ownerId, + ); + if (matches.length !== 1) { + throw new Error( + `module-state has ${matches.length} global snapshots for ` + + `${activationId}:${ownerId}; expected one`, + ); + } + return decodeForkGlobalSnapshot( + matches[0]!.payload, + `module-state global ${activationId}:${ownerId}`, + ); +} + +function validateRecordOwnership( + records: readonly ForkModuleStateRecord[], + ptrWidth?: 4 | 8, +): void { + const modules = new Set(); + const owned = new Set(); + const tables = new Map(); + const tablePageCounts = new Map(); + const lastTablePages = new Map(); + let replayEventsSeen = false; + let replayEventSegments = 0; + let referenceRecipeSeen = false; + let referenceRecipeSegments = 0; + let importedGlobalBindingsSeen = false; + let importedTableBindingsSeen = false; + let replayEventManifest: Uint8Array | null = null; + let activationContinuations: ForkActivationContinuation[] | null = null; + + for (const [recordIndex, record] of records.entries()) { + const context = `module-state record ${recordIndex}`; + checkedU32(record.activationId, `${context} activation id`); + checkedU32(record.ownerId, `${context} owner id`); + if (record.kind === ForkModuleStateRecordKind.ReferenceRecipeSegment) { + if (record.activationId !== 0 || record.ownerId !== 1) { + throw new Error( + `${context}: reference-recipe segments must use process ownership`, + ); + } + if (referenceRecipeSeen) { + throw new Error( + `${context}: reference-recipe segment follows its final manifest`, + ); + } + referenceRecipeSegments++; + continue; + } + if (record.kind === ForkModuleStateRecordKind.ReplayEventSegment) { + if ( + record.activationId !== 0 + || record.ownerId !== WPK_FORK_MODULE_STATE_REPLAY_EVENTS_OWNER + ) { + throw new Error( + `${context}: replay-event segments must use process ownership`, + ); + } + if (replayEventsSeen) { + throw new Error( + `${context}: replay-event segment follows its final manifest`, + ); + } + replayEventSegments++; + continue; + } + if (record.kind === ForkModuleStateRecordKind.ReplayEvents) { + if ( + record.activationId !== 0 + || record.ownerId !== WPK_FORK_MODULE_STATE_REPLAY_EVENTS_OWNER + ) { + throw new Error(`${context}: replay events must use process ownership`); + } + if (replayEventsSeen) { + throw new Error(`${context}: duplicate process replay-event record`); + } + replayEventManifest = record.payload; + replayEventsSeen = true; + continue; + } + if (record.kind === ForkModuleStateRecordKind.ImportedGlobalBindings) { + if ( + record.activationId !== 0 + || record.ownerId !== WPK_FORK_IMPORTED_GLOBAL_BINDINGS_OWNER + ) { + throw new Error( + `${context}: imported-global bindings must use process ownership`, + ); + } + if (importedGlobalBindingsSeen) { + throw new Error(`${context}: duplicate imported-global binding record`); + } + decodeForkImportedGlobalBindings(record.payload, context); + importedGlobalBindingsSeen = true; + continue; + } + if (record.kind === ForkModuleStateRecordKind.ImportedTableBindings) { + if ( + record.activationId !== 0 + || record.ownerId !== WPK_FORK_IMPORTED_TABLE_BINDINGS_OWNER + ) { + throw new Error( + `${context}: imported-table bindings must use process ownership`, + ); + } + if (importedTableBindingsSeen) { + throw new Error(`${context}: duplicate imported-table binding record`); + } + decodeForkImportedTableBindings(record.payload, context); + importedTableBindingsSeen = true; + continue; + } + if (record.kind === ForkModuleStateRecordKind.ActivationContinuations) { + if ( + record.activationId !== 0 + || record.ownerId !== WPK_FORK_ACTIVATION_CONTINUATIONS_OWNER + ) { + throw new Error( + `${context}: activation continuations must use process ownership`, + ); + } + if (activationContinuations) { + throw new Error(`${context}: duplicate activation-continuation record`); + } + activationContinuations = decodeForkActivationContinuations( + record.payload, + context, + ); + if (ptrWidth === 4) { + for (const continuation of activationContinuations) { + if (continuation.root > 0xffff_ffffn) { + throw new RangeError( + `${context}: activation ${continuation.activationId} root does not fit wasm32`, + ); + } + } + } + continue; + } + if (record.kind === ForkModuleStateRecordKind.Module) { + if (record.ownerId !== 0) { + throw new Error(`${context}: module record must use owner id zero`); + } + if (modules.has(record.activationId)) { + throw new Error(`${context}: duplicate module activation ${record.activationId}`); + } + decodeModulePayload(record.payload, context); + modules.add(record.activationId); + continue; + } + + if (!modules.has(record.activationId)) { + throw new Error( + `${context}: state refers to undeclared module activation ${record.activationId}`, + ); + } + if (record.ownerId === 0) { + throw new Error(`${context}: state record has no explicit owner`); + } + + if (record.kind === ForkModuleStateRecordKind.TablePage) { + const key = tableKey(record.activationId, record.ownerId); + const table = tables.get(key); + if (!table) { + throw new Error(`${context}: table page precedes its owning table descriptor`); + } + const page = validateTablePage(record, table, context); + const previous = lastTablePages.get(key); + if (previous !== undefined && page.pageIndex <= previous) { + throw new Error(`${context}: table pages are not strictly increasing`); + } + lastTablePages.set(key, page.pageIndex); + tablePageCounts.set(key, (tablePageCounts.get(key) ?? 0) + 1); + continue; + } + + const key = ownerKey(record.kind, record.activationId, record.ownerId); + if (owned.has(key)) { + throw new Error( + `${context}: duplicate owner ${record.ownerId} for record kind ${record.kind}`, + ); + } + owned.add(key); + + if (record.kind === ForkModuleStateRecordKind.ReferenceRecipe) { + referenceRecipeSeen = true; + } + + if (record.kind === ForkModuleStateRecordKind.Table) { + const table = decodeTableDescriptor(record, context); + const tableIdentity = tableKey(record.activationId, record.ownerId); + if (tables.has(tableIdentity)) { + throw new Error(`${context}: duplicate table owner ${tableIdentity}`); + } + tables.set(tableIdentity, table); + tablePageCounts.set(tableIdentity, 0); + } else if (record.kind === ForkModuleStateRecordKind.MutableGlobal) { + decodeForkGlobalSnapshot(record.payload, context); + } else if (record.kind === ForkModuleStateRecordKind.ElementSegments) { + decodeElementSegments(record.payload, context); + } else if (record.kind === ForkModuleStateRecordKind.DataSegments) { + decodeDataSegments(record.payload, context); + } + } + + if (referenceRecipeSegments !== 0 && !referenceRecipeSeen) { + throw new Error( + `module-state has ${referenceRecipeSegments} reference-recipe segment(s) ` + + "without a final manifest", + ); + } + if (replayEventSegments !== 0 && !replayEventsSeen) { + throw new Error( + `module-state has ${replayEventSegments} replay-event segment(s) ` + + "without a final manifest", + ); + } + const replayActivationIds = replayEventManifest + ? validateForkReplayEventWire({ + manifest: replayEventManifest, + segments: { + *[Symbol.iterator]() { + for (const record of records) { + if (record.kind === ForkModuleStateRecordKind.ReplayEventSegment) { + yield record.payload; + } + } + }, + }, + }).activationIds + : null; + + for (const [key, table] of tables) { + const actual = tablePageCounts.get(key) ?? 0; + if (actual !== table.pageCount) { + throw new Error( + `module-state table ${key} declares ${table.pageCount} sparse pages, found ${actual}`, + ); + } + } + if (modules.size === 0) { + throw new Error("module-state arena has no declared module activation"); + } + if (activationContinuations) { + if (!replayActivationIds) { + throw new Error( + "module-state activation continuations have no replay-event manifest", + ); + } + assertActivationContinuationSet( + activationContinuations, + replayActivationIds, + "module-state activation continuations", + ); + for (const { activationId } of activationContinuations) { + if (!modules.has(activationId)) { + throw new Error( + `module-state activation continuation ${activationId} has no module descriptor`, + ); + } + } + } +} + +function decodeSparseTables( + records: readonly ForkModuleStateRecord[], +): DecodedForkSparseTableSnapshot[] { + validateRecordOwnership(records); + const tables = new Map(); + for (const [recordIndex, record] of records.entries()) { + if (record.kind === ForkModuleStateRecordKind.Table) { + const descriptor = decodeTableDescriptor( + record, + `module-state record ${recordIndex}`, + ); + tables.set(tableKey(record.activationId, record.ownerId), { + activationId: descriptor.activationId, + ownerId: descriptor.ownerId, + indexWidth: descriptor.indexWidth, + pageShift: descriptor.pageShift, + length: descriptor.length, + baselineLength: descriptor.baselineLength, + baselineFingerprint: descriptor.baselineFingerprint, + pages: [], + }); + } else if (record.kind === ForkModuleStateRecordKind.TablePage) { + const table = tables.get(tableKey(record.activationId, record.ownerId)); + if (!table) { + throw new Error(`module-state record ${recordIndex}: missing table descriptor`); + } + const descriptor: DecodedTableDescriptor = { + ...table, + flags: TABLE_FLAG_SPARSE_OVERRIDES, + pageCount: 0, + }; + const page = decodeTablePage( + record, + descriptor, + `module-state record ${recordIndex}`, + ); + table.pages.push({ pageIndex: page.pageIndex, runs: page.runs }); + } + } + return Array.from(tables.values()); +} + +/** + * Owner for one activation's versioned module-state recipe arena. + * + * The arena contains bytes only. Any temporary JS reference-to-recipe maps + * belong to the future coordinator and must be cleared before calling + * `release`; this class never turns such references into hidden GC roots. + */ +export class ForkModuleStateArena { + private root = 0; + private tail = 0; + private chunks: ArenaChunk[] = []; + private sealed = false; + private pending: PendingRecord | null = null; + private readonly payloadIndex = new Map(); + + constructor( + private readonly memory: WebAssembly.Memory, + readonly ptrWidth: 4 | 8, + private readonly allocate: ContinuationAllocate, + private readonly deallocate: ContinuationDeallocate, + private readonly label: string, + ) { + if (ptrWidth !== 4 && ptrWidth !== 8) { + throw new RangeError(`${label}: unsupported module-state pointer width ${ptrWidth}`); + } + } + + begin(): number { + if (this.root !== 0) { + throw new Error(`${this.label}: module-state arena is already active`); + } + const root = this.allocateChunk(WASM_PAGE_SIZE, 0, true); + this.root = root; + this.tail = root; + return root; + } + + attach(root: number | bigint): void { + if (this.root !== 0) { + throw new Error(`${this.label}: module-state arena is already active`); + } + const rootNumber = checkedPointer( + root, + this.ptrWidth, + `${this.label}: module-state root`, + false, + ); + if (rootNumber % WASM_PAGE_SIZE !== 0) { + throw new Error(`${this.label}: module-state root is not page-aligned`); + } + const chunks = this.validateChunks(rootNumber, true); + const records = this.decodeRecords(chunks, false); + validateRecordOwnership(records, this.ptrWidth); + const payloadIndex = this.buildPayloadIndex(chunks); + // Publish ownership only after the complete guest-controlled arena passes + // structural and semantic validation. Failed attachment must not release + // mappings that this host never safely adopted. + this.root = rootNumber; + this.tail = chunks[chunks.length - 1]!.addr; + this.chunks = chunks; + this.payloadIndex.clear(); + for (const [key, addresses] of payloadIndex) { + this.payloadIndex.set(key, addresses); + } + this.sealed = true; + } + + /** + * Copy one exact record from a sealed, guest-owned arena without adopting + * or copying every other payload. + * + * This validates the chunk chain and every record envelope, but deliberately + * leaves record-kind semantics to the selected payload's decoder. It exists + * for pre-launch owners such as the externref broker: a large dirty table + * must not be copied into JavaScript merely to inspect the small KFRV graph. + */ + inspectSealedRecordPayload( + root: number | bigint, + kind: ForkModuleStateRecordKind, + activationId: number, + ownerId: number, + ): Uint8Array { + const matches = this.inspectSealedRecordViews( + root, + [kind], + activationId, + ownerId, + ); + if (matches.length !== 1) { + throw new Error( + `${this.label}: ${matches.length === 0 ? "missing" : "duplicate"} ` + + `module-state record ${kind}:${activationId}:${ownerId}`, + ); + } + return matches[0]!.payload.slice(); + } + + /** + * Validate a sealed arena and expose selected payloads as zero-copy views. + * + * WHY: process owners must sometimes inspect a segmented reference/event + * stream before the child Worker exists. Copying every selected segment (or + * unrelated table page) would recreate a whole-transaction allocation + * boundary. Returned views remain valid only while `memory` and the sealed + * arena mappings remain alive. + */ + inspectSealedRecordViews( + root: number | bigint, + kinds: readonly ForkModuleStateRecordKind[], + activationId?: number, + ownerId?: number, + ): readonly ForkModuleStateRecordView[] { + if (this.root !== 0) { + throw new Error( + `${this.label}: cannot inspect while the arena owns another root`, + ); + } + if (kinds.length === 0) { + throw new RangeError(`${this.label}: no module-state record kinds selected`); + } + const selectedKinds = new Set(); + for (const kind of kinds) { + if (!RECORD_KINDS.has(kind)) { + throw new RangeError(`${this.label}: unknown module-state record kind ${kind}`); + } + selectedKinds.add(kind); + } + if ((activationId === undefined) !== (ownerId === undefined)) { + throw new Error( + `${this.label}: record activation and owner filters must be paired`, + ); + } + if (activationId !== undefined && ownerId !== undefined) { + checkedU32(activationId, `${this.label}: record activation id`); + checkedU32(ownerId, `${this.label}: record owner id`); + } + const rootNumber = checkedPointer( + root, + this.ptrWidth, + `${this.label}: module-state root`, + false, + ); + if (rootNumber % WASM_PAGE_SIZE !== 0) { + throw new Error(`${this.label}: module-state root is not page-aligned`); + } + + const chunks = this.validateChunks(rootNumber, true); + return Object.freeze( + this.decodeRecords(chunks, false).filter((record) => + selectedKinds.has(record.kind) + && ( + activationId === undefined + || ( + record.activationId === activationId + && record.ownerId === ownerId + ) + ) + ), + ); + } + + appendModule(record: ForkModuleDescriptorRecord): void { + this.appendRecord({ + kind: ForkModuleStateRecordKind.Module, + activationId: record.activationId, + ownerId: 0, + payload: encodeModulePayload(record), + }); + } + + appendElementSegmentState(state: ForkElementSegmentState): void { + this.appendRecord({ + kind: ForkModuleStateRecordKind.ElementSegments, + activationId: state.activationId, + ownerId: state.ownerId, + payload: encodeElementSegments(state), + }); + } + + appendDataSegmentState(state: ForkDataSegmentState): void { + this.appendRecord({ + kind: ForkModuleStateRecordKind.DataSegments, + activationId: state.activationId, + ownerId: state.ownerId, + payload: encodeDataSegments(state), + }); + } + + appendImportedGlobalBindings( + bindings: readonly ForkImportedGlobalBinding[], + ): void { + this.appendRecord({ + kind: ForkModuleStateRecordKind.ImportedGlobalBindings, + activationId: 0, + ownerId: WPK_FORK_IMPORTED_GLOBAL_BINDINGS_OWNER, + payload: encodeForkImportedGlobalBindings(bindings), + }); + } + + appendImportedTableBindings( + bindings: readonly ForkImportedTableBinding[], + ): void { + this.appendRecord({ + kind: ForkModuleStateRecordKind.ImportedTableBindings, + activationId: 0, + ownerId: WPK_FORK_IMPORTED_TABLE_BINDINGS_OWNER, + payload: encodeForkImportedTableBindings(bindings), + }); + } + + appendSparseTable(snapshot: ForkSparseTableSnapshot): void { + checkedU32(snapshot.activationId, "table activation id"); + checkedU32(snapshot.ownerId, "table owner id", false); + const descriptorPayload = encodeTableDescriptor(snapshot); + const descriptor = decodeTableDescriptor({ + kind: ForkModuleStateRecordKind.Table, + activationId: snapshot.activationId, + ownerId: snapshot.ownerId, + payload: descriptorPayload, + }, "table snapshot"); + // Validate the complete logical snapshot before publishing its descriptor. + // This pass allocates no page payloads, so table size does not become a + // second whole-transaction memory requirement in the host. + let previousPageIndex: bigint | null = null; + for (const page of snapshot.pages) { + previousPageIndex = validateSparseTablePage( + descriptor, + page, + previousPageIndex, + ).pageIndex; + } + + this.appendRecord({ + kind: ForkModuleStateRecordKind.Table, + activationId: snapshot.activationId, + ownerId: snapshot.ownerId, + payload: descriptorPayload, + }); + previousPageIndex = null; + for (const page of snapshot.pages) { + const payload = encodeTablePage(descriptor, page, previousPageIndex); + this.appendRecord({ + kind: ForkModuleStateRecordKind.TablePage, + activationId: snapshot.activationId, + ownerId: snapshot.ownerId, + payload, + }); + previousPageIndex = checkedU64(page.pageIndex, "table page index"); + } + } + + appendRecord(record: ForkModuleStateRecord): void { + if (!(record.payload instanceof Uint8Array)) { + throw new TypeError(`${this.label}: module-state record payload must be Uint8Array`); + } + const payloadAddr = this.reserveRecord( + record.kind, + record.activationId, + record.ownerId, + record.payload.byteLength, + ); + const payloadNumber = checkedPointer( + payloadAddr, + this.ptrWidth, + `${this.label}: reserved module-state payload`, + false, + ); + new Uint8Array( + this.memory.buffer, + payloadNumber, + record.payload.byteLength, + ).set(record.payload); + this.commitRecord(payloadAddr); + } + + appendReplayEvents(source: ForkReplayEventCaptureSource): void { + // WHY: segments are committed before the small final manifest. The + // manifest is the transaction marker, so a failed page allocation cannot + // make a truncated journal appear complete in a copied child arena. + for (const payload of source.capturedSegmentPayloads()) { + this.appendRecord({ + kind: ForkModuleStateRecordKind.ReplayEventSegment, + activationId: 0, + ownerId: WPK_FORK_MODULE_STATE_REPLAY_EVENTS_OWNER, + payload, + }); + } + this.appendRecord({ + kind: ForkModuleStateRecordKind.ReplayEvents, + activationId: 0, + ownerId: WPK_FORK_MODULE_STATE_REPLAY_EVENTS_OWNER, + payload: source.capturedManifestPayload(), + }); + } + + appendActivationContinuations( + continuations: readonly ForkActivationContinuation[], + ): void { + this.appendRecord({ + kind: ForkModuleStateRecordKind.ActivationContinuations, + activationId: 0, + ownerId: WPK_FORK_ACTIVATION_CONTINUATIONS_OWNER, + payload: encodeForkActivationContinuations(continuations), + }); + } + + /** + * Reserve one record payload for an instrumented Wasm snapshot helper. + * + * The record is not reachable through a chunk's committed `used` boundary + * until `commitRecord` succeeds. A trap or encoding failure can therefore + * release the whole arena without publishing partial recipe bytes. + */ + reserveRecord( + kind: number, + activationId: number, + ownerId: number, + payloadSize: number | bigint, + ): number | bigint { + this.requireWritable(); + if (this.pending) { + throw new Error(`${this.label}: a module-state record reservation is already pending`); + } + if (!RECORD_KINDS.has(kind)) { + throw new RangeError(`${this.label}: unknown module-state record kind ${kind}`); + } + checkedU32(activationId, `${this.label}: record activation id`); + checkedU32(ownerId, `${this.label}: record owner id`); + const payloadBytes = checkedPointer( + payloadSize, + this.ptrWidth, + `${this.label}: module-state payload size`, + true, + ); + const totalSize = alignUp( + RECORD_HEADER_SIZE + payloadBytes, + FORK_MODULE_STATE_RECORD_ALIGNMENT, + ); + if (totalSize > 0xffff_ffff) { + throw new RangeError(`${this.label}: module-state record exceeds u32 size`); + } + let chunk = this.chunks[this.chunks.length - 1]!; + if (totalSize > chunk.size - chunk.used) { + const capacity = alignUp( + Math.max(WASM_PAGE_SIZE, chunkHeaderSize(this.ptrWidth) + totalSize), + WASM_PAGE_SIZE, + ); + const next = this.allocateChunk(capacity, this.tail, false); + writePointer( + this.memory, + this.ptrWidth, + this.tail + chunkOffset(this.ptrWidth, 2), + next, + ); + this.tail = next; + chunk = this.chunks[this.chunks.length - 1]!; + } + const recordAddr = chunk.addr + chunk.used; + checkedMemoryRange( + this.memory, + recordAddr, + totalSize, + `${this.label}: module-state record`, + ); + const view = new DataView(this.memory.buffer); + view.setUint32(recordAddr, RECORD_MAGIC, true); + view.setUint16(recordAddr + 4, FORK_MODULE_STATE_RECORD_VERSION, true); + view.setUint16(recordAddr + 6, kind, true); + view.setUint32(recordAddr + 8, totalSize, true); + view.setUint32(recordAddr + 12, payloadBytes, true); + view.setUint32(recordAddr + 16, activationId, true); + view.setUint32(recordAddr + 20, ownerId, true); + new Uint8Array( + this.memory.buffer, + recordAddr + RECORD_HEADER_SIZE, + totalSize - RECORD_HEADER_SIZE, + ).fill(0); + const payloadAddr = recordAddr + RECORD_HEADER_SIZE; + this.pending = { + chunk, + kind: kind as ForkModuleStateRecordKind, + activationId, + ownerId, + payloadAddr, + totalSize, + payloadSize: payloadBytes, + }; + return this.ptrWidth === 8 ? BigInt(payloadAddr) : payloadAddr; + } + + commitRecord(payloadAddr: number | bigint): void { + this.requireWritable(); + const payloadNumber = checkedPointer( + payloadAddr, + this.ptrWidth, + `${this.label}: module-state payload commit`, + false, + ); + const pending = this.pending; + if (!pending || pending.payloadAddr !== payloadNumber) { + throw new Error(`${this.label}: module-state commit does not match reservation`); + } + const chunk = this.chunks[this.chunks.length - 1]; + if (!chunk || chunk !== pending.chunk) { + throw new Error(`${this.label}: pending module-state record is not in the active chunk`); + } + // Publish used/count only after the complete record and zero padding are + // visible. A sealed root can therefore never expose a partial TLV. + const paddingSize = pending.totalSize - RECORD_HEADER_SIZE - pending.payloadSize; + requireZeroBytes( + new Uint8Array( + this.memory.buffer, + pending.payloadAddr + pending.payloadSize, + paddingSize, + ), + `${this.label}: pending module-state record`, + ); + chunk.used += pending.totalSize; + chunk.recordCount++; + writePointer( + this.memory, + this.ptrWidth, + chunk.addr + chunkOffset(this.ptrWidth, 4), + chunk.used, + ); + new DataView(this.memory.buffer).setUint32( + chunk.addr + chunkRecordCountOffset(this.ptrWidth), + chunk.recordCount, + true, + ); + const key = ownerKey( + pending.kind, + pending.activationId, + pending.ownerId, + ); + const addresses = this.payloadIndex.get(key) ?? []; + addresses.push(pending.payloadAddr); + this.payloadIndex.set(key, addresses); + this.pending = null; + } + + /** + * Resolve one committed record without copying its payload. + * + * The generated restore helper addresses table pages by ordinal, so keeping + * this index linearizes attachment once and makes large-table replay O(n) + * instead of rescanning the arena for every page. + */ + findRecord( + kind: number, + activationId: number, + ownerId: number, + ordinal: number, + ): number | bigint { + if (!this.sealed || this.root === 0) { + throw new Error(`${this.label}: cannot find a record in an unsealed arena`); + } + if (!RECORD_KINDS.has(kind)) { + throw new RangeError(`${this.label}: unknown module-state record kind ${kind}`); + } + checkedU32(activationId, `${this.label}: record activation id`); + checkedU32(ownerId, `${this.label}: record owner id`); + checkedU32(ordinal, `${this.label}: record ordinal`); + const addresses = this.payloadIndex.get(ownerKey( + kind as ForkModuleStateRecordKind, + activationId, + ownerId, + )); + const payload = addresses?.[ordinal]; + if (payload === undefined) { + throw new Error( + `${this.label}: missing module-state record ` + + `${kind}:${activationId}:${ownerId}:${ordinal}`, + ); + } + return this.ptrWidth === 8 ? BigInt(payload) : payload; + } + + seal(): number { + this.requireWritable(); + if (this.pending) { + throw new Error(`${this.label}: cannot seal with a pending module-state record`); + } + const validatedChunks = this.validateChunks(this.root, false); + if ( + validatedChunks.length !== this.chunks.length + || validatedChunks.some((chunk, index) => ( + chunk.addr !== this.chunks[index]!.addr + || chunk.size !== this.chunks[index]!.size + )) + ) { + throw new Error(`${this.label}: module-state chunk ownership changed before seal`); + } + const records = this.decodeRecords(validatedChunks, false); + validateRecordOwnership(records, this.ptrWidth); + for (let index = this.chunks.length - 1; index >= 1; index--) { + this.writeChunkFlags(this.chunks[index]!.addr, CHUNK_FLAG_SEALED); + } + // Root is the commit point. Child attachment refuses an arena unless this + // flag is present, so it cannot race a partially populated tail. + this.writeChunkFlags(this.root, CHUNK_FLAG_ROOT | CHUNK_FLAG_SEALED); + this.sealed = true; + return this.root; + } + + records(): ForkModuleStateRecord[] { + if (!this.sealed || this.root === 0) { + throw new Error(`${this.label}: module-state arena is not sealed`); + } + return this.decodeRecords(this.chunks); + } + + /** + * Zero-copy counterpart to `records()` for bounded streaming decoders. + * + * Callers must finish using these views before `release()` recycles the arena + * mappings. The returned record envelopes and underlying sealed bytes are + * immutable by contract. + */ + recordViews(): readonly ForkModuleStateRecordView[] { + if (!this.sealed || this.root === 0) { + throw new Error(`${this.label}: module-state arena is not sealed`); + } + return Object.freeze(this.decodeRecords(this.chunks, false)); + } + + /** + * Inspect committed records while the parent transaction is still writable. + * + * Process-owned manifests such as imported-global provenance are derived + * from module-owned records and must themselves be appended before `seal`. + * Pending reservations are excluded so callers never derive state from + * bytes that Wasm has not committed. + */ + recordsForCapture(): ForkModuleStateRecord[] { + this.requireWritable(); + if (this.pending) { + throw new Error( + `${this.label}: cannot inspect module state with a pending record`, + ); + } + const chunks = this.validateChunks(this.root, false); + const records = this.decodeRecords(chunks); + validateRecordOwnership(records, this.ptrWidth); + return records; + } + + sparseTables(): DecodedForkSparseTableSnapshot[] { + return decodeSparseTables(this.records()); + } + + rootAddress(): number { + if (this.root === 0) { + throw new Error(`${this.label}: no active module-state arena`); + } + return this.root; + } + + hasActiveArena(): boolean { + return this.root !== 0; + } + + isSealed(): boolean { + return this.sealed; + } + + release(): void { + if (this.root === 0) { + throw new Error(`${this.label}: no active module-state arena to release`); + } + const chunks = this.chunks.splice(0).reverse(); + this.pending = null; + this.payloadIndex.clear(); + this.root = 0; + this.tail = 0; + this.sealed = false; + let firstError: unknown; + for (const chunk of chunks) { + try { + this.deallocate(chunk.addr, chunk.size); + } catch (error) { + firstError ??= error; + } + } + if (firstError !== undefined) throw firstError; + } + + private requireWritable(): void { + if (this.root === 0) { + throw new Error(`${this.label}: module-state arena has not begun`); + } + if (this.sealed) { + throw new Error(`${this.label}: module-state arena is sealed`); + } + } + + private allocateChunk(capacity: number, previous: number, root: boolean): number { + let addr: number; + try { + addr = this.allocate(capacity); + } catch (error) { + if (error instanceof ContinuationAllocationError) throw error; + throw new Error( + `${this.label}: module-state allocation of ${capacity} bytes failed: ` + + `${error instanceof Error ? error.message : String(error)}`, + ); + } + const valid = Number.isSafeInteger(addr) + && addr > 0 + && addr % WASM_PAGE_SIZE === 0 + && capacity >= WASM_PAGE_SIZE + && capacity % WASM_PAGE_SIZE === 0 + && checkedEnd(addr, capacity, `${this.label}: module-state allocation`) + <= this.memory.buffer.byteLength + && (this.ptrWidth === 8 || addr + capacity <= 0x1_0000_0000); + if (!valid) { + if (Number.isSafeInteger(addr) && addr > 0) { + try { + this.deallocate(addr, capacity); + } catch { + // Preserve the allocator contract failure. + } + } + throw new Error(`${this.label}: allocator returned an invalid module-state chunk`); + } + const headerSize = chunkHeaderSize(this.ptrWidth); + const view = new DataView(this.memory.buffer); + view.setUint32(addr, CHUNK_MAGIC, true); + view.setUint16(addr + 4, FORK_MODULE_STATE_ARENA_VERSION, true); + view.setUint16(addr + 6, root ? CHUNK_FLAG_ROOT : 0, true); + writePointer(this.memory, this.ptrWidth, addr + chunkOffset(this.ptrWidth, 0), root ? addr : this.root); + writePointer(this.memory, this.ptrWidth, addr + chunkOffset(this.ptrWidth, 1), previous); + writePointer(this.memory, this.ptrWidth, addr + chunkOffset(this.ptrWidth, 2), 0); + writePointer(this.memory, this.ptrWidth, addr + chunkOffset(this.ptrWidth, 3), capacity); + writePointer(this.memory, this.ptrWidth, addr + chunkOffset(this.ptrWidth, 4), headerSize); + new DataView(this.memory.buffer).setUint32( + addr + chunkRecordCountOffset(this.ptrWidth), + 0, + true, + ); + new DataView(this.memory.buffer).setUint32( + addr + chunkReservedOffset(this.ptrWidth), + 0, + true, + ); + const fieldsEnd = chunkReservedOffset(this.ptrWidth) + 4; + new Uint8Array(this.memory.buffer, fieldsEnd, headerSize - fieldsEnd).fill(0); + this.chunks.push({ + addr, + size: capacity, + used: headerSize, + recordCount: 0, + }); + return addr; + } + + private writeChunkFlags(addr: number, flags: number): void { + new DataView(this.memory.buffer).setUint16(addr + 6, flags, true); + } + + private validateChunks(root: number, requireSealed: boolean): ArenaChunk[] { + const chunks: ArenaChunk[] = []; + const seen = new Set(); + const maxChunks = Math.floor(this.memory.buffer.byteLength / WASM_PAGE_SIZE); + let current = root; + let previous = 0; + for (;;) { + if (seen.has(current)) { + throw new Error(`${this.label}: module-state chunk cycle`); + } + if (seen.size >= maxChunks) { + throw new Error(`${this.label}: module-state chunk chain exceeds memory`); + } + seen.add(current); + const chunk = this.validateChunk( + current, + root, + previous, + chunks.length === 0, + requireSealed, + ); + chunks.push(chunk); + const next = readPointer( + this.memory, + this.ptrWidth, + current + chunkOffset(this.ptrWidth, 2), + `${this.label}: module-state next chunk`, + ); + if (next === 0) break; + previous = current; + current = next; + } + + const sorted = [...chunks].sort((left, right) => left.addr - right.addr); + for (let index = 1; index < sorted.length; index++) { + const prior = sorted[index - 1]!; + const currentChunk = sorted[index]!; + if ( + checkedEnd(prior.addr, prior.size, `${this.label}: module-state chunk`) + > currentChunk.addr + ) { + throw new Error(`${this.label}: module-state chunk ranges overlap`); + } + } + return chunks; + } + + private validateChunk( + addr: number, + root: number, + previous: number, + isRoot: boolean, + requireSealed: boolean, + ): ArenaChunk { + const headerSize = chunkHeaderSize(this.ptrWidth); + if ( + !Number.isSafeInteger(addr) + || addr <= 0 + || addr % WASM_PAGE_SIZE !== 0 + || checkedEnd(addr, headerSize, `${this.label}: module-state chunk header`) + > this.memory.buffer.byteLength + ) { + throw new Error(`${this.label}: invalid module-state chunk address`); + } + const view = new DataView(this.memory.buffer); + const expectedFlags = (requireSealed ? CHUNK_FLAG_SEALED : 0) + | (isRoot ? CHUNK_FLAG_ROOT : 0); + if ( + view.getUint32(addr, true) !== CHUNK_MAGIC + || view.getUint16(addr + 4, true) !== FORK_MODULE_STATE_ARENA_VERSION + || view.getUint16(addr + 6, true) !== expectedFlags + || readPointer( + this.memory, + this.ptrWidth, + addr + chunkOffset(this.ptrWidth, 0), + `${this.label}: module-state chunk root`, + ) !== root + || readPointer( + this.memory, + this.ptrWidth, + addr + chunkOffset(this.ptrWidth, 1), + `${this.label}: module-state previous chunk`, + ) !== previous + ) { + throw new Error(`${this.label}: invalid or unsealed module-state chunk`); + } + const capacity = readPointer( + this.memory, + this.ptrWidth, + addr + chunkOffset(this.ptrWidth, 3), + `${this.label}: module-state chunk capacity`, + ); + const used = readPointer( + this.memory, + this.ptrWidth, + addr + chunkOffset(this.ptrWidth, 4), + `${this.label}: module-state chunk used bytes`, + ); + const recordCount = view.getUint32( + addr + chunkRecordCountOffset(this.ptrWidth), + true, + ); + if ( + capacity < WASM_PAGE_SIZE + || capacity % WASM_PAGE_SIZE !== 0 + || checkedEnd(addr, capacity, `${this.label}: module-state chunk bounds`) + > this.memory.buffer.byteLength + || used < headerSize + || used > capacity + || (!isRoot && (used === headerSize || recordCount === 0)) + || view.getUint32(addr + chunkReservedOffset(this.ptrWidth), true) !== 0 + ) { + throw new Error(`${this.label}: invalid module-state chunk bounds or metadata`); + } + const fieldsEnd = chunkReservedOffset(this.ptrWidth) + 4; + requireZeroBytes( + new Uint8Array(this.memory.buffer, addr + fieldsEnd, headerSize - fieldsEnd), + `${this.label}: module-state chunk header`, + ); + return { addr, size: capacity, used, recordCount }; + } + + private decodeRecords( + chunks: readonly ArenaChunk[], + copyPayload = true, + ): ForkModuleStateRecord[] { + const records: ForkModuleStateRecord[] = []; + for (const [chunkIndex, chunk] of chunks.entries()) { + let offset = chunkHeaderSize(this.ptrWidth); + let recordCount = 0; + while (offset < chunk.used) { + const addr = chunk.addr + offset; + if (offset + RECORD_HEADER_SIZE > chunk.used) { + throw new Error( + `${this.label}: module-state chunk ${chunkIndex} has a truncated record header`, + ); + } + const view = new DataView(this.memory.buffer); + const kind = view.getUint16(addr + 6, true); + if ( + view.getUint32(addr, true) !== RECORD_MAGIC + || view.getUint16(addr + 4, true) !== FORK_MODULE_STATE_RECORD_VERSION + || !RECORD_KINDS.has(kind) + ) { + throw new Error( + `${this.label}: module-state chunk ${chunkIndex} has an invalid record header`, + ); + } + const totalSize = view.getUint32(addr + 8, true); + const payloadSize = view.getUint32(addr + 12, true); + const expectedTotal = alignUp( + RECORD_HEADER_SIZE + payloadSize, + FORK_MODULE_STATE_RECORD_ALIGNMENT, + ); + if ( + totalSize !== expectedTotal + || totalSize < RECORD_HEADER_SIZE + || offset + totalSize > chunk.used + ) { + throw new Error( + `${this.label}: module-state chunk ${chunkIndex} has invalid record bounds`, + ); + } + const paddingSize = totalSize - RECORD_HEADER_SIZE - payloadSize; + requireZeroBytes( + new Uint8Array( + this.memory.buffer, + addr + RECORD_HEADER_SIZE + payloadSize, + paddingSize, + ), + `${this.label}: module-state record ${records.length}`, + ); + records.push({ + kind: kind as ForkModuleStateRecordKind, + activationId: view.getUint32(addr + 16, true), + ownerId: view.getUint32(addr + 20, true), + payload: (() => { + const payload = new Uint8Array( + this.memory.buffer, + addr + RECORD_HEADER_SIZE, + payloadSize, + ); + // `records()` preserves the historical detached-lifetime contract; + // internal validation and streaming consumers opt into sealed + // arena views explicitly. + return copyPayload ? payload.slice() : payload; + })(), + }); + offset += totalSize; + recordCount++; + } + if (offset !== chunk.used || recordCount !== chunk.recordCount) { + throw new Error( + `${this.label}: module-state chunk ${chunkIndex} record count is inconsistent`, + ); + } + } + return records; + } + + private buildPayloadIndex( + chunks: readonly ArenaChunk[], + ): Map { + const index = new Map(); + for (const chunk of chunks) { + let offset = chunkHeaderSize(this.ptrWidth); + while (offset < chunk.used) { + const addr = chunk.addr + offset; + const view = new DataView(this.memory.buffer); + const kind = view.getUint16(addr + 6, true) as ForkModuleStateRecordKind; + const totalSize = view.getUint32(addr + 8, true); + const activationId = view.getUint32(addr + 16, true); + const ownerId = view.getUint32(addr + 20, true); + const key = ownerKey(kind, activationId, ownerId); + const addresses = index.get(key) ?? []; + addresses.push(addr + RECORD_HEADER_SIZE); + index.set(key, addresses); + offset += totalSize; + } + } + return index; + } +} diff --git a/host/src/fork-process-continuation.ts b/host/src/fork-process-continuation.ts new file mode 100644 index 0000000000..45f9a888d4 --- /dev/null +++ b/host/src/fork-process-continuation.ts @@ -0,0 +1,873 @@ +import { + WPK_FORK_FRAME_IMPORT_PEEK, + WPK_FORK_RESUME_IMPORT_PEEK, + WPK_FORK_RESUME_IMPORT_TABLE, +} from "./generated/abi"; +import { + type ForkActivationRegistration, + ForkActivationRegistry, +} from "./fork-activation-registry"; +import { + invokeForkContinuationBegin, + LinkedForkContinuation, +} from "./fork-continuation"; +import { + activationContinuationsForChild, + type ForkModuleStateArena, + replayEventsForChild, + writeForkModuleStateRoot, +} from "./fork-module-state"; +import { + type ForkReplayEvent, + ForkReplayEventJournal, + type ForkResumeTarget, + ForkResumeTable, +} from "./fork-replay-events"; +import { + checkedWasmGuestPointerOffset, + type WasmGuestPointer, +} from "./wasm-guest-pointer"; +import type { + DecodedSegmentedForkReferenceTransaction, +} from "./fork-reference-segments"; + +const WPK_FORK_NORMAL = 0; +const WPK_FORK_UNWINDING = 1; +const WPK_FORK_REWINDING = 2; +const WPK_FORK_ABORT_UNWINDING = 3; +// These names are already part of the Rust ABI table; generated named TS +// exports land with the ABI generator update that also publishes the fixed +// resume-boundary export names. +const WPK_FORK_FRAME_IMPORT_RESERVE = "__wpk_fork_frame_reserve"; +const WPK_FORK_FRAME_IMPORT_COMMIT = "__wpk_fork_frame_commit"; +const WPK_FORK_FRAME_IMPORT_NEXT = "__wpk_fork_frame_next"; + +type ProcessContinuationPhase = + | "idle" + | "capture" + | "sealed-parent" + | "parent-replay" + | "child-replay" + | "abort-replay"; + +export interface ForkProcessActivationBinding { + readonly activationId: number; + readonly continuation: LinkedForkContinuation; + /** + * Publish the process launch root in process-owned copied memory. + * + * Only activation zero owns this process-wide anchor. Its value may name + * any active activation's continuation: a side module can call the fork + * import without placing a main-module Wasm frame on the captured stack. + */ + readonly publishProcessLaunchRoot?: (address: number) => void; + /** Read the copied process launch root after fresh-child instantiation. */ + readonly readProcessLaunchRoot?: () => number; +} + +interface CompleteForkProcessActivation extends ForkProcessActivationBinding { + readonly registration: ForkActivationRegistration; + root: number; +} + +function assertActivationId(value: number): void { + if (!Number.isInteger(value) || value < 0 || value > 0xffff_ffff) { + throw new RangeError(`invalid fork process activation id ${value}`); + } +} + +function requireExportFunction( + activation: CompleteForkProcessActivation, + name: string, +): CallableFunction { + const value = activation.registration.instance.exports[name]; + if (typeof value !== "function") { + throw new Error( + `fork activation ${activation.activationId} is missing export ${name}`, + ); + } + return value as CallableFunction; +} + +function activationState(activation: CompleteForkProcessActivation): number { + return Number(requireExportFunction(activation, "wpk_fork_state")()); +} + +/** + * One process-worker transaction for linked frames from every module instance. + * + * Per-activation continuations still own their bytes because each artifact has + * its own fixed runtime prefix. The event journal is process-wide: it records + * the exact order in which frames from main and side modules commit, and + * replay consumes the reverse order. No module-instance table slot is durable + * state. + */ +export class ForkProcessContinuationCoordinator { + readonly resumeTable = new ForkResumeTable(); + + private readonly prepared = new Map(); + private readonly activations = new Map(); + private readonly events = new ForkReplayEventJournal(); + private phase: ProcessContinuationPhase = "idle"; + private arena: ForkModuleStateArena | null = null; + + constructor( + private readonly memory: WebAssembly.Memory, + private readonly registry: ForkActivationRegistry, + private readonly label: string, + ) {} + + /** + * Bind frame imports before the Wasm instance exists. + * + * Instantiation then calls `registerActivation` with the reflected exports + * and resume catalog. Splitting these steps avoids a circular dependency + * between imports and the instance that implements the codecs. + */ + prepareActivation(binding: ForkProcessActivationBinding): void { + this.requireIdle("prepare a module activation"); + assertActivationId(binding.activationId); + if ( + this.prepared.has(binding.activationId) + || this.activations.has(binding.activationId) + ) { + throw new Error( + `${this.label}: fork activation ${binding.activationId} is already prepared`, + ); + } + if ( + binding.activationId !== 0 + && ( + binding.publishProcessLaunchRoot !== undefined + || binding.readProcessLaunchRoot !== undefined + ) + ) { + throw new Error( + `${this.label}: only activation zero may own the process launch anchor`, + ); + } + if ( + (binding.publishProcessLaunchRoot === undefined) + !== (binding.readProcessLaunchRoot === undefined) + ) { + throw new Error( + `${this.label}: process launch anchor must provide both read and publish`, + ); + } + this.prepared.set(binding.activationId, binding); + } + + registerActivation( + registration: ForkActivationRegistration, + resumeTargets: readonly ForkResumeTarget[], + ): void { + this.requireIdle("register a module activation"); + const binding = this.prepared.get(registration.activationId); + if (!binding) { + throw new Error( + `${this.label}: fork activation ${registration.activationId} was not prepared`, + ); + } + this.registry.registerActivation(registration); + try { + this.resumeTable.registerActivation(registration.activationId, resumeTargets); + } catch (error) { + this.registry.unregisterActivation(registration.activationId); + throw error; + } + this.prepared.delete(registration.activationId); + this.activations.set(registration.activationId, { + ...binding, + registration, + root: 0, + }); + } + + unregisterActivation(activationId: number): void { + this.requireIdle("unregister a module activation"); + const activation = this.getActivation(activationId); + this.resumeTable.unregisterActivation(activationId); + this.registry.unregisterActivation(activationId); + this.activations.delete(activationId); + } + + discardPreparedActivation(activationId: number): void { + this.requireIdle("discard a prepared module activation"); + assertActivationId(activationId); + if (!this.prepared.delete(activationId)) { + throw new Error( + `${this.label}: fork activation ${activationId} is not prepared`, + ); + } + } + + /** + * Imports bound to one activation but coordinated by the process journal. + */ + continuationImports( + activationId: number, + onReservationAbort?: (errno: number) => void, + ): Record { + assertActivationId(activationId); + const binding = this.prepared.get(activationId) + ?? this.activations.get(activationId); + if (!binding) { + throw new Error( + `${this.label}: fork activation ${activationId} has no continuation binding`, + ); + } + const continuation = binding.continuation; + return { + [WPK_FORK_FRAME_IMPORT_RESERVE]: (size: number | bigint) => { + const payload = continuation.reserveFrame(size); + if ((payload === 0 || payload === 0n) && onReservationAbort) { + onReservationAbort(continuation.abortErrno()); + } + return payload; + }, + [WPK_FORK_FRAME_IMPORT_COMMIT]: (payload: WasmGuestPointer): void => { + continuation.commitFrame(payload); + const functionOrdinal = this.readFunctionOrdinal( + payload, + continuation, + "committed", + ); + this.events.recordCommit(activationId, functionOrdinal); + }, + [WPK_FORK_FRAME_IMPORT_PEEK]: (size: number | bigint) => { + const event = this.requireSelectedEvent(activationId, "peek"); + const payload = continuation.peekFrame(size); + const functionOrdinal = this.readFunctionOrdinal( + payload, + continuation, + "peeked", + ); + if (functionOrdinal !== event.functionOrdinal) { + throw new Error( + `${this.label}: replay selected ${activationId}:${event.functionOrdinal}, ` + + `but the frame belongs to ${activationId}:${functionOrdinal}`, + ); + } + return payload; + }, + [WPK_FORK_FRAME_IMPORT_NEXT]: (size: number | bigint) => { + const event = this.requireSelectedEvent(activationId, "consume"); + // WHY: validate identity through the non-consuming path first. A bad + // activation/function coordinate must not advance the linked cursor + // and turn a deterministic launch failure into later child corruption. + const peeked = continuation.peekFrame(size); + const functionOrdinal = this.readFunctionOrdinal( + peeked, + continuation, + "consumed", + ); + if (functionOrdinal !== event.functionOrdinal) { + throw new Error( + `${this.label}: replay selected ${activationId}:${event.functionOrdinal}, ` + + `but the frame belongs to ${activationId}:${functionOrdinal}`, + ); + } + const payload = continuation.nextFrame(size); + this.events.consume(activationId, functionOrdinal); + return payload; + }, + [WPK_FORK_RESUME_IMPORT_PEEK]: (_typeDiagnostic: number): number => + this.resumeTable.slotFor(this.events.peek()), + [WPK_FORK_RESUME_IMPORT_TABLE]: + this.resumeTable.table as unknown as WebAssembly.ImportValue, + }; + } + + /** + * Snapshot all activations, allocate their private runtime prefixes, and put + * every instance in UNWINDING before the private transport crosses modules. + */ + beginCapture(arena: ForkModuleStateArena): void { + this.requirePhase("idle", "begin process continuation capture"); + if (this.prepared.size !== 0) { + throw new Error( + `${this.label}: cannot fork with ${this.prepared.size} incomplete activation(s)`, + ); + } + this.events.beginCapture(); + this.arena = arena; + try { + this.publishProcessLaunchRoot(0); + this.registry.beginCapture(arena); + this.phase = "capture"; + for (const activation of this.orderedActivations()) { + const root = Number(activation.continuation.beginUnwind()); + activation.root = root; + // WHY: the main Wasm activation need not be on a side-module fork + // stack. Every activation prefix therefore carries the process arena + // root, allowing the deterministic launch root chosen after unwind + // to come from any active module without an archive-private side slot. + writeForkModuleStateRoot( + this.memory, + root, + activation.continuation.format.ptrWidth, + arena.rootAddress(), + ); + invokeForkContinuationBegin( + requireExportFunction(activation, "wpk_fork_unwind_begin"), + root, + activation.continuation.format.ptrWidth, + `${this.label}: activation ${activation.activationId} unwind`, + ); + this.requireActivationState(activation, WPK_FORK_UNWINDING, "unwind"); + } + } catch (error) { + this.cancelCapture(); + throw error; + } + } + + /** + * Close unwind, discard activations that were not on the captured stack, + * and seal references, module state, and global frame ordering together. + */ + sealCapture(): void { + this.requirePhase("capture", "seal process continuation capture"); + this.events.sealCapture(); + const active = this.events.capturedActivationIds(); + try { + for (const activation of this.orderedActivations()) { + requireExportFunction(activation, "wpk_fork_unwind_end")(); + this.requireActivationState(activation, WPK_FORK_NORMAL, "end unwind"); + if (active.has(activation.activationId)) { + activation.continuation.finishUnwind(); + } else { + activation.continuation.cancelUnwindAndRelease(); + activation.root = 0; + } + } + if (active.size === 0) { + throw new Error(`${this.label}: captured fork stack has no continuation frames`); + } + const continuations = this.orderedActivations() + .filter(({ activationId }) => active.has(activationId)) + .map(({ activationId, root }) => ({ + activationId, + root: BigInt(root), + })); + const arena = this.registry.currentArena(); + arena.appendReplayEvents(this.events); + arena.appendActivationContinuations(continuations); + this.registry.sealCapture(); + this.publishProcessLaunchRoot( + this.selectProcessLaunchRoot(continuations), + ); + this.phase = "sealed-parent"; + } catch (error) { + this.abort(); + throw error; + } + } + + beginParentReplay(): void { + this.requirePhase("sealed-parent", "begin parent process replay"); + this.registry.beginParentReplay(); + this.events.beginParentReplay(); + this.phase = "parent-replay"; + try { + this.registry.restoreModuleState(); + this.beginActivationReplay(WPK_FORK_REWINDING); + } catch (error) { + this.abort(); + throw error; + } + } + + /** + * Attach copied state only after every fresh child activation is registered. + */ + attachChild( + arena: ForkModuleStateArena, + adoptPreinstantiatedReferences?: () => void, + decodedReferences?: DecodedSegmentedForkReferenceTransaction, + ): void { + this.requirePhase("idle", "attach child process replay"); + if (this.prepared.size !== 0) { + throw new Error( + `${this.label}: child has ${this.prepared.size} incomplete activation(s)`, + ); + } + this.arena = arena; + try { + this.registry.attachChild(arena, decodedReferences); + // Imported immutable references may have forced a strict prefix of the + // recipe graph to be materialized while provider activations were still + // being instantiated. Adopt those exact identities after the full + // transaction validates the copied wire, but before module restore can + // request them again. + adoptPreinstantiatedReferences?.(); + const records = arena.recordViews(); + this.events.attachChild(replayEventsForChild(records)); + const continuations = activationContinuationsForChild( + records, + arena.ptrWidth, + ); + const copiedLaunchRoot = this.readProcessLaunchRoot(); + const expectedLaunchRoot = this.selectProcessLaunchRoot(continuations); + if (copiedLaunchRoot !== expectedLaunchRoot) { + throw new Error( + `${this.label}: copied process launch root ${copiedLaunchRoot} ` + + `does not match manifest root ${expectedLaunchRoot}`, + ); + } + this.phase = "child-replay"; + this.registry.restoreModuleState(); + const roots = new Map( + continuations.map(({ activationId, root }) => [ + activationId, + Number(root), + ]), + ); + for (const activation of this.orderedActivations()) { + const root = roots.get(activation.activationId) ?? 0; + if (root === 0) { + activation.root = 0; + continue; + } + if (!Number.isSafeInteger(root) || root <= 0) { + throw new Error( + `${this.label}: active child activation ${activation.activationId} ` + + "has no copied continuation root", + ); + } + activation.root = root; + activation.continuation.attachForReplay( + activation.continuation.format.ptrWidth === 8 ? BigInt(root) : root, + ); + } + this.beginActivationReplay(WPK_FORK_REWINDING, false); + } catch (error) { + this.abort(); + throw error; + } + } + + /** + * Switch a sealed parent transaction to allocation-failure replay. + * + * The caller first seals capture so every already-committed frame and + * reference recipe has one deterministic owner. No child is launched. + */ + beginAbortReplay(errno: number): void { + this.requirePhase("sealed-parent", "begin process abort replay"); + if (!Number.isInteger(errno) || errno <= 0) { + throw new RangeError(`${this.label}: invalid fork abort errno ${errno}`); + } + this.registry.beginParentReplay(); + this.events.beginParentReplay(); + this.phase = "abort-replay"; + try { + this.registry.restoreModuleState(); + for (const activation of this.activeActivations()) { + activation.continuation.beginAbortReplay(errno); + invokeForkContinuationBegin( + requireExportFunction(activation, "wpk_fork_abort_begin"), + activation.root, + activation.continuation.format.ptrWidth, + `${this.label}: activation ${activation.activationId} abort replay`, + ); + this.requireActivationState( + activation, + WPK_FORK_ABORT_UNWINDING, + "begin abort replay", + ); + } + } catch (error) { + this.abort(); + throw error; + } + } + + /** + * Turn a partially committed unwind into deterministic errno replay. + * + * A linked-continuation chunk allocation can fail from inside a generated + * frame postamble. JavaScript cannot return an errno through that postamble, + * so the null reservation asks the same activation stack to replay the + * already committed inner frames and return the allocation error from the + * original fork call. Every activation remains rooted until that replay + * reaches the leaf: an activation with no committed frame can still be an + * outer live caller whose runtime state must return to NORMAL. + */ + beginCaptureAbort(errno: number): void { + this.requirePhase("capture", "begin partial-capture abort replay"); + if (!Number.isInteger(errno) || errno <= 0) { + throw new RangeError(`${this.label}: invalid fork abort errno ${errno}`); + } + try { + this.events.sealCapture(); + this.registry.currentArena().appendReplayEvents(this.events); + this.registry.sealCapture(); + this.registry.beginParentReplay(); + this.events.beginParentReplay(); + this.phase = "abort-replay"; + for (const activation of this.orderedActivations()) { + // reserveFrame has already marked the failing continuation. Repeating + // the same errno is intentionally idempotent; the other activation + // owners need the identical replay cursor and failure result. + activation.continuation.beginAbortReplay(errno); + invokeForkContinuationBegin( + requireExportFunction(activation, "wpk_fork_abort_begin"), + activation.root, + activation.continuation.format.ptrWidth, + `${this.label}: activation ${activation.activationId} partial abort replay`, + ); + this.requireActivationState( + activation, + WPK_FORK_ABORT_UNWINDING, + "begin partial abort replay", + ); + } + } catch (error) { + this.abort(); + throw error; + } + } + + finishReplay(): void { + if (this.phase !== "parent-replay" && this.phase !== "child-replay") { + throw new Error( + `${this.label}: cannot finish process replay while coordinator is ${this.phase}`, + ); + } + this.finishTransaction(false); + } + + finishAbortReplay(): void { + this.requirePhase("abort-replay", "finish process abort replay"); + this.finishTransaction(true); + } + + phaseName(): ProcessContinuationPhase { + return this.phase; + } + + rootFor(activationId: number): number { + return this.getActivation(activationId).root; + } + + abort(): void { + let failure: unknown; + for (const activation of this.orderedActivations()) { + if (!activation.continuation.hasActiveContinuation()) continue; + try { + activation.continuation.cancelUnwindAndRelease(); + } catch (error) { + failure ??= error; + } + activation.root = 0; + } + try { + this.publishProcessLaunchRoot(0, false); + } catch (error) { + failure ??= error; + } + try { + this.events.abort(); + } catch (error) { + failure ??= error; + } + try { + this.registry.abort(); + } catch (error) { + failure ??= error; + } + try { + this.releaseArena(); + } catch (error) { + failure ??= error; + } + this.phase = "idle"; + if (failure !== undefined) throw failure; + } + + clear(): void { + this.abort(); + this.resumeTable.clear(); + this.activations.clear(); + this.prepared.clear(); + this.registry.clear(); + } + + private beginActivationReplay( + expectedState: typeof WPK_FORK_REWINDING, + beginContinuation = true, + ): void { + for (const activation of this.activeActivations()) { + if (beginContinuation) activation.continuation.beginReplay(); + invokeForkContinuationBegin( + requireExportFunction(activation, "wpk_fork_rewind_begin"), + activation.root, + activation.continuation.format.ptrWidth, + `${this.label}: activation ${activation.activationId} replay`, + ); + this.requireActivationState(activation, expectedState, "begin replay"); + } + } + + private finishTransaction(abortReplay: boolean): void { + let failure: unknown; + for (const activation of this.activeActivations()) { + try { + requireExportFunction( + activation, + abortReplay ? "wpk_fork_abort_end" : "wpk_fork_rewind_end", + )(); + this.requireActivationState(activation, WPK_FORK_NORMAL, "finish replay"); + } catch (error) { + failure ??= error; + } + } + try { + this.events.finishReplay(); + } catch (error) { + failure ??= error; + // The child journal may retain zero-copy views into the arena. Drop its + // cursor before the arena mappings are released even when replay ended + // early, or a later diagnostic could read recycled process memory. + this.events.abort(); + } + try { + this.registry.finishReplay(); + } catch (error) { + failure ??= error; + } + for (const activation of this.activeActivations()) { + try { + if (abortReplay) activation.continuation.finishAbortReplayAndRelease(); + else activation.continuation.finishReplayAndRelease(); + } catch (error) { + failure ??= error; + } + activation.root = 0; + } + try { + this.publishProcessLaunchRoot(0); + } catch (error) { + failure ??= error; + } + try { + this.releaseArena(); + } catch (error) { + failure ??= error; + } + this.phase = "idle"; + if (failure !== undefined) throw failure; + } + + private cancelCapture(): void { + let failure: unknown; + for (const activation of this.orderedActivations()) { + if (!activation.continuation.hasActiveContinuation()) continue; + try { + if (activationState(activation) === WPK_FORK_UNWINDING) { + requireExportFunction(activation, "wpk_fork_unwind_end")(); + } + } catch (error) { + failure ??= error; + } + try { + activation.continuation.cancelUnwindAndRelease(); + } catch (error) { + failure ??= error; + } + activation.root = 0; + } + try { + this.publishProcessLaunchRoot(0); + } catch (error) { + failure ??= error; + } + try { + this.events.abort(); + } catch (error) { + failure ??= error; + } + try { + this.registry.abort(); + } catch (error) { + failure ??= error; + } + try { + this.releaseArena(); + } catch (error) { + failure ??= error; + } + this.phase = "idle"; + if (failure !== undefined) throw failure; + } + + private activeActivationIds(): Set { + const ids = new Set(); + const phase = this.events.phaseName(); + if (phase === "capture" || phase === "sealed-parent") { + return this.events.capturedActivationIds(); + } + // During replay, selecting every remaining event would mutate the journal. + // Activation roots came from the exact-set-validated KFMS manifest. + for (const activation of this.activations.values()) { + if (activation.root !== 0) { + ids.add(activation.activationId); + } + } + return ids; + } + + private selectProcessLaunchRoot( + continuations: readonly { + activationId: number; + root: bigint; + }[], + ): number { + if (continuations.length === 0) { + throw new Error(`${this.label}: process continuation manifest is empty`); + } + const selected = continuations.find(({ activationId }) => activationId === 0) + ?? continuations[0]!; + const root = Number(selected.root); + if (!Number.isSafeInteger(root) || root <= 0) { + throw new RangeError( + `${this.label}: activation ${selected.activationId} launch root ` + + `${selected.root} is not a safe guest address`, + ); + } + return root; + } + + private publishProcessLaunchRoot( + root: number, + required = true, + ): void { + const owner = this.activations.get(0); + const publish = owner?.publishProcessLaunchRoot; + const read = owner?.readProcessLaunchRoot; + if (!owner || !publish || !read) { + if (!required) return; + throw new Error( + `${this.label}: activation zero has no process launch anchor`, + ); + } + publish(root); + } + + private readProcessLaunchRoot(): number { + const owner = this.activations.get(0); + const read = owner?.readProcessLaunchRoot; + if (!owner || !owner.publishProcessLaunchRoot || !read) { + throw new Error( + `${this.label}: activation zero has no process launch anchor`, + ); + } + const root = read(); + if (!Number.isSafeInteger(root) || root <= 0) { + throw new Error(`${this.label}: copied process launch root is invalid`); + } + return root; + } + + private releaseArena(): void { + const arena = this.arena; + if (!arena) return; + // Clear ownership first so a failing deallocator cannot make stale KFMS + // bytes look reusable by a later fork transaction. + this.arena = null; + arena.release(); + } + + private activeActivations(): CompleteForkProcessActivation[] { + const active = this.activeActivationIds(); + return this.orderedActivations().filter(({ activationId }) => + active.has(activationId) + ); + } + + private requireSelectedEvent( + activationId: number, + operation: string, + ): ForkReplayEvent { + const event = this.events.peek(); + if (!event) { + throw new Error( + `${this.label}: activation ${activationId} cannot ${operation}; ` + + "the replay event stream is exhausted", + ); + } + if (event.activationId !== activationId) { + throw new Error( + `${this.label}: activation ${activationId} cannot ${operation} frame for ` + + `activation ${event.activationId}`, + ); + } + return event; + } + + private readFunctionOrdinal( + payload: WasmGuestPointer, + continuation: LinkedForkContinuation, + operation: string, + ): number { + const offset = checkedWasmGuestPointerOffset( + payload, + continuation.format.ptrWidth, + `${this.label}: ${operation} frame`, + ); + if (offset > this.memory.buffer.byteLength - 4) { + throw new RangeError( + `${this.label}: ${operation} frame header escapes process memory`, + ); + } + return new DataView(this.memory.buffer).getUint32(offset, true); + } + + private getActivation(activationId: number): CompleteForkProcessActivation { + assertActivationId(activationId); + const activation = this.activations.get(activationId); + if (!activation) { + throw new Error( + `${this.label}: fork activation ${activationId} is not registered`, + ); + } + return activation; + } + + private orderedActivations(): CompleteForkProcessActivation[] { + return [...this.activations.values()].sort( + (left, right) => left.activationId - right.activationId, + ); + } + + private requireActivationState( + activation: CompleteForkProcessActivation, + expected: number, + operation: string, + ): void { + const actual = activationState(activation); + if (actual !== expected) { + throw new Error( + `${this.label}: activation ${activation.activationId} ${operation} ` + + `ended in state ${actual}, expected ${expected}`, + ); + } + } + + private requireIdle(operation: string): void { + this.requirePhase("idle", operation); + } + + private requirePhase( + expected: ProcessContinuationPhase, + operation: string, + ): void { + if (this.phase !== expected) { + throw new Error( + `${this.label}: cannot ${operation} while process continuation is ${this.phase}; ` + + `expected ${expected}`, + ); + } + } +} diff --git a/host/src/fork-reference-broker.ts b/host/src/fork-reference-broker.ts new file mode 100644 index 0000000000..c84f523d1e --- /dev/null +++ b/host/src/fork-reference-broker.ts @@ -0,0 +1,675 @@ +/** + * Process-independent ownership for opaque `externref` values. + * + * WebAssembly treats an externref as an opaque identity. A fork child runs in + * a fresh Worker, so copying a JavaScript object into that Worker is neither + * generally possible nor identity preserving. The host instead keeps the real + * value behind a stable handle and gives every Worker one canonical token for + * that handle. Host-import adapters can route token-bearing calls back to the + * owner without putting JavaScript heap objects in the Wasm continuation. + * + * This file deliberately owns only identity and lifetime. Dispatching a + * particular host import remains the responsibility of that import's adapter; + * the adapter resolves handles through the broker rather than receiving a + * best-effort structured clone. + */ + +const GENERATION_TOKEN = Symbol("kandelo.fork.externref-generation"); +const HANDLE_TOKEN = Symbol("kandelo.fork.externref-handle"); +const WORKER_GENERATION_TOKEN = + Symbol("kandelo.fork.externref-worker-generation"); +const MAX_WIRE_ID = 0xffff_ffff; + +export interface ForkExternrefToken { + readonly [HANDLE_TOKEN]: number; + readonly [WORKER_GENERATION_TOKEN]: number; +} + +/** + * Exact lifetime of one process Wasm image. + * + * A PID survives exec, so it is not sufficient authority for a host-owned + * externref. The broker issues a fresh token for every execution generation + * and rejects a token as soon as that generation is replaced or released. + */ +export interface ForkExternrefGeneration { + readonly id: number; + readonly pid: number; + readonly [GENERATION_TOKEN]: true; +} + +export interface ForkExternrefLease { + readonly generation: ForkExternrefGeneration; + readonly handleCount: number; + release(): void; +} + +export interface ForkExternrefBrokerOptions { + /** Test seam; production handles use the complete nonzero-u32 wire space. */ + readonly maxHandle?: number; + /** Test seam; production generations use the complete nonzero-u32 space. */ + readonly maxGeneration?: number; +} + +interface BrokerEntry { + value: unknown; + holders: Set; +} + +interface BrokerForkLeaseState { + readonly generation: BrokerGenerationState; + readonly handles: Set; + released: boolean; +} + +interface BrokerGenerationState { + readonly token: ForkExternrefGeneration; + readonly directHandles: Set; + readonly forkHandleCounts: Map; + readonly handles: Set; + readonly forkLeases: Set; + status: "active" | "released" | "replaced"; +} + +function assertProcessId(pid: number): void { + if (!Number.isInteger(pid) || pid <= 0 || pid > MAX_WIRE_ID) { + throw new RangeError(`invalid externref holder pid ${pid}`); + } +} + +function assertHandle(handle: number): void { + if (!Number.isInteger(handle) || handle <= 0 || handle > MAX_WIRE_ID) { + throw new RangeError(`invalid externref handle ${handle}`); + } +} + +function assertWireLimit(value: number, name: string): void { + if (!Number.isInteger(value) || value <= 0 || value > MAX_WIRE_ID) { + throw new RangeError(`${name} must be a positive unsigned 32-bit integer`); + } +} + +class BrokerForkExternrefLease implements ForkExternrefLease { + readonly handleCount: number; + + constructor( + readonly generation: ForkExternrefGeneration, + state: BrokerForkLeaseState, + private readonly releaseState: () => void, + ) { + // Do not duplicate the potentially large handle set merely for + // diagnostics; the broker-owned lease state is its sole lifetime owner. + this.handleCount = state.handles.size; + } + + release(): void { + this.releaseState(); + } +} + +/** + * Kernel-side owner for real JavaScript values. + * + * Ownership is deliberately generation-scoped and set-valued. Ten globals or + * graph edges that alias one externref require one strong owner entry, not ten + * reference counts. Separate successful fork transactions retain independent + * leases so rolling one transaction back cannot revoke another transaction or + * a host import's direct registration. + */ +export class ForkExternrefBroker { + private nextHandle = 1; + private nextGeneration = 1; + private readonly entries = new Map(); + private readonly objectHandles = new WeakMap(); + private readonly primitiveHandles = new Map(); + private readonly numberHandles = new Map(); + private readonly generations = + new WeakMap(); + private readonly currentGenerations = new Map(); + private readonly maxHandle: number; + private readonly maxGeneration: number; + + constructor(options: ForkExternrefBrokerOptions = {}) { + this.maxHandle = options.maxHandle ?? MAX_WIRE_ID; + this.maxGeneration = options.maxGeneration ?? MAX_WIRE_ID; + assertWireLimit(this.maxHandle, "externref broker maxHandle"); + assertWireLimit(this.maxGeneration, "externref broker maxGeneration"); + } + + /** + * Begin one exact process-image lifetime. + * + * Creating a replacement for the same PID retires the old generation before + * the new token is returned. WHY: delayed worker teardown must never use an + * old PID-only capability to resolve values for the post-exec image. + */ + createGeneration(pid: number): ForkExternrefGeneration { + assertProcessId(pid); + if (this.nextGeneration > this.maxGeneration) { + throw new RangeError("externref generation space exhausted"); + } + const id = this.nextGeneration++; + const token: ForkExternrefGeneration = Object.freeze({ + id, + pid, + [GENERATION_TOKEN]: true as const, + }); + const state: BrokerGenerationState = { + token, + directHandles: new Set(), + forkHandleCounts: new Map(), + handles: new Set(), + forkLeases: new Set(), + status: "active", + }; + + const previous = this.currentGenerations.get(pid); + if (previous) this.closeGeneration(previous, "replaced"); + this.generations.set(token, state); + this.currentGenerations.set(pid, state); + return token; + } + + register(generation: ForkExternrefGeneration, value: unknown): number { + const state = this.requireActiveGeneration(generation); + const known = this.lookupValueHandle(value); + if (known !== undefined) { + const entry = this.requireEntry(known); + this.acquireDirect(state, known, entry); + return known; + } + + if (this.nextHandle > this.maxHandle) { + throw new RangeError("externref handle space exhausted"); + } + // Reserve monotonically before publishing any map entry. Even a failed + // publication leaves a gap rather than making a stale wire handle alias a + // future value. + const handle = this.nextHandle++; + const entry: BrokerEntry = { + value, + holders: new Set(), + }; + this.entries.set(handle, entry); + try { + this.rememberValueHandle(value, handle); + state.directHandles.add(handle); + state.handles.add(handle); + entry.holders.add(state); + } catch (error) { + state.directHandles.delete(handle); + state.handles.delete(handle); + entry.holders.delete(state); + this.forget(handle, value); + throw error; + } + return handle; + } + + /** + * Grant a directly managed handle to a generation. + * + * Repeated acquisition is idempotent because aliases share the generation's + * one direct lease. + */ + acquire(generation: ForkExternrefGeneration, handle: number): void { + const state = this.requireActiveGeneration(generation); + assertHandle(handle); + this.acquireDirect(state, handle, this.requireEntry(handle)); + } + + /** + * Duplicate a parent's unique handle set for a fork child. + * + * WHY validation and mutation are separate passes: a corrupt recipe must not + * leave the child holding the valid prefix of an otherwise rejected + * snapshot. The mutation pass also has an explicit rollback so any future + * bookkeeping that can fail preserves that all-or-nothing boundary. + */ + acquireFork( + parentGeneration: ForkExternrefGeneration, + childGeneration: ForkExternrefGeneration, + uniqueHandles: Iterable, + ): ForkExternrefLease { + const parent = this.requireActiveGeneration(parentGeneration); + const child = this.requireActiveGeneration(childGeneration); + if (parent === child || parent.token.pid === child.token.pid) { + throw new Error("externref fork requires distinct process generations"); + } + + const handles = new Set(); + for (const handle of uniqueHandles) { + assertHandle(handle); + handles.add(handle); + } + + const validated: Array<[number, BrokerEntry]> = []; + for (const handle of handles) { + const entry = this.requireEntry(handle); + if (!parent.handles.has(handle) || !entry.holders.has(parent)) { + throw new Error( + `externref generation ${parent.token.id} for pid ${parent.token.pid} ` + + `does not own handle ${handle}`, + ); + } + if (child.handles.has(handle) !== entry.holders.has(child)) { + throw new Error( + `externref generation ${child.token.id} has inconsistent ownership ` + + `for handle ${handle}`, + ); + } + const childLeaseCount = child.forkHandleCounts.get(handle) ?? 0; + if (childLeaseCount >= Number.MAX_SAFE_INTEGER) { + throw new RangeError( + `externref fork lease count overflow for handle ${handle}`, + ); + } + validated.push([handle, entry]); + } + + const leaseState: BrokerForkLeaseState = { + generation: child, + handles: new Set(), + released: false, + }; + const applied: Array<[number, BrokerEntry, number, boolean]> = []; + try { + for (const [handle, entry] of validated) { + const previousCount = child.forkHandleCounts.get(handle) ?? 0; + const addedOwnership = !child.handles.has(handle); + // Record the old state before the first mutation so every partial step + // in this iteration is included in rollback. + applied.push([handle, entry, previousCount, addedOwnership]); + child.forkHandleCounts.set(handle, previousCount + 1); + if (addedOwnership) { + child.handles.add(handle); + entry.holders.add(child); + } + leaseState.handles.add(handle); + } + child.forkLeases.add(leaseState); + } catch (error) { + for (let index = applied.length - 1; index >= 0; index--) { + const [handle, entry, previousCount, addedOwnership] = applied[index]!; + if (previousCount === 0) child.forkHandleCounts.delete(handle); + else child.forkHandleCounts.set(handle, previousCount); + if (addedOwnership) { + child.handles.delete(handle); + entry.holders.delete(child); + } + } + throw error; + } + return new BrokerForkExternrefLease( + child.token, + leaseState, + () => this.releaseForkLease(leaseState), + ); + } + + /** Release one generation's direct (non-fork-transaction) lease. */ + release(generation: ForkExternrefGeneration, handle: number): void { + const state = this.requireActiveGeneration(generation); + assertHandle(handle); + const entry = this.requireEntry(handle); + if (!state.directHandles.has(handle)) { + throw new Error( + `externref generation ${state.token.id} for pid ${state.token.pid} ` + + `has no direct lease for handle ${handle}`, + ); + } + state.directHandles.delete(handle); + if ((state.forkHandleCounts.get(handle) ?? 0) === 0) { + this.removeGenerationHandle(state, handle, entry); + } + } + + /** Retire every handle and lease owned by one exact execution generation. */ + releaseGeneration(generation: ForkExternrefGeneration): boolean { + const state = this.generationState(generation); + if (state.status !== "active") return false; + this.closeGeneration(state, "released"); + return true; + } + + /** + * Validate one generation-scoped capability and return its opaque value. + * + * Host adapters must call this at dispatch time; possession of a numeric + * handle or a PID alone is not authority. + */ + authorize(generation: ForkExternrefGeneration, handle: number): unknown { + const state = this.requireActiveGeneration(generation); + assertHandle(handle); + const entry = this.requireEntry(handle); + if (!state.handles.has(handle) || !entry.holders.has(state)) { + throw new Error( + `externref generation ${state.token.id} for pid ${state.token.pid} ` + + `is not authorized for handle ${handle}`, + ); + } + return entry.value; + } + + /** Compatibility name for adapters that previously resolved PID ownership. */ + resolve(generation: ForkExternrefGeneration, handle: number): unknown { + return this.authorize(generation, handle); + } + + /** + * Permanently retire a handle after an explicit host-resource close. + * + * Every generation loses authorization, and registering the same JS value + * later receives a fresh monotonically larger handle. The monotonic allocator + * itself is the tombstone set: any issued-but-absent number is retired. + */ + tombstone(generation: ForkExternrefGeneration, handle: number): void { + const owner = this.requireActiveGeneration(generation); + assertHandle(handle); + const entry = this.requireEntry(handle); + if (!owner.handles.has(handle) || !entry.holders.has(owner)) { + throw new Error( + `externref generation ${owner.token.id} for pid ${owner.token.pid} ` + + `cannot tombstone unowned handle ${handle}`, + ); + } + + for (const holder of [...entry.holders]) { + holder.directHandles.delete(handle); + holder.forkHandleCounts.delete(handle); + holder.handles.delete(handle); + for (const lease of holder.forkLeases) lease.handles.delete(handle); + } + entry.holders.clear(); + this.forget(handle, entry.value); + } + + /** Set ownership is observable as either zero or one, never graph aliases. */ + holderCount( + handle: number, + generation: ForkExternrefGeneration, + ): 0 | 1 { + assertHandle(handle); + const state = this.generationState(generation); + if (state.status !== "active") return 0; + const entry = this.entries.get(handle); + return state.handles.has(handle) && entry?.holders.has(state) ? 1 : 0; + } + + private releaseForkLease(lease: BrokerForkLeaseState): void { + if (lease.released) { + throw new Error("externref fork lease is already released"); + } + const generation = lease.generation; + this.requireActiveGeneration(generation.token); + + // Verify the whole lease before removing anything. Lifecycle corruption + // must not release a valid prefix and retain the rest. + const entries = new Map(); + for (const handle of lease.handles) { + const entry = this.requireEntry(handle); + const count = generation.forkHandleCounts.get(handle) ?? 0; + if ( + count <= 0 + || !generation.handles.has(handle) + || !entry.holders.has(generation) + ) { + throw new Error( + `externref generation ${generation.token.id} no longer owns ` + + `fork lease handle ${handle}`, + ); + } + entries.set(handle, entry); + } + + for (const handle of lease.handles) { + const entry = entries.get(handle)!; + const count = generation.forkHandleCounts.get(handle)!; + if (count === 1) { + generation.forkHandleCounts.delete(handle); + if (!generation.directHandles.has(handle)) { + this.removeGenerationHandle(generation, handle, entry); + } + } else { + generation.forkHandleCounts.set(handle, count - 1); + } + } + lease.handles.clear(); + lease.released = true; + generation.forkLeases.delete(lease); + } + + private acquireDirect( + generation: BrokerGenerationState, + handle: number, + entry: BrokerEntry, + ): void { + if (generation.directHandles.has(handle)) return; + const addedOwnership = !generation.handles.has(handle); + generation.directHandles.add(handle); + try { + if (addedOwnership) { + generation.handles.add(handle); + entry.holders.add(generation); + } + } catch (error) { + generation.directHandles.delete(handle); + if (addedOwnership) { + generation.handles.delete(handle); + entry.holders.delete(generation); + } + throw error; + } + } + + private removeGenerationHandle( + generation: BrokerGenerationState, + handle: number, + entry: BrokerEntry, + ): void { + generation.handles.delete(handle); + entry.holders.delete(generation); + if (entry.holders.size === 0) this.forget(handle, entry.value); + } + + private closeGeneration( + generation: BrokerGenerationState, + status: "released" | "replaced", + ): void { + if (generation.status !== "active") return; + generation.status = status; + if (this.currentGenerations.get(generation.token.pid) === generation) { + this.currentGenerations.delete(generation.token.pid); + } + for (const lease of generation.forkLeases) { + lease.handles.clear(); + lease.released = true; + } + generation.forkLeases.clear(); + generation.directHandles.clear(); + generation.forkHandleCounts.clear(); + for (const handle of generation.handles) { + const entry = this.entries.get(handle); + if (!entry) continue; + entry.holders.delete(generation); + if (entry.holders.size === 0) this.forget(handle, entry.value); + } + generation.handles.clear(); + } + + private generationState( + generation: ForkExternrefGeneration, + ): BrokerGenerationState { + if ( + typeof generation !== "object" + || generation === null + || generation[GENERATION_TOKEN] !== true + ) { + throw new Error("unknown externref generation token"); + } + const state = this.generations.get(generation); + if (!state) throw new Error("externref generation belongs to another broker"); + return state; + } + + private requireActiveGeneration( + generation: ForkExternrefGeneration, + ): BrokerGenerationState { + const state = this.generationState(generation); + if ( + state.status !== "active" + || this.currentGenerations.get(state.token.pid) !== state + ) { + throw new Error( + `stale externref generation ${state.token.id} for pid ${state.token.pid}`, + ); + } + return state; + } + + private requireEntry(handle: number): BrokerEntry { + const entry = this.entries.get(handle); + if (entry) return entry; + if (handle < this.nextHandle) { + throw new Error(`retired externref handle ${handle}`); + } + throw new Error(`unknown externref handle ${handle}`); + } + + private lookupValueHandle(value: unknown): number | undefined { + if ((typeof value === "object" && value !== null) || typeof value === "function") { + return this.objectHandles.get(value as object); + } + if (typeof value === "number") { + return this.numberHandles.get(exactNumberBits(value)); + } + return this.primitiveHandles.get(value); + } + + private rememberValueHandle(value: unknown, handle: number): void { + if ((typeof value === "object" && value !== null) || typeof value === "function") { + this.objectHandles.set(value as object, handle); + } else if (typeof value === "number") { + this.numberHandles.set(exactNumberBits(value), handle); + } else { + this.primitiveHandles.set(value, handle); + } + } + + private forget(handle: number, value: unknown): void { + this.entries.delete(handle); + if ((typeof value === "object" && value !== null) || typeof value === "function") { + // WeakMap has no conditional delete. Deleting is safe because a handle + // is removed only after every process holder released the strong entry. + this.objectHandles.delete(value as object); + } else if (typeof value === "number") { + const bits = exactNumberBits(value); + if (this.numberHandles.get(bits) === handle) { + this.numberHandles.delete(bits); + } + } else if (this.primitiveHandles.get(value) === handle) { + this.primitiveHandles.delete(value); + } + } +} + +function exactNumberBits(value: number): bigint { + const bytes = new ArrayBuffer(Float64Array.BYTES_PER_ELEMENT); + const view = new DataView(bytes); + view.setFloat64(0, value, true); + return view.getBigUint64(0, true); +} + +/** + * Worker-local canonical tokens for broker handles. + * + * A child never receives the parent's token object. It recreates exactly one + * local token per handle, which preserves all identity observations available + * to Wasm while keeping the actual object under broker ownership. + */ +export class ForkExternrefTokenCache { + private readonly tokens = new Map>(); + + constructor(readonly generationId: number) { + assertWireLimit(generationId, "externref worker generation"); + } + + materialize(handle: number): ForkExternrefToken { + assertHandle(handle); + let token = this.tokens.get(handle)?.deref(); + if (!token) { + token = Object.freeze({ + [HANDLE_TOKEN]: handle, + [WORKER_GENERATION_TOKEN]: this.generationId, + }); + this.tokens.set(handle, new WeakRef(token)); + } + return token; + } + + encode(value: unknown): number | null { + if ( + typeof value !== "object" + || value === null + || !(HANDLE_TOKEN in value) + || !(WORKER_GENERATION_TOKEN in value) + ) { + return null; + } + if ( + (value as ForkExternrefToken)[WORKER_GENERATION_TOKEN] + !== this.generationId + ) { + return null; + } + const handle = (value as ForkExternrefToken)[HANDLE_TOKEN]; + assertHandle(handle); + return handle; + } + + clear(): void { + // Weak references do not own the tokens; clearing merely forgets canonical + // lookup entries at exec/process teardown. + this.tokens.clear(); + } +} + +/** + * Worker-facing recipe provider for externrefs already adapted by the process + * owner. + * + * Host imports that create opaque values must register them with the + * process-wide owner and return this Worker's canonical token. Consequently + * the continuation encoder never needs to clone or inspect the real value. + */ +export class ForkExternrefTokenRecipeProvider { + constructor( + private readonly tokens: ForkExternrefTokenCache, + /** + * Late owner adoption for an exact Worker-local value that has never + * needed to cross a process boundary before this fork. + */ + private readonly normalizeUnclaimed?: ( + value: unknown, + ) => ForkExternrefToken, + ) {} + + capture(value: unknown): number { + let handle = this.tokens.encode(value); + if (handle === null && this.normalizeUnclaimed) { + // WHY: the transaction separately retains `value` for parent replay. + // Only the fresh-child recipe uses this canonical owner token. + handle = this.tokens.encode(this.normalizeUnclaimed(value)); + } + if (handle === null) { + throw new Error( + "externref reached fork without passing through the process reference owner", + ); + } + return handle; + } + + materialize(handle: number): ForkExternrefToken { + return this.tokens.materialize(handle); + } +} diff --git a/host/src/fork-reference-recipes.ts b/host/src/fork-reference-recipes.ts new file mode 100644 index 0000000000..ef702c7408 --- /dev/null +++ b/host/src/fork-reference-recipes.ts @@ -0,0 +1,1316 @@ +/** + * Versioned, activation-owned reconstruction recipes for Wasm references. + * + * The wire image contains only integers and graph edges. JavaScript/Wasm + * objects stay under an explicit reconstruction owner and never become + * accidental evidence that a fresh fork Worker inherited module state. + */ + +import { + ForkFunctionCatalog, + type ForkFunctionRecipe, +} from "./fork-function-catalog"; +import { + ForkExternrefBroker, + type ForkExternrefGeneration, + type ForkExternrefLease, +} from "./fork-reference-broker"; +import { + ForkStaticRootCatalog, + type ForkStaticRootRecipe, +} from "./fork-static-root-catalog"; + +export const FORK_REFERENCE_RECIPE_VERSION = 1; + +const WIRE_MAGIC = 0x5252_464b; // "KFRR", little endian. +const HEADER_SIZE = 40; +const NODE_SIZE = 32; +const MAX_I31 = 0x3fff_ffff; +const MIN_I31 = -0x4000_0000; + +// Version 1 is: a 40-byte header, 32-byte node records in ascending recipe-ID +// order, ordered root IDs, one canonical edge vector, then exact scalar +// payload bytes. Fixed records make bounds validation O(1) per node and let +// decoders reject overlapping or reordered edge/blob ranges rather than +// accepting multiple encodings of one graph. + +const enum WireNodeKind { + Null = 0, + Funcref = 1, + Externref = 2, + Exnref = 3, + I31 = 4, + Struct = 5, + Array = 6, + StaticRoot = 7, +} + +export interface ForkReferenceRecipeLimits { + readonly maxWireBytes: number; + readonly maxNodes: number; + readonly maxRoots: number; + readonly maxEdges: number; +} + +const MAX_WIRE_U32 = 0xffff_ffff; + +export const DEFAULT_FORK_REFERENCE_RECIPE_LIMITS: ForkReferenceRecipeLimits = + Object.freeze({ + // These are the version-1 wire fields' representational bounds, not + // smaller policy quotas. The canonical byte-length equation below is the + // tighter combined bound and allocation failure remains the truthful + // resource boundary for a valid process graph. + maxWireBytes: MAX_WIRE_U32, + maxNodes: MAX_WIRE_U32, + maxRoots: MAX_WIRE_U32, + maxEdges: MAX_WIRE_U32, + }); + +export interface ForkNullRecipe { + readonly kind: "null"; +} + +export interface ForkFuncrefRecipe { + readonly kind: "funcref"; + readonly moduleActivation: number; + readonly functionOrdinal: number; +} + +export interface ForkExternrefRecipe { + readonly kind: "externref"; + readonly handle: number; +} + +export interface ForkExnrefRecipe { + readonly kind: "exnref"; + readonly moduleActivation: number; + readonly tagOrdinal: number; + /** Stable artifact-emitted payload layout for this tag. */ + readonly layoutId?: number; + /** Exact scalar payload bits; reference payloads remain graph edges. */ + readonly scalars?: Uint8Array; + readonly payloads: readonly number[]; +} + +export interface ForkI31Recipe { + readonly kind: "i31"; + readonly value: number; +} + +export interface ForkStructRecipe { + readonly kind: "struct"; + readonly moduleActivation: number; + readonly typeOrdinal: number; + readonly layoutId?: number; + /** Exact packed/non-reference field bits in artifact-catalog order. */ + readonly scalars?: Uint8Array; + readonly fields: readonly number[]; +} + +export interface ForkArrayRecipe { + readonly kind: "array"; + readonly moduleActivation: number; + readonly typeOrdinal: number; + readonly layoutId?: number; + /** Exact element bits for scalar arrays; empty for reference arrays. */ + readonly scalars?: Uint8Array; + readonly elements: readonly number[]; +} + +export interface ForkStaticReferenceRootRecipe { + readonly kind: "static-root"; + readonly moduleActivation: number; + readonly staticRootOrdinal: number; +} + +export type ForkReferenceRecipeNode = + | ForkNullRecipe + | ForkFuncrefRecipe + | ForkExternrefRecipe + | ForkExnrefRecipe + | ForkI31Recipe + | ForkStructRecipe + | ForkArrayRecipe + | ForkStaticReferenceRootRecipe; + +export interface ForkReferenceRecipeEntry { + /** Graph-local identity. Aggregate edges and roots refer to this value. */ + readonly id: number; + readonly node: ForkReferenceRecipeNode; +} + +export interface ForkReferenceRecipeGraph { + readonly roots: readonly number[]; + readonly nodes: readonly ForkReferenceRecipeEntry[]; +} + +export interface ForkReferenceModuleTypes { + readonly tags?: readonly { + readonly ordinal: number; + readonly payloadCount: number; + }[]; + readonly structs?: readonly { + readonly ordinal: number; + readonly fieldCount: number; + }[]; + readonly arrays?: readonly { + readonly ordinal: number; + }[]; +} + +interface RegisteredReferenceTypes { + tags: Map; + structs: Map; + arrays: Set; +} + +/** + * Instance-local ownership for exception tags and Wasm GC type identities. + * + * Ordinals are emitted from deterministic artifact catalogs. A child registers + * the same module-activation/type coordinates after instantiation, before a + * recipe is allowed to allocate anything. + */ +export class ForkReferenceTypeCatalog { + private readonly modules = new Map(); + + register(moduleActivation: number, types: ForkReferenceModuleTypes): void { + assertU32(moduleActivation, "module activation"); + if (this.modules.has(moduleActivation)) { + throw new Error( + `reference type catalog ${moduleActivation} is already registered`, + ); + } + + const registered: RegisteredReferenceTypes = { + tags: new Map(), + structs: new Map(), + arrays: new Set(), + }; + for (const tag of types.tags ?? []) { + assertU32(tag.ordinal, "exception tag ordinal"); + assertU32(tag.payloadCount, "exception tag payload count"); + if (registered.tags.has(tag.ordinal)) { + throw new Error(`duplicate exception tag ordinal ${tag.ordinal}`); + } + registered.tags.set(tag.ordinal, tag.payloadCount); + } + for (const struct of types.structs ?? []) { + assertU32(struct.ordinal, "struct type ordinal"); + assertU32(struct.fieldCount, "struct field count"); + if (registered.structs.has(struct.ordinal)) { + throw new Error(`duplicate struct type ordinal ${struct.ordinal}`); + } + registered.structs.set(struct.ordinal, struct.fieldCount); + } + for (const array of types.arrays ?? []) { + assertU32(array.ordinal, "array type ordinal"); + if (registered.arrays.has(array.ordinal)) { + throw new Error(`duplicate array type ordinal ${array.ordinal}`); + } + registered.arrays.add(array.ordinal); + } + this.modules.set(moduleActivation, registered); + } + + validateTag( + moduleActivation: number, + tagOrdinal: number, + payloadCount: number, + ): void { + const types = this.requireModule(moduleActivation); + assertU32(tagOrdinal, "exception tag ordinal"); + const expected = types.tags.get(tagOrdinal); + if (expected === undefined) { + throw new Error( + `exception tag ${moduleActivation}:${tagOrdinal} is not registered`, + ); + } + if (expected !== payloadCount) { + throw new Error( + `exception tag ${moduleActivation}:${tagOrdinal} expects ` + + `${expected} reference payloads, found ${payloadCount}`, + ); + } + } + + validateStruct( + moduleActivation: number, + typeOrdinal: number, + fieldCount: number, + ): void { + const types = this.requireModule(moduleActivation); + assertU32(typeOrdinal, "struct type ordinal"); + const expected = types.structs.get(typeOrdinal); + if (expected === undefined) { + throw new Error( + `struct type ${moduleActivation}:${typeOrdinal} is not registered`, + ); + } + if (expected !== fieldCount) { + throw new Error( + `struct type ${moduleActivation}:${typeOrdinal} expects ` + + `${expected} reference fields, found ${fieldCount}`, + ); + } + } + + validateArray(moduleActivation: number, typeOrdinal: number): void { + const types = this.requireModule(moduleActivation); + assertU32(typeOrdinal, "array type ordinal"); + if (!types.arrays.has(typeOrdinal)) { + throw new Error( + `array type ${moduleActivation}:${typeOrdinal} is not registered`, + ); + } + } + + clear(): void { + this.modules.clear(); + } + + private requireModule(moduleActivation: number): RegisteredReferenceTypes { + assertU32(moduleActivation, "module activation"); + const types = this.modules.get(moduleActivation); + if (!types) { + throw new Error( + `reference type catalog ${moduleActivation} is not registered`, + ); + } + return types; + } +} + +/** + * One fresh-instance staging arena. + * + * Aggregate allocation and edge initialization are separate so the target can + * preserve cycles and shared identity. `commit` must atomically install the + * roots and release its staging roots. `abort` must discard every staged root. + */ +export interface ForkReferenceReplayArena { + materializeExternref(handle: number): unknown; + materializeI31(value: number): unknown; + allocateException( + moduleActivation: number, + tagOrdinal: number, + payloadCount: number, + ): unknown; + allocateStruct( + moduleActivation: number, + typeOrdinal: number, + fieldCount: number, + ): unknown; + allocateArray( + moduleActivation: number, + typeOrdinal: number, + length: number, + ): unknown; + setExceptionPayload(exception: unknown, index: number, value: unknown): void; + setStructField(struct: unknown, index: number, value: unknown): void; + setArrayElement(array: unknown, index: number, value: unknown): void; + commit(roots: readonly unknown[]): void; + abort(): void; +} + +export interface ForkReferenceReplayTarget { + readonly functions: ForkFunctionCatalog; + readonly types: ForkReferenceTypeCatalog; + readonly staticRoots?: ForkStaticRootCatalog; + beginReferenceReplay(nodeCount: number): ForkReferenceReplayArena; +} + +export interface ForkReferenceReplayRequest { + readonly parentGeneration: ForkExternrefGeneration; + readonly childGeneration: ForkExternrefGeneration; + readonly wire: Uint8Array; + readonly target: ForkReferenceReplayTarget; +} + +/** + * Numeric broker ownership transferred to the child by a successful replay. + * + * The coordinator retains no JS/Wasm references after `replay` returns. The + * process/activation owner releases this lease when the reconstructed values + * can no longer reach host externrefs. + */ +export interface ForkReferenceReplayOwnership { + readonly childGeneration: ForkExternrefGeneration; + readonly childPid: number; + release(): void; +} + +class BrokerReferenceReplayOwnership implements ForkReferenceReplayOwnership { + private released = false; + readonly childPid: number; + + constructor( + readonly childGeneration: ForkExternrefGeneration, + private readonly lease: ForkExternrefLease, + ) { + this.childPid = childGeneration.pid; + } + + release(): void { + if (this.released) { + throw new Error("reference replay ownership is already released"); + } + this.lease.release(); + this.released = true; + } +} + +export class ForkReferenceRecipeCoordinator { + constructor( + private readonly sourceFunctions: ForkFunctionCatalog, + private readonly sourceTypes: ForkReferenceTypeCatalog, + private readonly broker: ForkExternrefBroker, + private readonly limits: ForkReferenceRecipeLimits = + DEFAULT_FORK_REFERENCE_RECIPE_LIMITS, + private readonly sourceStaticRoots?: ForkStaticRootCatalog, + ) { + validateLimits(limits); + } + + replay(request: ForkReferenceReplayRequest): ForkReferenceReplayOwnership { + if (request.parentGeneration === request.childGeneration) { + throw new Error( + "fork reference replay requires distinct parent and child generations", + ); + } + + const graph = decodeForkReferenceRecipes(request.wire, this.limits); + validateCatalogOwnership( + graph, + this.sourceFunctions, + this.sourceTypes, + this.sourceStaticRoots, + "source", + ); + validateCatalogOwnership( + graph, + request.target.functions, + request.target.types, + request.target.staticRoots, + "target", + ); + const handles = externrefOwnershipSet(graph); + + let lease: ForkExternrefLease | undefined; + let arena: ForkReferenceReplayArena | undefined; + const values: unknown[] = new Array(graph.nodes.length); + let roots: unknown[] = []; + try { + lease = this.broker.acquireFork( + request.parentGeneration, + request.childGeneration, + handles, + ); + arena = request.target.beginReferenceReplay(graph.nodes.length); + + for (const entry of graph.nodes) { + values[entry.id] = allocateRecipeNode( + entry.node, + request.target.functions, + request.target.staticRoots, + arena, + ); + } + for (const entry of graph.nodes) { + connectRecipeNode(entry, values, arena); + } + roots = graph.roots.map((id) => values[id]); + arena.commit(roots); + + // WHY: successful installation transfers the only strong roots to the + // child activation. The coordinator keeps only numeric broker ownership. + values.fill(undefined); + roots.fill(undefined); + return new BrokerReferenceReplayOwnership( + request.childGeneration, + lease, + ); + } catch (error) { + const rollbackErrors: unknown[] = [error]; + if (arena) { + try { + arena.abort(); + } catch (abortError) { + rollbackErrors.push(abortError); + } + } + values.fill(undefined); + roots.fill(undefined); + if (lease) { + try { + lease.release(); + } catch (releaseError) { + rollbackErrors.push(releaseError); + } + } + if (rollbackErrors.length === 1) throw error; + throw new AggregateError( + rollbackErrors, + "reference replay failed and rollback was incomplete", + ); + } + } +} + +export function encodeForkReferenceRecipes( + graph: ForkReferenceRecipeGraph, + limits: ForkReferenceRecipeLimits = + DEFAULT_FORK_REFERENCE_RECIPE_LIMITS, +): Uint8Array { + validateLimits(limits); + if (graph.nodes.length > limits.maxNodes) { + throw new RangeError( + `reference recipe has ${graph.nodes.length} nodes; limit is ${limits.maxNodes}`, + ); + } + if (graph.roots.length > limits.maxRoots) { + throw new RangeError( + `reference recipe has ${graph.roots.length} roots; limit is ${limits.maxRoots}`, + ); + } + + const ordered = graph.nodes.slice().sort((left, right) => left.id - right.id); + const remap = new Map(); + for (const [wireId, entry] of ordered.entries()) { + assertU32(entry.id, `reference recipe node ${wireId} id`); + if (remap.has(entry.id)) { + throw new Error(`duplicate reference recipe node id ${entry.id}`); + } + remap.set(entry.id, wireId); + validateNodeScalars(entry.node, `reference recipe node ${entry.id}`); + } + + const canonicalNodes: ForkReferenceRecipeEntry[] = ordered.map( + (entry, id) => ({ + id, + node: remapNodeEdges(entry.node, remap, `reference recipe node ${entry.id}`), + }), + ); + const roots = graph.roots.map((id, index) => + remapRequiredId(remap, id, `reference recipe root ${index}`) + ); + const canonical: ForkReferenceRecipeGraph = { roots, nodes: canonicalNodes }; + validateReachability(canonical); + + const edgeCount = canonicalNodes.reduce( + (count, entry) => checkedAdd(count, nodeEdges(entry.node).length, "edge count"), + 0, + ); + if (edgeCount > limits.maxEdges) { + throw new RangeError( + `reference recipe has ${edgeCount} edges; limit is ${limits.maxEdges}`, + ); + } + const blobByteLength = canonicalNodes.reduce( + (count, entry) => + checkedAdd(count, nodeScalarBytes(entry.node).byteLength, "scalar blob byte length"), + 0, + ); + const totalBytes = wireByteLength( + canonicalNodes.length, + roots.length, + edgeCount, + blobByteLength, + ); + if (totalBytes > limits.maxWireBytes) { + throw new RangeError( + `reference recipe needs ${totalBytes} bytes; limit is ${limits.maxWireBytes}`, + ); + } + + const bytes = new Uint8Array(totalBytes); + const view = new DataView(bytes.buffer); + view.setUint32(0, WIRE_MAGIC, true); + view.setUint16(4, FORK_REFERENCE_RECIPE_VERSION, true); + view.setUint16(6, HEADER_SIZE, true); + view.setUint32(8, totalBytes, true); + view.setUint32(12, canonicalNodes.length, true); + view.setUint32(16, roots.length, true); + view.setUint32(20, edgeCount, true); + view.setUint32(24, blobByteLength, true); + view.setUint32(28, NODE_SIZE, true); + view.setUint32(32, 0, true); + view.setUint32(36, 0, true); + + const rootsOffset = HEADER_SIZE + canonicalNodes.length * NODE_SIZE; + const edgesOffset = rootsOffset + roots.length * 4; + const blobsOffset = edgesOffset + edgeCount * 4; + let nextEdge = 0; + let nextBlobByte = 0; + for (const entry of canonicalNodes) { + const offset = HEADER_SIZE + entry.id * NODE_SIZE; + const edges = nodeEdges(entry.node); + const blob = nodeScalarBytes(entry.node); + encodeNodeRecord( + view, + offset, + entry.node, + nextEdge, + edges.length, + nextBlobByte, + blob.byteLength, + ); + for (const edge of edges) { + view.setUint32(edgesOffset + nextEdge * 4, edge, true); + nextEdge++; + } + bytes.set(blob, blobsOffset + nextBlobByte); + nextBlobByte += blob.byteLength; + } + for (const [index, root] of roots.entries()) { + view.setUint32(rootsOffset + index * 4, root, true); + } + return bytes; +} + +export function decodeForkReferenceRecipes( + bytes: Uint8Array, + limits: ForkReferenceRecipeLimits = + DEFAULT_FORK_REFERENCE_RECIPE_LIMITS, +): ForkReferenceRecipeGraph { + validateLimits(limits); + if (bytes.byteLength < HEADER_SIZE) { + throw new Error("reference recipe header is truncated"); + } + if (bytes.byteLength > limits.maxWireBytes) { + throw new RangeError( + `reference recipe has ${bytes.byteLength} bytes; limit is ${limits.maxWireBytes}`, + ); + } + const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); + if (view.getUint32(0, true) !== WIRE_MAGIC) { + throw new Error("reference recipe has invalid magic"); + } + const version = view.getUint16(4, true); + if (version !== FORK_REFERENCE_RECIPE_VERSION) { + throw new Error(`unsupported reference recipe version ${version}`); + } + if (view.getUint16(6, true) !== HEADER_SIZE) { + throw new Error("reference recipe declares an invalid header size"); + } + if (view.getUint32(8, true) !== bytes.byteLength) { + throw new Error("reference recipe declared byte length does not match its buffer"); + } + const nodeCount = view.getUint32(12, true); + const rootCount = view.getUint32(16, true); + const edgeCount = view.getUint32(20, true); + const blobByteLength = view.getUint32(24, true); + if (nodeCount > limits.maxNodes) { + throw new RangeError( + `reference recipe has ${nodeCount} nodes; limit is ${limits.maxNodes}`, + ); + } + if (rootCount > limits.maxRoots) { + throw new RangeError( + `reference recipe has ${rootCount} roots; limit is ${limits.maxRoots}`, + ); + } + if (edgeCount > limits.maxEdges) { + throw new RangeError( + `reference recipe has ${edgeCount} edges; limit is ${limits.maxEdges}`, + ); + } + if (view.getUint32(28, true) !== NODE_SIZE) { + throw new Error("reference recipe declares an invalid node record size"); + } + if (view.getUint32(32, true) !== 0 || view.getUint32(36, true) !== 0) { + throw new Error("reference recipe reserved header fields are nonzero"); + } + const expectedBytes = wireByteLength( + nodeCount, + rootCount, + edgeCount, + blobByteLength, + ); + if (expectedBytes !== bytes.byteLength) { + throw new Error( + `reference recipe layout needs ${expectedBytes} bytes, ` + + `found ${bytes.byteLength}`, + ); + } + + const rootsOffset = HEADER_SIZE + nodeCount * NODE_SIZE; + const edgesOffset = rootsOffset + rootCount * 4; + const blobsOffset = edgesOffset + edgeCount * 4; + const edgeIds = new Uint32Array(edgeCount); + for (let index = 0; index < edgeCount; index++) { + const id = view.getUint32(edgesOffset + index * 4, true); + if (id >= nodeCount) { + throw new Error(`reference recipe edge ${index} targets missing node ${id}`); + } + edgeIds[index] = id; + } + + let expectedEdgeStart = 0; + let expectedBlobStart = 0; + const nodes: ForkReferenceRecipeEntry[] = []; + for (let id = 0; id < nodeCount; id++) { + const offset = HEADER_SIZE + id * NODE_SIZE; + const decoded = decodeNodeRecord( + view, + offset, + id, + edgeIds, + expectedEdgeStart, + bytes.subarray(blobsOffset, blobsOffset + blobByteLength), + expectedBlobStart, + ); + expectedEdgeStart += nodeEdges(decoded).length; + expectedBlobStart += nodeScalarBytes(decoded).byteLength; + nodes.push(Object.freeze({ id, node: decoded })); + } + if (expectedEdgeStart !== edgeCount) { + throw new Error( + `reference recipe node records consume ${expectedEdgeStart} edges, ` + + `header declares ${edgeCount}`, + ); + } + if (expectedBlobStart !== blobByteLength) { + throw new Error( + `reference recipe node records consume ${expectedBlobStart} scalar bytes, ` + + `header declares ${blobByteLength}`, + ); + } + + const roots: number[] = []; + for (let index = 0; index < rootCount; index++) { + const id = view.getUint32(rootsOffset + index * 4, true); + if (id >= nodeCount) { + throw new Error(`reference recipe root ${index} targets missing node ${id}`); + } + roots.push(id); + } + const graph: ForkReferenceRecipeGraph = Object.freeze({ + roots: Object.freeze(roots), + nodes: Object.freeze(nodes), + }); + validateReachability(graph); + return graph; +} + +function validateCatalogOwnership( + graph: ForkReferenceRecipeGraph, + functions: ForkFunctionCatalog, + types: ForkReferenceTypeCatalog, + staticRoots: ForkStaticRootCatalog | undefined, + side: "source" | "target", +): void { + for (const entry of graph.nodes) { + const node = entry.node; + try { + switch (node.kind) { + case "funcref": + functions.decode(functionRecipe(node)); + break; + case "exnref": + types.validateTag( + node.moduleActivation, + node.tagOrdinal, + node.payloads.length, + ); + break; + case "struct": + types.validateStruct( + node.moduleActivation, + node.typeOrdinal, + node.fields.length, + ); + break; + case "array": + types.validateArray(node.moduleActivation, node.typeOrdinal); + break; + case "static-root": + if (!staticRoots) { + throw new Error("static-root catalog is not registered"); + } + staticRoots.decode(staticRootRecipe(node)); + break; + case "null": + case "externref": + case "i31": + break; + } + } catch (error) { + throw new Error( + `${side} catalog rejected reference recipe node ${entry.id}: ` + + `${errorMessage(error)}`, + { cause: error }, + ); + } + } +} + +function allocateRecipeNode( + node: ForkReferenceRecipeNode, + functions: ForkFunctionCatalog, + staticRoots: ForkStaticRootCatalog | undefined, + arena: ForkReferenceReplayArena, +): unknown { + switch (node.kind) { + case "null": + return null; + case "funcref": + return functions.decode(functionRecipe(node)); + case "externref": + return arena.materializeExternref(node.handle); + case "exnref": + return arena.allocateException( + node.moduleActivation, + node.tagOrdinal, + node.payloads.length, + ); + case "i31": + return arena.materializeI31(node.value); + case "struct": + return arena.allocateStruct( + node.moduleActivation, + node.typeOrdinal, + node.fields.length, + ); + case "array": + return arena.allocateArray( + node.moduleActivation, + node.typeOrdinal, + node.elements.length, + ); + case "static-root": + if (!staticRoots) { + throw new Error("static-root catalog is not registered"); + } + return staticRoots.decode(staticRootRecipe(node)); + } +} + +function connectRecipeNode( + entry: ForkReferenceRecipeEntry, + values: readonly unknown[], + arena: ForkReferenceReplayArena, +): void { + const value = values[entry.id]; + switch (entry.node.kind) { + case "exnref": + entry.node.payloads.forEach((payload, index) => { + arena.setExceptionPayload(value, index, values[payload]); + }); + break; + case "struct": + entry.node.fields.forEach((field, index) => { + arena.setStructField(value, index, values[field]); + }); + break; + case "array": + entry.node.elements.forEach((element, index) => { + arena.setArrayElement(value, index, values[element]); + }); + break; + case "null": + case "funcref": + case "externref": + case "i31": + case "static-root": + break; + } +} + +function externrefOwnershipSet( + graph: ForkReferenceRecipeGraph, +): Set { + const handles = new Set(); + // WHY: reference-graph multiplicity expresses Wasm aliasing, not independent + // host lifetime. One process execution generation owns one lease per opaque + // identity no matter how many roots or aggregate fields point at it. + const addIfExternref = (id: number): void => { + const node = graph.nodes[id]?.node; + if (node?.kind !== "externref") return; + handles.add(node.handle); + }; + graph.roots.forEach(addIfExternref); + for (const entry of graph.nodes) { + nodeEdges(entry.node).forEach(addIfExternref); + } + return handles; +} + +function functionRecipe(node: ForkFuncrefRecipe): ForkFunctionRecipe { + return { + moduleActivation: node.moduleActivation, + ordinal: node.functionOrdinal, + }; +} + +function staticRootRecipe( + node: ForkStaticReferenceRootRecipe, +): ForkStaticRootRecipe { + return { + moduleActivation: node.moduleActivation, + ordinal: node.staticRootOrdinal, + }; +} + +function encodeNodeRecord( + view: DataView, + offset: number, + node: ForkReferenceRecipeNode, + edgeStart: number, + edgeCount: number, + blobStart: number, + blobByteLength: number, +): void { + let kind: WireNodeKind; + let first = 0; + let second = 0; + let third = 0; + switch (node.kind) { + case "null": + kind = WireNodeKind.Null; + break; + case "funcref": + kind = WireNodeKind.Funcref; + first = node.moduleActivation; + second = node.functionOrdinal; + break; + case "externref": { + kind = WireNodeKind.Externref; + const handle = BigInt(node.handle); + first = Number(handle & 0xffff_ffffn); + second = Number(handle >> 32n); + break; + } + case "exnref": + kind = WireNodeKind.Exnref; + first = node.moduleActivation; + second = node.tagOrdinal; + third = node.layoutId ?? 0; + break; + case "i31": + kind = WireNodeKind.I31; + first = node.value >>> 0; + break; + case "struct": + kind = WireNodeKind.Struct; + first = node.moduleActivation; + second = node.typeOrdinal; + third = node.layoutId ?? 0; + break; + case "array": + kind = WireNodeKind.Array; + first = node.moduleActivation; + second = node.typeOrdinal; + third = node.layoutId ?? 0; + break; + case "static-root": + kind = WireNodeKind.StaticRoot; + first = node.moduleActivation; + second = node.staticRootOrdinal; + break; + } + const aggregate = + node.kind === "exnref" || node.kind === "struct" || node.kind === "array"; + const recordEdgeStart = aggregate ? edgeStart : 0; + const recordBlobStart = aggregate ? blobStart : 0; + view.setUint8(offset, kind); + view.setUint8(offset + 1, 0); + view.setUint16(offset + 2, 0, true); + view.setUint32(offset + 4, first, true); + view.setUint32(offset + 8, second, true); + view.setUint32(offset + 12, third, true); + view.setUint32(offset + 16, recordEdgeStart, true); + view.setUint32(offset + 20, edgeCount, true); + view.setUint32(offset + 24, recordBlobStart, true); + view.setUint32(offset + 28, blobByteLength, true); +} + +function decodeNodeRecord( + view: DataView, + offset: number, + id: number, + edges: Uint32Array, + expectedEdgeStart: number, + blobs: Uint8Array, + expectedBlobStart: number, +): ForkReferenceRecipeNode { + const context = `reference recipe node ${id}`; + const kind = view.getUint8(offset); + if ( + view.getUint8(offset + 1) !== 0 + || view.getUint16(offset + 2, true) !== 0 + ) { + throw new Error(`${context} has nonzero flags or reserved fields`); + } + const first = view.getUint32(offset + 4, true); + const second = view.getUint32(offset + 8, true); + const third = view.getUint32(offset + 12, true); + const edgeStart = view.getUint32(offset + 16, true); + const edgeCount = view.getUint32(offset + 20, true); + const blobStart = view.getUint32(offset + 24, true); + const blobByteLength = view.getUint32(offset + 28, true); + const edgeEnd = checkedAdd(edgeStart, edgeCount, `${context} edge range`); + if (edgeEnd > edges.length) { + throw new Error(`${context} edge range exceeds the shared edge vector`); + } + const blobEnd = checkedAdd( + blobStart, + blobByteLength, + `${context} scalar blob range`, + ); + if (blobEnd > blobs.byteLength) { + throw new Error(`${context} scalar blob range exceeds the shared blob vector`); + } + + const requireNoAggregateData = (): void => { + if ( + edgeStart !== 0 + || edgeCount !== 0 + || blobStart !== 0 + || blobByteLength !== 0 + ) { + throw new Error(`${context} scalar record declares graph edges or payload bytes`); + } + }; + const requireZeroScalars = (): void => { + if (first !== 0 || second !== 0 || third !== 0) { + throw new Error(`${context} has noncanonical scalar fields`); + } + }; + const aggregateEdges = (): readonly number[] => { + if (edgeStart !== expectedEdgeStart) { + throw new Error( + `${context} has noncanonical edge start ${edgeStart}; ` + + `expected ${expectedEdgeStart}`, + ); + } + return Object.freeze(Array.from(edges.subarray(edgeStart, edgeEnd))); + }; + const aggregateBlob = (): Uint8Array => { + if (blobStart !== expectedBlobStart) { + throw new Error( + `${context} has noncanonical scalar blob start ${blobStart}; ` + + `expected ${expectedBlobStart}`, + ); + } + return blobs.slice(blobStart, blobEnd); + }; + + let node: ForkReferenceRecipeNode; + switch (kind) { + case WireNodeKind.Null: + requireNoAggregateData(); + requireZeroScalars(); + node = { kind: "null" }; + break; + case WireNodeKind.Funcref: + requireNoAggregateData(); + if (third !== 0) { + throw new Error(`${context} funcref reserved scalar field is nonzero`); + } + node = { + kind: "funcref", + moduleActivation: first, + functionOrdinal: second, + }; + break; + case WireNodeKind.Externref: { + requireNoAggregateData(); + if (third !== 0) { + throw new Error(`${context} externref reserved scalar field is nonzero`); + } + const handle = Number((BigInt(second) << 32n) | BigInt(first)); + assertHandle(handle, `${context} externref handle`); + node = { kind: "externref", handle }; + break; + } + case WireNodeKind.Exnref: + node = { + kind: "exnref", + moduleActivation: first, + tagOrdinal: second, + layoutId: third, + scalars: aggregateBlob(), + payloads: aggregateEdges(), + }; + break; + case WireNodeKind.I31: + requireNoAggregateData(); + if (second !== 0 || third !== 0) { + throw new Error(`${context} i31 reserved scalar field is nonzero`); + } + node = { kind: "i31", value: first | 0 }; + break; + case WireNodeKind.Struct: + node = { + kind: "struct", + moduleActivation: first, + typeOrdinal: second, + layoutId: third, + scalars: aggregateBlob(), + fields: aggregateEdges(), + }; + break; + case WireNodeKind.Array: + node = { + kind: "array", + moduleActivation: first, + typeOrdinal: second, + layoutId: third, + scalars: aggregateBlob(), + elements: aggregateEdges(), + }; + break; + case WireNodeKind.StaticRoot: + requireNoAggregateData(); + if (third !== 0) { + throw new Error(`${context} static-root reserved scalar field is nonzero`); + } + node = { + kind: "static-root", + moduleActivation: first, + staticRootOrdinal: second, + }; + break; + default: + throw new Error(`${context} has unknown kind ${kind}`); + } + validateNodeScalars(node, context); + return Object.freeze(node); +} + +function remapNodeEdges( + node: ForkReferenceRecipeNode, + remap: ReadonlyMap, + context: string, +): ForkReferenceRecipeNode { + switch (node.kind) { + case "exnref": + return { + ...node, + scalars: nodeScalarBytes(node).slice(), + payloads: node.payloads.map((id, index) => + remapRequiredId(remap, id, `${context} payload ${index}`) + ), + }; + case "struct": + return { + ...node, + scalars: nodeScalarBytes(node).slice(), + fields: node.fields.map((id, index) => + remapRequiredId(remap, id, `${context} field ${index}`) + ), + }; + case "array": + return { + ...node, + scalars: nodeScalarBytes(node).slice(), + elements: node.elements.map((id, index) => + remapRequiredId(remap, id, `${context} element ${index}`) + ), + }; + case "null": + case "funcref": + case "externref": + case "i31": + case "static-root": + return { ...node }; + } +} + +function remapRequiredId( + remap: ReadonlyMap, + id: number, + context: string, +): number { + assertU32(id, context); + const mapped = remap.get(id); + if (mapped === undefined) { + throw new Error(`${context} targets missing node ${id}`); + } + return mapped; +} + +function nodeEdges(node: ForkReferenceRecipeNode): readonly number[] { + switch (node.kind) { + case "exnref": + return node.payloads; + case "struct": + return node.fields; + case "array": + return node.elements; + case "null": + case "funcref": + case "externref": + case "i31": + case "static-root": + return []; + } +} + +function nodeScalarBytes(node: ForkReferenceRecipeNode): Uint8Array { + switch (node.kind) { + case "exnref": + case "struct": + case "array": + return node.scalars ?? new Uint8Array(); + case "null": + case "funcref": + case "externref": + case "i31": + case "static-root": + return new Uint8Array(); + } +} + +function validateNodeScalars( + node: ForkReferenceRecipeNode, + context: string, +): void { + switch (node.kind) { + case "null": + return; + case "funcref": + assertU32(node.moduleActivation, `${context} module activation`); + assertU32(node.functionOrdinal, `${context} function ordinal`); + return; + case "externref": + assertHandle(node.handle, `${context} externref handle`); + return; + case "exnref": + assertU32(node.moduleActivation, `${context} module activation`); + assertU32(node.tagOrdinal, `${context} tag ordinal`); + assertU32(node.layoutId ?? 0, `${context} layout id`); + if (!(nodeScalarBytes(node) instanceof Uint8Array)) { + throw new TypeError(`${context} scalar payload is not a Uint8Array`); + } + assertU32(node.payloads.length, `${context} payload count`); + return; + case "i31": + if ( + !Number.isInteger(node.value) + || node.value < MIN_I31 + || node.value > MAX_I31 + ) { + throw new RangeError(`${context} has invalid i31 value ${node.value}`); + } + return; + case "struct": + assertU32(node.moduleActivation, `${context} module activation`); + assertU32(node.typeOrdinal, `${context} type ordinal`); + assertU32(node.layoutId ?? 0, `${context} layout id`); + if (!(nodeScalarBytes(node) instanceof Uint8Array)) { + throw new TypeError(`${context} scalar payload is not a Uint8Array`); + } + assertU32(node.fields.length, `${context} field count`); + return; + case "array": + assertU32(node.moduleActivation, `${context} module activation`); + assertU32(node.typeOrdinal, `${context} type ordinal`); + assertU32(node.layoutId ?? 0, `${context} layout id`); + if (!(nodeScalarBytes(node) instanceof Uint8Array)) { + throw new TypeError(`${context} scalar payload is not a Uint8Array`); + } + assertU32(node.elements.length, `${context} element count`); + return; + case "static-root": + assertU32(node.moduleActivation, `${context} module activation`); + assertU32(node.staticRootOrdinal, `${context} static-root ordinal`); + return; + } +} + +function validateReachability(graph: ForkReferenceRecipeGraph): void { + const reached = new Uint8Array(graph.nodes.length); + const pending = [...graph.roots]; + while (pending.length > 0) { + const id = pending.pop()!; + if (reached[id] !== 0) continue; + reached[id] = 1; + for (const edge of nodeEdges(graph.nodes[id]!.node)) pending.push(edge); + } + const unreachable = reached.findIndex((value) => value === 0); + if (unreachable !== -1) { + throw new Error( + `reference recipe node ${unreachable} is unreachable from every root`, + ); + } +} + +function wireByteLength( + nodeCount: number, + rootCount: number, + edgeCount: number, + blobByteLength: number, +): number { + const nodesEnd = checkedAdd( + HEADER_SIZE, + checkedMultiply(nodeCount, NODE_SIZE, "node byte length"), + "node section end", + ); + const rootsEnd = checkedAdd( + nodesEnd, + checkedMultiply(rootCount, 4, "root byte length"), + "root section end", + ); + const edgesEnd = checkedAdd( + rootsEnd, + checkedMultiply(edgeCount, 4, "edge byte length"), + "edge section end", + ); + return checkedAdd(edgesEnd, blobByteLength, "wire byte length"); +} + +function validateLimits(limits: ForkReferenceRecipeLimits): void { + for (const [label, value] of [ + ["wire byte", limits.maxWireBytes], + ["node", limits.maxNodes], + ["root", limits.maxRoots], + ["edge", limits.maxEdges], + ] as const) { + if ( + !Number.isSafeInteger(value) + || value < 0 + || value > 0xffff_ffff + ) { + throw new RangeError(`invalid reference recipe ${label} limit ${value}`); + } + } + if (limits.maxWireBytes < HEADER_SIZE) { + throw new RangeError( + `reference recipe wire byte limit must be at least ${HEADER_SIZE}`, + ); + } +} + +function checkedAdd(left: number, right: number, context: string): number { + const value = left + right; + if (!Number.isSafeInteger(value)) { + throw new RangeError(`${context} exceeds the host safe integer range`); + } + return value; +} + +function checkedMultiply(left: number, right: number, context: string): number { + const value = left * right; + if (!Number.isSafeInteger(value)) { + throw new RangeError(`${context} exceeds the host safe integer range`); + } + return value; +} + +function assertU32(value: number, context: string): void { + if (!Number.isInteger(value) || value < 0 || value > 0xffff_ffff) { + throw new RangeError(`${context} is not an unsigned 32-bit integer`); + } +} + +function assertHandle(value: number, context: string): void { + if (!Number.isInteger(value) || value <= 0 || value > 0xffff_ffff) { + throw new RangeError(`${context} is not a positive unsigned 32-bit integer`); + } +} + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} diff --git a/host/src/fork-reference-segments.ts b/host/src/fork-reference-segments.ts new file mode 100644 index 0000000000..3bf63ac902 --- /dev/null +++ b/host/src/fork-reference-segments.ts @@ -0,0 +1,2131 @@ +import { + ForkModuleStateRecordKind, + type ForkModuleStateArena, + type ForkModuleStateRecord, + type ForkModuleStateRecordView, +} from "./fork-module-state"; +import type { + ForkReferenceRecipeEntry, + ForkReferenceRecipeNode, +} from "./fork-reference-recipes"; +import { + WPK_FORK_REFERENCE_NODE_RECORD_SIZE, + WPK_FORK_REFERENCE_SECTION_EDGES, + WPK_FORK_REFERENCE_SECTION_NODES, + WPK_FORK_REFERENCE_SECTION_SCALARS, + WPK_FORK_REFERENCE_SECTION_VECTOR_ENTRIES, + WPK_FORK_REFERENCE_SECTION_VECTOR_INDEX, + WPK_FORK_REFERENCE_SEGMENT_HEADER_SIZE, + WPK_FORK_REFERENCE_SEGMENT_KNOWN_FLAGS, + WPK_FORK_REFERENCE_SEGMENT_MAGIC, + WPK_FORK_REFERENCE_TRANSACTION_FLAG_SEALED, + WPK_FORK_REFERENCE_TRANSACTION_KNOWN_FLAGS, + WPK_FORK_REFERENCE_TRANSACTION_MAGIC, + WPK_FORK_REFERENCE_TRANSACTION_MANIFEST_SIZE, + WPK_FORK_REFERENCE_TRANSACTION_VERSION, + WPK_FORK_REFERENCE_VECTOR_INDEX_SIZE, +} from "./generated/abi"; + +const KFRV_MAGIC = littleEndianMagic(WPK_FORK_REFERENCE_TRANSACTION_MAGIC); +const KFRS_MAGIC = littleEndianMagic(WPK_FORK_REFERENCE_SEGMENT_MAGIC); +export const FORK_REFERENCE_TRANSACTION_VERSION = + WPK_FORK_REFERENCE_TRANSACTION_VERSION; +export const FORK_REFERENCE_MANIFEST_SIZE = + WPK_FORK_REFERENCE_TRANSACTION_MANIFEST_SIZE; +export const FORK_REFERENCE_SEGMENT_HEADER_SIZE = + WPK_FORK_REFERENCE_SEGMENT_HEADER_SIZE; +export const FORK_REFERENCE_NODE_RECORD_SIZE = + WPK_FORK_REFERENCE_NODE_RECORD_SIZE; +export const FORK_REFERENCE_VECTOR_INDEX_SIZE = + WPK_FORK_REFERENCE_VECTOR_INDEX_SIZE; +export const DEFAULT_FORK_REFERENCE_SEGMENT_DATA_BYTES = 1024 * 1024; +const FORK_REFERENCE_MANIFEST_FLAG_SEALED = + WPK_FORK_REFERENCE_TRANSACTION_FLAG_SEALED; +const FORK_REFERENCE_MANIFEST_KNOWN_FLAGS = + WPK_FORK_REFERENCE_TRANSACTION_KNOWN_FLAGS; +const MAX_U64 = 0xffff_ffff_ffff_ffffn; +const MAX_U32 = 0xffff_ffff; +const MAX_U32_DIRECTORY_LENGTH = 0x1_0000_0000; +const VECTOR_PAGE_ENTRIES = 4096; + +const enum ReferenceSection { + Nodes = WPK_FORK_REFERENCE_SECTION_NODES, + Edges = WPK_FORK_REFERENCE_SECTION_EDGES, + Scalars = WPK_FORK_REFERENCE_SECTION_SCALARS, + VectorIndex = WPK_FORK_REFERENCE_SECTION_VECTOR_INDEX, + VectorEntries = WPK_FORK_REFERENCE_SECTION_VECTOR_ENTRIES, +} + +const SECTION_COUNT = 5; + +const enum WireNodeKind { + Null = 0, + Funcref = 1, + Externref = 2, + Exnref = 3, + I31 = 4, + Struct = 5, + Array = 6, + StaticRoot = 7, +} + +/** + * Read-only paged storage for one activation recipe-ID vector. + * + * A vector may grow to the generated Wasm u32 index boundary without asking + * JavaScript for one equally large dense Array. Pages are private and become + * immutable when the builder finishes. + */ +export interface ForkReferenceVector extends Iterable { + readonly length: number; + get(index: number): number | undefined; + forEach(callback: (value: number, index: number) => void): void; +} + +/** + * Random-access paged directory whose index namespace is the complete u32. + * + * Unlike a JavaScript Array, it can represent 2^32 entries (indices zero + * through 0xffff_ffff) without one engine-level contiguous backing store. + */ +export interface ForkReferenceDirectory extends Iterable { + readonly length: number; + get(index: number): T | undefined; + has(index: number): boolean; + forEach(callback: (value: T, index: number) => void): void; + some(predicate: (value: T, index: number) => boolean): boolean; +} + +type ForkReferenceSequence = + | ForkReferenceDirectory + | readonly T[]; + +export class PagedForkReferenceDirectory + implements ForkReferenceDirectory +{ + private pages = new Map>(); + private count = 0; + + get length(): number { + return this.count; + } + + get(index: number): T | undefined { + if ( + !Number.isInteger(index) + || index < 0 + || index > MAX_U32 + || index >= this.count + ) { + return undefined; + } + return this.pages.get(Math.floor(index / VECTOR_PAGE_ENTRIES))?.[ + index % VECTOR_PAGE_ENTRIES + ]; + } + + has(index: number): boolean { + if ( + !Number.isInteger(index) + || index < 0 + || index > MAX_U32 + || index >= this.count + ) { + return false; + } + const page = this.pages.get(Math.floor(index / VECTOR_PAGE_ENTRIES)); + return !!page && (index % VECTOR_PAGE_ENTRIES) in page; + } + + push(value: T): number { + if (this.count >= MAX_U32_DIRECTORY_LENGTH) { + throw new RangeError("fork reference u32 directory is exhausted"); + } + const index = this.count; + const pageIndex = Math.floor(index / VECTOR_PAGE_ENTRIES); + const page = this.pages.get(pageIndex) ?? []; + page[index % VECTOR_PAGE_ENTRIES] = value; + this.pages.set(pageIndex, page); + this.count++; + return this.count; + } + + set(index: number, value: T): void { + if (!Number.isInteger(index) || index < 0 || index >= this.count) { + throw new RangeError(`fork reference directory index ${index} is out of bounds`); + } + const pageIndex = Math.floor(index / VECTOR_PAGE_ENTRIES); + const page = this.pages.get(pageIndex) ?? []; + page[index % VECTOR_PAGE_ENTRIES] = value; + this.pages.set(pageIndex, page); + } + + clear(): void { + this.pages.clear(); + this.count = 0; + } + + forEach(callback: (value: T, index: number) => void): void { + let index = 0; + const pageCount = Math.ceil(this.count / VECTOR_PAGE_ENTRIES); + for (let pageIndex = 0; pageIndex < pageCount; pageIndex++) { + const page = this.pages.get(pageIndex); + if (!page) { + throw new Error( + `fork reference directory has no page ${pageIndex}`, + ); + } + const count = Math.min(page.length, this.count - index); + for (let local = 0; local < count; local++, index++) { + if (!(local in page)) { + throw new Error(`fork reference directory has a hole at ${index}`); + } + callback(page[local]!, index); + } + } + if (index !== this.count) { + throw new Error(`fork reference directory ends at ${index}; expected ${this.count}`); + } + } + + some(predicate: (value: T, index: number) => boolean): boolean { + for (const [index, value] of this.indexed()) { + if (predicate(value, index)) return true; + } + return false; + } + + *[Symbol.iterator](): Iterator { + for (const [, value] of this.indexed()) yield value; + } + + private *indexed(): IterableIterator { + let index = 0; + const pageCount = Math.ceil(this.count / VECTOR_PAGE_ENTRIES); + for (let pageIndex = 0; pageIndex < pageCount; pageIndex++) { + const page = this.pages.get(pageIndex); + if (!page) { + throw new Error( + `fork reference directory has no page ${pageIndex}`, + ); + } + const count = Math.min(page.length, this.count - index); + for (let local = 0; local < count; local++, index++) { + if (!(local in page)) { + throw new Error(`fork reference directory has a hole at ${index}`); + } + yield [index, page[local]!] as const; + } + } + if (index !== this.count) { + throw new Error(`fork reference directory ends at ${index}; expected ${this.count}`); + } + } +} + +/** + * Mutable tail layered over one immutable decoded directory. + * + * Early replay and ordinary replay both need to intern short-lived codec + * vectors after decoding KFRV. Keeping the decoded directory as the base avoids + * copying every vector reference into a second page tree merely to make the + * tail appendable. + */ +export class ForkReferenceDirectoryOverlay + implements ForkReferenceDirectory +{ + private base: ForkReferenceDirectory = + new PagedForkReferenceDirectory(); + private readonly extension = new PagedForkReferenceDirectory(); + + constructor(base?: ForkReferenceDirectory) { + if (base) this.base = base; + } + + get length(): number { + return this.base.length + this.extension.length; + } + + get(index: number): T | undefined { + if (!Number.isInteger(index) || index < 0 || index >= this.length) { + return undefined; + } + return index < this.base.length + ? this.base.get(index) + : this.extension.get(index - this.base.length); + } + + has(index: number): boolean { + if (!Number.isInteger(index) || index < 0 || index >= this.length) { + return false; + } + return index < this.base.length + ? this.base.has(index) + : this.extension.has(index - this.base.length); + } + + push(value: T): number { + if (this.length >= MAX_U32_DIRECTORY_LENGTH) { + throw new RangeError("fork reference u32 directory is exhausted"); + } + this.extension.push(value); + return this.length; + } + + reset(base?: ForkReferenceDirectory): void { + this.extension.clear(); + this.base = base ?? new PagedForkReferenceDirectory(); + } + + clear(): void { + this.reset(); + } + + forEach(callback: (value: T, index: number) => void): void { + this.base.forEach(callback); + const baseLength = this.base.length; + this.extension.forEach((value, index) => { + callback(value, baseLength + index); + }); + } + + some(predicate: (value: T, index: number) => boolean): boolean { + if (this.base.some(predicate)) return true; + const baseLength = this.base.length; + return this.extension.some((value, index) => + predicate(value, baseLength + index) + ); + } + + *[Symbol.iterator](): Iterator { + yield* this.base; + yield* this.extension; + } +} + +export class PagedForkReferenceVector implements ForkReferenceVector { + static readonly empty = new PagedForkReferenceVector(new Map(), 0); + + constructor( + private readonly pages: ReadonlyMap, + readonly length: number, + ) { + if (!Number.isInteger(length) || length < 0 || length > MAX_U32) { + throw new RangeError(`fork reference vector length ${length} is not a u32`); + } + } + + get(index: number): number | undefined { + if (!Number.isInteger(index) || index < 0 || index >= this.length) { + return undefined; + } + return this.pages.get(Math.floor(index / VECTOR_PAGE_ENTRIES))![ + index % VECTOR_PAGE_ENTRIES + ]; + } + + forEach(callback: (value: number, index: number) => void): void { + let index = 0; + const pageCount = Math.ceil(this.length / VECTOR_PAGE_ENTRIES); + for (let pageIndex = 0; pageIndex < pageCount; pageIndex++) { + const page = this.pages.get(pageIndex)!; + const remaining = this.length - index; + const count = Math.min(page.length, remaining); + for (let local = 0; local < count; local++, index++) { + callback(page[local]!, index); + } + } + } + + *[Symbol.iterator](): Iterator { + let emitted = 0; + const pageCount = Math.ceil(this.length / VECTOR_PAGE_ENTRIES); + for (let pageIndex = 0; pageIndex < pageCount; pageIndex++) { + const page = this.pages.get(pageIndex)!; + const count = Math.min(page.length, this.length - emitted); + for (let index = 0; index < count; index++, emitted++) { + yield page[index]!; + } + } + } +} + +export class ForkReferenceVectorBuilder { + private readonly pages = new Map(); + private count = 0; + + constructor(readonly expectedLength: number) { + if ( + !Number.isInteger(expectedLength) + || expectedLength <= 0 + || expectedLength > MAX_U32 + ) { + throw new RangeError( + `fork reference vector length ${expectedLength} is not a nonzero u32`, + ); + } + } + + get length(): number { + return this.count; + } + + append(recipeId: number): void { + assertRecipeId(recipeId, "fork reference vector recipe"); + if (this.count >= this.expectedLength) { + throw new Error("fork reference vector exceeds its declared length"); + } + const pageIndex = Math.floor(this.count / VECTOR_PAGE_ENTRIES); + let page = this.pages.get(pageIndex); + if (!page) { + page = new Uint32Array( + Math.min(VECTOR_PAGE_ENTRIES, this.expectedLength - this.count), + ); + this.pages.set(pageIndex, page); + } + page[this.count % VECTOR_PAGE_ENTRIES] = recipeId; + this.count++; + } + + finish(): PagedForkReferenceVector { + if (this.count !== this.expectedLength) { + throw new Error( + `fork reference vector has ${this.count} entries; ` + + `expected ${this.expectedLength}`, + ); + } + return new PagedForkReferenceVector(new Map(this.pages), this.count); + } +} + +export function forkReferenceVectorFrom( + values: Iterable, + expectedLength?: number, +): PagedForkReferenceVector { + if (expectedLength === 0) return PagedForkReferenceVector.empty; + if (expectedLength !== undefined) { + const builder = new ForkReferenceVectorBuilder(expectedLength); + for (const value of values) builder.append(value); + return builder.finish(); + } + const pages = new Map(); + let count = 0; + for (const value of values) { + assertRecipeId(value, "fork reference vector recipe"); + if (count === MAX_U32) { + throw new RangeError("fork reference vector length exceeds u32"); + } + const pageIndex = Math.floor(count / VECTOR_PAGE_ENTRIES); + let page = pages.get(pageIndex); + if (!page) { + page = new Uint32Array(VECTOR_PAGE_ENTRIES); + pages.set(pageIndex, page); + } + page[count % VECTOR_PAGE_ENTRIES] = value; + count++; + } + return count === 0 + ? PagedForkReferenceVector.empty + : new PagedForkReferenceVector(pages, count); +} + +export function forkReferenceVectorsEqual( + left: ForkReferenceVector, + right: ForkReferenceVector, +): boolean { + if (left.length !== right.length) return false; + for (let index = 0; index < left.length; index++) { + if (left.get(index) !== right.get(index)) return false; + } + return true; +} + +export function forkReferenceVectorInternKey( + values: ForkReferenceVector, +): string { + let first = (0x811c_9dc5 ^ values.length) >>> 0; + let second = (0x9e37_79b9 ^ values.length) >>> 0; + values.forEach((value) => { + first = Math.imul(first ^ value, 0x0100_0193) >>> 0; + const rotated = ((value << 16) | (value >>> 16)) >>> 0; + second = Math.imul(second ^ rotated, 0x85eb_ca6b) >>> 0; + second = (second ^ (first >>> 13)) >>> 0; + }); + return `${values.length}:${first}:${second}`; +} + +export type ForkReferenceVectorInternIndex = + ReadonlyMap>; + +export type MutableForkReferenceVectorInternIndex = + Map>; + +export function indexForkReferenceVector( + index: MutableForkReferenceVectorInternIndex, + values: ForkReferenceVector, + ordinal: number, +): void { + assertU32(ordinal, "fork reference vector ordinal"); + const key = forkReferenceVectorInternKey(values); + let candidates = index.get(key); + if (!candidates) { + candidates = new PagedForkReferenceDirectory(); + index.set(key, candidates); + } + candidates.push(ordinal); +} + +export function findForkReferenceVectorOrdinal( + indexes: Iterable, + directory: ForkReferenceDirectory, + values: ForkReferenceVector, +): number | undefined { + const key = forkReferenceVectorInternKey(values); + for (const index of indexes) { + const candidates = index.get(key); + if (!candidates) continue; + for (const ordinal of candidates) { + const candidate = directory.get(ordinal); + if (candidate && forkReferenceVectorsEqual(candidate, values)) { + return ordinal; + } + } + } + return undefined; +} + +export interface DecodedSegmentedForkReferenceTransaction { + /** + * One object is shared by pre-instantiation and ordinary child replay. + * Object identity is the adoption proof; no complete wire copy is retained. + */ + readonly identity: object; + readonly graph: { + readonly roots: readonly number[]; + readonly nodes: ForkReferenceDirectory; + }; + /** Index zero is the canonical empty-vector sentinel. */ + readonly vectors: ForkReferenceDirectory; + /** Canonical hash candidates shared by early and ordinary replay. */ + readonly vectorIntern: ForkReferenceVectorInternIndex; +} + +export interface ForkReferenceSegmentEncodingOptions { + /** + * Transient writer target, not a total-state limit. Smaller values are useful + * for boundary tests; production uses a near-1-MiB bounded copy window. + */ + readonly segmentDataBytes?: number; +} + +interface SectionTotals { + readonly nodes: bigint; + readonly edges: bigint; + readonly scalars: bigint; + readonly vectorIndex: bigint; + readonly vectorEntries: bigint; +} + +interface SegmentState { + ordinal: bigint; +} + +class SegmentWriter { + private readonly buffer: Uint8Array; + private used = 0; + private logicalOffset = 0n; + + constructor( + private readonly arena: ForkModuleStateArena, + private readonly ownerId: number, + private readonly section: ReferenceSection, + private readonly state: SegmentState, + segmentDataBytes: number, + ) { + this.buffer = new Uint8Array(segmentDataBytes); + } + + write(bytes: Uint8Array): void { + let source = 0; + while (source < bytes.byteLength) { + const count = Math.min( + bytes.byteLength - source, + this.buffer.byteLength - this.used, + ); + this.buffer.set(bytes.subarray(source, source + count), this.used); + this.used += count; + source += count; + if (this.used === this.buffer.byteLength) this.flush(); + } + } + + finish(): void { + this.flush(); + } + + private flush(): void { + if (this.used === 0) return; + const payload = new Uint8Array( + FORK_REFERENCE_SEGMENT_HEADER_SIZE + this.used, + ); + const view = new DataView(payload.buffer); + view.setUint32(0, KFRS_MAGIC, true); + view.setUint16(4, FORK_REFERENCE_TRANSACTION_VERSION, true); + view.setUint16(6, FORK_REFERENCE_SEGMENT_HEADER_SIZE, true); + view.setUint16(8, this.section, true); + view.setUint16(10, WPK_FORK_REFERENCE_SEGMENT_KNOWN_FLAGS, true); + view.setUint32(12, 0, true); + view.setBigUint64(16, this.state.ordinal, true); + view.setBigUint64(24, this.logicalOffset, true); + view.setUint32(32, this.used, true); + view.setUint32(36, 0, true); + payload.set(this.buffer.subarray(0, this.used), FORK_REFERENCE_SEGMENT_HEADER_SIZE); + this.arena.appendRecord({ + kind: ForkModuleStateRecordKind.ReferenceRecipeSegment, + activationId: 0, + ownerId: this.ownerId, + payload, + }); + this.logicalOffset = addU64( + this.logicalOffset, + BigInt(this.used), + "fork reference section offset", + ); + this.state.ordinal = addU64( + this.state.ordinal, + 1n, + "fork reference segment ordinal", + ); + this.used = 0; + } +} + +/** + * Stream canonical production KFRV v2 records directly into the KFMS arena. + * + * The transaction already assigns dense recipe IDs as values are discovered, + * so production must not sort/remap the graph or manufacture an all-node root + * list merely to serialize it. + */ +export function appendSegmentedForkReferenceTransaction( + arena: ForkModuleStateArena, + ownerId: number, + nodes: ForkReferenceSequence, + vectors: ForkReferenceSequence, + options: ForkReferenceSegmentEncodingOptions = {}, +): Uint8Array { + assertOwner(ownerId); + const segmentDataBytes = + options.segmentDataBytes ?? DEFAULT_FORK_REFERENCE_SEGMENT_DATA_BYTES; + if ( + !Number.isInteger(segmentDataBytes) + || segmentDataBytes <= 0 + || segmentDataBytes > MAX_U32 - FORK_REFERENCE_SEGMENT_HEADER_SIZE + ) { + throw new RangeError( + `fork reference segment data size ${segmentDataBytes} is invalid`, + ); + } + validateCanonicalCapture(nodes, vectors); + const totals = computeSectionTotals(nodes, vectors); + const state: SegmentState = { ordinal: 0n }; + + const nodeWriter = new SegmentWriter( + arena, + ownerId, + ReferenceSection.Nodes, + state, + segmentDataBytes, + ); + let edgeStart = 0n; + let scalarStart = 0n; + const nodeRecord = new Uint8Array(FORK_REFERENCE_NODE_RECORD_SIZE); + for (const { node } of nodes) { + nodeRecord.fill(0); + const edges = nodeEdges(node); + const scalars = nodeScalars(node); + encodeNodeRecordV2( + new DataView(nodeRecord.buffer), + node, + edgeStart, + BigInt(edges.length), + scalarStart, + BigInt(scalars.byteLength), + ); + nodeWriter.write(nodeRecord); + edgeStart = addU64(edgeStart, BigInt(edges.length), "fork reference edge count"); + scalarStart = addU64( + scalarStart, + BigInt(scalars.byteLength), + "fork reference scalar byte count", + ); + } + nodeWriter.finish(); + + const u32 = new Uint8Array(4); + const u32View = new DataView(u32.buffer); + const edgeWriter = new SegmentWriter( + arena, + ownerId, + ReferenceSection.Edges, + state, + segmentDataBytes, + ); + for (const { node } of nodes) { + for (const edge of nodeEdges(node)) { + u32View.setUint32(0, edge, true); + edgeWriter.write(u32); + } + } + edgeWriter.finish(); + + const scalarWriter = new SegmentWriter( + arena, + ownerId, + ReferenceSection.Scalars, + state, + segmentDataBytes, + ); + for (const { node } of nodes) scalarWriter.write(nodeScalars(node)); + scalarWriter.finish(); + + const vectorIndexWriter = new SegmentWriter( + arena, + ownerId, + ReferenceSection.VectorIndex, + state, + segmentDataBytes, + ); + const vectorIndex = new Uint8Array(FORK_REFERENCE_VECTOR_INDEX_SIZE); + const vectorIndexView = new DataView(vectorIndex.buffer); + let vectorEntryStart = 0n; + for (let ordinal = 1; ordinal < vectors.length; ordinal++) { + const vector = requiredDirectoryEntry( + vectors, + ordinal, + "fork reference vector", + ); + vectorIndexView.setBigUint64(0, vectorEntryStart, true); + vectorIndexView.setBigUint64(8, BigInt(vector.length), true); + vectorIndexWriter.write(vectorIndex); + vectorEntryStart = addU64( + vectorEntryStart, + BigInt(vector.length), + "fork reference vector entry count", + ); + } + vectorIndexWriter.finish(); + + const vectorEntryWriter = new SegmentWriter( + arena, + ownerId, + ReferenceSection.VectorEntries, + state, + segmentDataBytes, + ); + for (let ordinal = 1; ordinal < vectors.length; ordinal++) { + for (const recipeId of requiredDirectoryEntry( + vectors, + ordinal, + "fork reference vector", + )) { + u32View.setUint32(0, recipeId, true); + vectorEntryWriter.write(u32); + } + } + vectorEntryWriter.finish(); + + const manifest = encodeManifest( + state.ordinal, + BigInt(nodes.length), + BigInt(vectors.length - 1), + totals, + ); + // WHY: the manifest is the transaction commit point. A failed segment + // allocation leaves no authoritative KFRV record, and arena sealing rejects + // the incomplete stream instead of exposing a truncated graph to a child. + arena.appendRecord({ + kind: ForkModuleStateRecordKind.ReferenceRecipe, + activationId: 0, + ownerId, + payload: manifest, + }); + return manifest; +} + +/** + * Test/helper encoder that preserves record segmentation without allocating a + * whole KFRV transaction. Production capture writes to a real KFMS arena. + */ +export function encodeSegmentedForkReferenceRecords( + ownerId: number, + nodes: ForkReferenceSequence, + vectors: ForkReferenceSequence, + options: ForkReferenceSegmentEncodingOptions = {}, +): ForkModuleStateRecord[] { + const records: ForkModuleStateRecord[] = []; + const sink = { + appendRecord(record: ForkModuleStateRecord): void { + records.push({ + ...record, + payload: record.payload.slice(), + }); + }, + } as Pick; + appendSegmentedForkReferenceTransaction( + sink as ForkModuleStateArena, + ownerId, + nodes, + vectors, + options, + ); + return records; +} + +function computeSectionTotals( + nodes: ForkReferenceSequence, + vectors: ForkReferenceSequence, +): SectionTotals { + let edgeCount = 0n; + let scalarBytes = 0n; + for (const { node } of nodes) { + edgeCount = addU64( + edgeCount, + BigInt(nodeEdges(node).length), + "fork reference edge count", + ); + scalarBytes = addU64( + scalarBytes, + BigInt(nodeScalars(node).byteLength), + "fork reference scalar byte count", + ); + } + let vectorEntries = 0n; + for (let ordinal = 1; ordinal < vectors.length; ordinal++) { + vectorEntries = addU64( + vectorEntries, + BigInt(requiredDirectoryEntry( + vectors, + ordinal, + "fork reference vector", + ).length), + "fork reference vector entry count", + ); + } + return { + nodes: multiplyU64( + BigInt(nodes.length), + BigInt(FORK_REFERENCE_NODE_RECORD_SIZE), + "fork reference node bytes", + ), + edges: multiplyU64(edgeCount, 4n, "fork reference edge bytes"), + scalars: scalarBytes, + vectorIndex: multiplyU64( + BigInt(vectors.length - 1), + BigInt(FORK_REFERENCE_VECTOR_INDEX_SIZE), + "fork reference vector-index bytes", + ), + vectorEntries: multiplyU64( + vectorEntries, + 4n, + "fork reference vector-entry bytes", + ), + }; +} + +function encodeManifest( + segmentCount: bigint, + nodeCount: bigint, + vectorCount: bigint, + totals: SectionTotals, +): Uint8Array { + const totalLogical = sectionTotalsArray(totals).reduce( + (sum, value) => addU64(sum, value, "fork reference logical bytes"), + 0n, + ); + const manifest = new Uint8Array(FORK_REFERENCE_MANIFEST_SIZE); + const view = new DataView(manifest.buffer); + view.setUint32(0, KFRV_MAGIC, true); + view.setUint16(4, FORK_REFERENCE_TRANSACTION_VERSION, true); + view.setUint16(6, FORK_REFERENCE_MANIFEST_SIZE, true); + view.setUint32(8, FORK_REFERENCE_MANIFEST_FLAG_SEALED, true); + view.setUint32(12, FORK_REFERENCE_NODE_RECORD_SIZE, true); + view.setUint32(16, FORK_REFERENCE_VECTOR_INDEX_SIZE, true); + view.setUint32(20, 0, true); + view.setBigUint64(24, segmentCount, true); + view.setBigUint64(32, nodeCount, true); + view.setBigUint64(40, vectorCount, true); + view.setBigUint64(48, totals.nodes, true); + view.setBigUint64(56, totals.edges, true); + view.setBigUint64(64, totals.scalars, true); + view.setBigUint64(72, totals.vectorIndex, true); + view.setBigUint64(80, totals.vectorEntries, true); + view.setBigUint64(88, totalLogical, true); + return manifest; +} + +interface ReferenceSegment { + readonly ordinal: bigint; + readonly offset: bigint; + readonly data: Uint8Array; +} + +const SEGMENT_DIRECTORY_PAGE_ENTRIES = 4096n; + +/** + * Segment metadata uses bigint indexes as well as u64 wire ordinals. + * + * A JavaScript Array (including a paged directory addressed by `number`) would + * reintroduce a 2^32-segment ceiling even though KFRV v2 deliberately carries + * u64 segment counts. Pages remain ordinary small arrays; only their sparse + * page keys and the logical length are bigint. + */ +class BigIntPagedDirectory implements Iterable { + private readonly pages = new Map>(); + private count = 0n; + + get length(): bigint { + return this.count; + } + + get(index: bigint): T | undefined { + if (index < 0n || index >= this.count) return undefined; + const page = this.pages.get(index / SEGMENT_DIRECTORY_PAGE_ENTRIES); + return page?.[Number(index % SEGMENT_DIRECTORY_PAGE_ENTRIES)]; + } + + push(value: T): void { + if (this.count === MAX_U64) { + throw new RangeError("fork reference segment directory exceeds u64"); + } + const pageIndex = this.count / SEGMENT_DIRECTORY_PAGE_ENTRIES; + let page = this.pages.get(pageIndex); + if (!page) { + page = []; + this.pages.set(pageIndex, page); + } + page[Number(this.count % SEGMENT_DIRECTORY_PAGE_ENTRIES)] = value; + this.count++; + } + + *[Symbol.iterator](): Iterator { + for (let index = 0n; index < this.count; index++) { + const value = this.get(index); + if (value === undefined) { + throw new Error(`fork reference segment directory has a hole at ${index}`); + } + yield value; + } + } +} + +interface ParsedReferenceManifest { + readonly segmentCount: bigint; + readonly nodeCount: number; + readonly vectorCount: number; + readonly totals: SectionTotals; +} + +interface ParsedSegmentedForkReferenceTransaction { + readonly manifest: ParsedReferenceManifest; + readonly sections: ReadonlyMap; +} + +interface ValidatedVectorRange { + readonly start: bigint; + readonly length: number; +} + +interface ValidatedReferenceSemantics { + readonly vectors: ForkReferenceDirectory; + readonly vectorIntern: ForkReferenceVectorInternIndex; +} + +/** + * One logical section backed by ordered KFMS record payloads. + * + * `totalBytes` and offsets remain bigint all the way through validation. A + * transaction may therefore cross the 4-GiB boundary without requiring a + * JavaScript ArrayBuffer of that size. + */ +class SegmentedSection { + constructor( + readonly segments: BigIntPagedDirectory, + readonly totalBytes: bigint, + ) {} + + reader(): SegmentedSectionReader { + return new SegmentedSectionReader(this); + } + + readU32At(offset: bigint): number { + if (offset < 0n || addU64(offset, 4n, "fork reference u32 end") > this.totalBytes) { + throw new Error(`fork reference section offset ${offset} is out of bounds`); + } + return ( + this.byteAt(offset) + | (this.byteAt(offset + 1n) << 8) + | (this.byteAt(offset + 2n) << 16) + | (this.byteAt(offset + 3n) << 24) + ) >>> 0; + } + + private byteAt(offset: bigint): number { + let low = 0n; + let high = this.segments.length; + while (low < high) { + const middle = low + ((high - low) >> 1n); + const segment = this.segments.get(middle)!; + const end = segment.offset + BigInt(segment.data.byteLength); + if (offset < segment.offset) { + high = middle; + } else if (offset >= end) { + low = middle + 1n; + } else { + return segment.data[Number(offset - segment.offset)]!; + } + } + throw new Error(`fork reference section offset ${offset} has no segment`); + } +} + +class SegmentedSectionReader { + private segmentIndex = 0n; + private localOffset = 0; + private consumed = 0n; + private readonly numberBytes = new Uint8Array(8); + private readonly numberView = new DataView(this.numberBytes.buffer); + + constructor(private readonly section: SegmentedSection) {} + + get position(): bigint { + return this.consumed; + } + + readInto(target: Uint8Array): void { + let targetOffset = 0; + while (targetOffset < target.byteLength) { + const segment = this.section.segments.get(this.segmentIndex); + if (!segment) { + throw new Error( + `fork reference section is truncated at logical byte ${this.consumed}`, + ); + } + const available = segment.data.byteLength - this.localOffset; + const count = Math.min(available, target.byteLength - targetOffset); + target.set( + segment.data.subarray(this.localOffset, this.localOffset + count), + targetOffset, + ); + this.advance(count, segment); + targetOffset += count; + } + } + + readBytes(length: number): Uint8Array { + if (!Number.isInteger(length) || length < 0 || length > MAX_U32) { + throw new RangeError(`fork reference byte count ${length} is not a u32`); + } + const bytes = new Uint8Array(length); + this.readInto(bytes); + return bytes; + } + + readU32(): number { + this.readInto(this.numberBytes.subarray(0, 4)); + return this.numberView.getUint32(0, true); + } + + skip(length: bigint): void { + if (length < 0n) { + throw new RangeError("fork reference skip length is negative"); + } + let remaining = length; + while (remaining !== 0n) { + const segment = this.section.segments.get(this.segmentIndex); + if (!segment) { + throw new Error( + `fork reference section is truncated at logical byte ${this.consumed}`, + ); + } + const available = segment.data.byteLength - this.localOffset; + const count = remaining < BigInt(available) + ? Number(remaining) + : available; + this.advance(count, segment); + remaining -= BigInt(count); + } + } + + requireEnd(context: string): void { + if (this.consumed !== this.section.totalBytes) { + throw new Error( + `${context} consumed ${this.consumed} bytes; ` + + `section contains ${this.section.totalBytes}`, + ); + } + } + + private advance(count: number, segment: ReferenceSegment): void { + this.localOffset += count; + this.consumed += BigInt(count); + if (this.localOffset === segment.data.byteLength) { + this.segmentIndex++; + this.localOffset = 0; + } + } +} + +/** + * Validate and decode the production KFRV v2 stream without concatenating it. + * + * Structural and semantic validation completes before graph nodes or reference + * vectors are materialized. This prevents malformed guest bytes from driving + * partial Wasm reconstruction. + */ +export function decodeSegmentedForkReferenceTransaction( + records: readonly ForkModuleStateRecordView[], + ownerId: number, +): DecodedSegmentedForkReferenceTransaction { + const parsed = parseSegmentedForkReferenceTransaction(records, ownerId); + const semantics = validateReferenceSemantics(parsed); + const graph = materializeReferenceGraph(parsed); + const vectors = materializeReferenceVectors(parsed, semantics); + const identity = Object.freeze({}); + return Object.freeze({ + identity, + graph, + vectors, + vectorIntern: semantics.vectorIntern, + }); +} + +/** + * Scan only opaque externref handles for the pre-launch process owner. + * + * This uses the exact production parser and semantic validator but deliberately + * avoids constructing graph/vector objects or retaining a whole wire image. + */ +export function scanSegmentedForkReferenceExternrefHandles( + records: readonly ForkModuleStateRecordView[], + ownerId: number, +): ReadonlySet { + const parsed = parseSegmentedForkReferenceTransaction(records, ownerId); + validateReferenceSemantics(parsed); + const handles = new Set(); + const reader = requiredSection(parsed, ReferenceSection.Nodes).reader(); + const recordBytes = new Uint8Array(FORK_REFERENCE_NODE_RECORD_SIZE); + for (let id = 0; id < parsed.manifest.nodeCount; id++) { + reader.readInto(recordBytes); + const view = new DataView(recordBytes.buffer); + if (view.getUint8(0) !== WireNodeKind.Externref) continue; + handles.add(decodeHandle(view.getUint32(4, true), view.getUint32(8, true), id)); + } + reader.requireEnd("fork reference node scan"); + return handles; +} + +function parseSegmentedForkReferenceTransaction( + records: readonly ForkModuleStateRecordView[], + ownerId: number, +): ParsedSegmentedForkReferenceTransaction { + assertOwner(ownerId); + const bySection = new Map< + ReferenceSection, + BigIntPagedDirectory + >(); + const observed = new Map(); + let selectedCount = 0n; + let segmentCount = 0n; + let manifestCount = 0; + let manifest: ParsedReferenceManifest | undefined; + let previousSection = 0; + + for (const record of records) { + if ( + record.kind !== ForkModuleStateRecordKind.ReferenceRecipeSegment + && record.kind !== ForkModuleStateRecordKind.ReferenceRecipe + ) { + continue; + } + const selectedIndex = selectedCount; + selectedCount = addU64( + selectedCount, + 1n, + "fork reference record count", + ); + if (record.activationId !== 0 || record.ownerId !== ownerId) { + throw new Error( + `fork reference record ${selectedIndex} has invalid process ownership ` + + `${record.activationId}:${record.ownerId}`, + ); + } + + if (record.kind === ForkModuleStateRecordKind.ReferenceRecipe) { + manifestCount++; + if (manifestCount === 1) manifest = decodeManifest(record.payload); + continue; + } + if (manifestCount !== 0) { + throw new Error("fork reference segment follows its final manifest"); + } + const segment = decodeSegment(record.payload, segmentCount); + if (segment.section < previousSection) { + throw new Error( + `fork reference segment ${segmentCount} reorders section ${segment.section}`, + ); + } + previousSection = segment.section; + const expectedOffset = observed.get(segment.section) ?? 0n; + if (segment.offset !== expectedOffset) { + throw new Error( + `fork reference segment ${segmentCount} starts at ${segment.offset}; ` + + `expected ${expectedOffset} (gap, overlap, or duplicate)`, + ); + } + observed.set( + segment.section, + addU64( + expectedOffset, + BigInt(segment.data.byteLength), + `fork reference section ${segment.section} bytes`, + ), + ); + let sectionSegments = bySection.get(segment.section); + if (!sectionSegments) { + sectionSegments = new BigIntPagedDirectory(); + bySection.set(segment.section, sectionSegments); + } + sectionSegments.push({ + ordinal: segmentCount, + offset: segment.offset, + data: segment.data, + }); + segmentCount = addU64( + segmentCount, + 1n, + "fork reference segment count", + ); + } + + if (selectedCount === 0n) { + throw new Error("fork module state has no process reference transaction"); + } + if (manifestCount !== 1 || manifest === undefined) { + throw new Error( + `fork module state has ${manifestCount} process reference manifests; ` + + "expected one", + ); + } + if (segmentCount !== manifest.segmentCount) { + throw new Error( + `fork reference manifest declares ${manifest.segmentCount} segments; ` + + `found ${segmentCount}`, + ); + } + + const totals = sectionTotalsArray(manifest.totals); + const sections = new Map(); + for (let ordinal = 0; ordinal < SECTION_COUNT; ordinal++) { + const section = (ordinal + 1) as ReferenceSection; + const expected = totals[ordinal]!; + const actual = observed.get(section) ?? 0n; + if (actual !== expected) { + throw new Error( + `fork reference section ${section} contains ${actual} bytes; ` + + `manifest declares ${expected}`, + ); + } + sections.set( + section, + new SegmentedSection( + bySection.get(section) ?? new BigIntPagedDirectory(), + expected, + ), + ); + } + return { manifest, sections }; +} + +function decodeManifest(payload: Uint8Array): ParsedReferenceManifest { + if (payload.byteLength !== FORK_REFERENCE_MANIFEST_SIZE) { + throw new Error( + `fork reference manifest has ${payload.byteLength} bytes; ` + + `expected ${FORK_REFERENCE_MANIFEST_SIZE}`, + ); + } + const view = new DataView( + payload.buffer, + payload.byteOffset, + payload.byteLength, + ); + if (view.getUint32(0, true) !== KFRV_MAGIC) { + throw new Error("fork reference manifest has invalid KFRV magic"); + } + const version = view.getUint16(4, true); + if (version !== FORK_REFERENCE_TRANSACTION_VERSION) { + throw new Error(`unsupported fork reference transaction version ${version}`); + } + if (view.getUint16(6, true) !== FORK_REFERENCE_MANIFEST_SIZE) { + throw new Error("fork reference manifest declares an invalid header size"); + } + const flags = view.getUint32(8, true); + if ( + flags !== FORK_REFERENCE_MANIFEST_FLAG_SEALED + || (flags & ~FORK_REFERENCE_MANIFEST_KNOWN_FLAGS) !== 0 + ) { + throw new Error(`fork reference manifest has invalid flags 0x${flags.toString(16)}`); + } + if ( + view.getUint32(12, true) !== FORK_REFERENCE_NODE_RECORD_SIZE + || view.getUint32(16, true) !== FORK_REFERENCE_VECTOR_INDEX_SIZE + ) { + throw new Error("fork reference manifest declares invalid record sizes"); + } + if (view.getUint32(20, true) !== 0) { + throw new Error("fork reference manifest reserved field is nonzero"); + } + + const nodeCount64 = view.getBigUint64(32, true); + const vectorCount64 = view.getBigUint64(40, true); + if (nodeCount64 === 0n || nodeCount64 > BigInt(MAX_U32_DIRECTORY_LENGTH)) { + throw new RangeError(`fork reference node count ${nodeCount64} is invalid`); + } + if (vectorCount64 > BigInt(MAX_U32)) { + throw new RangeError(`fork reference vector count ${vectorCount64} is invalid`); + } + const totals: SectionTotals = { + nodes: view.getBigUint64(48, true), + edges: view.getBigUint64(56, true), + scalars: view.getBigUint64(64, true), + vectorIndex: view.getBigUint64(72, true), + vectorEntries: view.getBigUint64(80, true), + }; + const expectedNodeBytes = multiplyU64( + nodeCount64, + BigInt(FORK_REFERENCE_NODE_RECORD_SIZE), + "fork reference node bytes", + ); + const expectedVectorIndexBytes = multiplyU64( + vectorCount64, + BigInt(FORK_REFERENCE_VECTOR_INDEX_SIZE), + "fork reference vector-index bytes", + ); + if (totals.nodes !== expectedNodeBytes) { + throw new Error( + `fork reference node section has ${totals.nodes} bytes; ` + + `expected ${expectedNodeBytes}`, + ); + } + if (totals.vectorIndex !== expectedVectorIndexBytes) { + throw new Error( + `fork reference vector-index section has ${totals.vectorIndex} bytes; ` + + `expected ${expectedVectorIndexBytes}`, + ); + } + if (totals.edges % 4n !== 0n || totals.vectorEntries % 4n !== 0n) { + throw new Error("fork reference u32 section byte length is not divisible by four"); + } + const expectedTotal = sectionTotalsArray(totals).reduce( + (sum, value) => addU64(sum, value, "fork reference logical bytes"), + 0n, + ); + const declaredTotal = view.getBigUint64(88, true); + if (declaredTotal !== expectedTotal) { + throw new Error( + `fork reference manifest declares ${declaredTotal} logical bytes; ` + + `sections contain ${expectedTotal}`, + ); + } + return { + segmentCount: view.getBigUint64(24, true), + nodeCount: Number(nodeCount64), + vectorCount: Number(vectorCount64), + totals, + }; +} + +function decodeSegment( + payload: Uint8Array, + expectedOrdinal: bigint, +): { + readonly section: ReferenceSection; + readonly offset: bigint; + readonly data: Uint8Array; +} { + if (payload.byteLength < FORK_REFERENCE_SEGMENT_HEADER_SIZE) { + throw new Error(`fork reference segment ${expectedOrdinal} header is truncated`); + } + const view = new DataView( + payload.buffer, + payload.byteOffset, + payload.byteLength, + ); + if (view.getUint32(0, true) !== KFRS_MAGIC) { + throw new Error(`fork reference segment ${expectedOrdinal} has invalid magic`); + } + if (view.getUint16(4, true) !== FORK_REFERENCE_TRANSACTION_VERSION) { + throw new Error( + `fork reference segment ${expectedOrdinal} has unsupported version`, + ); + } + if (view.getUint16(6, true) !== FORK_REFERENCE_SEGMENT_HEADER_SIZE) { + throw new Error( + `fork reference segment ${expectedOrdinal} declares an invalid header size`, + ); + } + const section = view.getUint16(8, true); + if (section < ReferenceSection.Nodes || section > ReferenceSection.VectorEntries) { + throw new Error( + `fork reference segment ${expectedOrdinal} has unknown section ${section}`, + ); + } + if ( + view.getUint16(10, true) !== WPK_FORK_REFERENCE_SEGMENT_KNOWN_FLAGS + || view.getUint32(12, true) !== 0 + || view.getUint32(36, true) !== 0 + ) { + throw new Error( + `fork reference segment ${expectedOrdinal} flags or reserved fields are nonzero`, + ); + } + const ordinal = view.getBigUint64(16, true); + if (ordinal !== expectedOrdinal) { + throw new Error( + `fork reference segment ordinal ${ordinal} is not expected ${expectedOrdinal}`, + ); + } + const dataLength = view.getUint32(32, true); + if ( + dataLength === 0 + || payload.byteLength !== FORK_REFERENCE_SEGMENT_HEADER_SIZE + dataLength + ) { + throw new Error( + `fork reference segment ${expectedOrdinal} has invalid data length`, + ); + } + return { + section: section as ReferenceSection, + offset: view.getBigUint64(24, true), + data: payload.subarray(FORK_REFERENCE_SEGMENT_HEADER_SIZE), + }; +} + +function validateReferenceSemantics( + parsed: ParsedSegmentedForkReferenceTransaction, +): ValidatedReferenceSemantics { + validateNodeSemantics(parsed); + return validateVectorSemantics(parsed); +} + +function validateNodeSemantics( + parsed: ParsedSegmentedForkReferenceTransaction, +): void { + const nodeReader = requiredSection(parsed, ReferenceSection.Nodes).reader(); + const edgeReader = requiredSection(parsed, ReferenceSection.Edges).reader(); + const scalarReader = requiredSection(parsed, ReferenceSection.Scalars).reader(); + const recordBytes = new Uint8Array(FORK_REFERENCE_NODE_RECORD_SIZE); + let expectedEdgeStart = 0n; + let expectedScalarStart = 0n; + + for (let id = 0; id < parsed.manifest.nodeCount; id++) { + nodeReader.readInto(recordBytes); + const header = decodeNodeHeader(recordBytes, id); + validateNodeHeader( + header, + id, + expectedEdgeStart, + expectedScalarStart, + ); + for (let edgeIndex = 0n; edgeIndex < header.edgeCount; edgeIndex++) { + const recipeId = edgeReader.readU32(); + if (recipeId >= parsed.manifest.nodeCount) { + throw new Error( + `fork reference node ${id} edge ${edgeIndex} names ` + + `missing recipe ${recipeId}`, + ); + } + } + scalarReader.skip(header.scalarLength); + expectedEdgeStart = addU64( + expectedEdgeStart, + header.edgeCount, + "fork reference consumed edge count", + ); + expectedScalarStart = addU64( + expectedScalarStart, + header.scalarLength, + "fork reference consumed scalar bytes", + ); + } + nodeReader.requireEnd("fork reference node records"); + edgeReader.requireEnd("fork reference graph edges"); + scalarReader.requireEnd("fork reference scalar payloads"); +} + +function validateVectorSemantics( + parsed: ParsedSegmentedForkReferenceTransaction, +): ValidatedReferenceSemantics { + const indexReader = requiredSection( + parsed, + ReferenceSection.VectorIndex, + ).reader(); + const entries = requiredSection(parsed, ReferenceSection.VectorEntries); + const entryReader = entries.reader(); + const indexBytes = new Uint8Array(FORK_REFERENCE_VECTOR_INDEX_SIZE); + const vectors = new PagedForkReferenceDirectory(); + const vectorIntern: MutableForkReferenceVectorInternIndex = new Map(); + let expectedStart = 0n; + + for (let ordinal = 1; ordinal <= parsed.manifest.vectorCount; ordinal++) { + indexReader.readInto(indexBytes); + const view = new DataView(indexBytes.buffer); + const start = view.getBigUint64(0, true); + const length64 = view.getBigUint64(8, true); + if (start !== expectedStart) { + throw new Error( + `fork reference vector ${ordinal} starts at ${start}; ` + + `expected ${expectedStart}`, + ); + } + if (length64 === 0n || length64 > BigInt(MAX_U32)) { + throw new RangeError( + `fork reference vector ${ordinal} length ${length64} is invalid`, + ); + } + const length = Number(length64); + let first = (0x811c_9dc5 ^ length) >>> 0; + let second = (0x9e37_79b9 ^ length) >>> 0; + for (let index = 0; index < length; index++) { + const recipeId = entryReader.readU32(); + if (recipeId >= parsed.manifest.nodeCount) { + throw new Error( + `fork reference vector ${ordinal} entry ${index} names ` + + `missing recipe ${recipeId}`, + ); + } + first = Math.imul(first ^ recipeId, 0x0100_0193) >>> 0; + const rotated = ((recipeId << 16) | (recipeId >>> 16)) >>> 0; + second = Math.imul(second ^ rotated, 0x85eb_ca6b) >>> 0; + second = (second ^ (first >>> 13)) >>> 0; + } + const key = `${length}:${first}:${second}`; + for (const previousOrdinal of vectorIntern.get(key) ?? []) { + const previous = requiredDirectoryEntry( + vectors, + previousOrdinal - 1, + "fork reference vector range", + ); + if (vectorRangesEqual(entries, previous, { start, length })) { + throw new Error( + `fork reference vector ${ordinal} duplicates canonical vector ` + + `${previousOrdinal}`, + ); + } + } + let bucket = vectorIntern.get(key); + if (!bucket) { + bucket = new PagedForkReferenceDirectory(); + vectorIntern.set(key, bucket); + } + bucket.push(ordinal); + vectors.push(Object.freeze({ start, length })); + expectedStart = addU64( + expectedStart, + length64, + "fork reference vector entry count", + ); + } + indexReader.requireEnd("fork reference vector indexes"); + entryReader.requireEnd("fork reference vector entries"); + return { vectors, vectorIntern }; +} + +function vectorRangesEqual( + entries: SegmentedSection, + left: ValidatedVectorRange, + right: ValidatedVectorRange, +): boolean { + if (left.length !== right.length) return false; + for (let index = 0; index < left.length; index++) { + const local = BigInt(index) * 4n; + if ( + entries.readU32At(left.start * 4n + local) + !== entries.readU32At(right.start * 4n + local) + ) { + return false; + } + } + return true; +} + +function materializeReferenceGraph( + parsed: ParsedSegmentedForkReferenceTransaction, +): DecodedSegmentedForkReferenceTransaction["graph"] { + const nodeReader = requiredSection(parsed, ReferenceSection.Nodes).reader(); + const edgeReader = requiredSection(parsed, ReferenceSection.Edges).reader(); + const scalarReader = requiredSection(parsed, ReferenceSection.Scalars).reader(); + const recordBytes = new Uint8Array(FORK_REFERENCE_NODE_RECORD_SIZE); + const nodes = new PagedForkReferenceDirectory(); + + for (let id = 0; id < parsed.manifest.nodeCount; id++) { + nodeReader.readInto(recordBytes); + const header = decodeNodeHeader(recordBytes, id); + const edges: number[] = []; + for (let edgeIndex = 0; edgeIndex < Number(header.edgeCount); edgeIndex++) { + edges.push(edgeReader.readU32()); + } + const scalars = scalarReader.readBytes(Number(header.scalarLength)); + nodes.push(Object.freeze({ + id, + node: decodeRecipeNode(header, edges, scalars, id), + })); + } + return Object.freeze({ + // Recipe IDs are direct roots from frames/globals/tables. A redundant + // all-node root vector would double graph bookkeeping and force remapping. + roots: Object.freeze([]), + nodes, + }); +} + +function materializeReferenceVectors( + parsed: ParsedSegmentedForkReferenceTransaction, + semantics: ValidatedReferenceSemantics, +): ForkReferenceDirectory { + const reader = requiredSection( + parsed, + ReferenceSection.VectorEntries, + ).reader(); + const vectors = new PagedForkReferenceDirectory(); + vectors.push(PagedForkReferenceVector.empty); + for (const range of semantics.vectors) { + const builder = new ForkReferenceVectorBuilder(range.length); + for (let index = 0; index < range.length; index++) { + builder.append(reader.readU32()); + } + vectors.push(builder.finish()); + } + reader.requireEnd("fork reference vector materialization"); + return vectors; +} + +interface DecodedNodeHeader { + readonly kind: WireNodeKind; + readonly first: number; + readonly second: number; + readonly third: number; + readonly edgeStart: bigint; + readonly edgeCount: bigint; + readonly scalarStart: bigint; + readonly scalarLength: bigint; +} + +function decodeNodeHeader( + bytes: Uint8Array, + id: number, +): DecodedNodeHeader { + const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); + if (view.getUint8(1) !== 0 || view.getUint16(2, true) !== 0) { + throw new Error(`fork reference node ${id} has nonzero flags or reserved fields`); + } + const kind = view.getUint8(0); + if (kind < WireNodeKind.Null || kind > WireNodeKind.StaticRoot) { + throw new Error(`fork reference node ${id} has unknown kind ${kind}`); + } + return { + kind: kind as WireNodeKind, + first: view.getUint32(4, true), + second: view.getUint32(8, true), + third: view.getUint32(12, true), + edgeStart: view.getBigUint64(16, true), + edgeCount: view.getBigUint64(24, true), + scalarStart: view.getBigUint64(32, true), + scalarLength: view.getBigUint64(40, true), + }; +} + +function validateNodeHeader( + header: DecodedNodeHeader, + id: number, + expectedEdgeStart: bigint, + expectedScalarStart: bigint, +): void { + const context = `fork reference node ${id}`; + const aggregate = + header.kind === WireNodeKind.Exnref + || header.kind === WireNodeKind.Struct + || header.kind === WireNodeKind.Array; + if (aggregate) { + if ( + header.edgeStart !== expectedEdgeStart + || header.scalarStart !== expectedScalarStart + ) { + throw new Error( + `${context} has noncanonical edge or scalar start`, + ); + } + if ( + header.edgeCount > BigInt(MAX_U32) + || header.scalarLength > BigInt(MAX_U32) + ) { + throw new RangeError(`${context} aggregate length exceeds u32`); + } + } else if ( + header.edgeStart !== 0n + || header.edgeCount !== 0n + || header.scalarStart !== 0n + || header.scalarLength !== 0n + ) { + throw new Error(`${context} scalar record declares aggregate data`); + } + + switch (header.kind) { + case WireNodeKind.Null: + if ( + id !== 0 + || header.first !== 0 + || header.second !== 0 + || header.third !== 0 + ) { + throw new Error(`${context} is not the canonical null recipe`); + } + break; + case WireNodeKind.Funcref: + case WireNodeKind.StaticRoot: + if (header.third !== 0) { + throw new Error(`${context} reserved scalar field is nonzero`); + } + break; + case WireNodeKind.Externref: + decodeHandle(header.first, header.second, id); + if (header.third !== 0) { + throw new Error(`${context} externref reserved scalar field is nonzero`); + } + break; + case WireNodeKind.I31: { + if (header.second !== 0 || header.third !== 0) { + throw new Error(`${context} i31 reserved scalar field is nonzero`); + } + const value = header.first | 0; + if (value < -0x4000_0000 || value > 0x3fff_ffff) { + throw new RangeError(`${context} has invalid i31 payload ${value}`); + } + break; + } + case WireNodeKind.Exnref: + case WireNodeKind.Struct: + case WireNodeKind.Array: + break; + } +} + +function decodeRecipeNode( + header: DecodedNodeHeader, + edges: readonly number[], + scalars: Uint8Array, + id: number, +): ForkReferenceRecipeNode { + switch (header.kind) { + case WireNodeKind.Null: + return Object.freeze({ kind: "null" }); + case WireNodeKind.Funcref: + return Object.freeze({ + kind: "funcref", + moduleActivation: header.first, + functionOrdinal: header.second, + }); + case WireNodeKind.Externref: + return Object.freeze({ + kind: "externref", + handle: decodeHandle(header.first, header.second, id), + }); + case WireNodeKind.Exnref: + return Object.freeze({ + kind: "exnref", + moduleActivation: header.first, + tagOrdinal: header.second, + layoutId: header.third, + scalars, + payloads: Object.freeze([...edges]), + }); + case WireNodeKind.I31: + return Object.freeze({ kind: "i31", value: header.first | 0 }); + case WireNodeKind.Struct: + return Object.freeze({ + kind: "struct", + moduleActivation: header.first, + typeOrdinal: header.second, + layoutId: header.third, + scalars, + fields: Object.freeze([...edges]), + }); + case WireNodeKind.Array: + return Object.freeze({ + kind: "array", + moduleActivation: header.first, + typeOrdinal: header.second, + layoutId: header.third, + scalars, + elements: Object.freeze([...edges]), + }); + case WireNodeKind.StaticRoot: + return Object.freeze({ + kind: "static-root", + moduleActivation: header.first, + staticRootOrdinal: header.second, + }); + } +} + +function decodeHandle(first: number, second: number, id: number): number { + const handle = Number((BigInt(second) << 32n) | BigInt(first)); + if ( + !Number.isInteger(handle) + || handle <= 0 + || handle > MAX_U32 + ) { + throw new RangeError( + `fork reference node ${id} externref handle ${handle} is invalid`, + ); + } + return handle; +} + +function requiredSection( + parsed: ParsedSegmentedForkReferenceTransaction, + section: ReferenceSection, +): SegmentedSection { + const value = parsed.sections.get(section); + if (!value) { + throw new Error(`fork reference section ${section} is absent`); + } + return value; +} + +function requiredDirectoryEntry( + directory: ForkReferenceSequence, + index: number, + context: string, +): T { + const present = Array.isArray(directory) + ? index >= 0 && index < directory.length && index in directory + : (directory as ForkReferenceDirectory).has(index); + if (!present) { + throw new Error(`${context} ${index} is absent`); + } + return ( + Array.isArray(directory) + ? directory[index] + : (directory as ForkReferenceDirectory).get(index) + )!; +} + +function validateCanonicalCapture( + nodes: ForkReferenceSequence, + vectors: ForkReferenceSequence, +): void { + if (nodes.length === 0 || nodes.length > MAX_U32_DIRECTORY_LENGTH) { + throw new RangeError(`fork reference node count ${nodes.length} is invalid`); + } + if (vectors.length === 0 || vectors.length - 1 > MAX_U32) { + throw new RangeError(`fork reference vector count ${vectors.length - 1} is invalid`); + } + if (requiredDirectoryEntry(vectors, 0, "fork reference vector").length !== 0) { + throw new Error("fork reference vector zero is not the empty sentinel"); + } + let id = 0; + for (const entry of nodes) { + if (entry.id !== id) { + throw new Error( + `fork reference recipe node ${entry.id} is out of canonical order at ${id}`, + ); + } + if ((id === 0) !== (entry.node.kind === "null")) { + throw new Error( + id === 0 + ? "fork reference recipe node zero is not null" + : `fork reference recipe node ${id} duplicates null`, + ); + } + validateCaptureNode(entry.node, id, nodes.length); + id++; + } + const intern: MutableForkReferenceVectorInternIndex = new Map(); + for (let ordinal = 1; ordinal < vectors.length; ordinal++) { + const vector = requiredDirectoryEntry( + vectors, + ordinal, + "fork reference vector", + ); + if (vector.length === 0) { + throw new Error( + `fork reference vector ${ordinal} duplicates the empty sentinel`, + ); + } + vector.forEach((recipeId, index) => { + assertRecipeId(recipeId, `fork reference vector ${ordinal} entry ${index}`); + if (recipeId >= nodes.length) { + throw new Error( + `fork reference vector ${ordinal} entry ${index} names ` + + `missing recipe ${recipeId}`, + ); + } + }); + const key = forkReferenceVectorInternKey(vector); + for (const previous of intern.get(key) ?? []) { + if (forkReferenceVectorsEqual( + requiredDirectoryEntry(vectors, previous, "fork reference vector"), + vector, + )) { + throw new Error( + `fork reference vector ${ordinal} duplicates canonical vector ${previous}`, + ); + } + } + let bucket = intern.get(key); + if (!bucket) { + bucket = new PagedForkReferenceDirectory(); + intern.set(key, bucket); + } + bucket.push(ordinal); + } +} + +function validateCaptureNode( + node: ForkReferenceRecipeNode, + id: number, + nodeCount: number, +): void { + const context = `fork reference node ${id}`; + switch (node.kind) { + case "null": + return; + case "funcref": + assertU32(node.moduleActivation, `${context} module activation`); + assertU32(node.functionOrdinal, `${context} function ordinal`); + return; + case "externref": + if (!Number.isInteger(node.handle) || node.handle <= 0 || node.handle > MAX_U32) { + throw new RangeError(`${context} externref handle is invalid`); + } + return; + case "i31": + if ( + !Number.isInteger(node.value) + || node.value < -0x4000_0000 + || node.value > 0x3fff_ffff + ) { + throw new RangeError(`${context} has invalid i31 value ${node.value}`); + } + return; + case "exnref": + assertU32(node.moduleActivation, `${context} module activation`); + assertU32(node.tagOrdinal, `${context} tag ordinal`); + assertU32(node.layoutId ?? 0, `${context} layout id`); + validateCaptureAggregate(node.payloads, node.scalars, context, nodeCount); + return; + case "struct": + assertU32(node.moduleActivation, `${context} module activation`); + assertU32(node.typeOrdinal, `${context} type ordinal`); + assertU32(node.layoutId ?? 0, `${context} layout id`); + validateCaptureAggregate(node.fields, node.scalars, context, nodeCount); + return; + case "array": + assertU32(node.moduleActivation, `${context} module activation`); + assertU32(node.typeOrdinal, `${context} type ordinal`); + assertU32(node.layoutId ?? 0, `${context} layout id`); + validateCaptureAggregate(node.elements, node.scalars, context, nodeCount); + return; + case "static-root": + assertU32(node.moduleActivation, `${context} module activation`); + assertU32(node.staticRootOrdinal, `${context} static-root ordinal`); + return; + } +} + +function validateCaptureAggregate( + edges: readonly number[], + scalars: Uint8Array | undefined, + context: string, + nodeCount: number, +): void { + if (edges.length > MAX_U32) { + throw new RangeError(`${context} edge count exceeds u32`); + } + if (scalars !== undefined && !(scalars instanceof Uint8Array)) { + throw new TypeError(`${context} scalar payload is not a Uint8Array`); + } + edges.forEach((recipeId, index) => { + assertRecipeId(recipeId, `${context} edge ${index}`); + if (recipeId >= nodeCount) { + throw new Error(`${context} edge ${index} names missing recipe ${recipeId}`); + } + }); +} + +function encodeNodeRecordV2( + view: DataView, + node: ForkReferenceRecipeNode, + edgeStart: bigint, + edgeCount: bigint, + scalarStart: bigint, + scalarLength: bigint, +): void { + let kind: WireNodeKind; + let first = 0; + let second = 0; + let third = 0; + switch (node.kind) { + case "null": + kind = WireNodeKind.Null; + break; + case "funcref": + kind = WireNodeKind.Funcref; + first = node.moduleActivation; + second = node.functionOrdinal; + break; + case "externref": + kind = WireNodeKind.Externref; + first = node.handle >>> 0; + second = Math.floor(node.handle / 0x1_0000_0000); + break; + case "exnref": + kind = WireNodeKind.Exnref; + first = node.moduleActivation; + second = node.tagOrdinal; + third = node.layoutId ?? 0; + break; + case "i31": + kind = WireNodeKind.I31; + first = node.value >>> 0; + break; + case "struct": + kind = WireNodeKind.Struct; + first = node.moduleActivation; + second = node.typeOrdinal; + third = node.layoutId ?? 0; + break; + case "array": + kind = WireNodeKind.Array; + first = node.moduleActivation; + second = node.typeOrdinal; + third = node.layoutId ?? 0; + break; + case "static-root": + kind = WireNodeKind.StaticRoot; + first = node.moduleActivation; + second = node.staticRootOrdinal; + break; + } + const aggregate = + node.kind === "exnref" || node.kind === "struct" || node.kind === "array"; + view.setUint8(0, kind); + view.setUint8(1, 0); + view.setUint16(2, 0, true); + view.setUint32(4, first, true); + view.setUint32(8, second, true); + view.setUint32(12, third, true); + view.setBigUint64(16, aggregate ? edgeStart : 0n, true); + view.setBigUint64(24, aggregate ? edgeCount : 0n, true); + view.setBigUint64(32, aggregate ? scalarStart : 0n, true); + view.setBigUint64(40, aggregate ? scalarLength : 0n, true); +} + +function nodeEdges(node: ForkReferenceRecipeNode): readonly number[] { + switch (node.kind) { + case "exnref": + return node.payloads; + case "struct": + return node.fields; + case "array": + return node.elements; + case "null": + case "funcref": + case "externref": + case "i31": + case "static-root": + return []; + } +} + +function nodeScalars(node: ForkReferenceRecipeNode): Uint8Array { + switch (node.kind) { + case "exnref": + case "struct": + case "array": + return node.scalars ?? new Uint8Array(); + case "null": + case "funcref": + case "externref": + case "i31": + case "static-root": + return new Uint8Array(); + } +} + +function sectionTotalsArray(totals: SectionTotals): readonly bigint[] { + return [ + totals.nodes, + totals.edges, + totals.scalars, + totals.vectorIndex, + totals.vectorEntries, + ]; +} + +function addU64(left: bigint, right: bigint, context: string): bigint { + if (left < 0n || right < 0n || left > MAX_U64 - right) { + throw new RangeError(`${context} exceeds u64`); + } + return left + right; +} + +function multiplyU64(left: bigint, right: bigint, context: string): bigint { + if (left < 0n || right < 0n || (right !== 0n && left > MAX_U64 / right)) { + throw new RangeError(`${context} exceeds u64`); + } + return left * right; +} + +/** Pure u64 helper used by boundary tests without allocating a 4-GiB buffer. */ +export function advanceForkReferenceLogicalOffset( + offset: bigint, + byteLength: number, +): bigint { + if (!Number.isSafeInteger(byteLength) || byteLength < 0) { + throw new RangeError("fork reference logical byte length is invalid"); + } + return addU64(offset, BigInt(byteLength), "fork reference logical offset"); +} + +function assertRecipeId(value: number, context: string): void { + if (!Number.isInteger(value) || value < 0 || value > MAX_U32) { + throw new RangeError(`${context} ${value} is not a u32 recipe id`); + } +} + +function assertU32(value: number, context: string): void { + if (!Number.isInteger(value) || value < 0 || value > MAX_U32) { + throw new RangeError(`${context} is not an unsigned 32-bit integer`); + } +} + +function assertOwner(ownerId: number): void { + assertU32(ownerId, "fork reference owner id"); + if (ownerId === 0) { + throw new RangeError("fork reference owner id must be nonzero"); + } +} + +function littleEndianMagic(bytes: readonly number[]): number { + if (bytes.length !== 4) { + throw new Error("fork reference ABI magic must contain four bytes"); + } + return ( + bytes[0]! + | (bytes[1]! << 8) + | (bytes[2]! << 16) + | (bytes[3]! << 24) + ) >>> 0; +} diff --git a/host/src/fork-reference-transaction.ts b/host/src/fork-reference-transaction.ts new file mode 100644 index 0000000000..8d8ec0eb33 --- /dev/null +++ b/host/src/fork-reference-transaction.ts @@ -0,0 +1,2054 @@ +import { + type ForkModuleStateArena, +} from "./fork-module-state"; +import { + type ForkReferenceRecipeEntry, + type ForkReferenceRecipeNode, +} from "./fork-reference-recipes"; +import { + appendSegmentedForkReferenceTransaction, + decodeSegmentedForkReferenceTransaction, + findForkReferenceVectorOrdinal, + forkReferenceVectorFrom, + ForkReferenceDirectoryOverlay, + ForkReferenceVectorBuilder, + indexForkReferenceVector, + PagedForkReferenceDirectory, + PagedForkReferenceVector, + type DecodedSegmentedForkReferenceTransaction, + type ForkReferenceDirectory, + type ForkReferenceVector, + type MutableForkReferenceVectorInternIndex, +} from "./fork-reference-segments"; +import { ForkFunctionCatalog } from "./fork-function-catalog"; +import { + FORK_GC_FIELD_ALLOCATION_DEPENDENCY, + FORK_GC_FIELD_MUTABLE, + FORK_GC_FIELD_REFERENCE, + FORK_GC_LAYOUT_DEFAULTABLE_SHELL, + FORK_GC_LAYOUT_REQUIRES_PROVENANCE, + ForkGcConstructorKind, + type ForkGcCodecDescriptor, + type ForkGcCodecProvider, + type ForkGcConstructorProvenance, + type ForkGcLayoutDescriptor, +} from "./fork-gc-codec"; +import { ForkStaticRootCatalog } from "./fork-static-root-catalog"; +import { + WPK_FORK_REFERENCE_TRANSACTION_OWNER, +} from "./generated/abi"; + +/** + * The owner id is local to the ReferenceRecipe record kind. One process fork + * has one identity space shared by main-module frames, side-module frames, + * globals, and tables, so aliases never become module-local by accident. + */ +export const FORK_REFERENCE_TRANSACTION_OWNER_ID = + WPK_FORK_REFERENCE_TRANSACTION_OWNER; +export const FORK_HOST_EXCEPTION_ACTIVATION_ID = 0xffff_ffff; +// Generated Wasm carries recipe IDs and vector ordinals as raw i32 bits. The +// host import boundary normalizes signed JavaScript arguments with `>>> 0`, so +// the complete u32 namespace is available; zero alone is the empty sentinel. +const MAX_REFERENCE_VECTOR_ORDINAL = 0xffff_ffff; + +export interface ForkExternrefRecipeProvider { + /** + * Move an opaque value under process-owned lifetime management and return + * the scalar handle that is safe to copy through the continuation. + */ + capture(value: unknown): number; + + /** + * Produce this Worker's canonical token for a process-owned handle. + * The real host value remains with the provider's owner. + */ + materialize(handle: number): unknown; +} + +export interface ForkExceptionSlotProvider { + /** Throw the exact exnref held in a Wasm-only scratch slot. */ + throwSlot(slot: number): never; + /** Release every Wasm scratch-table root after replay or abort. */ + clearSlots(): void; +} + +export type ForkReferenceScratchAllocate = (size: number) => number; +export type ForkReferenceScratchDeallocate = (addr: number, size: number) => void; + +/** + * Late-bound process owner used only in a fresh child. Keeping this scalar + * interface on the transaction prevents GC/exn references from crossing the + * JavaScript boundary while still allowing graph-wide validation and ordering. + */ +export interface ForkTypedReferenceReplayOwner { + prepareTransit(maxRecipeId: number): void; + /** + * Publish an instantiation-owned GC root at its canonical recipe slot. + * + * Generated codecs publish reconstructed struct/array/i31 identities + * themselves. Static roots already exist in the fresh activation, so the + * host must route that exact identity before a constructor or field fill + * consumes it as a graph edge. + */ + publishTransit(recipeId: number, value: unknown): void; + /** + * Publish an opaque process-owned leaf through a generated + * `any.convert_extern`, preserving the canonical token without asking + * JavaScript to manufacture an anyref. + */ + publishExternref(recipeId: number, value: unknown): void; + provider(activationId: number): ForkGcCodecProvider; + providers(): readonly ForkGcCodecProvider[]; + validateExceptionOwner(activationId: number): void; + materializeException(recipeId: number, activationId: number): void; +} + +/** + * Canonical identities materialized before every fresh activation exists. + * + * Imported immutable reference globals must be supplied while their consumer + * is instantiated. The early child provider therefore reconstructs a strict + * prefix of the process graph before the full replay transaction can attach. + * Adoption transfers those identities and typed-codec milestones so replay + * never allocates a second object for a recipe already visible to Wasm. + */ +export interface ForkReferenceChildReplayAdoption { + /** Exact decoded KFRV transaction object that produced this state. */ + readonly transaction: DecodedSegmentedForkReferenceTransaction; + /** + * Sparse canonical values. A Map is required because `undefined` is a valid + * externref and must remain distinct from an unmaterialized recipe. + */ + readonly materializedValues: ReadonlyMap; + readonly allocatedTypedRecipes: ReadonlySet; + readonly filledTypedRecipes: ReadonlySet; + readonly materializedExceptionRecipes: ReadonlySet; +} + +export interface ForkGcDefinitionProvenance { + readonly record: ForkGcConstructorProvenance; + readonly recipeIds: readonly number[]; +} + +interface ScratchChunk { + readonly addr: number; + readonly size: number; + used: number; +} + +interface ScratchReservation { + readonly addr: number; + readonly requestedSize: number; + readonly alignedSize: number; + readonly previousUsed: number; + readonly chunk: ScratchChunk; +} + +interface CaptureReferenceVector { + readonly expectedLength: number; + readonly builder: ForkReferenceVectorBuilder; +} + +type CanonicalReferenceVector = ForkReferenceVector; + +type TransactionPhase = + | "idle" + | "capture" + | "sealed-parent" + | "parent-replay" + | "child-replay"; + +function assertRecipeId(value: number): void { + if (!Number.isInteger(value) || value < 0 || value > 0xffff_ffff) { + throw new RangeError(`invalid fork reference recipe id ${value}`); + } +} + +export type DecodedForkReferenceTransaction = + DecodedSegmentedForkReferenceTransaction; + +/** + * Validate and expose the one process-wide reference graph in a sealed arena. + * + * This is shared by fresh-child reconstruction and the kernel-side externref + * owner. WHY: the owner must acquire the child's numeric handle lease before + * starting its Worker, but it must validate the exact same KFRV bytes that the + * child will later consume rather than maintaining a second permissive parser. + */ +export function decodeForkReferenceTransactionRecord( + records: Parameters[0], +): DecodedForkReferenceTransaction { + return decodeSegmentedForkReferenceTransaction( + records, + FORK_REFERENCE_TRANSACTION_OWNER_ID, + ); +} + +/** + * Per-fork owner for activation reference recipes. + * + * Recipe id zero is node zero and the canonical null value. Every nonnull + * recipe id is its graph node id. Seeding the graph this way keeps reference + * payload vectors lossless: a null field is an ordinary edge to node zero, + * while zero remains cheap in every generated typed codec. + */ +export class ForkReferenceTransaction { + private phase: TransactionPhase = "idle"; + private readonly nodes = + new PagedForkReferenceDirectory(); + private readonly capturedValues = + new PagedForkReferenceDirectory(); + private objectIds = new WeakMap(); + private readonly primitiveIds = new Map(); + private decodedNodes: ForkReferenceDirectory = + new PagedForkReferenceDirectory(); + private readonly materializedValues = new Map(); + private readonly pendingExceptions = new Set(); + private readonly pendingGc = new Set(); + private readonly i31Ids = new Map(); + private readonly exceptionCacheIndexes = new Map(); + private exceptionSlots: ForkExceptionSlotProvider | undefined; + private readonly scratchChunks: ScratchChunk[] = []; + private readonly scratchReservations: ScratchReservation[] = []; + /** Index zero is the canonical empty-vector sentinel. */ + private readonly referenceVectors = + new PagedForkReferenceDirectory(); + private readonly pendingReferenceVectors = new Map< + number, + CaptureReferenceVector + >(); + private readonly freeReferenceVectorHandles: number[] = []; + private nextReferenceVectorHandle = 1; + private readonly referenceVectorIntern: + MutableForkReferenceVectorInternIndex = new Map(); + private readonly decodedReferenceVectors = + new ForkReferenceDirectoryOverlay(); + private readonly decodedReferenceVectorIntern: + MutableForkReferenceVectorInternIndex = new Map(); + private readonly replayGcVectors = new Map(); + private typedMaterialized = false; + private childTransaction: DecodedSegmentedForkReferenceTransaction | null = null; + private childReplayAdopted = false; + private readonly adoptedAllocatedTypedRecipes = new Set(); + private readonly adoptedFilledTypedRecipes = new Set(); + private readonly adoptedMaterializedExceptionRecipes = new Set(); + + constructor( + readonly functions: ForkFunctionCatalog, + private readonly externrefs: ForkExternrefRecipeProvider, + private readonly memory?: WebAssembly.Memory, + private readonly allocateScratch?: ForkReferenceScratchAllocate, + private readonly deallocateScratch?: ForkReferenceScratchDeallocate, + private readonly label = "fork reference transaction", + private readonly staticRoots?: ForkStaticRootCatalog, + private readonly typedReplay?: ForkTypedReferenceReplayOwner, + ) {} + + setExceptionSlotProvider(provider: ForkExceptionSlotProvider): void { + if (this.exceptionSlots && this.exceptionSlots !== provider) { + throw new Error("fork exception slot provider is already registered"); + } + this.exceptionSlots = provider; + } + + beginCapture(): void { + if (this.phase !== "idle") { + throw new Error(`cannot begin reference capture while transaction is ${this.phase}`); + } + this.nodes.push({ id: 0, node: { kind: "null" } }); + this.capturedValues.push(null); + this.referenceVectors.push(PagedForkReferenceVector.empty); + this.phase = "capture"; + } + + encodeFuncref(value: unknown): number { + this.requirePhase("capture", "encode a funcref"); + if (value === null) return 0; + if (typeof value !== "function") { + throw new TypeError("fork funcref encoder received a non-function value"); + } + return this.intern(value, () => { + const recipe = this.functions.encode(value); + if (!recipe) throw new Error("non-null funcref produced a null catalog recipe"); + return { + kind: "funcref", + moduleActivation: recipe.moduleActivation, + functionOrdinal: recipe.ordinal, + }; + }); + } + + encodeExternref(value: unknown): number { + this.requirePhase("capture", "encode an externref"); + if (value === null) return 0; + return this.intern(value, () => { + // A WebAssembly function can cross an externref conversion. Keep one + // node for it so converting back in the child observes the same fresh + // function identity as a funcref slot. + if (typeof value === "function") { + const recipe = this.functions.encode(value); + if (recipe) { + return { + kind: "funcref", + moduleActivation: recipe.moduleActivation, + functionOrdinal: recipe.ordinal, + }; + } + } + return { + kind: "externref", + handle: this.externrefs.capture(value), + }; + }); + } + + lookupGcSlot(table: WebAssembly.Table, slot: number): number { + this.requirePhase("capture", "look up a Wasm-GC identity"); + const value = this.gcSlotValue(table, slot); + const known = this.lookupId(value); + if (known !== undefined) return known; + const staticRoot = this.staticRoots?.encode(value); + if (!staticRoot) return 0; + const id = this.nodes.length; + if (id > 0xffff_ffff) { + throw new RangeError("fork reference recipe id space exhausted"); + } + this.nodes.push({ + id, + node: { + kind: "static-root", + moduleActivation: staticRoot.moduleActivation, + staticRootOrdinal: staticRoot.ordinal, + }, + }); + this.capturedValues.push(value); + this.rememberId(value, id); + return id; + } + + claimGcSlot(table: WebAssembly.Table, slot: number): number { + this.requirePhase("capture", "claim a Wasm-GC identity"); + const value = this.gcSlotValue(table, slot); + const known = this.lookupId(value); + if (known !== undefined) return known; + const id = this.nodes.length; + if (id > 0xffff_ffff) { + throw new RangeError("fork reference recipe id space exhausted"); + } + // WHY: publish graph identity before recursively encoding fields. The + // placeholder is never serializable; `sealInto` rejects it via pendingGc + // unless the generated codec completes `defineGc`. + this.nodes.push({ + id, + node: { + kind: "struct", + moduleActivation: 0, + typeOrdinal: 0, + layoutId: 0, + scalars: new Uint8Array(), + fields: [], + }, + }); + this.capturedValues.push(value); + this.rememberId(value, id); + this.pendingGc.add(id); + return id; + } + + encodeI31(value: number): number { + this.requirePhase("capture", "encode an i31ref"); + if ( + !Number.isInteger(value) + || value < -0x4000_0000 + || value > 0x3fff_ffff + ) { + throw new RangeError(`invalid signed i31 payload ${value}`); + } + const known = this.i31Ids.get(value); + if (known !== undefined) return known; + const id = this.nodes.length; + if (id > 0xffff_ffff) { + throw new RangeError("fork reference recipe id space exhausted"); + } + this.nodes.push({ id, node: { kind: "i31", value } }); + // i31 identity is defined by its 31-bit payload. Keep a scalar here so + // parent replay and graph aliases use the same canonical recipe id. + this.capturedValues.push(value); + this.i31Ids.set(value, id); + return id; + } + + capturedGcValue(recipeId: number): unknown { + this.requirePhase("capture", "read a captured Wasm-GC identity"); + assertRecipeId(recipeId); + if (recipeId === 0 || recipeId >= this.capturedValues.length) { + throw new Error(`fork Wasm-GC recipe ${recipeId} is out of bounds`); + } + return this.capturedValues.get(recipeId); + } + + defineGc( + recipeId: number, + moduleActivation: number, + typeOrdinal: number, + layoutId: number, + kind: number, + scalarPointer: number | bigint, + scalarByteLength: number, + referenceVectorOrdinal: number, + descriptor: ForkGcCodecDescriptor, + provenance: ForkGcDefinitionProvenance | null, + ): void { + this.requirePhase("capture", "define a Wasm-GC recipe"); + assertRecipeId(recipeId); + this.assertU32(moduleActivation, "GC module activation"); + this.assertU32(typeOrdinal, "GC type ordinal"); + this.assertU31(layoutId, "GC layout id", false); + this.assertU32(referenceVectorOrdinal, "GC reference vector ordinal"); + if (!this.pendingGc.has(recipeId)) { + throw new Error(`fork Wasm-GC recipe ${recipeId} is not pending definition`); + } + const layout = descriptor.require(layoutId); + if ( + layout.typeOrdinal !== typeOrdinal + || layout.kind !== kind + ) { + throw new Error( + `fork Wasm-GC recipe ${recipeId} coordinate does not match ` + + `${moduleActivation}:${typeOrdinal}:${layoutId}:${kind}`, + ); + } + const snapshotScalars = this.readBytes( + scalarPointer, + scalarByteLength, + "fork Wasm-GC scalar payload", + ); + const vector = this.referenceVectors.get(referenceVectorOrdinal); + if (!vector) { + throw new Error( + `fork Wasm-GC recipe ${recipeId} names an unavailable reference vector`, + ); + } + this.validateGcSnapshot( + layout, + snapshotScalars, + vector, + `fork Wasm-GC recipe ${recipeId}`, + ); + + const requiresProvenance = + (layout.flags & FORK_GC_LAYOUT_REQUIRES_PROVENANCE) !== 0; + if (requiresProvenance !== (provenance !== null)) { + throw new Error( + `fork Wasm-GC recipe ${recipeId} has ` + + `${provenance ? "unexpected" : "missing"} constructor provenance`, + ); + } + let provenanceScalars: Uint8Array = new Uint8Array(); + let provenanceIds: readonly number[] = []; + if (provenance) { + const selected = descriptor.requireCaptureLayout( + layout.baseLayoutId, + provenance.record.layoutId, + ); + if ( + selected.id !== layout.id + || provenance.record.activationId !== moduleActivation + || provenance.record.baseLayoutId !== layout.baseLayoutId + || provenance.record.scalars.byteLength + !== layout.provenanceScalarLength + || provenance.record.references.length + !== layout.provenanceReferenceCount + || provenance.recipeIds.length + !== layout.provenanceReferenceCount + ) { + throw new Error( + `fork Wasm-GC recipe ${recipeId} provenance does not match ` + + `layout ${layout.id}`, + ); + } + provenance.recipeIds.forEach((id, index) => { + assertRecipeId(id); + if (id >= this.nodes.length) { + throw new Error( + `fork Wasm-GC provenance ${index} names missing recipe ${id}`, + ); + } + }); + provenanceScalars = provenance.record.scalars; + provenanceIds = provenance.recipeIds; + } + + const scalars = new Uint8Array( + provenanceScalars.byteLength + snapshotScalars.byteLength, + ); + scalars.set(provenanceScalars); + scalars.set(snapshotScalars, provenanceScalars.byteLength); + const references = [...provenanceIds, ...vector]; + const node: ForkReferenceRecipeNode = + layout.kind === 1 + ? { + kind: "struct", + moduleActivation, + typeOrdinal, + layoutId, + scalars, + fields: references, + } + : { + kind: "array", + moduleActivation, + typeOrdinal, + layoutId, + scalars, + elements: references, + }; + this.nodes.set(recipeId, { id: recipeId, node }); + this.pendingGc.delete(recipeId); + } + + sealInto(arena: ForkModuleStateArena): Uint8Array { + this.requirePhase("capture", "seal reference capture"); + if (this.scratchReservations.length !== 0) { + throw new Error( + `cannot seal with ${this.scratchReservations.length} live reference scratch reservation(s)`, + ); + } + if (this.pendingExceptions.size !== 0) { + throw new Error( + `cannot seal ${this.pendingExceptions.size} incomplete exception recipe(s)`, + ); + } + if (this.pendingGc.size !== 0) { + throw new Error( + `cannot seal ${this.pendingGc.size} incomplete Wasm-GC recipe(s)`, + ); + } + if (this.pendingReferenceVectors.size !== 0) { + throw new Error( + `cannot seal ${this.pendingReferenceVectors.size} unfinished reference vector(s)`, + ); + } + const manifest = appendSegmentedForkReferenceTransaction( + arena, + FORK_REFERENCE_TRANSACTION_OWNER_ID, + this.nodes, + this.referenceVectors, + ); + this.phase = "sealed-parent"; + return manifest; + } + + beginParentReplay(): void { + this.requirePhase("sealed-parent", "begin parent reference replay"); + this.phase = "parent-replay"; + } + + attachChild( + source: + | Parameters[0] + | DecodedSegmentedForkReferenceTransaction, + ): void { + if (this.phase !== "idle") { + throw new Error(`cannot attach child reference state while transaction is ${this.phase}`); + } + const decoded = "identity" in source + ? source + : decodeForkReferenceTransactionRecord(source); + const { graph } = decoded; + this.decodedNodes = graph.nodes; + // WHY: generated GC codecs need an appendable tail, but copying every + // decoded vector into a mutable directory would retain a redundant + // transaction-wide index. The immutable decoded directory remains the + // shared base used by both early and ordinary replay. + this.decodedReferenceVectors.reset(decoded.vectors); + this.decodedReferenceVectorIntern.clear(); + for (const entry of graph.nodes) { + if (entry.node.kind === "exnref") { + this.exceptionCacheIndexes.set( + entry.id, + this.exceptionCacheIndexes.size + 1, + ); + } + } + this.materializedValues.clear(); + for (const entry of graph.nodes) { + if (entry.node.kind !== "static-root") continue; + if (!this.staticRoots) { + throw new Error( + `fork recipe ${entry.id} requires a static-root catalog`, + ); + } + // WHY: module-state restore can overwrite a template table and the + // second restore phase can drop its element segment before activation + // locals decode. Pin only roots actually named by this continuation, + // then release them with the rest of the transaction after replay. + this.materializedValues.set(entry.id, this.staticRoots.decode({ + moduleActivation: entry.node.moduleActivation, + ordinal: entry.node.staticRootOrdinal, + })); + } + this.childTransaction = decoded; + this.phase = "child-replay"; + } + + /** + * Take ownership of identities reconstructed for imported globals. + * + * This is deliberately a separate one-shot step after `attachChild`: the + * transaction first validates the authoritative wire itself, then accepts + * only state proven to come from those exact bytes. + */ + adoptChildReplay(adoption: ForkReferenceChildReplayAdoption): void { + this.requirePhase("child-replay", "adopt early child reference replay"); + if (this.typedMaterialized) { + throw new Error("cannot adopt child references after typed materialization"); + } + if (this.childReplayAdopted) { + throw new Error("early child reference replay was adopted twice"); + } + if (!this.childTransaction || adoption.transaction !== this.childTransaction) { + throw new Error( + "early child reference replay does not share the attached transaction", + ); + } + + const requireNodeKind = ( + recipeId: number, + kinds: readonly ForkReferenceRecipeNode["kind"][], + context: string, + ): ForkReferenceRecipeNode => { + assertRecipeId(recipeId); + const node = this.decodedNodes.get(recipeId)?.node; + if (!node) { + throw new Error(`${context} names missing recipe ${recipeId}`); + } + if (!kinds.includes(node.kind)) { + throw new Error( + `${context} recipe ${recipeId} is ${node.kind}, expected ` + + kinds.join(" or "), + ); + } + return node; + }; + + const stagedValues = new Map(); + for (const [recipeId, value] of adoption.materializedValues) { + const node = requireNodeKind( + recipeId, + ["null", "funcref", "externref", "i31", "struct", "array", "static-root"], + "early materialized value", + ); + if ( + (node.kind === "null" && value !== null) + || (node.kind === "funcref" && typeof value !== "function") + ) { + throw new TypeError( + `early materialized recipe ${recipeId} has an invalid ${node.kind} value`, + ); + } + if (this.materializedValues.has(recipeId)) { + if (!Object.is(this.materializedValues.get(recipeId), value)) { + throw new Error( + `early materialized recipe ${recipeId} conflicts with its child catalog`, + ); + } + continue; + } + stagedValues.set(recipeId, value); + } + + const stagedAllocated = new Set(); + for (const recipeId of adoption.allocatedTypedRecipes) { + requireNodeKind( + recipeId, + ["i31", "struct", "array"], + "early allocated typed reference", + ); + stagedAllocated.add(recipeId); + } + const stagedFilled = new Set(); + for (const recipeId of adoption.filledTypedRecipes) { + requireNodeKind( + recipeId, + ["struct", "array"], + "early filled typed reference", + ); + if (!stagedAllocated.has(recipeId)) { + throw new Error( + `early filled typed recipe ${recipeId} was not allocated`, + ); + } + stagedFilled.add(recipeId); + } + const stagedExceptions = new Set(); + for (const recipeId of adoption.materializedExceptionRecipes) { + requireNodeKind( + recipeId, + ["exnref"], + "early materialized exception", + ); + stagedExceptions.add(recipeId); + } + for (const recipeId of stagedAllocated) { + if ( + !stagedValues.has(recipeId) + && !this.materializedValues.has(recipeId) + ) { + throw new Error( + `early allocated typed recipe ${recipeId} has no canonical value`, + ); + } + } + for (const [recipeId] of stagedValues) { + const kind = this.decodedNodes.get(recipeId)!.node.kind; + if ( + (kind === "i31" || kind === "struct" || kind === "array") + && !stagedAllocated.has(recipeId) + ) { + throw new Error( + `early materialized typed recipe ${recipeId} was not allocated`, + ); + } + } + + for (const [recipeId, value] of stagedValues) { + // Assign even when value is undefined: the own property is the cache bit. + this.materializedValues.set(recipeId, value); + } + stagedAllocated.forEach((id) => this.adoptedAllocatedTypedRecipes.add(id)); + stagedFilled.forEach((id) => this.adoptedFilledTypedRecipes.add(id)); + stagedExceptions.forEach((id) => + this.adoptedMaterializedExceptionRecipes.add(id) + ); + this.childReplayAdopted = true; + } + + decodeFuncref(recipeId: number): CallableFunction | null { + const value = this.decode(recipeId, "funcref"); + if (value !== null && typeof value !== "function") { + throw new TypeError(`fork recipe ${recipeId} did not reconstruct a funcref`); + } + return value as CallableFunction | null; + } + + decodeExternref(recipeId: number): unknown { + return this.decode(recipeId, "externref"); + } + + beginReferenceVector(expectedLength: number): number { + this.requirePhase("capture", "begin a reference vector"); + this.assertU32(expectedLength, "reference vector length"); + if (expectedLength === 0) { + throw new RangeError("reference vector zero is the canonical empty vector"); + } + let handle = this.freeReferenceVectorHandles.pop(); + if (handle === undefined) { + if (this.nextReferenceVectorHandle > MAX_REFERENCE_VECTOR_ORDINAL) { + throw new RangeError("fork reference vector builder handle space exhausted"); + } + handle = this.nextReferenceVectorHandle++; + } + this.pendingReferenceVectors.set(handle, { + expectedLength, + builder: new ForkReferenceVectorBuilder(expectedLength), + }); + return handle; + } + + appendReferenceVector(handle: number, recipeId: number): void { + this.requirePhase("capture", "append a reference vector"); + this.assertU32(handle, "reference vector builder handle"); + assertRecipeId(recipeId); + const vector = this.pendingReferenceVectors.get(handle); + if (!vector) { + throw new Error(`fork reference vector builder ${handle} is not allocated`); + } + if (recipeId >= this.nodes.length) { + throw new Error( + `fork reference vector builder ${handle} names missing recipe ${recipeId}`, + ); + } + vector.builder.append(recipeId); + } + + /** + * Intern one complete activation vector and return its stable wire ordinal. + * + * WHY: recursive activations commonly carry the same reference-recipe + * sequence. Frames keep only this canonical ordinal in their existing header + * word, so recursion grows the linked continuation without duplicating the + * process-owned vector payload. + */ + finishReferenceVector(handle: number): number { + this.requirePhase("capture", "finish a reference vector"); + this.assertU32(handle, "reference vector builder handle"); + const pending = this.pendingReferenceVectors.get(handle); + if (!pending) { + throw new Error(`fork reference vector builder ${handle} is not allocated`); + } + if (pending.builder.length !== pending.expectedLength) { + throw new Error( + `fork reference vector builder ${handle} has ${pending.builder.length} entries; ` + + `expected ${pending.expectedLength}`, + ); + } + const values = pending.builder.finish(); + + const existing = findForkReferenceVectorOrdinal( + [this.referenceVectorIntern], + this.referenceVectors, + values, + ); + if (existing !== undefined) { + this.releaseReferenceVectorHandle(handle); + return existing; + } + const ordinal = this.referenceVectors.length; + if (ordinal > MAX_REFERENCE_VECTOR_ORDINAL) { + throw new RangeError("fork reference vector ordinal space exhausted"); + } + const canonical = values; + this.referenceVectors.push(canonical); + indexForkReferenceVector(this.referenceVectorIntern, canonical, ordinal); + this.releaseReferenceVectorHandle(handle); + return ordinal; + } + + getReferenceVector(ordinal: number, index: number): number { + this.assertU32(ordinal, "reference vector ordinal"); + this.assertU32(index, "reference vector index"); + let vector: ForkReferenceVector | undefined; + if (this.phase === "parent-replay") { + vector = this.referenceVectors.get(ordinal); + } else { + this.requirePhase("child-replay", "read a reference vector"); + vector = this.decodedReferenceVectors.get(ordinal); + } + if (!vector) { + throw new Error(`fork reference vector ${ordinal} is not available`); + } + const recipeId = vector.get(index); + if (recipeId === undefined) { + throw new Error( + `fork reference vector ${ordinal} index ${index} is out of bounds`, + ); + } + return recipeId; + } + + /** + * Reserve transient bytes in the one process memory copied by fork. + * + * Reservations are stack-disciplined because generated codecs recurse while + * walking reference payloads. A retained page amortizes the common case; + * unusually large or deeply nested payloads allocate extra page-rounded + * chunks and release those chunks as soon as their nested scope returns. + */ + reserveScratch(size: number | bigint): number { + this.requireActivePhase("reserve reference scratch"); + const requestedSize = this.checkedScratchSize(size); + const alignedSize = this.alignScratch(requestedSize); + let chunk = this.scratchChunks[this.scratchChunks.length - 1]; + if (!chunk || alignedSize > chunk.size - chunk.used) { + const allocate = this.allocateScratch; + if (!allocate || !this.deallocateScratch) { + throw new Error(`${this.label} has no scratch mapping owner`); + } + const chunkSize = this.alignScratch(Math.max(65_536, alignedSize), 65_536); + const addr = allocate(chunkSize); + if ( + !Number.isSafeInteger(addr) + || addr <= 0 + || addr % 16 !== 0 + || addr > this.requireMemory().buffer.byteLength - chunkSize + ) { + if (Number.isSafeInteger(addr) && addr > 0) { + try { + this.deallocateScratch(addr, chunkSize); + } catch { + // Preserve the allocator contract violation. + } + } + throw new RangeError( + `${this.label} scratch allocator returned an invalid mapping`, + ); + } + chunk = { addr, size: chunkSize, used: 0 }; + this.scratchChunks.push(chunk); + } + const previousUsed = chunk.used; + const addr = chunk.addr + previousUsed; + chunk.used += alignedSize; + new Uint8Array(this.requireMemory().buffer, addr, alignedSize).fill(0); + this.scratchReservations.push({ + addr, + requestedSize, + alignedSize, + previousUsed, + chunk, + }); + return addr; + } + + releaseScratch(pointer: number | bigint, size: number | bigint): void { + this.requireActivePhase("release reference scratch"); + const addr = this.checkedScratchPointer(pointer); + const requestedSize = this.checkedScratchSize(size); + const reservation = this.scratchReservations.pop(); + if ( + !reservation + || reservation.addr !== addr + || reservation.requestedSize !== requestedSize + ) { + if (reservation) this.scratchReservations.push(reservation); + throw new Error( + `${this.label} scratch release is not the most recent reservation`, + ); + } + new Uint8Array( + this.requireMemory().buffer, + reservation.addr, + reservation.alignedSize, + ).fill(0); + reservation.chunk.used = reservation.previousUsed; + + const tail = this.scratchChunks[this.scratchChunks.length - 1]; + if ( + tail === reservation.chunk + && tail.used === 0 + && this.scratchChunks.length > 1 + ) { + this.scratchChunks.pop(); + this.deallocateScratch!(tail.addr, tail.size); + } + } + + lookupExceptionSlot( + slot: number, + provider: ForkExceptionSlotProvider = this.requireExceptionSlotProvider(), + ): number { + this.requirePhase("capture", "look up an exception identity"); + const value = this.exceptionValue(provider, slot); + const known = this.lookupExceptionId(value); + return known !== undefined && this.nodes.get(known)?.node.kind === "exnref" + ? known + : 0; + } + + claimExceptionSlot( + slot: number, + provider: ForkExceptionSlotProvider = this.requireExceptionSlotProvider(), + ): number { + this.requirePhase("capture", "claim an exception identity"); + const value = this.exceptionValue(provider, slot); + const known = this.lookupExceptionId(value); + if (known !== undefined) { + const existing = this.nodes.get(known)?.node; + if (existing?.kind === "exnref") return known; + // WHY: one value may cross the embedding first as externref/anyref and + // later as exnref. Keep its recipe ID and upgrade the node in place so + // every view retains one graph identity. + this.nodes.set(known, { + id: known, + node: { + kind: "exnref", + moduleActivation: 0, + tagOrdinal: 0, + layoutId: 0, + scalars: new Uint8Array(), + payloads: [], + }, + }); + this.pendingExceptions.add(known); + this.exceptionCacheIndexes.set( + known, + this.exceptionCacheIndexes.size + 1, + ); + return known; + } + const id = this.nodes.length; + if (id > 0xffff_ffff) { + throw new RangeError("fork reference recipe id space exhausted"); + } + this.nodes.push({ + id, + node: { + kind: "exnref", + moduleActivation: 0, + tagOrdinal: 0, + layoutId: 0, + scalars: new Uint8Array(), + payloads: [], + }, + }); + this.capturedValues.push(value); + this.rememberExceptionId(value, id); + this.pendingExceptions.add(id); + this.exceptionCacheIndexes.set(id, this.exceptionCacheIndexes.size + 1); + return id; + } + + /** + * Encode a raw JavaScript exception caught through `WebAssembly.JSTag`. + * + * The exception itself is an exnref recipe; its JS payload is a separate + * externref node so process-wide broker ownership remains visible to the + * wire graph and the child receives its canonical local token. + */ + captureHostException( + value: unknown, + childPayloadValue: unknown = value, + ): number { + this.requirePhase("capture", "capture a host exception"); + const known = this.lookupExceptionId(value); + if (known !== undefined && this.nodes.get(known)?.node.kind === "exnref") { + return known; + } + const recipeId = known ?? this.nodes.length; + if (recipeId >= 0x7fff_fffe) { + throw new RangeError("fork reference recipe id space exhausted"); + } + const payloadId = this.nodes.length + (known === undefined ? 1 : 0); + const existing = known === undefined + ? undefined + : this.nodes.get(known)?.node; + const handle = ( + existing?.kind === "externref" + && Object.is(value, childPayloadValue) + ) + ? existing.handle + : this.externrefs.capture(childPayloadValue); + const exceptionEntry: ForkReferenceRecipeEntry = { + id: recipeId, + node: { + kind: "exnref", + moduleActivation: FORK_HOST_EXCEPTION_ACTIVATION_ID, + tagOrdinal: 0, + layoutId: 0, + scalars: new Uint8Array(), + payloads: [payloadId], + }, + }; + if (known === undefined) { + this.nodes.push(exceptionEntry); + this.capturedValues.push(value); + this.rememberId(value, recipeId); + } else { + this.nodes.set(known, exceptionEntry); + } + this.nodes.push({ + id: payloadId, + node: { + kind: "externref", + handle, + }, + }); + this.capturedValues.push(childPayloadValue); + this.exceptionCacheIndexes.set( + recipeId, + this.exceptionCacheIndexes.size + 1, + ); + return recipeId; + } + + exceptionOwner(recipeId: number): number { + assertRecipeId(recipeId); + const node = this.recipeNode(recipeId); + if (node?.kind !== "exnref") { + throw new Error(`fork recipe ${recipeId} is not an exception`); + } + return node.moduleActivation; + } + + materializeHostException(recipeId: number): unknown { + const owner = this.exceptionOwner(recipeId); + if (owner !== FORK_HOST_EXCEPTION_ACTIVATION_ID) { + throw new Error(`fork exception recipe ${recipeId} is not host-owned`); + } + const node = this.recipeNode(recipeId); + if (node?.kind !== "exnref" || node.payloads.length !== 1) { + throw new Error(`fork host exception recipe ${recipeId} is malformed`); + } + if (this.phase === "parent-replay") { + // WHY: the parent still owns the original JavaScript/Wasm exception and + // must retain its exact tag and identity. Only a fresh child consumes + // the owner-normalized externref payload. + if (recipeId >= this.capturedValues.length) { + throw new Error(`fork host exception recipe ${recipeId} is out of bounds`); + } + return this.capturedValues.get(recipeId); + } + return this.decodeExternref(node.payloads[0]!); + } + + exceptionCacheIndex(recipeId: number): number { + assertRecipeId(recipeId); + const index = this.exceptionCacheIndexes.get(recipeId); + if (index === undefined) { + throw new Error(`fork recipe ${recipeId} has no exception cache index`); + } + return index; + } + + defineException( + recipeId: number, + moduleActivation: number, + tagOrdinal: number, + layoutId: number, + scalarPointer: number | bigint, + scalarByteLength: number, + referenceIdsPointer: number | bigint, + referenceCount: number, + ): void { + this.requirePhase("capture", "define an exception recipe"); + this.assertExceptionCoordinate( + recipeId, + moduleActivation, + tagOrdinal, + layoutId, + ); + if (!this.pendingExceptions.has(recipeId)) { + throw new Error(`fork exception recipe ${recipeId} is not pending definition`); + } + const scalars = this.readBytes( + scalarPointer, + scalarByteLength, + "fork exception scalar payload", + ); + const payloads = this.readRecipeIds( + referenceIdsPointer, + referenceCount, + "fork exception reference payloads", + ); + this.nodes.set(recipeId, { + id: recipeId, + node: { + kind: "exnref", + moduleActivation, + tagOrdinal, + layoutId, + scalars, + payloads, + }, + }); + this.pendingExceptions.delete(recipeId); + } + + routeException(recipeId: number, expectedActivation: number): number { + assertRecipeId(recipeId); + this.assertU32(expectedActivation, "exception route activation"); + const node = this.recipeNode(recipeId); + if (node?.kind !== "exnref" || node.moduleActivation !== expectedActivation) { + return -1; + } + const layoutId = node.layoutId ?? 0; + if (layoutId > 0x7fff_ffff) { + throw new Error(`fork exception recipe ${recipeId} has a non-routable layout id`); + } + return layoutId; + } + + loadException( + recipeId: number, + moduleActivation: number, + tagOrdinal: number, + layoutId: number, + scalarDestination: number | bigint, + scalarByteLength: number, + referenceIdsDestination: number | bigint, + referenceCount: number, + ): number { + this.assertExceptionCoordinate( + recipeId, + moduleActivation, + tagOrdinal, + layoutId, + ); + const node = this.recipeNode(recipeId); + if (node?.kind !== "exnref") { + throw new Error(`fork recipe ${recipeId} is not an exception`); + } + const scalars = node.scalars ?? new Uint8Array(); + if ( + scalars.byteLength !== scalarByteLength + || node.payloads.length !== referenceCount + ) { + throw new Error( + `fork exception recipe ${recipeId} payload layout does not match ` + + `the generated codec`, + ); + } + this.writeBytes( + scalarDestination, + scalars, + "fork exception scalar destination", + ); + this.writeRecipeIds( + referenceIdsDestination, + node.payloads, + "fork exception reference destination", + ); + return 1; + } + + routeGc(recipeId: number, expectedActivation: number): number { + assertRecipeId(recipeId); + this.assertU32(expectedActivation, "GC route activation"); + const node = this.recipeNode(recipeId); + if (node?.kind === "i31") return 0; + if ( + (node?.kind !== "struct" && node?.kind !== "array") + || node.moduleActivation !== expectedActivation + ) { + return -1; + } + const layoutId = node.layoutId ?? 0; + this.assertU31(layoutId, `fork Wasm-GC recipe ${recipeId} layout`, false); + return layoutId; + } + + gcPayloadLength( + recipeId: number, + expectedActivation: number, + expectedLayoutId: number, + ): number { + assertRecipeId(recipeId); + this.assertU32(expectedActivation, "GC payload activation"); + this.assertU31(expectedLayoutId, "GC payload layout"); + const node = this.recipeNode(recipeId); + if (node?.kind === "i31") { + if (expectedLayoutId !== 0) { + throw new Error(`fork i31 recipe ${recipeId} has a nonzero layout`); + } + return 4; + } + if ( + (node?.kind !== "struct" && node?.kind !== "array") + || node.moduleActivation !== expectedActivation + || (node.layoutId ?? 0) !== expectedLayoutId + ) { + throw new Error( + `fork Wasm-GC recipe ${recipeId} does not match payload route ` + + `${expectedActivation}:${expectedLayoutId}`, + ); + } + return (node.scalars ?? new Uint8Array()).byteLength; + } + + loadGc( + recipeId: number, + moduleActivation: number, + typeOrdinal: number, + layoutId: number, + kind: number, + scalarDestination: number | bigint, + scalarByteLength: number, + ): number { + this.requirePhase("child-replay", "load a Wasm-GC recipe"); + assertRecipeId(recipeId); + this.assertU32(moduleActivation, "GC load activation"); + this.assertU32(typeOrdinal, "GC load type ordinal"); + this.assertU31(layoutId, "GC load layout id"); + this.assertU32(kind, "GC load kind"); + const node = this.recipeNode(recipeId); + if (node?.kind === "i31") { + if ( + layoutId !== 0 + || typeOrdinal !== 0xffff_ffff + || kind !== 0 + || scalarByteLength !== 4 + ) { + throw new Error(`fork i31 recipe ${recipeId} has an invalid load coordinate`); + } + const bytes = new Uint8Array(4); + new DataView(bytes.buffer).setInt32(0, node.value, true); + this.writeBytes( + scalarDestination, + bytes, + "fork i31 scalar destination", + ); + return 0; + } + if (node?.kind !== "struct" && node?.kind !== "array") { + throw new Error(`fork recipe ${recipeId} is not a Wasm-GC aggregate`); + } + const nodeKind = node.kind === "struct" ? 1 : 2; + const scalars = node.scalars ?? new Uint8Array(); + if ( + node.moduleActivation !== moduleActivation + || node.typeOrdinal !== typeOrdinal + || (node.layoutId ?? 0) !== layoutId + || nodeKind !== kind + || scalars.byteLength !== scalarByteLength + ) { + throw new Error( + `fork Wasm-GC recipe ${recipeId} payload does not match ` + + `the generated codec`, + ); + } + this.writeBytes( + scalarDestination, + scalars, + "fork Wasm-GC scalar destination", + ); + const edges = node.kind === "struct" ? node.fields : node.elements; + if (edges.length === 0) return 0; + const known = this.replayGcVectors.get(recipeId); + if (known !== undefined) return known; + const existing = findForkReferenceVectorOrdinal( + [ + this.childTransaction!.vectorIntern, + this.decodedReferenceVectorIntern, + ], + this.decodedReferenceVectors, + forkReferenceVectorFrom(edges, edges.length), + ); + if (existing !== undefined) { + this.replayGcVectors.set(recipeId, existing); + return existing; + } + const ordinal = this.decodedReferenceVectors.length; + if (ordinal > MAX_REFERENCE_VECTOR_ORDINAL) { + throw new RangeError("fork reference vector ordinal space exhausted"); + } + const canonical = forkReferenceVectorFrom(edges, edges.length); + this.decodedReferenceVectors.push(canonical); + indexForkReferenceVector( + this.decodedReferenceVectorIntern, + canonical, + ordinal, + ); + this.replayGcVectors.set(recipeId, ordinal); + return ordinal; + } + + /** + * Eager fresh-child barrier for all Wasm-only reference identities. + * + * The graph is validated in full before allocation. Defaultable shells are + * then allocated globally, remaining constructors follow their exact + * dependency edges, exceptions are cached in their owning activation, and + * mutable fields are filled only after every identity exists. + */ + materializeAllTyped(): void { + this.requirePhase("child-replay", "materialize typed references"); + if (this.typedMaterialized) { + throw new Error("typed fork references were materialized twice"); + } + const owner = this.typedReplay; + if (!owner) { + if ( + this.decodedNodes.some(({ node }) => + node.kind === "i31" + || node.kind === "struct" + || node.kind === "array" + || node.kind === "exnref" + ) + ) { + throw new Error(`${this.label} has no typed-reference replay owner`); + } + this.typedMaterialized = true; + return; + } + const providers = owner.providers(); + const layouts = new Map(); + for (const entry of this.decodedNodes) { + switch (entry.node.kind) { + case "struct": + case "array": { + const provider = owner.provider(entry.node.moduleActivation); + layouts.set( + entry.id, + this.validateGcRecipeNode( + entry.node, + provider.descriptor, + `fork Wasm-GC recipe ${entry.id}`, + ), + ); + break; + } + case "exnref": + owner.validateExceptionOwner(entry.node.moduleActivation); + break; + case "i31": + if (providers.length === 0) { + throw new Error("fork i31 replay has no generated GC codec"); + } + break; + case "null": + case "funcref": + case "externref": + case "static-root": + break; + } + } + + owner.prepareTransit(Math.max(0, this.decodedNodes.length - 1)); + for (const entry of this.decodedNodes) { + if (entry.node.kind !== "static-root") continue; + if (!this.materializedValues.has(entry.id)) { + throw new Error( + `fork static-root recipe ${entry.id} was not pinned during child attach`, + ); + } + // WHY: generated GC constructors and field fills decode reference edges + // from recipe+1 in the shared anyref table. Instantiation recreated this + // identity instead of a codec, so publish the pinned child root before + // any dynamic object can consume it. + owner.publishTransit(entry.id, this.materializedValues.get(entry.id)); + } + for (const entry of this.decodedNodes) { + if (entry.node.kind !== "externref") continue; + // Externref leaves must exist before immutable GC constructors or + // exception payload decoders consume their recipe edges. Materializing + // only the JavaScript token is insufficient: the shared transit table + // stores anyref, so a generated Wasm helper performs the conversion. + owner.publishExternref(entry.id, this.decodeExternref(entry.id)); + } + const allocated = new Set(this.adoptedAllocatedTypedRecipes); + const completedExceptions = new Set( + this.adoptedMaterializedExceptionRecipes, + ); + for (const entry of this.decodedNodes) { + if (entry.node.kind === "i31") { + if (allocated.has(entry.id)) continue; + providers[0]!.allocate(entry.id); + allocated.add(entry.id); + continue; + } + if ( + (entry.node.kind === "struct" || entry.node.kind === "array") + && ( + layouts.get(entry.id)!.flags + & FORK_GC_LAYOUT_DEFAULTABLE_SHELL + ) !== 0 + ) { + if (allocated.has(entry.id)) continue; + owner.provider(entry.node.moduleActivation).allocate(entry.id); + allocated.add(entry.id); + } + } + + type PendingTypedRecipe = { + readonly recipeId: number; + readonly node: Extract< + ForkReferenceRecipeNode, + { kind: "exnref" | "struct" | "array" } + >; + readonly dependencies: readonly number[]; + nextDependency: number; + }; + // A sparse set tracks only the active constructor path. A graph-sized + // Uint8Array would reintroduce one contiguous allocation cliff. + const visiting = new Set(); + const beginTypedRecipe = ( + recipeId: number, + ): PendingTypedRecipe | null => { + if (allocated.has(recipeId) || completedExceptions.has(recipeId)) { + return null; + } + const node = this.decodedNodes.get(recipeId)?.node; + if (!node) { + throw new Error(`typed replay dependency ${recipeId} is missing`); + } + if ( + node.kind === "null" + || node.kind === "funcref" + || node.kind === "externref" + || node.kind === "static-root" + || node.kind === "i31" + ) { + return null; + } + if (visiting.has(recipeId)) { + throw new Error( + `typed replay has an unallocatable constructor cycle at recipe ` + + `${recipeId}`, + ); + } + visiting.add(recipeId); + return { + recipeId, + node, + dependencies: node.kind === "exnref" + ? node.payloads + : this.gcAllocationDependencies(node, layouts.get(recipeId)!), + nextDependency: 0, + }; + }; + const materialize = (recipeId: number): void => { + const first = beginTypedRecipe(recipeId); + if (!first) return; + const pending: PendingTypedRecipe[] = [first]; + while (pending.length !== 0) { + const current = pending[pending.length - 1]!; + let descended = false; + while (current.nextDependency < current.dependencies.length) { + const dependency = + current.dependencies[current.nextDependency++]!; + const child = beginTypedRecipe(dependency); + if (!child) continue; + pending.push(child); + descended = true; + break; + } + if (descended) continue; + + if (current.node.kind === "exnref") { + owner.materializeException( + current.recipeId, + current.node.moduleActivation, + ); + completedExceptions.add(current.recipeId); + } else { + owner.provider(current.node.moduleActivation).allocate( + current.recipeId, + ); + allocated.add(current.recipeId); + } + visiting.delete(current.recipeId); + pending.pop(); + } + }; + this.decodedNodes.forEach(({ id }) => materialize(id)); + + for (const entry of this.decodedNodes) { + if (entry.node.kind !== "struct" && entry.node.kind !== "array") { + continue; + } + if (this.adoptedFilledTypedRecipes.has(entry.id)) continue; + owner.provider(entry.node.moduleActivation).fill(entry.id); + } + this.typedMaterialized = true; + } + + /** + * Drop every temporary strong root after the outermost frame has restored. + * Module globals/tables/locals now own live reconstructed values; this + * transaction must not extend their lifetime across a later fork. + */ + finishReplay(): void { + if (this.phase !== "parent-replay" && this.phase !== "child-replay") { + throw new Error(`cannot finish reference replay while transaction is ${this.phase}`); + } + this.clear(); + } + + abort(): void { + if (this.phase === "idle") return; + this.clear(); + } + + private decode(recipeId: number, expected: "funcref" | "externref"): unknown { + assertRecipeId(recipeId); + if (recipeId === 0) return null; + if (this.phase === "parent-replay") { + if (recipeId >= this.capturedValues.length) { + throw new Error(`fork ${expected} recipe ${recipeId} is out of bounds`); + } + return this.capturedValues.get(recipeId); + } + this.requirePhase("child-replay", `decode a ${expected}`); + const index = recipeId; + const node = this.decodedNodes.get(index)?.node; + if (!node) { + throw new Error(`fork ${expected} recipe ${recipeId} is out of bounds`); + } + if (this.materializedValues.has(index)) { + return this.materializedValues.get(index); + } + const value = this.materializeNode(node, recipeId); + this.materializedValues.set(index, value); + return value; + } + + private recipeNode(recipeId: number): ForkReferenceRecipeNode | undefined { + if (this.phase === "capture" || this.phase === "sealed-parent" || this.phase === "parent-replay") { + return this.nodes.get(recipeId)?.node; + } + if (this.phase === "child-replay") { + return this.decodedNodes.get(recipeId)?.node; + } + throw new Error("fork reference transaction has no active recipe graph"); + } + + private requireExceptionSlotProvider(): ForkExceptionSlotProvider { + if (!this.exceptionSlots) { + throw new Error("fork exception scratch provider is not registered"); + } + return this.exceptionSlots; + } + + private exceptionValue(provider: ForkExceptionSlotProvider, slot: number): unknown { + this.assertU32(slot, "exception scratch slot"); + try { + provider.throwSlot(slot); + } catch (value) { + return value; + } + throw new Error(`fork exception scratch slot ${slot} returned without throwing`); + } + + private assertExceptionCoordinate( + recipeId: number, + moduleActivation: number, + tagOrdinal: number, + layoutId: number, + ): void { + assertRecipeId(recipeId); + if (recipeId === 0) { + throw new Error("the null recipe cannot be defined or loaded as an exception"); + } + this.assertU32(moduleActivation, "exception module activation"); + this.assertU32(tagOrdinal, "exception tag ordinal"); + this.assertU32(layoutId, "exception layout id"); + if (layoutId > 0x7fff_ffff) { + throw new RangeError(`exception layout id ${layoutId} is not routable`); + } + const node = this.recipeNode(recipeId); + if (!node) { + throw new Error(`fork exception recipe ${recipeId} is out of bounds`); + } + if ( + !this.pendingExceptions.has(recipeId) + && ( + node.kind !== "exnref" + || node.moduleActivation !== moduleActivation + || node.tagOrdinal !== tagOrdinal + || (node.layoutId ?? 0) !== layoutId + ) + ) { + throw new Error( + `fork exception recipe ${recipeId} coordinate does not match ` + + `${moduleActivation}:${tagOrdinal}:${layoutId}`, + ); + } + } + + private gcSlotValue(table: WebAssembly.Table, slot: number): unknown { + this.assertU32(slot, "Wasm-GC transit slot"); + if (slot >= table.length) { + throw new RangeError(`Wasm-GC transit slot ${slot} is out of bounds`); + } + const value = table.get(slot); + if ( + (typeof value !== "object" || value === null) + && typeof value !== "function" + ) { + throw new TypeError("Wasm-GC transit slot is not a non-null reference"); + } + return value; + } + + private validateGcSnapshot( + layout: ForkGcLayoutDescriptor, + scalars: Uint8Array, + references: readonly number[] | ForkReferenceVector, + context: string, + ): void { + references.forEach((id, index) => { + assertRecipeId(id); + if (id >= this.nodes.length && id >= this.decodedNodes.length) { + throw new Error(`${context} reference ${index} names missing recipe ${id}`); + } + }); + const referenceFieldCount = layout.fields.filter( + ({ flags }) => (flags & FORK_GC_FIELD_REFERENCE) !== 0, + ).length; + if (layout.kind === 1) { + if ( + scalars.byteLength !== layout.scalarLengthOrStride + || references.length !== referenceFieldCount + ) { + throw new Error(`${context} does not match struct layout ${layout.id}`); + } + return; + } + if (scalars.byteLength < 4) { + throw new Error(`${context} array length is truncated`); + } + const length = new DataView( + scalars.buffer, + scalars.byteOffset, + scalars.byteLength, + ).getUint32(0, true); + const referenceElements = + (layout.fields[0]!.flags & FORK_GC_FIELD_REFERENCE) !== 0; + const expectedScalarLength = referenceElements + ? 4 + : 4 + length * layout.scalarLengthOrStride; + if ( + !Number.isSafeInteger(expectedScalarLength) + || expectedScalarLength > 0xffff_ffff + || scalars.byteLength !== expectedScalarLength + || references.length !== (referenceElements ? length : 0) + || ( + layout.constructor === ForkGcConstructorKind.ArrayFixed + && layout.auxiliary !== length + ) + ) { + throw new Error(`${context} does not match array layout ${layout.id}`); + } + } + + private validateGcRecipeNode( + node: Extract, + descriptor: ForkGcCodecDescriptor, + context: string, + ): ForkGcLayoutDescriptor { + const layout = descriptor.require(node.layoutId ?? 0); + if ( + layout.typeOrdinal !== node.typeOrdinal + || (node.kind === "struct" ? 1 : 2) !== layout.kind + ) { + throw new Error(`${context} has an invalid type/layout coordinate`); + } + const scalars = node.scalars ?? new Uint8Array(); + const references = node.kind === "struct" ? node.fields : node.elements; + if ( + scalars.byteLength < layout.provenanceScalarLength + || references.length < layout.provenanceReferenceCount + ) { + throw new Error(`${context} has truncated constructor provenance`); + } + this.validateGcSnapshot( + layout, + scalars.subarray(layout.provenanceScalarLength), + references.slice(layout.provenanceReferenceCount), + context, + ); + return layout; + } + + private gcAllocationDependencies( + node: Extract, + layout: ForkGcLayoutDescriptor, + ): readonly number[] { + const edges = node.kind === "struct" ? node.fields : node.elements; + const dependencies = edges.slice(0, layout.provenanceReferenceCount); + const snapshotStart = layout.provenanceReferenceCount; + if (node.kind === "struct") { + for (const field of layout.fields) { + if ( + (field.flags & FORK_GC_FIELD_ALLOCATION_DEPENDENCY) === 0 + || field.referenceOrdinal === null + ) { + continue; + } + dependencies.push(edges[snapshotStart + field.referenceOrdinal]!); + } + return dependencies; + } + if ((layout.fields[0]!.flags & FORK_GC_FIELD_REFERENCE) === 0) { + return dependencies; + } + const snapshot = edges.slice(snapshotStart); + if (layout.constructor === ForkGcConstructorKind.ArrayFixed) { + // Mutable non-null internal arrays use constructor provenance as their + // seed; immutable arrays use their final (and therefore original) + // elements directly. + if (layout.provenanceReferenceCount === 0) { + dependencies.push(...snapshot); + } + } else if ( + layout.constructor === ForkGcConstructorKind.ArrayNew + && layout.provenanceReferenceCount === 0 + && snapshot.length !== 0 + ) { + dependencies.push(snapshot[0]!); + } + return dependencies; + } + + private readBytes( + pointer: number | bigint, + byteLength: number, + context: string, + ): Uint8Array { + const { offset, length } = this.memoryRange(pointer, byteLength, context); + return new Uint8Array( + new Uint8Array(this.requireMemory().buffer, offset, length), + ); + } + + private writeBytes( + pointer: number | bigint, + bytes: Uint8Array, + context: string, + ): void { + const { offset } = this.memoryRange(pointer, bytes.byteLength, context); + new Uint8Array(this.requireMemory().buffer, offset, bytes.byteLength).set(bytes); + } + + private readRecipeIds( + pointer: number | bigint, + count: number, + context: string, + ): number[] { + this.assertU32(count, `${context} count`); + const bytes = this.readBytes(pointer, count * 4, context); + const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); + const ids: number[] = []; + for (let index = 0; index < count; index++) { + const id = view.getUint32(index * 4, true); + assertRecipeId(id); + if (id >= this.nodes.length) { + throw new Error(`${context} entry ${index} names missing recipe ${id}`); + } + ids.push(id); + } + return ids; + } + + private writeRecipeIds( + pointer: number | bigint, + ids: readonly number[], + context: string, + ): void { + const { offset } = this.memoryRange(pointer, ids.length * 4, context); + const view = new DataView(this.requireMemory().buffer); + ids.forEach((id, index) => { + assertRecipeId(id); + view.setUint32(offset + index * 4, id, true); + }); + } + + private memoryRange( + pointer: number | bigint, + byteLength: number, + context: string, + ): { offset: number; length: number } { + this.assertU32(byteLength, `${context} byte length`); + const offset = typeof pointer === "bigint" ? Number(pointer) : pointer; + if ( + !Number.isSafeInteger(offset) + || offset < 0 + || (typeof pointer === "bigint" && BigInt(offset) !== pointer) + ) { + throw new RangeError(`${context} has an invalid guest pointer`); + } + const memoryLength = this.requireMemory().buffer.byteLength; + if (offset > memoryLength || byteLength > memoryLength - offset) { + throw new RangeError(`${context} exceeds WebAssembly memory`); + } + return { offset, length: byteLength }; + } + + private requireMemory(): WebAssembly.Memory { + if (!this.memory) { + throw new Error("fork reference transaction has no staging memory"); + } + return this.memory; + } + + private assertU32(value: number, context: string): void { + if (!Number.isInteger(value) || value < 0 || value > 0xffff_ffff) { + throw new RangeError(`${context} is not a u32`); + } + } + + private assertU31( + value: number, + context: string, + allowZero = true, + ): void { + if ( + !Number.isInteger(value) + || value < (allowZero ? 0 : 1) + || value > 0x7fff_ffff + ) { + throw new RangeError(`${context} is not ${allowZero ? "a" : "a nonzero"} u31`); + } + } + + private requireActivePhase(operation: string): void { + if ( + this.phase !== "capture" + && this.phase !== "parent-replay" + && this.phase !== "child-replay" + ) { + throw new Error(`cannot ${operation} while reference transaction is ${this.phase}`); + } + } + + private checkedScratchPointer(value: number | bigint): number { + const result = typeof value === "bigint" ? Number(value) : value; + if ( + !Number.isSafeInteger(result) + || result <= 0 + || (typeof value === "bigint" && BigInt(result) !== value) + ) { + throw new RangeError(`${this.label} scratch pointer is invalid`); + } + return result; + } + + private checkedScratchSize(value: number | bigint): number { + const result = typeof value === "bigint" ? Number(value) : value; + if ( + !Number.isSafeInteger(result) + || result <= 0 + || result > 0xffff_ffff + || (typeof value === "bigint" && BigInt(result) !== value) + ) { + throw new RangeError(`${this.label} scratch size is not a nonzero u32`); + } + return result; + } + + private alignScratch(value: number, alignment = 16): number { + const result = Math.ceil(value / alignment) * alignment; + if (!Number.isSafeInteger(result) || result < value) { + throw new RangeError(`${this.label} scratch alignment overflow`); + } + return result; + } + + private materializeNode( + node: ForkReferenceRecipeNode, + recipeId: number, + ): unknown { + switch (node.kind) { + case "funcref": + return this.functions.decode({ + moduleActivation: node.moduleActivation, + ordinal: node.functionOrdinal, + }); + case "externref": + return this.externrefs.materialize(node.handle); + case "static-root": + if (!this.staticRoots) { + throw new Error( + `fork recipe ${recipeId} requires a static-root catalog`, + ); + } + return this.staticRoots.decode({ + moduleActivation: node.moduleActivation, + ordinal: node.staticRootOrdinal, + }); + case "null": + return null; + case "exnref": + case "i31": + case "struct": + case "array": + // Those nodes are materialized by generated Wasm codecs. Reaching one + // through an abstract JS-compatible import is an ABI/provider mismatch, + // not a value-shape policy decision. + throw new Error( + `fork recipe ${recipeId} requires its generated ${node.kind} codec`, + ); + } + } + + private intern( + value: unknown, + createNode: () => ForkReferenceRecipeNode, + ): number { + const known = this.lookupId(value); + if (known !== undefined) return known; + const id = this.nodes.length; + if (id > 0xffff_ffff) { + throw new RangeError("fork reference recipe id space exhausted"); + } + const recipeId = id; + const staticRoot = this.staticRoots?.encode(value); + const node: ForkReferenceRecipeNode = staticRoot + ? { + kind: "static-root", + moduleActivation: staticRoot.moduleActivation, + staticRootOrdinal: staticRoot.ordinal, + } + : createNode(); + this.nodes.push({ id, node }); + this.capturedValues.push(value); + this.rememberId(value, recipeId); + return recipeId; + } + + private lookupId(value: unknown): number | undefined { + return (typeof value === "object" && value !== null) || typeof value === "function" + ? this.objectIds.get(value as object) + : this.primitiveIds.get(value); + } + + private rememberId(value: unknown, recipeId: number): void { + if ((typeof value === "object" && value !== null) || typeof value === "function") { + this.objectIds.set(value as object, recipeId); + } else { + this.primitiveIds.set(value, recipeId); + } + } + + private lookupExceptionId(value: unknown): number | undefined { + return this.lookupId(value); + } + + private rememberExceptionId(value: unknown, recipeId: number): void { + this.rememberId(value, recipeId); + } + + private releaseReferenceVectorHandle(handle: number): void { + if (!this.pendingReferenceVectors.delete(handle)) { + throw new Error(`fork reference vector builder ${handle} is not allocated`); + } + // Builder handles are transaction-local and never enter sealed bytes. + // Reuse keeps deeply repetitive capture bounded by simultaneous nesting + // instead of total activation count. + this.freeReferenceVectorHandles.push(handle); + } + + private requirePhase(expected: TransactionPhase, operation: string): void { + if (this.phase !== expected) { + throw new Error( + `cannot ${operation} while reference transaction is ${this.phase}; expected ${expected}`, + ); + } + } + + private clear(): void { + this.exceptionSlots?.clearSlots(); + // An exception or host callback may have aborted between reserve/release. + // Zero every transaction-owned byte before returning its mappings. + for (const chunk of this.scratchChunks) { + new Uint8Array(this.requireMemory().buffer, chunk.addr, chunk.size).fill(0); + } + this.scratchReservations.length = 0; + const chunks = this.scratchChunks.splice(0).reverse(); + let firstScratchError: unknown; + for (const chunk of chunks) { + try { + this.deallocateScratch?.(chunk.addr, chunk.size); + } catch (error) { + firstScratchError ??= error; + } + } + this.pendingExceptions.clear(); + this.pendingGc.clear(); + this.i31Ids.clear(); + this.exceptionCacheIndexes.clear(); + this.nodes.clear(); + this.capturedValues.clear(); + this.objectIds = new WeakMap(); + this.primitiveIds.clear(); + this.decodedNodes = new PagedForkReferenceDirectory(); + this.referenceVectors.clear(); + this.pendingReferenceVectors.clear(); + this.freeReferenceVectorHandles.length = 0; + this.nextReferenceVectorHandle = 1; + this.referenceVectorIntern.clear(); + this.decodedReferenceVectors.clear(); + this.decodedReferenceVectorIntern.clear(); + this.replayGcVectors.clear(); + this.typedMaterialized = false; + this.childTransaction = null; + this.childReplayAdopted = false; + this.adoptedAllocatedTypedRecipes.clear(); + this.adoptedFilledTypedRecipes.clear(); + this.adoptedMaterializedExceptionRecipes.clear(); + this.materializedValues.clear(); + this.phase = "idle"; + if (firstScratchError !== undefined) throw firstScratchError; + } +} diff --git a/host/src/fork-replay-events.ts b/host/src/fork-replay-events.ts new file mode 100644 index 0000000000..571f021cca --- /dev/null +++ b/host/src/fork-replay-events.ts @@ -0,0 +1,738 @@ +import { + WPK_FORK_MODULE_STATE_REPLAY_EVENT_SEGMENT_CAPACITY, + WPK_FORK_MODULE_STATE_REPLAY_EVENT_SEGMENT_HEADER_SIZE, + WPK_FORK_MODULE_STATE_REPLAY_EVENT_SEGMENT_KNOWN_FLAGS, + WPK_FORK_MODULE_STATE_REPLAY_EVENT_SEGMENT_VERSION, + WPK_FORK_MODULE_STATE_REPLAY_EVENTS_HEADER_SIZE, + WPK_FORK_MODULE_STATE_REPLAY_EVENTS_KNOWN_FLAGS, + WPK_FORK_MODULE_STATE_REPLAY_EVENTS_MAGIC, + WPK_FORK_MODULE_STATE_REPLAY_EVENTS_VERSION, + WPK_FORK_MODULE_STATE_REPLAY_EVENT_SIZE, +} from "./generated/abi"; + +function littleEndianMagic(bytes: readonly number[]): number { + return bytes.reduce( + (magic, byte, index) => magic | (byte << (index * 8)), + 0, + ) >>> 0; +} + +const REPLAY_EVENT_MAGIC = littleEndianMagic( + WPK_FORK_MODULE_STATE_REPLAY_EVENTS_MAGIC, +); +export const FORK_REPLAY_EVENT_VERSION = + WPK_FORK_MODULE_STATE_REPLAY_EVENTS_VERSION; +export const FORK_REPLAY_EVENT_HEADER_SIZE = + WPK_FORK_MODULE_STATE_REPLAY_EVENTS_HEADER_SIZE; +export const FORK_REPLAY_EVENT_SEGMENT_VERSION = + WPK_FORK_MODULE_STATE_REPLAY_EVENT_SEGMENT_VERSION; +export const FORK_REPLAY_EVENT_SEGMENT_HEADER_SIZE = + WPK_FORK_MODULE_STATE_REPLAY_EVENT_SEGMENT_HEADER_SIZE; +export const FORK_REPLAY_EVENT_ENTRY_SIZE = + WPK_FORK_MODULE_STATE_REPLAY_EVENT_SIZE; +/** + * Allocation geometry, not a continuation-depth limit. + * + * A roughly 32-KiB event page fits twice in the arena's normal 64-KiB chunks, + * including two record envelopes and the larger wasm64 chunk header. + */ +export const FORK_REPLAY_EVENT_SEGMENT_CAPACITY = + WPK_FORK_MODULE_STATE_REPLAY_EVENT_SEGMENT_CAPACITY; +const REPLAY_EVENT_KNOWN_FLAGS = + WPK_FORK_MODULE_STATE_REPLAY_EVENTS_KNOWN_FLAGS; +const REPLAY_EVENT_SEGMENT_KNOWN_FLAGS = + WPK_FORK_MODULE_STATE_REPLAY_EVENT_SEGMENT_KNOWN_FLAGS; +const MAX_U32 = 0xffff_ffff; +const MAX_U64 = 0xffff_ffff_ffff_ffffn; + +export interface ForkReplayEvent { + readonly activationId: number; + readonly functionOrdinal: number; +} + +export interface ForkResumeTarget { + readonly functionOrdinal: number; + /** No-parameter Wasm thunk that restores params from the unconsumed frame. */ + readonly thunk: CallableFunction; +} + +type JournalPhase = "idle" | "capture" | "sealed-parent" | "replay"; + +function assertU32(value: number, context: string): void { + if (!Number.isInteger(value) || value < 0 || value > MAX_U32) { + throw new RangeError(`${context} is not a u32: ${value}`); + } +} + +function exactU64(value: bigint | number, context: string): bigint { + if ( + typeof value === "number" + && (!Number.isSafeInteger(value) || value < 0) + ) { + throw new RangeError(`${context} is not an exact nonnegative integer`); + } + const exact = typeof value === "bigint" ? value : BigInt(value); + if (exact < 0n || exact > MAX_U64) { + throw new RangeError(`${context} is not representable as u64`); + } + return exact; +} + +function eventKey(activationId: number, functionOrdinal: number): string { + return `${activationId}:${functionOrdinal}`; +} + +interface CapturedEventPage { + readonly words: Uint32Array; + count: number; + previous: ReplayEventPage | null; + next: ReplayEventPage | null; +} + +interface ChildEventPage { + readonly payload: Uint8Array; + readonly view: DataView; + readonly count: number; + previous: ReplayEventPage | null; + next: ReplayEventPage | null; +} + +type ReplayEventPage = CapturedEventPage | ChildEventPage; + +export interface ForkReplayEventWire { + readonly manifest: Uint8Array; + /** Restartable ordered source; callers may validate before attaching. */ + readonly segments: Iterable; +} + +export interface ForkReplayEventCaptureSource { + capturedSegmentPayloads(): Iterable; + capturedManifestPayload(): Uint8Array; +} + +export interface ForkReplayEventWireSummary { + readonly eventCount: bigint; + readonly segmentCount: bigint; + readonly activationIds: ReadonlySet; +} + +function segmentCountForEvents(eventCount: bigint): bigint { + if (eventCount === 0n) return 0n; + return ( + eventCount + BigInt(FORK_REPLAY_EVENT_SEGMENT_CAPACITY) - 1n + ) / BigInt(FORK_REPLAY_EVENT_SEGMENT_CAPACITY); +} + +export function encodeForkReplayEventManifest( + eventCount: bigint, + segmentCount: bigint | number, +): Uint8Array { + const exactEventCount = exactU64(eventCount, "fork replay event count"); + const exactSegmentCount = exactU64( + segmentCount, + "fork replay event segment count", + ); + if (segmentCountForEvents(exactEventCount) !== exactSegmentCount) { + throw new RangeError( + "fork replay event segment count is inconsistent with event count", + ); + } + const bytes = new Uint8Array(FORK_REPLAY_EVENT_HEADER_SIZE); + const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); + view.setUint32(0, REPLAY_EVENT_MAGIC, true); + view.setUint16(4, FORK_REPLAY_EVENT_VERSION, true); + view.setUint16(6, FORK_REPLAY_EVENT_HEADER_SIZE, true); + view.setUint16(8, FORK_REPLAY_EVENT_ENTRY_SIZE, true); + view.setUint16(10, FORK_REPLAY_EVENT_SEGMENT_HEADER_SIZE, true); + view.setUint32(12, FORK_REPLAY_EVENT_SEGMENT_CAPACITY, true); + view.setUint16(16, REPLAY_EVENT_KNOWN_FLAGS, true); + view.setUint16(18, 0, true); + view.setUint32(20, 0, true); + view.setBigUint64(24, exactSegmentCount, true); + view.setBigUint64(32, exactEventCount, true); + return bytes; +} + +export function encodeForkReplayEventSegment( + words: Uint32Array, + count: number, + sequence: bigint | number, +): Uint8Array { + const exactSequence = exactU64( + sequence, + "fork replay event segment sequence", + ); + if ( + !Number.isInteger(count) + || count <= 0 + || count > FORK_REPLAY_EVENT_SEGMENT_CAPACITY + || words.length < count * 2 + ) { + throw new RangeError(`invalid fork replay event segment count ${count}`); + } + const bytes = new Uint8Array( + FORK_REPLAY_EVENT_SEGMENT_HEADER_SIZE + + count * FORK_REPLAY_EVENT_ENTRY_SIZE, + ); + const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); + view.setUint16(0, FORK_REPLAY_EVENT_SEGMENT_VERSION, true); + view.setUint16(2, FORK_REPLAY_EVENT_SEGMENT_HEADER_SIZE, true); + view.setUint16(4, FORK_REPLAY_EVENT_ENTRY_SIZE, true); + view.setUint16(6, REPLAY_EVENT_SEGMENT_KNOWN_FLAGS, true); + view.setBigUint64(8, exactSequence, true); + view.setUint32(16, count, true); + view.setUint32(20, 0, true); + for (let index = 0; index < count; index++) { + const offset = + FORK_REPLAY_EVENT_SEGMENT_HEADER_SIZE + + index * FORK_REPLAY_EVENT_ENTRY_SIZE; + view.setUint32(offset, words[index * 2]!, true); + view.setUint32(offset + 4, words[index * 2 + 1]!, true); + } + return bytes; +} + +interface DecodedReplayEventManifest { + readonly eventCount: bigint; + readonly segmentCount: bigint; +} + +function decodeForkReplayEventManifest( + bytes: Uint8Array, +): DecodedReplayEventManifest { + if ( + !(bytes instanceof Uint8Array) + || bytes.byteLength < FORK_REPLAY_EVENT_HEADER_SIZE + ) { + throw new Error("fork replay event manifest is truncated"); + } + if (bytes.byteLength !== FORK_REPLAY_EVENT_HEADER_SIZE) { + throw new Error("fork replay event manifest has inconsistent bounds"); + } + const view = new DataView( + bytes.buffer, + bytes.byteOffset, + bytes.byteLength, + ); + if (view.getUint32(0, true) !== REPLAY_EVENT_MAGIC) { + throw new Error("fork replay event manifest has invalid magic"); + } + if (view.getUint16(4, true) !== FORK_REPLAY_EVENT_VERSION) { + throw new Error( + `fork replay event manifest has version ${view.getUint16(4, true)}`, + ); + } + if (view.getUint16(6, true) !== FORK_REPLAY_EVENT_HEADER_SIZE) { + throw new Error("fork replay event manifest has an invalid header size"); + } + if ( + view.getUint16(8, true) !== FORK_REPLAY_EVENT_ENTRY_SIZE + || view.getUint16(10, true) !== FORK_REPLAY_EVENT_SEGMENT_HEADER_SIZE + ) { + throw new Error("fork replay event manifest has an invalid entry size"); + } + if (view.getUint32(12, true) !== FORK_REPLAY_EVENT_SEGMENT_CAPACITY) { + throw new Error("fork replay event manifest has an invalid segment capacity"); + } + if ( + (view.getUint16(16, true) & ~REPLAY_EVENT_KNOWN_FLAGS) !== 0 + || view.getUint16(18, true) !== 0 + || view.getUint32(20, true) !== 0 + ) { + throw new Error("fork replay event manifest has nonzero reserved fields"); + } + const segmentCount = view.getBigUint64(24, true); + const eventCount = view.getBigUint64(32, true); + if (segmentCountForEvents(eventCount) !== segmentCount) { + throw new Error( + "fork replay event manifest segment count is inconsistent with event count", + ); + } + return { eventCount, segmentCount }; +} + +function decodeForkReplayEventSegment( + payload: Uint8Array, + sequence: bigint, + expectedCount: number, +): ChildEventPage { + if ( + !(payload instanceof Uint8Array) + || payload.byteLength < FORK_REPLAY_EVENT_SEGMENT_HEADER_SIZE + ) { + throw new Error(`fork replay event segment ${sequence} is truncated`); + } + const view = new DataView( + payload.buffer, + payload.byteOffset, + payload.byteLength, + ); + if ( + view.getUint16(0, true) !== FORK_REPLAY_EVENT_SEGMENT_VERSION + || view.getUint16(2, true) !== FORK_REPLAY_EVENT_SEGMENT_HEADER_SIZE + || view.getUint16(4, true) !== FORK_REPLAY_EVENT_ENTRY_SIZE + ) { + throw new Error( + `fork replay event segment ${sequence} has an invalid version or layout`, + ); + } + if ( + (view.getUint16(6, true) & ~REPLAY_EVENT_SEGMENT_KNOWN_FLAGS) !== 0 + ) { + throw new Error(`fork replay event segment ${sequence} has unknown flags`); + } + if (view.getBigUint64(8, true) !== sequence) { + throw new Error( + `fork replay event segment sequence ${view.getBigUint64(8, true)} ` + + `is out of order; expected ${sequence}`, + ); + } + const count = view.getUint32(16, true); + if (view.getUint32(20, true) !== 0) { + throw new Error(`fork replay event segment ${sequence} has reserved data`); + } + if (count !== expectedCount) { + throw new Error( + `fork replay event segment ${sequence} has ${count} entries; ` + + `expected ${expectedCount}`, + ); + } + const expectedSize = + FORK_REPLAY_EVENT_SEGMENT_HEADER_SIZE + + count * FORK_REPLAY_EVENT_ENTRY_SIZE; + if (payload.byteLength !== expectedSize) { + throw new Error( + `fork replay event segment ${sequence} has inconsistent bounds`, + ); + } + return { + payload, + view, + count, + previous: null, + next: null, + }; +} + +function inspectForkReplayEventWire( + wire: ForkReplayEventWire, + retainPages: boolean, +): { + readonly summary: ForkReplayEventWireSummary; + readonly firstPage: ChildEventPage | null; + readonly lastPage: ChildEventPage | null; +} { + const manifest = decodeForkReplayEventManifest(wire.manifest); + const iterator = wire.segments[Symbol.iterator](); + const activationIds = new Set(); + let firstPage: ChildEventPage | null = null; + let lastPage: ChildEventPage | null = null; + let remaining = manifest.eventCount; + let sequence = 0n; + while (sequence < manifest.segmentCount) { + const item = iterator.next(); + if (item.done) { + throw new Error( + `fork replay event wire ended after ${sequence} segments; ` + + `expected ${manifest.segmentCount}`, + ); + } + const count = Number( + remaining > BigInt(FORK_REPLAY_EVENT_SEGMENT_CAPACITY) + ? BigInt(FORK_REPLAY_EVENT_SEGMENT_CAPACITY) + : remaining, + ); + const page = decodeForkReplayEventSegment( + item.value, + sequence, + count, + ); + for (let index = 0; index < page.count; index++) { + activationIds.add(readPageWord(page, index, 0)); + } + if (retainPages) { + page.previous = lastPage; + if (lastPage) lastPage.next = page; + firstPage ??= page; + lastPage = page; + } + remaining -= BigInt(page.count); + sequence++; + } + if (!iterator.next().done) { + throw new Error( + "fork replay event wire has segments after its declared segment count " + + `${manifest.segmentCount}`, + ); + } + if (remaining !== 0n) { + throw new Error("fork replay event wire ended before its declared event count"); + } + return { + summary: { + eventCount: manifest.eventCount, + segmentCount: manifest.segmentCount, + activationIds, + }, + firstPage, + lastPage, + }; +} + +/** + * Validate segmented replay-event bytes and derive only their active modules. + * + * The caller does not receive per-frame objects, so exact-set validation stays + * proportional to the number of activations rather than continuation depth. + */ +export function validateForkReplayEventWire( + wire: ForkReplayEventWire, +): ForkReplayEventWireSummary { + return inspectForkReplayEventWire(wire, false).summary; +} + +function readPageWord( + page: ReplayEventPage, + eventIndex: number, + wordIndex: 0 | 1, +): number { + if ("words" in page) { + return page.words[eventIndex * 2 + wordIndex]!; + } + return page.view.getUint32( + FORK_REPLAY_EVENT_SEGMENT_HEADER_SIZE + + eventIndex * FORK_REPLAY_EVENT_ENTRY_SIZE + + wordIndex * 4, + true, + ); +} + +/** + * Global ordering for frames committed into per-module linked continuations. + * + * Unwind commits the innermost activation first. Replay therefore consumes + * the exact reverse order. `peek` is non-consuming so a resume thunk can be + * selected before the original function preamble atomically validates and + * consumes the same event alongside `frame_next`. + */ +export class ForkReplayEventJournal { + private phase: JournalPhase = "idle"; + private capturedFirstPage: CapturedEventPage | null = null; + private capturedLastPage: CapturedEventPage | null = null; + private capturedPageCount = 0n; + private capturedCount = 0n; + private replayPage: ReplayEventPage | null = null; + private replayEventIndex = -1; + private replayRemaining = 0n; + private selected: ForkReplayEvent | null = null; + + beginCapture(): void { + this.requirePhase("idle", "begin replay-event capture"); + this.capturedFirstPage = null; + this.capturedLastPage = null; + this.capturedPageCount = 0n; + this.capturedCount = 0n; + this.phase = "capture"; + } + + recordCommit(activationId: number, functionOrdinal: number): void { + this.requirePhase("capture", "record a replay event"); + assertU32(activationId, "fork replay activation id"); + assertU32(functionOrdinal, "fork replay function ordinal"); + let page = this.capturedLastPage; + if (!page || page.count === FORK_REPLAY_EVENT_SEGMENT_CAPACITY) { + page = { + words: new Uint32Array(FORK_REPLAY_EVENT_SEGMENT_CAPACITY * 2), + count: 0, + previous: this.capturedLastPage, + next: null, + }; + if (this.capturedLastPage) this.capturedLastPage.next = page; + this.capturedFirstPage ??= page; + this.capturedLastPage = page; + this.capturedPageCount++; + } + page.words[page.count * 2] = activationId; + page.words[page.count * 2 + 1] = functionOrdinal; + page.count++; + this.capturedCount++; + } + + sealCapture(): void { + this.requirePhase("capture", "seal replay-event capture"); + this.phase = "sealed-parent"; + } + + capturedEventCount(): bigint { + if (this.phase !== "capture" && this.phase !== "sealed-parent") { + throw new Error( + `cannot read captured replay events while replay-event journal is ${this.phase}`, + ); + } + return this.capturedCount; + } + + capturedActivationIds(): Set { + if (this.phase !== "capture" && this.phase !== "sealed-parent") { + throw new Error( + `cannot read captured replay events while replay-event journal is ${this.phase}`, + ); + } + const ids = new Set(); + for ( + let page: ReplayEventPage | null = this.capturedFirstPage; + page; + page = page.next + ) { + if (!("words" in page)) { + throw new Error("captured replay-event chain contains a child page"); + } + for (let index = 0; index < page.count; index++) { + ids.add(page.words[index * 2]!); + } + } + return ids; + } + + *capturedSegmentPayloads(): IterableIterator { + if (this.phase !== "capture" && this.phase !== "sealed-parent") { + throw new Error( + `cannot encode captured replay events while replay-event journal is ${this.phase}`, + ); + } + let sequence = 0n; + for ( + let page: ReplayEventPage | null = this.capturedFirstPage; + page; + page = page.next + ) { + if (!("words" in page)) { + throw new Error("captured replay-event chain contains a child page"); + } + yield encodeForkReplayEventSegment(page.words, page.count, sequence); + sequence++; + } + } + + capturedManifestPayload(): Uint8Array { + if (this.phase !== "capture" && this.phase !== "sealed-parent") { + throw new Error( + `cannot encode captured replay events while replay-event journal is ${this.phase}`, + ); + } + return encodeForkReplayEventManifest( + this.capturedCount, + this.capturedPageCount, + ); + } + + beginParentReplay(): void { + this.requirePhase("sealed-parent", "begin parent replay events"); + this.beginReplayFrom(this.capturedLastPage, this.capturedCount); + } + + attachChild(wire: ForkReplayEventWire): void { + this.requirePhase("idle", "attach child replay events"); + const { summary, lastPage } = inspectForkReplayEventWire(wire, true); + this.beginReplayFrom(lastPage, summary.eventCount); + } + + peek(): ForkReplayEvent | null { + this.requirePhase("replay", "peek a replay event"); + if (this.selected) return this.selected; + const page = this.replayPage; + if (!page) return null; + this.selected = { + activationId: readPageWord(page, this.replayEventIndex, 0), + functionOrdinal: readPageWord(page, this.replayEventIndex, 1), + }; + return this.selected; + } + + consume(activationId: number, functionOrdinal: number): void { + this.requirePhase("replay", "consume a replay event"); + const event = this.selected; + if (!event) { + throw new Error( + "fork replay frame was consumed without selecting its resume target", + ); + } + if ( + event.activationId !== activationId + || event.functionOrdinal !== functionOrdinal + ) { + throw new Error( + `fork replay event expected ${event.activationId}:${event.functionOrdinal}, ` + + `found ${activationId}:${functionOrdinal}`, + ); + } + this.replayRemaining--; + this.replayEventIndex--; + if (this.replayEventIndex < 0) { + this.replayPage = this.replayPage?.previous ?? null; + this.replayEventIndex = this.replayPage + ? this.replayPage.count - 1 + : -1; + } + this.selected = null; + } + + finishReplay(): void { + this.requirePhase("replay", "finish replay events"); + if (this.replayRemaining !== 0n || this.selected !== null) { + throw new Error( + `fork replay event stream has ${this.replayRemaining} unconsumed entries`, + ); + } + this.clear(); + } + + abort(): void { + this.clear(); + } + + phaseName(): JournalPhase { + return this.phase; + } + + private beginReplayFrom( + lastPage: ReplayEventPage | null, + eventCount: bigint, + ): void { + this.replayPage = lastPage; + this.replayEventIndex = lastPage ? lastPage.count - 1 : -1; + this.replayRemaining = eventCount; + this.selected = null; + this.phase = "replay"; + } + + private clear(): void { + this.capturedFirstPage = null; + this.capturedLastPage = null; + this.capturedPageCount = 0n; + this.capturedCount = 0n; + this.replayPage = null; + this.replayEventIndex = -1; + this.replayRemaining = 0n; + this.selected = null; + this.phase = "idle"; + } + + private requirePhase(expected: JournalPhase, operation: string): void { + if (this.phase !== expected) { + throw new Error( + `cannot ${operation} while replay-event journal is ${this.phase}; ` + + `expected ${expected}`, + ); + } + } +} + +interface RegisteredResumeTarget extends ForkResumeTarget { + readonly activationId: number; + readonly slot: number; +} + +/** + * Host reconstruction owner for the private heterogeneous resume table. + * + * The table is not guest state. Every fresh worker populates it from exact + * artifact catalogs after all main/side activations instantiate. Slots may + * differ across workers because continuation bytes name activation/function + * coordinates, never raw table indexes. + */ +export class ForkResumeTable { + readonly table = new WebAssembly.Table({ + element: "anyfunc", + initial: 1, + }); + + private readonly targets = new Map(); + private readonly activationKeys = new Map(); + private freeSlots: number[] = []; + + registerActivation( + activationId: number, + targets: readonly ForkResumeTarget[], + ): void { + assertU32(activationId, "resume-table activation id"); + if (this.activationKeys.has(activationId)) { + throw new Error(`resume-table activation ${activationId} is already registered`); + } + const ordered = [...targets].sort( + (left, right) => left.functionOrdinal - right.functionOrdinal, + ); + const keys: string[] = []; + let previous: number | undefined; + for (const target of ordered) { + assertU32(target.functionOrdinal, "resume function ordinal"); + if (typeof target.thunk !== "function") { + throw new TypeError("resume target thunk is not a Wasm function"); + } + if (previous === target.functionOrdinal) { + throw new Error( + `resume-table activation ${activationId} repeats function ordinal ${previous}`, + ); + } + previous = target.functionOrdinal; + const slot = this.allocateSlot(); + this.table.set(slot, target.thunk); + const key = eventKey(activationId, target.functionOrdinal); + this.targets.set(key, { ...target, activationId, slot }); + keys.push(key); + } + this.activationKeys.set(activationId, keys); + } + + unregisterActivation(activationId: number): void { + assertU32(activationId, "resume-table activation id"); + const keys = this.activationKeys.get(activationId); + if (!keys) { + throw new Error(`resume-table activation ${activationId} is not registered`); + } + for (const key of keys) { + const target = this.targets.get(key)!; + this.table.set(target.slot, null); + this.targets.delete(key); + this.freeSlots.push(target.slot); + } + this.freeSlots.sort((left, right) => left - right); + this.activationKeys.delete(activationId); + } + + slotFor(event: ForkReplayEvent | null): number { + if (!event) return 0; + const target = this.targets.get( + eventKey(event.activationId, event.functionOrdinal), + ); + if (!target) { + throw new Error( + `fork replay target ${event.activationId}:${event.functionOrdinal} ` + + `is not registered`, + ); + } + // WHY: recursive/reference type equality belongs to the Wasm engine. The + // caller invokes this heterogeneous slot through a statically typed + // call_indirect, which validates the exact result type before the thunk + // can consume its frame. Reimplementing canonical recursive types in JS + // would create a second, weaker type system. + return target.slot; + } + + clear(): void { + for (const activationId of [...this.activationKeys.keys()].sort( + (left, right) => right - left, + )) { + this.unregisterActivation(activationId); + } + this.freeSlots = []; + } + + private allocateSlot(): number { + const reused = this.freeSlots.shift(); + if (reused !== undefined) return reused; + const slot = this.table.length; + this.table.grow(1); + return slot; + } +} diff --git a/host/src/fork-replay-gate.ts b/host/src/fork-replay-gate.ts new file mode 100644 index 0000000000..6e563b6a78 --- /dev/null +++ b/host/src/fork-replay-gate.ts @@ -0,0 +1,222 @@ +import type { WorkerHandle } from "./worker-adapter"; +import type { WorkerToHostMessage } from "./worker-protocol"; + +const FORK_REPLAY_PENDING = 0; +const FORK_REPLAY_COMMITTED = 1; +const FORK_REPLAY_CANCELLED = -1; +const FORK_REPLAY_GATE_BYTES = Int32Array.BYTES_PER_ELEMENT; + +function gateView(buffer: SharedArrayBuffer): Int32Array { + if ( + !(buffer instanceof SharedArrayBuffer) + || buffer.byteLength !== FORK_REPLAY_GATE_BYTES + ) { + throw new TypeError("fork replay gate must be one shared i32"); + } + return new Int32Array(buffer); +} + +export function createForkReplayGate(): SharedArrayBuffer { + return new SharedArrayBuffer(FORK_REPLAY_GATE_BYTES); +} + +/** + * Release a child that proved reconstruction reached the inherited fork site. + */ +export function commitForkReplayGate(buffer: SharedArrayBuffer): void { + const gate = gateView(buffer); + if ( + Atomics.compareExchange( + gate, + 0, + FORK_REPLAY_PENDING, + FORK_REPLAY_COMMITTED, + ) !== FORK_REPLAY_PENDING + ) { + throw new Error("fork replay gate is no longer pending"); + } + Atomics.notify(gate, 0); +} + +/** + * Wake a reconstruction Worker that the kernel host is rolling back. + */ +export function cancelForkReplayGate(buffer: SharedArrayBuffer): void { + const gate = gateView(buffer); + if ( + Atomics.compareExchange( + gate, + 0, + FORK_REPLAY_PENDING, + FORK_REPLAY_CANCELLED, + ) === FORK_REPLAY_PENDING + ) { + Atomics.notify(gate, 0); + } +} + +/** + * Stop the child immediately before the inherited fork() returns zero. + * + * WHY this is a blocking shared-memory gate: a JavaScript promise cannot be + * awaited inside a synchronous Wasm import. Process Workers already execute + * off the main thread and use Atomics.wait for syscall channels, so the same + * primitive gives Node and browsers one exact two-phase commit boundary. + */ +export function waitForForkReplayCommit( + buffer: SharedArrayBuffer, + context: string, +): void { + const gate = gateView(buffer); + for (;;) { + const state = Atomics.load(gate, 0); + if (state === FORK_REPLAY_COMMITTED) return; + if (state === FORK_REPLAY_CANCELLED) { + throw new Error(`${context}: fork replay was cancelled before commit`); + } + if (state !== FORK_REPLAY_PENDING) { + throw new Error(`${context}: invalid fork replay gate state ${state}`); + } + Atomics.wait(gate, 0, FORK_REPLAY_PENDING); + } +} + +type ForkReplayCoordinatorPhase = + | "pending" + | "ready" + | "committed" + | "cancelled"; + +function cancellationError(context: string, reason: unknown): Error { + if (reason instanceof Error) return reason; + return new Error( + `${context}: ${reason === undefined ? "fork replay was cancelled" : String(reason)}`, + ); +} + +/** + * Host-side half of the fork replay two-phase commit. + * + * `ready()` records the child Worker proving it reconstructed the inherited + * fork site, but deliberately leaves the shared gate closed. `commit()` is a + * separate operation performed only after the entrypoint has revalidated the + * exact child generation. Any launch, protocol, error, or exit path can call + * `cancel()` idempotently; a child already blocked in its synchronous Wasm + * import is woken with cancellation rather than leaked forever. + */ +export class ForkReplayGateCoordinator { + readonly gate = createForkReplayGate(); + private phase: ForkReplayCoordinatorPhase = "pending"; + private cancelledWith: Error | null = null; + private readonly readyPromise: Promise; + private resolveReady!: () => void; + private rejectReady!: (reason: Error) => void; + + constructor(readonly context: string) { + this.readyPromise = new Promise((resolve, reject) => { + this.resolveReady = resolve; + this.rejectReady = reject; + }); + // A Worker constructor can fail before handleFork reaches its await. Keep + // that synchronous rollback from producing an unhandled rejection while + // preserving the rejected promise for any later waiter. + void this.readyPromise.catch(() => {}); + } + + get currentPhase(): ForkReplayCoordinatorPhase { + return this.phase; + } + + ready(): void { + if (this.phase === "pending") { + this.phase = "ready"; + this.resolveReady(); + } + } + + waitUntilReady(): Promise { + return this.readyPromise; + } + + commit(): void { + if (this.phase === "cancelled") { + throw this.cancelledWith + ?? new Error(`${this.context}: fork replay was cancelled before commit`); + } + if (this.phase !== "ready") { + throw new Error( + `${this.context}: cannot commit fork replay while ${this.phase}`, + ); + } + commitForkReplayGate(this.gate); + this.phase = "committed"; + } + + cancel(reason?: unknown): void { + if (this.phase === "cancelled" || this.phase === "committed") return; + const error = cancellationError(this.context, reason); + this.cancelledWith = error; + this.phase = "cancelled"; + cancelForkReplayGate(this.gate); + this.rejectReady(error); + } +} + +/** + * Bind readiness and every premature Worker terminal path to one coordinator. + * + * Entry points still own normal process teardown. This observer only controls + * the launch transaction and is intentionally host-neutral so Node and browser + * cannot drift in which events release or cancel a fork. + */ +export function observeForkReplayWorker( + coordinator: ForkReplayGateCoordinator, + worker: WorkerHandle, + pid: number, + isCurrentGeneration: () => boolean, +): void { + const protocolFailure = (detail: string): void => { + coordinator.cancel(new Error(`${coordinator.context}: ${detail}`)); + }; + + worker.on("message", (raw: unknown) => { + const message = raw as Partial; + if (message.type === "fork_replay_ready") { + if (message.pid !== pid) { + protocolFailure( + `Worker reported replay readiness for pid=${String(message.pid)}, expected pid=${pid}`, + ); + } else if (!isCurrentGeneration()) { + protocolFailure("stale Worker generation reported replay readiness"); + } else { + coordinator.ready(); + } + return; + } + + if (message.type === "error" || message.type === "exit") { + if (message.pid !== pid) { + protocolFailure( + `Worker reported ${message.type} for pid=${String(message.pid)}, expected pid=${pid}`, + ); + } else if (message.type === "error") { + protocolFailure( + `Worker failed before replay readiness: ${message.message ?? "unknown error"}`, + ); + } else { + protocolFailure( + `Worker exited before replay readiness (status=${String(message.status)})`, + ); + } + } + }); + + worker.on("error", (error: Error) => { + protocolFailure( + `Worker error before replay readiness: ${error.message || String(error)}`, + ); + }); + worker.on("exit", (code: number) => { + protocolFailure(`Worker exited before replay readiness (code=${code})`); + }); +} diff --git a/host/src/fork-resume-catalog.ts b/host/src/fork-resume-catalog.ts new file mode 100644 index 0000000000..591b0bd285 --- /dev/null +++ b/host/src/fork-resume-catalog.ts @@ -0,0 +1,134 @@ +import type { ForkResumeTarget } from "./fork-replay-events"; + +export const FORK_RESUME_CATALOG_SECTION = + "kandelo.wpk_fork.resume_catalog"; +export const FORK_RESUME_CATALOG_EXPORT = "__wpk_fork_resume_catalog"; +export const FORK_RESUME_CATALOG_VERSION = 1; +export const FORK_RESUME_CATALOG_HEADER_SIZE = 12; +export const FORK_RESUME_CATALOG_RECORD_SIZE = 8; + +const FORK_RESUME_CATALOG_MAGIC = 0x4352_464b; // "KFRC", little endian. + +export interface ForkResumeCatalogRecord { + readonly functionOrdinal: number; + readonly localCatalogSlot: number; +} + +export interface ForkResumeCatalogTarget extends ForkResumeTarget { + readonly localCatalogSlot: number; +} + +function requireCatalogTable(instance: WebAssembly.Instance): WebAssembly.Table { + const value = instance.exports[FORK_RESUME_CATALOG_EXPORT]; + if (!(value instanceof WebAssembly.Table)) { + throw new Error( + `fork resume catalog is missing table export ${FORK_RESUME_CATALOG_EXPORT}`, + ); + } + return value; +} + +/** + * Parse the deterministic function-ordinal to local-table-slot metadata. + * + * Result types are deliberately absent: the exact module template chooses the + * target and the generated Wasm `call_indirect` performs the authoritative + * recursive/reference-type compatibility check before consuming a frame. + */ +export function readForkResumeCatalog( + module: WebAssembly.Module, +): readonly ForkResumeCatalogRecord[] { + const sections = WebAssembly.Module.customSections( + module, + FORK_RESUME_CATALOG_SECTION, + ); + if (sections.length !== 1) { + throw new Error( + `expected one ${FORK_RESUME_CATALOG_SECTION} section, found ${sections.length}`, + ); + } + const bytes = new Uint8Array(sections[0]!); + if (bytes.byteLength < FORK_RESUME_CATALOG_HEADER_SIZE) { + throw new Error("fork resume catalog is truncated"); + } + const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); + if (view.getUint32(0, true) !== FORK_RESUME_CATALOG_MAGIC) { + throw new Error("fork resume catalog has invalid magic"); + } + const version = view.getUint16(4, true); + if (version !== FORK_RESUME_CATALOG_VERSION) { + throw new Error(`unsupported fork resume catalog version ${version}`); + } + if (view.getUint16(6, true) !== FORK_RESUME_CATALOG_HEADER_SIZE) { + throw new Error("fork resume catalog has an invalid header size"); + } + const count = view.getUint32(8, true); + const expected = + FORK_RESUME_CATALOG_HEADER_SIZE + count * FORK_RESUME_CATALOG_RECORD_SIZE; + if (!Number.isSafeInteger(expected) || bytes.byteLength !== expected) { + throw new Error("fork resume catalog has an invalid size"); + } + + const records: ForkResumeCatalogRecord[] = []; + const slots = new Set(); + let previousOrdinal: number | undefined; + for (let index = 0; index < count; index++) { + const offset = + FORK_RESUME_CATALOG_HEADER_SIZE + index * FORK_RESUME_CATALOG_RECORD_SIZE; + const functionOrdinal = view.getUint32(offset, true); + const localCatalogSlot = view.getUint32(offset + 4, true); + if ( + previousOrdinal !== undefined + && functionOrdinal <= previousOrdinal + ) { + throw new Error( + `fork resume catalog function ordinal ${functionOrdinal} is not strictly ordered`, + ); + } + if (slots.has(localCatalogSlot)) { + throw new Error( + `fork resume catalog repeats local slot ${localCatalogSlot}`, + ); + } + previousOrdinal = functionOrdinal; + slots.add(localCatalogSlot); + records.push({ functionOrdinal, localCatalogSlot }); + } + return records; +} + +/** + * Resolve one fresh module instance's local catalog to process registration + * targets. No function object is serialized; each child performs this pairing + * again after instantiation. + */ +export function forkResumeTargetsFromInstance( + module: WebAssembly.Module, + instance: WebAssembly.Instance, +): readonly ForkResumeCatalogTarget[] { + const records = readForkResumeCatalog(module); + const table = requireCatalogTable(instance); + if (table.length !== records.length) { + throw new Error( + `fork resume catalog table has length ${table.length}, expected ${records.length}`, + ); + } + return records.map(({ functionOrdinal, localCatalogSlot }) => { + if (localCatalogSlot >= table.length) { + throw new Error( + `fork resume catalog slot ${localCatalogSlot} is out of bounds`, + ); + } + const thunk = table.get(localCatalogSlot); + if (typeof thunk !== "function") { + throw new Error( + `fork resume catalog slot ${localCatalogSlot} is not a Wasm function`, + ); + } + return { + functionOrdinal, + localCatalogSlot, + thunk: thunk as CallableFunction, + }; + }); +} diff --git a/host/src/fork-static-root-catalog.ts b/host/src/fork-static-root-catalog.ts new file mode 100644 index 0000000000..e500d32aea --- /dev/null +++ b/host/src/fork-static-root-catalog.ts @@ -0,0 +1,236 @@ +/** + * Deterministic identities for GC references recreated by instantiation. + * + * A structurally cloned child object is not interchangeable with an immutable + * global or static element root: `ref.eq` would see two identities. Each + * instrumented activation therefore exposes an instantiation-time harvest + * table. The host records only weak identities and immediately clears every + * table entry. Recipes name roots by `(activationId, ordinal)` and pin only + * referenced child roots for the duration of replay. + */ + +import { + WPK_FORK_STATIC_ROOT_CATALOG_EXPORT, + WPK_FORK_STATIC_ROOT_CATALOG_HEADER_SIZE, + WPK_FORK_STATIC_ROOT_CATALOG_MAGIC, + WPK_FORK_STATIC_ROOT_CATALOG_SECTION, + WPK_FORK_STATIC_ROOT_CATALOG_VERSION, + WPK_FORK_STATIC_ROOT_HARVEST_EXPORT, +} from "./generated/abi"; + +export const FORK_STATIC_ROOT_CATALOG_EXPORT = + WPK_FORK_STATIC_ROOT_CATALOG_EXPORT; +export const FORK_STATIC_ROOT_HARVEST_EXPORT = + WPK_FORK_STATIC_ROOT_HARVEST_EXPORT; +export const FORK_STATIC_ROOT_CATALOG_SECTION = + WPK_FORK_STATIC_ROOT_CATALOG_SECTION; +export const FORK_STATIC_ROOT_CATALOG_VERSION = + WPK_FORK_STATIC_ROOT_CATALOG_VERSION; +export const FORK_STATIC_ROOT_CATALOG_HEADER_SIZE = + WPK_FORK_STATIC_ROOT_CATALOG_HEADER_SIZE; + +const FORMAT_MAGIC = Uint8Array.from(WPK_FORK_STATIC_ROOT_CATALOG_MAGIC); + +export interface ForkStaticRootRecipe { + readonly moduleActivation: number; + readonly ordinal: number; +} + +interface RegisteredStaticRoots { + readonly entries: readonly StaticRootHandle[]; +} + +type StaticRootHandle = + | { readonly kind: "object"; readonly value: WeakRef } + | { readonly kind: "primitive"; readonly value: unknown }; + +function assertU32(value: number, label: string): void { + if (!Number.isInteger(value) || value < 0 || value > 0xffff_ffff) { + throw new RangeError(`invalid ${label} ${value}`); + } +} + +function isObjectIdentity(value: unknown): value is object { + return ( + (typeof value === "object" && value !== null) + || typeof value === "function" + ); +} + +export function readForkStaticRootCatalogCount( + module: WebAssembly.Module, +): number { + const sections = WebAssembly.Module.customSections( + module, + FORK_STATIC_ROOT_CATALOG_SECTION, + ); + if (sections.length !== 1) { + throw new Error( + `expected one ${FORK_STATIC_ROOT_CATALOG_SECTION} section, ` + + `found ${sections.length}`, + ); + } + const bytes = new Uint8Array(sections[0]!); + if (bytes.byteLength !== FORK_STATIC_ROOT_CATALOG_HEADER_SIZE) { + throw new Error("fork static-root catalog descriptor has an invalid size"); + } + if (FORMAT_MAGIC.some((byte, index) => bytes[index] !== byte)) { + throw new Error("fork static-root catalog descriptor has invalid magic"); + } + const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); + const version = view.getUint16(4, true); + if (version !== FORK_STATIC_ROOT_CATALOG_VERSION) { + throw new Error(`unsupported fork static-root catalog version ${version}`); + } + if (view.getUint16(6, true) !== FORK_STATIC_ROOT_CATALOG_HEADER_SIZE) { + throw new Error("fork static-root catalog descriptor has an invalid header size"); + } + return view.getUint32(8, true); +} + +export function forkStaticRootTableFromInstance( + module: WebAssembly.Module, + instance: WebAssembly.Instance, +): WebAssembly.Table { + const count = readForkStaticRootCatalogCount(module); + const table = instance.exports[FORK_STATIC_ROOT_CATALOG_EXPORT]; + if (!(table instanceof WebAssembly.Table)) { + throw new Error( + `fork activation is missing table export ${FORK_STATIC_ROOT_CATALOG_EXPORT}`, + ); + } + if (table.length !== count) { + throw new Error( + `fork static-root catalog has ${table.length} entries; descriptor declares ${count}`, + ); + } + return table; +} + +export function clearForkStaticRootTable(table: WebAssembly.Table): void { + for (let ordinal = 0; ordinal < table.length; ordinal++) { + table.set(ordinal, null); + } +} + +/** + * Process-worker view of every activation's static roots. + * + * Aliases are canonicalized when registrations are read. Keeping the first + * coordinate makes recipe selection deterministic even when an imported + * immutable global exposes the same object through multiple activations. The + * weak reverse handles preserve later-fork identity without extending object + * lifetime after its guest-owned global/table/segment/local releases it. + */ +export class ForkStaticRootCatalog { + private readonly catalogs = new Map(); + private objectRecipes = new WeakMap(); + private readonly primitiveRecipes = new Map(); + + register(moduleActivation: number, table: WebAssembly.Table): void { + assertU32(moduleActivation, "static-root module activation"); + if (this.catalogs.has(moduleActivation)) { + throw new Error( + `static-root catalog ${moduleActivation} is already registered`, + ); + } + const entries: StaticRootHandle[] = []; + try { + for (let ordinal = 0; ordinal < table.length; ordinal++) { + const value = table.get(ordinal); + entries.push( + isObjectIdentity(value) + ? { kind: "object", value: new WeakRef(value) } + : { kind: "primitive", value }, + ); + } + } finally { + // WHY: this table is only an instantiation-time observation window. + // Retaining entries here would recreate the retired module stash after + // table mutation or elem.drop. + clearForkStaticRootTable(table); + } + this.catalogs.set(moduleActivation, { + entries: Object.freeze(entries), + }); + this.rebuildIndexes(); + } + + unregister(moduleActivation: number): void { + assertU32(moduleActivation, "static-root module activation"); + if (!this.catalogs.delete(moduleActivation)) return; + this.rebuildIndexes(); + } + + encode(value: unknown): ForkStaticRootRecipe | null { + if (value === null) return null; + return this.lookup(value) ?? null; + } + + decode(recipe: ForkStaticRootRecipe): unknown { + assertU32(recipe.moduleActivation, "static-root module activation"); + assertU32(recipe.ordinal, "static-root ordinal"); + const catalog = this.catalogs.get(recipe.moduleActivation); + if (!catalog) { + throw new Error( + `static-root catalog ${recipe.moduleActivation} is not registered`, + ); + } + if (recipe.ordinal >= catalog.entries.length) { + throw new Error( + `static-root recipe ${recipe.moduleActivation}:${recipe.ordinal} ` + + "is out of bounds", + ); + } + const handle = catalog.entries[recipe.ordinal]!; + if (handle.kind === "primitive") return handle.value; + const value = handle.value.deref(); + if (value === undefined) { + throw new Error( + `static-root recipe ${recipe.moduleActivation}:${recipe.ordinal} ` + + "was collected before replay pinned it", + ); + } + return value; + } + + clear(): void { + this.catalogs.clear(); + this.objectRecipes = new WeakMap(); + this.primitiveRecipes.clear(); + } + + private lookup(value: unknown): ForkStaticRootRecipe | undefined { + return isObjectIdentity(value) + ? this.objectRecipes.get(value) + : this.primitiveRecipes.get(value); + } + + private rebuildIndexes(): void { + this.objectRecipes = new WeakMap(); + this.primitiveRecipes.clear(); + const catalogs = [...this.catalogs].sort( + ([left], [right]) => left - right, + ); + for (const [moduleActivation, catalog] of catalogs) { + catalog.entries.forEach((handle, ordinal) => { + const value = handle.kind === "object" + ? handle.value.deref() + : handle.value; + if ( + value === undefined + || value === null + || this.lookup(value) !== undefined + ) { + return; + } + const recipe = Object.freeze({ moduleActivation, ordinal }); + if (isObjectIdentity(value)) { + this.objectRecipes.set(value, recipe); + } else { + this.primitiveRecipes.set(value, recipe); + } + }); + } + } +} diff --git a/host/src/fork-unwind-transport.ts b/host/src/fork-unwind-transport.ts new file mode 100644 index 0000000000..eaad6891ed --- /dev/null +++ b/host/src/fork-unwind-transport.ts @@ -0,0 +1,47 @@ +/** + * Private Wasm-EH transport used while serializing a fork continuation. + * + * This tag is process-owned and shared by the main instance and every loaded + * side module in the same Worker. It is not a program exception: instrumented + * catch-all clauses rethrow it, each activation commits its frame while it + * propagates, and only the worker entry boundary consumes it. + */ + +export { + WPK_FORK_UNWIND_TAG_IMPORT_MODULE as FORK_UNWIND_TAG_IMPORT_MODULE, + WPK_FORK_UNWIND_TAG_IMPORT_NAME as FORK_UNWIND_TAG_IMPORT_NAME, + WPK_FORK_UNWIND_TRANSPORT_PAYLOAD_ARITY as FORK_UNWIND_TRANSPORT_PAYLOAD_ARITY, + WPK_FORK_UNWIND_TRANSPORT_SECTION as FORK_UNWIND_TRANSPORT_SECTION, + WPK_FORK_UNWIND_TRANSPORT_VERSION as FORK_UNWIND_TRANSPORT_VERSION, +} from "./generated/abi"; + +export function createForkUnwindTag(): WebAssembly.Tag { + if (typeof WebAssembly.Tag !== "function") { + throw new Error("WebAssembly exception tags are required for fork instrumentation"); + } + return new WebAssembly.Tag({ parameters: [] }); +} + +export function requireForkUnwindTag( + tag: unknown, + context: string, +): WebAssembly.Tag { + if (typeof WebAssembly.Tag !== "function") { + throw new Error(`${context}: WebAssembly exception tags are unavailable`); + } + if (!(tag instanceof WebAssembly.Tag)) { + throw new TypeError(`${context}: missing valid process-owned fork unwind tag`); + } + return tag; +} + +export function isForkUnwindException( + value: unknown, + tag: WebAssembly.Tag, +): value is WebAssembly.Exception { + return ( + typeof WebAssembly.Exception === "function" + && value instanceof WebAssembly.Exception + && value.is(tag) + ); +} diff --git a/host/src/fork-worker-exception-capability.ts b/host/src/fork-worker-exception-capability.ts new file mode 100644 index 0000000000..b37b153d7b --- /dev/null +++ b/host/src/fork-worker-exception-capability.ts @@ -0,0 +1,134 @@ +/** + * Durable owner-realm representation of a JavaScript value thrown by a + * deliberately Worker-local Wasm import. + * + * A raw object or function cannot move from a process Worker into the durable + * process owner, and a fork child cannot inherit the Worker's JavaScript heap. + * Ordinary imports and WebAssembly.JSTag first retain exact local behavior. + * If such a value is still live at fork and no activation-local exception + * codec owns it, capture normalizes the child's payload into this explicit + * capability. Primitive values are retained exactly behind the capability and + * are unwrapped when a later owner-side host import consumes the handle. + */ + +const WORKER_EXCEPTION_CAPABILITY = + Symbol("kandelo.fork.worker-exception-capability"); + +export const FORK_WORKER_EXCEPTION_RECIPE_VERSION = 1; + +export type ForkWorkerExceptionKind = + | "undefined" + | "null" + | "boolean" + | "number" + | "bigint" + | "string" + | "symbol" + | "error" + | "object" + | "function"; + +export interface ForkWorkerExceptionCapability { + readonly recipeVersion: typeof FORK_WORKER_EXCEPTION_RECIPE_VERSION; + readonly sourceImportOrdinal: number; + readonly kind: ForkWorkerExceptionKind; + /** + * Worker-local Error objects have no transferable original. Preserve the + * standard observable fields on their stable owner capability. + */ + readonly name?: string; + readonly message?: string; + readonly [WORKER_EXCEPTION_CAPABILITY]: true; +} + +class WorkerExceptionCapability + implements ForkWorkerExceptionCapability +{ + readonly recipeVersion = FORK_WORKER_EXCEPTION_RECIPE_VERSION; + readonly [WORKER_EXCEPTION_CAPABILITY] = true as const; + + constructor( + readonly sourceImportOrdinal: number, + readonly kind: ForkWorkerExceptionKind, + readonly name?: string, + readonly message?: string, + /** + * Exact owner-side value exposed to a later owner-routed host import. + * Opaque Worker objects/functions and Worker Errors use this capability + * itself because no durable original exists outside the Worker. + */ + private boundaryValue?: unknown, + ) { + if (boundaryValue === undefined && kind === "undefined") { + // Undefined is a real exact boundary value, not an omitted initializer. + this.boundaryValue = undefined; + } + Object.freeze(this); + } + + unwrap(): unknown { + if ( + this.kind === "error" + || this.kind === "object" + || this.kind === "function" + ) { + return this; + } + return this.boundaryValue; + } +} + +export interface CreateForkWorkerExceptionCapabilityOptions { + readonly sourceImportOrdinal: number; + readonly kind: ForkWorkerExceptionKind; + readonly name?: string; + readonly message?: string; + readonly boundaryValue?: unknown; +} + +export function createForkWorkerExceptionCapability( + options: CreateForkWorkerExceptionCapabilityOptions, +): ForkWorkerExceptionCapability { + if ( + !Number.isInteger(options.sourceImportOrdinal) + || options.sourceImportOrdinal < 0 + || options.sourceImportOrdinal > 0x7fff_ffff + ) { + throw new RangeError( + `invalid Worker exception source import ordinal ` + + `${options.sourceImportOrdinal}`, + ); + } + return new WorkerExceptionCapability( + options.sourceImportOrdinal, + options.kind, + options.name, + options.message, + options.boundaryValue, + ); +} + +export function isForkWorkerExceptionCapability( + value: unknown, +): value is ForkWorkerExceptionCapability { + return ( + typeof value === "object" + && value !== null + && (value as Partial)[ + WORKER_EXCEPTION_CAPABILITY + ] === true + ); +} + +/** + * Restore exact primitives at owner-side host boundaries. For values that + * never had a durable owner-realm original, return the stable capability + * itself; callers can inspect Error name/message and otherwise treat it as the + * opaque identity WebAssembly exposes. + */ +export function unwrapForkWorkerExceptionCapability( + value: unknown, +): unknown { + if (!isForkWorkerExceptionCapability(value)) return value; + return (value as WorkerExceptionCapability).unwrap(); +} diff --git a/host/src/fork-worker-import-exceptions.ts b/host/src/fork-worker-import-exceptions.ts new file mode 100644 index 0000000000..e28e1c7477 --- /dev/null +++ b/host/src/fork-worker-import-exceptions.ts @@ -0,0 +1,855 @@ +import { + defineForkExternrefImport, + type ForkExternrefImportBinding, + type ForkExternrefImportDescriptor, + ForkExternrefImportOwnerCatalog, + type ForkExternrefImportValue, + ForkExternrefImportWorkerCaller, +} from "./fork-externref-import-mailbox"; +import { + createForkWorkerExceptionCapability, + FORK_WORKER_EXCEPTION_RECIPE_VERSION, + type ForkWorkerExceptionCapability, + type ForkWorkerExceptionKind, +} from "./fork-worker-exception-capability"; +import { + type ForkExternrefToken, + ForkExternrefTokenCache, +} from "./fork-reference-broker"; + +const NORMALIZE_BEGIN_ORDINAL = 0xffff_fffc; +const NORMALIZE_CHUNK_ORDINAL = 0xffff_fffd; +const NORMALIZE_COMMIT_ORDINAL = 0xffff_fffe; +const NORMALIZE_ABORT_ORDINAL = 0xffff_ffff; +const CHUNK_WORDS = 13; +const CODE_UNITS_PER_WORD = 4; +const CHUNK_CODE_UNITS = CHUNK_WORDS * CODE_UNITS_PER_WORD; +const MAX_SESSION_ID = 0x7fff_fffe; +export const FORK_WORKER_EXCEPTION_FORK_CAPTURE_ORDINAL = 0x7fff_ffff; + +export const FORK_WORKER_EXCEPTION_RESERVED_ORDINAL_START = + NORMALIZE_BEGIN_ORDINAL; + +export const FORK_WORKER_EXCEPTION_BEGIN_DESCRIPTOR = + defineForkExternrefImport( + NORMALIZE_BEGIN_ORDINAL, + ["i32", "i32", "i32", "i32", "i64", "i32", "i32"], + ["i32"], + ); + +export const FORK_WORKER_EXCEPTION_CHUNK_DESCRIPTOR = + defineForkExternrefImport( + NORMALIZE_CHUNK_ORDINAL, + [ + "i32", + "i32", + "i32", + ...Array(CHUNK_WORDS).fill("i64"), + ] as const, + [], + ); + +export const FORK_WORKER_EXCEPTION_COMMIT_DESCRIPTOR = + defineForkExternrefImport( + NORMALIZE_COMMIT_ORDINAL, + ["i32"], + ["externref"], + ); + +export const FORK_WORKER_EXCEPTION_ABORT_DESCRIPTOR = + defineForkExternrefImport( + NORMALIZE_ABORT_ORDINAL, + ["i32"], + [], + ); + +const enum ExceptionKindCode { + Undefined = 1, + Null = 2, + Boolean = 3, + Number = 4, + BigInt = 5, + String = 6, + Symbol = 7, + Error = 8, + Object = 9, + Function = 10, +} + +const FLAG_BOOLEAN_TRUE = 1 << 0; +const FLAG_SYMBOL_GLOBAL = 1 << 0; +const FLAG_SYMBOL_HAS_DESCRIPTION = 1 << 1; + +interface WorkerExceptionRecipe { + readonly kind: ExceptionKindCode; + readonly flags: number; + readonly scalarBits: bigint; + readonly fields: readonly [string, string]; +} + +interface OwnerExceptionSession { + readonly binding: ForkExternrefImportBinding; + readonly sourceImportOrdinal: number; + readonly kind: ExceptionKindCode; + readonly flags: number; + readonly scalarBits: bigint; + readonly expectedLengths: readonly [number, number]; + readonly chunks: [string[], string[]]; + receivedLengths: [number, number]; +} + +function sameBinding( + left: ForkExternrefImportBinding, + right: ForkExternrefImportBinding, +): boolean { + return ( + left.pid === right.pid + && left.generationId === right.generationId + && left.senderId === right.senderId + ); +} + +function assertI32(value: unknown, label: string): number { + if ( + typeof value !== "number" + || !Number.isInteger(value) + || value < -0x8000_0000 + || value > 0x7fff_ffff + ) { + throw new TypeError(`${label} must be an i32`); + } + return value; +} + +function assertNonnegativeI32(value: unknown, label: string): number { + const checked = assertI32(value, label); + if (checked < 0) throw new RangeError(`${label} must be nonnegative`); + return checked; +} + +function assertI64(value: unknown, label: string): bigint { + if (typeof value !== "bigint") { + throw new TypeError(`${label} must be an i64`); + } + return value; +} + +function kindName(code: ExceptionKindCode): ForkWorkerExceptionKind { + switch (code) { + case ExceptionKindCode.Undefined: + return "undefined"; + case ExceptionKindCode.Null: + return "null"; + case ExceptionKindCode.Boolean: + return "boolean"; + case ExceptionKindCode.Number: + return "number"; + case ExceptionKindCode.BigInt: + return "bigint"; + case ExceptionKindCode.String: + return "string"; + case ExceptionKindCode.Symbol: + return "symbol"; + case ExceptionKindCode.Error: + return "error"; + case ExceptionKindCode.Object: + return "object"; + case ExceptionKindCode.Function: + return "function"; + default: + throw new RangeError(`unknown Worker exception kind ${code}`); + } +} + +function exactNumberBits(value: number): bigint { + const bytes = new ArrayBuffer(8); + const view = new DataView(bytes); + view.setFloat64(0, value, true); + return view.getBigInt64(0, true); +} + +function numberFromExactBits(bits: bigint): number { + const bytes = new ArrayBuffer(8); + const view = new DataView(bytes); + view.setBigInt64(0, bits, true); + return view.getFloat64(0, true); +} + +function readStringDataProperty( + value: object, + name: "name" | "message", +): string | undefined { + let current: object | null = value; + // Avoid invoking arbitrary getters while normalizing an already failing + // import. Standard Error name/message properties are data descriptors. + for (let depth = 0; current !== null && depth < 32; depth++) { + try { + const descriptor = Object.getOwnPropertyDescriptor(current, name); + if (descriptor && "value" in descriptor) { + return typeof descriptor.value === "string" + ? descriptor.value + : undefined; + } + current = Object.getPrototypeOf(current) as object | null; + } catch { + return undefined; + } + } + return undefined; +} + +function isErrorObject(value: object): boolean { + try { + return value instanceof Error; + } catch { + return false; + } +} + +function describeThrown(value: unknown): WorkerExceptionRecipe { + switch (typeof value) { + case "undefined": + return { + kind: ExceptionKindCode.Undefined, + flags: 0, + scalarBits: 0n, + fields: ["", ""], + }; + case "boolean": + return { + kind: ExceptionKindCode.Boolean, + flags: value ? FLAG_BOOLEAN_TRUE : 0, + scalarBits: 0n, + fields: ["", ""], + }; + case "number": + return { + kind: ExceptionKindCode.Number, + flags: 0, + scalarBits: exactNumberBits(value), + fields: ["", ""], + }; + case "bigint": + return { + kind: ExceptionKindCode.BigInt, + flags: 0, + scalarBits: 0n, + fields: [value.toString(10), ""], + }; + case "string": + return { + kind: ExceptionKindCode.String, + flags: 0, + scalarBits: 0n, + fields: [value, ""], + }; + case "symbol": { + const globalKey = Symbol.keyFor(value); + const description = value.description; + return { + kind: ExceptionKindCode.Symbol, + flags: + (globalKey === undefined ? 0 : FLAG_SYMBOL_GLOBAL) + | (description === undefined ? 0 : FLAG_SYMBOL_HAS_DESCRIPTION), + scalarBits: 0n, + fields: [globalKey ?? description ?? "", ""], + }; + } + case "function": + return { + kind: ExceptionKindCode.Function, + flags: 0, + scalarBits: 0n, + fields: ["", ""], + }; + case "object": + if (value === null) { + return { + kind: ExceptionKindCode.Null, + flags: 0, + scalarBits: 0n, + fields: ["", ""], + }; + } + if (isErrorObject(value)) { + return { + kind: ExceptionKindCode.Error, + flags: 0, + scalarBits: 0n, + fields: [ + readStringDataProperty(value, "name") ?? "Error", + readStringDataProperty(value, "message") ?? "", + ], + }; + } + return { + kind: ExceptionKindCode.Object, + flags: 0, + scalarBits: 0n, + fields: ["", ""], + }; + } +} + +function packCodeUnits( + value: string, + offset: number, +): bigint[] { + const words: bigint[] = []; + for (let wordIndex = 0; wordIndex < CHUNK_WORDS; wordIndex++) { + let word = 0n; + for ( + let codeUnitIndex = 0; + codeUnitIndex < CODE_UNITS_PER_WORD; + codeUnitIndex++ + ) { + const index = + offset + wordIndex * CODE_UNITS_PER_WORD + codeUnitIndex; + const codeUnit = index < value.length ? value.charCodeAt(index) : 0; + word |= BigInt(codeUnit) << BigInt(codeUnitIndex * 16); + } + words.push(BigInt.asIntN(64, word)); + } + return words; +} + +function unpackCodeUnits( + words: readonly bigint[], + count: number, +): string { + const codeUnits: number[] = []; + for (const signedWord of words) { + const word = BigInt.asUintN(64, signedWord); + for (let index = 0; index < CODE_UNITS_PER_WORD; index++) { + codeUnits.push( + Number((word >> BigInt(index * 16)) & 0xffffn), + ); + } + } + for (let index = count; index < codeUnits.length; index++) { + if (codeUnits[index] !== 0) { + throw new Error("Worker exception chunk has nonzero padding"); + } + } + return String.fromCharCode(...codeUnits.slice(0, count)); +} + +function validateRecipeShape( + kind: ExceptionKindCode, + flags: number, + fieldLengths: readonly [number, number], +): void { + kindName(kind); + if (!Number.isInteger(flags) || flags < 0) { + throw new RangeError("Worker exception recipe flags are invalid"); + } + const [first, second] = fieldLengths; + switch (kind) { + case ExceptionKindCode.Boolean: + if ((flags & ~FLAG_BOOLEAN_TRUE) !== 0 || first !== 0 || second !== 0) { + throw new Error("malformed boolean Worker exception recipe"); + } + return; + case ExceptionKindCode.Symbol: + if ( + (flags & ~(FLAG_SYMBOL_GLOBAL | FLAG_SYMBOL_HAS_DESCRIPTION)) !== 0 + || second !== 0 + || ( + (flags & FLAG_SYMBOL_GLOBAL) !== 0 + && (flags & FLAG_SYMBOL_HAS_DESCRIPTION) === 0 + ) + ) { + throw new Error("malformed symbol Worker exception recipe"); + } + return; + case ExceptionKindCode.BigInt: + case ExceptionKindCode.String: + if (flags !== 0 || second !== 0) { + throw new Error("malformed scalar Worker exception recipe"); + } + return; + case ExceptionKindCode.Error: + if (flags !== 0) { + throw new Error("malformed Error Worker exception recipe"); + } + return; + default: + if (flags !== 0 || first !== 0 || second !== 0) { + throw new Error("malformed opaque Worker exception recipe"); + } + } +} + +/** + * Owner-side state for the exceptional, chunked normalization protocol. + * + * Sessions contain only scalar code units. They own no Worker object and are + * cleared explicitly with the Worker binding on teardown. + */ +export class ForkWorkerExceptionCapabilityOwner { + private readonly sessions = new Map(); + private nextSessionId = 1; + private installed = false; + + install(catalog: ForkExternrefImportOwnerCatalog): void { + if (this.installed) { + throw new Error("Worker exception owner was installed twice"); + } + this.installed = true; + catalog.register( + FORK_WORKER_EXCEPTION_BEGIN_DESCRIPTOR, + (context, ...args) => this.begin(context, args), + ); + catalog.register( + FORK_WORKER_EXCEPTION_CHUNK_DESCRIPTOR, + (context, ...args) => this.append(context, args), + ); + catalog.register( + FORK_WORKER_EXCEPTION_COMMIT_DESCRIPTOR, + (context, sessionId) => this.commit(context, sessionId), + ); + catalog.register( + FORK_WORKER_EXCEPTION_ABORT_DESCRIPTOR, + (context, sessionId) => { + this.abort(context, sessionId); + }, + ); + } + + clearBinding(binding: ForkExternrefImportBinding): void { + for (const [sessionId, session] of this.sessions) { + if (sameBinding(session.binding, binding)) { + this.sessions.delete(sessionId); + } + } + } + + /** Test/diagnostic visibility without exposing mutable session contents. */ + get activeSessionCount(): number { + return this.sessions.size; + } + + private begin( + binding: ForkExternrefImportBinding, + args: readonly unknown[], + ): number { + const version = assertI32(args[0], "Worker exception recipe version"); + if (version !== FORK_WORKER_EXCEPTION_RECIPE_VERSION) { + throw new Error(`unsupported Worker exception recipe version ${version}`); + } + const sourceImportOrdinal = assertNonnegativeI32( + args[1], + "Worker exception source import ordinal", + ); + const kind = assertI32( + args[2], + "Worker exception kind", + ) as ExceptionKindCode; + const flags = assertNonnegativeI32( + args[3], + "Worker exception flags", + ); + const scalarBits = assertI64(args[4], "Worker exception scalar bits"); + const fieldLengths: [number, number] = [ + assertNonnegativeI32(args[5], "Worker exception field 0 length"), + assertNonnegativeI32(args[6], "Worker exception field 1 length"), + ]; + validateRecipeShape(kind, flags, fieldLengths); + + const sessionId = this.allocateSessionId(); + this.sessions.set(sessionId, { + binding: { + pid: binding.pid, + generationId: binding.generationId, + senderId: binding.senderId, + }, + sourceImportOrdinal, + kind, + flags, + scalarBits, + expectedLengths: fieldLengths, + chunks: [[], []], + receivedLengths: [0, 0], + }); + return sessionId; + } + + private append( + binding: ForkExternrefImportBinding, + args: readonly unknown[], + ): undefined { + const sessionId = assertNonnegativeI32( + args[0], + "Worker exception session", + ); + const field = assertNonnegativeI32( + args[1], + "Worker exception field", + ); + const offset = assertNonnegativeI32( + args[2], + "Worker exception field offset", + ); + if (field > 1) throw new RangeError(`invalid Worker exception field ${field}`); + const session = this.requireSession(binding, sessionId); + if (offset !== session.receivedLengths[field]) { + throw new Error( + `Worker exception field ${field} expected offset ` + + `${session.receivedLengths[field]}, received ${offset}`, + ); + } + const remaining = session.expectedLengths[field] - offset; + if (remaining <= 0) { + throw new Error(`Worker exception field ${field} is already complete`); + } + const count = Math.min(remaining, CHUNK_CODE_UNITS); + const words = args.slice(3).map((value, index) => + assertI64(value, `Worker exception chunk word ${index}`) + ); + if (words.length !== CHUNK_WORDS) { + throw new Error("Worker exception chunk has the wrong word count"); + } + session.chunks[field].push(unpackCodeUnits(words, count)); + session.receivedLengths[field] += count; + return undefined; + } + + private commit( + binding: ForkExternrefImportBinding, + rawSessionId: unknown, + ): ForkWorkerExceptionCapability { + const sessionId = assertNonnegativeI32( + rawSessionId, + "Worker exception session", + ); + const session = this.requireSession(binding, sessionId); + if ( + session.receivedLengths[0] !== session.expectedLengths[0] + || session.receivedLengths[1] !== session.expectedLengths[1] + ) { + throw new Error("Worker exception recipe was committed before completion"); + } + this.sessions.delete(sessionId); + const fields: [string, string] = [ + session.chunks[0].join(""), + session.chunks[1].join(""), + ]; + return this.materializeCapability(session, fields); + } + + private abort( + binding: ForkExternrefImportBinding, + rawSessionId: unknown, + ): void { + const sessionId = assertNonnegativeI32( + rawSessionId, + "Worker exception session", + ); + this.requireSession(binding, sessionId); + this.sessions.delete(sessionId); + } + + private materializeCapability( + session: OwnerExceptionSession, + fields: readonly [string, string], + ): ForkWorkerExceptionCapability { + let boundaryValue: unknown; + switch (session.kind) { + case ExceptionKindCode.Undefined: + boundaryValue = undefined; + break; + case ExceptionKindCode.Null: + boundaryValue = null; + break; + case ExceptionKindCode.Boolean: + boundaryValue = (session.flags & FLAG_BOOLEAN_TRUE) !== 0; + break; + case ExceptionKindCode.Number: + boundaryValue = numberFromExactBits(session.scalarBits); + break; + case ExceptionKindCode.BigInt: + boundaryValue = BigInt(fields[0]); + break; + case ExceptionKindCode.String: + boundaryValue = fields[0]; + break; + case ExceptionKindCode.Symbol: + boundaryValue = (session.flags & FLAG_SYMBOL_GLOBAL) !== 0 + ? Symbol.for(fields[0]) + : (session.flags & FLAG_SYMBOL_HAS_DESCRIPTION) !== 0 + ? Symbol(fields[0]) + : Symbol(); + break; + case ExceptionKindCode.Error: + case ExceptionKindCode.Object: + case ExceptionKindCode.Function: + boundaryValue = undefined; + break; + default: + kindName(session.kind); + } + return createForkWorkerExceptionCapability({ + sourceImportOrdinal: session.sourceImportOrdinal, + kind: kindName(session.kind), + name: session.kind === ExceptionKindCode.Error ? fields[0] : undefined, + message: session.kind === ExceptionKindCode.Error + ? fields[1] + : undefined, + boundaryValue, + }); + } + + private requireSession( + binding: ForkExternrefImportBinding, + sessionId: number, + ): OwnerExceptionSession { + const session = this.sessions.get(sessionId); + if (!session || !sameBinding(session.binding, binding)) { + throw new Error( + `unknown Worker exception session ${sessionId} for ` + + `pid=${binding.pid} generation=${binding.generationId} ` + + `sender=${binding.senderId}`, + ); + } + return session; + } + + private allocateSessionId(): number { + const start = this.nextSessionId; + do { + const candidate = this.nextSessionId++; + if (this.nextSessionId > MAX_SESSION_ID) this.nextSessionId = 1; + if (!this.sessions.has(candidate)) return candidate; + } while (this.nextSessionId !== start); + throw new RangeError("Worker exception normalization session space exhausted"); + } +} + +function buildFatalTrap(): () => never { + // (module (func (export "trap") unreachable)) + const bytes = new Uint8Array([ + 0x00, 0x61, 0x73, 0x6d, 0x01, 0x00, 0x00, 0x00, + 0x01, 0x04, 0x01, 0x60, 0x00, 0x00, + 0x03, 0x02, 0x01, 0x00, + 0x07, 0x08, 0x01, 0x04, 0x74, 0x72, 0x61, 0x70, 0x00, 0x00, + 0x0a, 0x05, 0x01, 0x03, 0x00, 0x00, 0x0b, + ]); + const instance = new WebAssembly.Instance(new WebAssembly.Module(bytes)); + const trap = instance.exports.trap; + if (typeof trap !== "function") { + throw new Error("failed to construct Worker exception fatal trap"); + } + return (): never => { + trap(); + throw new Error("unreachable Worker exception fatal trap returned"); + }; +} + +const fatalTrap = buildFatalTrap(); + +export interface ForkWorkerLocalImportExceptionNormalizerOptions { + readonly onFatal?: ( + error: unknown, + sourceImportOrdinal: number, + ) => void; +} + +/** + * Exception-only adapter for imports that must execute beside their Wasm + * instance (memory, activation, syscall, and dynamic-linker intrinsics). + * + * Normal returns and ordinary throws perform no owner RPC, preserving exact + * JavaScript/Wasm exception behavior. A nested Wasm RuntimeError is re-trapped + * so it cannot become CatchAllRef-visible merely by crossing this JS frame. + * Values that remain live at fork are normalized separately, after exact tag + * codecs have had the opportunity to claim them. + */ +export class ForkWorkerLocalImportExceptionNormalizer { + private readonly objectTokens = + new WeakMap(); + private readonly symbolTokens = + new Map(); + + constructor( + private readonly caller: ForkExternrefImportWorkerCaller, + private readonly tokens: ForkExternrefTokenCache, + private readonly options: + ForkWorkerLocalImportExceptionNormalizerOptions = {}, + ) {} + + wrap( + sourceImportOrdinal: number, + implementation: T, + ): T { + if ( + !Number.isInteger(sourceImportOrdinal) + || sourceImportOrdinal < 0 + || sourceImportOrdinal > 0x7fff_ffff + ) { + throw new RangeError( + `invalid Worker-local import ordinal ${sourceImportOrdinal}`, + ); + } + const normalizer = this; + return function ( + this: unknown, + ...args: unknown[] + ): unknown { + try { + return Reflect.apply(implementation, this, args); + } catch (thrown) { + return normalizer.replaceThrown( + sourceImportOrdinal, + thrown, + ); + } + } as unknown as T; + } + + clear(): void { + this.symbolTokens.clear(); + // WeakMap keys do not keep Worker-local objects alive. The whole + // normalizer becomes unreachable on exec/Worker teardown. + } + + /** + * Normalize a Worker-local value only when fork capture proves a fresh child + * needs it. This is shared by raw externrefs and by exceptions that every + * activation-local exact-tag codec has declined. + * + * Before that point the value remains exact, preserving ordinary host-import + * and exception behavior in the parent. + */ + normalizeUnclaimedForkValue(value: unknown): ForkExternrefToken { + const existingHandle = this.tokens.encode(value); + if (existingHandle !== null) return value as ForkExternrefToken; + const cached = this.cachedToken(value); + if (cached) return cached; + try { + const token = this.normalize( + FORK_WORKER_EXCEPTION_FORK_CAPTURE_ORDINAL, + value, + ); + this.rememberToken(value, token); + return token; + } catch (error) { + try { + this.options.onFatal?.( + error, + FORK_WORKER_EXCEPTION_FORK_CAPTURE_ORDINAL, + ); + } catch { + // Diagnostics cannot replace the capture failure. + } + throw error; + } + } + + normalizeUnclaimedForkException(thrown: unknown): ForkExternrefToken { + return this.normalizeUnclaimedForkValue(thrown); + } + + private replaceThrown( + _sourceImportOrdinal: number, + thrown: unknown, + ): never { + if (thrown instanceof WebAssembly.RuntimeError) { + // A nested Wasm call can surface a trap as a RuntimeError in this JS + // frame. Re-entering Wasm by throwing that JS object would turn it into + // a catchable JSTag exception, so preserve trap semantics explicitly. + return fatalTrap(); + } + // WHY: eager normalization would change ordinary CatchAllRef/rethrow + // behavior even when fork is never called. The broker normalizes only if + // this exact value remains live at fork and no activation codec owns it. + throw thrown; + } + + private normalize( + sourceImportOrdinal: number, + thrown: unknown, + ): ForkExternrefToken { + const recipe = describeThrown(thrown); + let sessionId: number | undefined; + try { + sessionId = this.caller.call( + FORK_WORKER_EXCEPTION_BEGIN_DESCRIPTOR, + [ + FORK_WORKER_EXCEPTION_RECIPE_VERSION, + sourceImportOrdinal, + recipe.kind, + recipe.flags, + recipe.scalarBits, + recipe.fields[0].length, + recipe.fields[1].length, + ], + ) as number; + for (let field = 0; field < recipe.fields.length; field++) { + const value = recipe.fields[field]!; + for (let offset = 0; offset < value.length; offset += CHUNK_CODE_UNITS) { + this.caller.call( + FORK_WORKER_EXCEPTION_CHUNK_DESCRIPTOR, + [ + sessionId, + field, + offset, + ...packCodeUnits(value, offset), + ] as ForkExternrefImportValue[], + ); + } + } + const token = this.caller.call( + FORK_WORKER_EXCEPTION_COMMIT_DESCRIPTOR, + [sessionId], + ); + const handle = this.tokens.encode(token); + if (handle === null) { + throw new Error( + "Worker exception owner returned a noncanonical externref token", + ); + } + sessionId = undefined; + return token as ForkExternrefToken; + } catch (error) { + if (sessionId !== undefined) { + try { + this.caller.call( + FORK_WORKER_EXCEPTION_ABORT_DESCRIPTOR, + [sessionId], + ); + } catch { + // Preserve the original normalization failure. Endpoint teardown + // clears any scalar-only abandoned session. + } + } + throw error; + } + } + + private cachedToken(thrown: unknown): ForkExternrefToken | undefined { + if ( + (typeof thrown === "object" && thrown !== null) + || typeof thrown === "function" + ) { + return this.objectTokens.get(thrown as object); + } + if (typeof thrown === "symbol") { + return this.symbolTokens.get(thrown); + } + return undefined; + } + + private rememberToken( + thrown: unknown, + token: ForkExternrefToken, + ): void { + if ( + (typeof thrown === "object" && thrown !== null) + || typeof thrown === "function" + ) { + this.objectTokens.set(thrown as object, token); + } else if (typeof thrown === "symbol") { + this.symbolTokens.set(thrown, token); + } + } +} diff --git a/host/src/generated/abi.ts b/host/src/generated/abi.ts index 0c8c2498bd..659383a267 100644 --- a/host/src/generated/abi.ts +++ b/host/src/generated/abi.ts @@ -1,7 +1,7 @@ /* GENERATED by `cargo xtask dump-abi`. Do not edit by hand. */ /* Regenerated by scripts/check-abi-version.sh; drift is a CI failure. */ -export const ABI_VERSION = 42 as const; +export const ABI_VERSION = 43 as const; export const ABI_CUSTOM_SECTION = "wasm-posix-abi" as const; export const ABI_KERNEL_EXPORT = "__abi_version" as const; @@ -15,14 +15,325 @@ export const WPK_FORK_LINKED_FRAME_POINTER_WIDTHS = [ { bytes: 4, chunkHeaderSize: 32, nodeHeaderSize: 24 }, { bytes: 8, chunkHeaderSize: 56, nodeHeaderSize: 32 }, ] as const; +export const WPK_FORK_MODULE_STATE_FORMAT_SECTION = "kandelo.wpk_fork.module_state" as const; +export const WPK_FORK_MODULE_STATE_FORMAT_VERSION = 1 as const; +export const WPK_FORK_MODULE_STATE_FORMAT_MAGIC = [75, 70, 77, 68] as const; +export const WPK_FORK_MODULE_STATE_DESCRIPTOR_SIZE = 24 as const; +export const WPK_FORK_MODULE_STATE_RECORD_ALIGNMENT = 8 as const; +export const WPK_FORK_MODULE_STATE_FLAG_ROOT_PREFIX_POINTER = 1 as const; +export const WPK_FORK_MODULE_STATE_FLAG_EXPLICIT_OWNERS = 2 as const; +export const WPK_FORK_MODULE_STATE_FLAG_SPARSE_TABLES = 4 as const; +export const WPK_FORK_MODULE_STATE_REQUIRED_FLAGS = 7 as const; +export const WPK_FORK_MODULE_STATE_KNOWN_FLAGS = 7 as const; +export const WPK_FORK_MODULE_STATE_ARENA_VERSION = 1 as const; +export const WPK_FORK_MODULE_STATE_RECORD_VERSION = 1 as const; +export const WPK_FORK_MODULE_STATE_ROOT_POINTER_WORD_OFFSET = 1 as const; +export const WPK_FORK_MODULE_STATE_CHUNK_MAGIC = [75, 70, 77, 67] as const; +export const WPK_FORK_MODULE_STATE_CHUNK_FLAG_ROOT = 1 as const; +export const WPK_FORK_MODULE_STATE_CHUNK_FLAG_SEALED = 2 as const; +export const WPK_FORK_MODULE_STATE_CHUNK_KNOWN_FLAGS = 3 as const; +export const WPK_FORK_MODULE_STATE_RECORD_MAGIC = [75, 70, 77, 82] as const; +export const WPK_FORK_MODULE_STATE_RECORD_HEADER_SIZE = 24 as const; +export const WPK_FORK_MODULE_STATE_RECORD_KIND_MODULE = 1 as const; +export const WPK_FORK_MODULE_STATE_RECORD_KIND_REFERENCE_RECIPE = 2 as const; +export const WPK_FORK_MODULE_STATE_RECORD_KIND_MUTABLE_GLOBAL = 3 as const; +export const WPK_FORK_MODULE_STATE_RECORD_KIND_TABLE = 4 as const; +export const WPK_FORK_MODULE_STATE_RECORD_KIND_TABLE_PAGE = 5 as const; +export const WPK_FORK_MODULE_STATE_RECORD_KIND_ELEMENT_SEGMENTS = 6 as const; +export const WPK_FORK_MODULE_STATE_RECORD_KIND_DATA_SEGMENTS = 7 as const; +export const WPK_FORK_MODULE_STATE_RECORD_KIND_REPLAY_EVENTS = 8 as const; +export const WPK_FORK_MODULE_STATE_RECORD_KIND_IMPORTED_GLOBAL_BINDINGS = 9 as const; +export const WPK_FORK_MODULE_STATE_RECORD_KIND_ACTIVATION_CONTINUATIONS = 10 as const; +export const WPK_FORK_MODULE_STATE_RECORD_KIND_IMPORTED_TABLE_BINDINGS = 11 as const; +export const WPK_FORK_MODULE_STATE_RECORD_KIND_REFERENCE_RECIPE_SEGMENT = 12 as const; +export const WPK_FORK_MODULE_STATE_RECORD_KIND_REPLAY_EVENT_SEGMENT = 13 as const; +export const WPK_FORK_MODULE_STATE_RECORD_KINDS = [ + { number: 1, name: "module" }, + { number: 2, name: "reference_recipe" }, + { number: 3, name: "mutable_global" }, + { number: 4, name: "table" }, + { number: 5, name: "table_page" }, + { number: 6, name: "element_segments" }, + { number: 7, name: "data_segments" }, + { number: 8, name: "replay_events" }, + { number: 9, name: "imported_global_bindings" }, + { number: 10, name: "activation_continuations" }, + { number: 11, name: "imported_table_bindings" }, + { number: 12, name: "reference_recipe_segment" }, + { number: 13, name: "replay_event_segment" }, +] as const; +export const WPK_FORK_MODULE_STATE_POINTER_WIDTHS = [ + { bytes: 4, chunkHeaderSize: 40 }, + { bytes: 8, chunkHeaderSize: 56 }, +] as const; +export const WPK_FORK_MODULE_STATE_REPLAY_EVENTS_MAGIC = [75, 70, 82, 69] as const; +export const WPK_FORK_REFERENCE_TRANSACTION_MAGIC = [75, 70, 82, 86] as const; +export const WPK_FORK_REFERENCE_SEGMENT_MAGIC = [75, 70, 82, 83] as const; +export const WPK_FORK_REFERENCE_TRANSACTION_OWNER = 1 as const; +export const WPK_FORK_REFERENCE_TRANSACTION_VERSION = 2 as const; +export const WPK_FORK_REFERENCE_TRANSACTION_MANIFEST_SIZE = 96 as const; +export const WPK_FORK_REFERENCE_TRANSACTION_FLAG_SEALED = 1 as const; +export const WPK_FORK_REFERENCE_TRANSACTION_KNOWN_FLAGS = 1 as const; +export const WPK_FORK_REFERENCE_SEGMENT_HEADER_SIZE = 40 as const; +export const WPK_FORK_REFERENCE_SEGMENT_KNOWN_FLAGS = 0 as const; +export const WPK_FORK_REFERENCE_NODE_RECORD_SIZE = 48 as const; +export const WPK_FORK_REFERENCE_VECTOR_INDEX_SIZE = 16 as const; +export const WPK_FORK_REFERENCE_SECTION_NODES = 1 as const; +export const WPK_FORK_REFERENCE_SECTION_EDGES = 2 as const; +export const WPK_FORK_REFERENCE_SECTION_SCALARS = 3 as const; +export const WPK_FORK_REFERENCE_SECTION_VECTOR_INDEX = 4 as const; +export const WPK_FORK_REFERENCE_SECTION_VECTOR_ENTRIES = 5 as const; +export const WPK_FORK_MODULE_STATE_MODULE_TEMPLATE_ID_SIZE = 32 as const; +export const WPK_FORK_MODULE_STATE_MODULE_RECORD_PAYLOAD_SIZE = 40 as const; +export const WPK_FORK_MODULE_STATE_MODULE_RECORD_KNOWN_FLAGS = 0 as const; +export const WPK_FORK_MODULE_STATE_GLOBAL_HEADER_SIZE = 8 as const; +export const WPK_FORK_MODULE_STATE_TABLE_BASELINE_FINGERPRINT_SIZE = 32 as const; +export const WPK_FORK_MODULE_STATE_TABLE_DESCRIPTOR_PAYLOAD_SIZE = 56 as const; +export const WPK_FORK_MODULE_STATE_TABLE_FLAG_SPARSE_OVERRIDES = 1 as const; +export const WPK_FORK_MODULE_STATE_TABLE_KNOWN_FLAGS = 1 as const; +export const WPK_FORK_MODULE_STATE_TABLE_PAGE_HEADER_SIZE = 16 as const; +export const WPK_FORK_MODULE_STATE_TABLE_RUN_HEADER_SIZE = 8 as const; +export const WPK_FORK_MODULE_STATE_ELEMENT_SEGMENT_HEADER_SIZE = 8 as const; +export const WPK_FORK_MODULE_STATE_DATA_SEGMENT_HEADER_SIZE = 8 as const; +export const WPK_FORK_MODULE_STATE_REPLAY_EVENTS_OWNER = 1 as const; +export const WPK_FORK_MODULE_STATE_REPLAY_EVENTS_VERSION = 2 as const; +export const WPK_FORK_MODULE_STATE_REPLAY_EVENTS_HEADER_SIZE = 40 as const; +export const WPK_FORK_MODULE_STATE_REPLAY_EVENT_SIZE = 8 as const; +export const WPK_FORK_MODULE_STATE_REPLAY_EVENTS_KNOWN_FLAGS = 0 as const; +export const WPK_FORK_MODULE_STATE_REPLAY_EVENT_SEGMENT_VERSION = 1 as const; +export const WPK_FORK_MODULE_STATE_REPLAY_EVENT_SEGMENT_HEADER_SIZE = 24 as const; +export const WPK_FORK_MODULE_STATE_REPLAY_EVENT_SEGMENT_CAPACITY = 4080 as const; +export const WPK_FORK_MODULE_STATE_REPLAY_EVENT_SEGMENT_KNOWN_FLAGS = 0 as const; +export const WPK_FORK_MODULE_STATE_MIN_TABLE_PAGE_SHIFT = 4 as const; +export const WPK_FORK_MODULE_STATE_MAX_TABLE_PAGE_SHIFT = 20 as const; +export const WPK_FORK_MODULE_STATE_TABLE_PAGE_SHIFT = 10 as const; +export const WPK_FORK_IMPORTED_GLOBAL_BINDINGS_MAGIC = [75, 70, 66, 71] as const; +export const WPK_FORK_IMPORTED_GLOBAL_BINDINGS_OWNER = 2 as const; +export const WPK_FORK_IMPORTED_GLOBAL_BINDINGS_VERSION = 1 as const; +export const WPK_FORK_IMPORTED_GLOBAL_BINDINGS_HEADER_SIZE = 24 as const; +export const WPK_FORK_IMPORTED_GLOBAL_BINDINGS_ENTRY_SIZE = 40 as const; +export const WPK_FORK_IMPORTED_GLOBAL_BINDINGS_KNOWN_FLAGS = 0 as const; +export const WPK_FORK_IMPORTED_GLOBAL_BINDING_RAW_NUMBER = 1 as const; +export const WPK_FORK_IMPORTED_GLOBAL_BINDING_RAW_BIGINT = 2 as const; +export const WPK_FORK_IMPORTED_GLOBAL_BINDING_RAW_REFERENCE = 3 as const; +export const WPK_FORK_IMPORTED_GLOBAL_BINDING_ACTIVATION_GLOBAL = 4 as const; +export const WPK_FORK_IMPORTED_GLOBAL_BINDING_BASE_IMPORT = 5 as const; +export const WPK_FORK_GLOBAL_CATALOG_EXPORT_PREFIX = "__wpk_fork_global_" as const; +export const WPK_FORK_ACTIVATION_CONTINUATIONS_MAGIC = [75, 70, 65, 67] as const; +export const WPK_FORK_ACTIVATION_CONTINUATIONS_OWNER = 3 as const; +export const WPK_FORK_ACTIVATION_CONTINUATIONS_VERSION = 1 as const; +export const WPK_FORK_ACTIVATION_CONTINUATIONS_HEADER_SIZE = 24 as const; +export const WPK_FORK_ACTIVATION_CONTINUATIONS_ENTRY_SIZE = 16 as const; +export const WPK_FORK_ACTIVATION_CONTINUATIONS_KNOWN_FLAGS = 0 as const; +export const WPK_FORK_ACTIVATION_CONTINUATIONS_ENTRY_KNOWN_FLAGS = 0 as const; +export const WPK_FORK_IMPORTED_TABLE_BINDINGS_MAGIC = [75, 70, 66, 84] as const; +export const WPK_FORK_IMPORTED_TABLE_BINDINGS_OWNER = 4 as const; +export const WPK_FORK_IMPORTED_TABLE_BINDINGS_VERSION = 1 as const; +export const WPK_FORK_IMPORTED_TABLE_BINDINGS_HEADER_SIZE = 24 as const; +export const WPK_FORK_IMPORTED_TABLE_BINDINGS_ENTRY_SIZE = 24 as const; +export const WPK_FORK_IMPORTED_TABLE_BINDINGS_KNOWN_FLAGS = 0 as const; +export const WPK_FORK_IMPORTED_TABLE_BINDING_ACTIVATION_TABLE = 1 as const; +export const WPK_FORK_IMPORTED_TABLE_BINDING_BASE_IMPORT = 2 as const; +export const WPK_FORK_TABLE_CATALOG_EXPORT_PREFIX = "__wpk_fork_table_" as const; +export const WPK_FORK_MODULE_STATE_GLOBAL_TYPE_I32 = 1 as const; +export const WPK_FORK_MODULE_STATE_GLOBAL_TYPE_I64 = 2 as const; +export const WPK_FORK_MODULE_STATE_GLOBAL_TYPE_F32 = 3 as const; +export const WPK_FORK_MODULE_STATE_GLOBAL_TYPE_F64 = 4 as const; +export const WPK_FORK_MODULE_STATE_GLOBAL_TYPE_V128 = 5 as const; +export const WPK_FORK_MODULE_STATE_GLOBAL_TYPE_FUNCREF = 6 as const; +export const WPK_FORK_MODULE_STATE_GLOBAL_TYPE_EXTERNREF = 7 as const; +export const WPK_FORK_MODULE_STATE_GLOBAL_TYPE_EXNREF = 8 as const; +export const WPK_FORK_MODULE_STATE_GLOBAL_TYPE_ANYREF = 9 as const; +export const WPK_FORK_CAPABILITIES_SECTION = "kandelo.wpk_fork.capabilities" as const; +export const WPK_FORK_CAPABILITIES_VERSION = 1 as const; +export const WPK_FORK_CAP_SIDE_ENTRY = 1 as const; +export const WPK_FORK_CAP_DYLINK_MAIN = 2 as const; +export const WPK_FORK_CAP_ACTIVATION_STATE_SAFE = 4 as const; +export const WPK_FORK_CAP_KNOWN_MASK = 7 as const; +export const WPK_FORK_CAP_REQUIRED_FLAGS = 4 as const; +export const WPK_FORK_EXCEPTION_CODEC_SECTION = "kandelo.wpk_fork.exception_codec" as const; +export const WPK_FORK_EXCEPTION_CODEC_VERSION = 1 as const; +export const WPK_FORK_EXCEPTION_CODEC_HEADER_SIZE = 8 as const; +export const WPK_FORK_EXCEPTION_CODEC_TAG_RECORD_SIZE = 16 as const; +export const WPK_FORK_GC_CODEC_SECTION = "kandelo.wpk_fork.gc_codec" as const; +export const WPK_FORK_GC_CODEC_MAGIC = [75, 70, 71, 67] as const; +export const WPK_FORK_GC_CODEC_VERSION = 1 as const; +export const WPK_FORK_GC_CODEC_HEADER_SIZE = 16 as const; +export const WPK_FORK_GC_CODEC_LAYOUT_RECORD_SIZE = 44 as const; +export const WPK_FORK_GC_CODEC_FIELD_RECORD_SIZE = 12 as const; +export const WPK_FORK_UNWIND_TAG_IMPORT_MODULE = "env" as const; +export const WPK_FORK_UNWIND_TAG_IMPORT_NAME = "__wpk_fork_unwind" as const; +export const WPK_FORK_UNWIND_TRANSPORT_SECTION = "kandelo.wpk_fork.unwind_transport" as const; +export const WPK_FORK_STATIC_ROOT_CATALOG_EXPORT = "__wpk_fork_static_root_catalog" as const; +export const WPK_FORK_STATIC_ROOT_CATALOG_SECTION = "kandelo.wpk_fork.static_root_catalog" as const; +export const WPK_FORK_STATIC_ROOT_HARVEST_EXPORT = "__wpk_fork_static_root_harvest" as const; +export const WPK_FORK_UNWIND_TRANSPORT_VERSION = 1 as const; +export const WPK_FORK_UNWIND_TRANSPORT_PAYLOAD_ARITY = 0 as const; +export const WPK_FORK_STATIC_ROOT_CATALOG_VERSION = 1 as const; +export const WPK_FORK_STATIC_ROOT_CATALOG_HEADER_SIZE = 12 as const; +export const WPK_FORK_STATIC_ROOT_CATALOG_MAGIC = [75, 70, 83, 82] as const; +export const WPK_FORK_IMPORTED_GLOBALS_SECTION = "kandelo.wpk_fork.imported_globals" as const; +export const WPK_FORK_FRAME_IMPORT_COMMIT = "__wpk_fork_frame_commit" as const; +export const WPK_FORK_FRAME_IMPORT_NEXT = "__wpk_fork_frame_next" as const; +export const WPK_FORK_FRAME_IMPORT_PEEK = "__wpk_fork_frame_peek" as const; +export const WPK_FORK_FRAME_IMPORT_RESERVE = "__wpk_fork_frame_reserve" as const; +export const WPK_FORK_RESUME_IMPORT_PEEK = "__wpk_fork_resume_peek" as const; +export const WPK_FORK_RESUME_IMPORT_TABLE = "__wpk_fork_resume_table" as const; +export const WPK_FORK_IMPORTED_GLOBALS_MAGIC = [75, 70, 73, 71] as const; +export const WPK_FORK_IMPORTED_GLOBALS_VERSION = 1 as const; +export const WPK_FORK_IMPORTED_GLOBALS_HEADER_SIZE = 16 as const; +export const WPK_FORK_IMPORTED_GLOBALS_RECORD_HEADER_SIZE = 24 as const; +export const WPK_FORK_IMPORTED_GLOBALS_FLAG_MUTABLE = 1 as const; +export const WPK_FORK_IMPORTED_GLOBALS_FLAG_SHARED = 2 as const; +export const WPK_FORK_IMPORTED_GLOBALS_KNOWN_FLAGS = 3 as const; +export const WPK_FORK_IMPORTED_TABLES_SECTION = "kandelo.wpk_fork.imported_tables" as const; +export const WPK_FORK_IMPORTED_TABLES_MAGIC = [75, 70, 73, 84] as const; +export const WPK_FORK_IMPORTED_TABLES_VERSION = 1 as const; +export const WPK_FORK_IMPORTED_TABLES_HEADER_SIZE = 16 as const; +export const WPK_FORK_IMPORTED_TABLES_RECORD_HEADER_SIZE = 24 as const; +export const WPK_FORK_IMPORTED_TABLES_FLAG_TABLE64 = 1 as const; +export const WPK_FORK_IMPORTED_TABLES_KNOWN_FLAGS = 1 as const; +export const WPK_FORK_EXCEPTION_CODEC_IMPORT_MODULE = "env" as const; +export const WPK_FORK_EXCEPTION_IMPORT_ACTIVATION = "__wpk_fork_module_activation" as const; +export const WPK_FORK_EXCEPTION_IMPORT_BROKER_ENCODE = "__wpk_fork_ref_exn_broker_encode" as const; +export const WPK_FORK_EXCEPTION_IMPORT_BROKER_THROW_RECIPE = "__wpk_fork_ref_exn_broker_throw_recipe" as const; +export const WPK_FORK_EXCEPTION_IMPORT_CACHE_INDEX = "__wpk_fork_ref_exn_cache_index" as const; +export const WPK_FORK_EXCEPTION_IMPORT_CLAIM = "__wpk_fork_ref_exn_claim" as const; +export const WPK_FORK_EXCEPTION_IMPORT_DEFINE = "__wpk_fork_ref_exn_define" as const; +export const WPK_FORK_EXCEPTION_IMPORT_INGRESS_THROW = "__wpk_fork_ref_exn_ingress_throw" as const; +export const WPK_FORK_EXCEPTION_IMPORT_LOAD = "__wpk_fork_ref_exn_load" as const; +export const WPK_FORK_EXCEPTION_IMPORT_LOOKUP = "__wpk_fork_ref_exn_lookup" as const; +export const WPK_FORK_EXCEPTION_IMPORT_ROUTE = "__wpk_fork_ref_exn_route" as const; +export const WPK_FORK_EXCEPTION_EXPORT_ABORT = "__wpk_fork_ref_exn_abort" as const; +export const WPK_FORK_EXCEPTION_EXPORT_CLEAR = "__wpk_fork_ref_exn_clear" as const; +export const WPK_FORK_EXCEPTION_EXPORT_DECODE = "__wpk_fork_ref_decode_exnref" as const; +export const WPK_FORK_EXCEPTION_EXPORT_ENCODE = "__wpk_fork_ref_encode_exnref" as const; +export const WPK_FORK_EXCEPTION_EXPORT_ENCODE_INGRESS = "__wpk_fork_ref_exn_encode_ingress" as const; +export const WPK_FORK_EXCEPTION_EXPORT_MATERIALIZE = "__wpk_fork_exception_materialize" as const; +export const WPK_FORK_EXCEPTION_EXPORT_THROW_RECIPE = "__wpk_fork_ref_exn_throw_recipe" as const; +export const WPK_FORK_EXCEPTION_EXPORT_THROW_SLOT = "__wpk_fork_ref_exn_throw_slot" as const; +export const WPK_FORK_MODULE_STATE_IMPORT_RECORD_COMMIT = "__wpk_fork_module_state_record_commit" as const; +export const WPK_FORK_MODULE_STATE_IMPORT_RECORD_FIND = "__wpk_fork_module_state_record_find" as const; +export const WPK_FORK_MODULE_STATE_IMPORT_RECORD_RESERVE = "__wpk_fork_module_state_record_reserve" as const; +export const WPK_FORK_MODULE_STATE_IMPORT_TABLE_DIRTY_COUNT = "__wpk_fork_module_state_table_dirty_count" as const; +export const WPK_FORK_MODULE_STATE_IMPORT_TABLE_DIRTY_MARK = "__wpk_fork_module_state_table_dirty_mark" as const; +export const WPK_FORK_MODULE_STATE_IMPORT_TABLE_DIRTY_PAGE = "__wpk_fork_module_state_table_dirty_page" as const; +export const WPK_FORK_MODULE_STATE_IMPORT_TABLE_STATE_OWNED = "__wpk_fork_module_state_table_state_owned" as const; +export const WPK_FORK_EXPORT_MODULE_BOOTSTRAP = "wpk_fork_module_bootstrap" as const; +export const WPK_FORK_EXPORT_MODULE_STATE_FINISH_RESTORE = "wpk_fork_module_state_finish_restore" as const; +export const WPK_FORK_EXPORT_MODULE_STATE_RESTORE = "wpk_fork_module_state_restore" as const; +export const WPK_FORK_EXPORT_MODULE_STATE_SAVE = "wpk_fork_module_state_save" as const; +export const WPK_FORK_EXPORT_MODULE_THREAD_BOOTSTRAP = "wpk_fork_module_thread_bootstrap" as const; +export const WPK_FORK_EXPORT_RESUME_START = "wpk_fork_resume_start" as const; +export const WPK_FORK_EXPORT_RESUME_THREAD = "wpk_fork_resume_thread" as const; +export const WPK_FORK_REFERENCE_IMPORT_DECODE_ANYREF = "__wpk_fork_ref_decode_anyref" as const; +export const WPK_FORK_REFERENCE_IMPORT_DECODE_EXNREF = "__wpk_fork_ref_decode_exnref" as const; +export const WPK_FORK_REFERENCE_IMPORT_DECODE_EXTERNREF = "__wpk_fork_ref_decode_externref" as const; +export const WPK_FORK_REFERENCE_IMPORT_DECODE_FUNCREF = "__wpk_fork_ref_decode_funcref" as const; +export const WPK_FORK_REFERENCE_IMPORT_ENCODE_ANYREF = "__wpk_fork_ref_encode_anyref" as const; +export const WPK_FORK_REFERENCE_IMPORT_ENCODE_EXNREF = "__wpk_fork_ref_encode_exnref" as const; +export const WPK_FORK_REFERENCE_IMPORT_ENCODE_EXTERNREF = "__wpk_fork_ref_encode_externref" as const; +export const WPK_FORK_REFERENCE_IMPORT_ENCODE_FUNCREF = "__wpk_fork_ref_encode_funcref" as const; +export const WPK_FORK_REFERENCE_CODEC_IMPORT_MODULE = "env" as const; +export const WPK_FORK_REFERENCE_IMPORT_GC_BROKER_ENCODE = "__wpk_fork_ref_gc_broker_encode" as const; +export const WPK_FORK_REFERENCE_IMPORT_GC_CAPTURE_LAYOUT = "__wpk_fork_ref_gc_capture_layout" as const; +export const WPK_FORK_REFERENCE_IMPORT_GC_CLAIM = "__wpk_fork_ref_gc_claim" as const; +export const WPK_FORK_REFERENCE_IMPORT_GC_DEFINE = "__wpk_fork_ref_gc_define" as const; +export const WPK_FORK_REFERENCE_IMPORT_GC_I31 = "__wpk_fork_ref_gc_i31" as const; +export const WPK_FORK_REFERENCE_IMPORT_GC_LOAD = "__wpk_fork_ref_gc_load" as const; +export const WPK_FORK_REFERENCE_IMPORT_GC_LOOKUP = "__wpk_fork_ref_gc_lookup" as const; +export const WPK_FORK_REFERENCE_IMPORT_GC_PAYLOAD_LEN = "__wpk_fork_ref_gc_payload_len" as const; +export const WPK_FORK_REFERENCE_IMPORT_GC_PROVENANCE_BEGIN = "__wpk_fork_ref_gc_provenance_begin" as const; +export const WPK_FORK_REFERENCE_IMPORT_GC_PROVENANCE_END = "__wpk_fork_ref_gc_provenance_end" as const; +export const WPK_FORK_REFERENCE_IMPORT_GC_PROVENANCE_REF = "__wpk_fork_ref_gc_provenance_ref" as const; +export const WPK_FORK_REFERENCE_IMPORT_GC_ROUTE = "__wpk_fork_ref_gc_route" as const; +export const WPK_FORK_REFERENCE_IMPORT_GC_TRANSIT = "__wpk_fork_ref_gc_transit" as const; +export const WPK_FORK_REFERENCE_EXPORT_GC_ALLOCATE = "__wpk_fork_ref_gc_allocate" as const; +export const WPK_FORK_REFERENCE_EXPORT_GC_ENCODE_SLOT = "__wpk_fork_ref_gc_encode_slot" as const; +export const WPK_FORK_REFERENCE_EXPORT_GC_FILL = "__wpk_fork_ref_gc_fill" as const; +export const WPK_FORK_REFERENCE_EXPORT_GC_PUBLISH_EXTERNREF = "__wpk_fork_ref_gc_publish_externref" as const; +export const WPK_FORK_REFERENCE_EXPORT_GC_PROBE = "__wpk_fork_ref_gc_probe" as const; +export const WPK_FORK_REFERENCE_IMPORT_SCRATCH_RELEASE = "__wpk_fork_ref_scratch_release" as const; +export const WPK_FORK_REFERENCE_IMPORT_SCRATCH_RESERVE = "__wpk_fork_ref_scratch_reserve" as const; +export const WPK_FORK_REFERENCE_IMPORT_VECTOR_APPEND = "__wpk_fork_ref_vector_append" as const; +export const WPK_FORK_REFERENCE_IMPORT_VECTOR_BEGIN = "__wpk_fork_ref_vector_begin" as const; +export const WPK_FORK_REFERENCE_IMPORT_VECTOR_FINISH = "__wpk_fork_ref_vector_finish" as const; +export const WPK_FORK_REFERENCE_IMPORT_VECTOR_GET = "__wpk_fork_ref_vector_get" as const; export const WPK_FORK_REQUIRED_IMPORTS = [ { module: "env", name: "__wpk_fork_frame_commit", params: ["ptr"], results: [] }, { module: "env", name: "__wpk_fork_frame_next", params: ["ptr"], results: ["ptr"] }, + { module: "env", name: "__wpk_fork_frame_peek", params: ["ptr"], results: ["ptr"] }, { module: "env", name: "__wpk_fork_frame_reserve", params: ["ptr"], results: ["ptr"] }, + { module: "env", name: "__wpk_fork_module_state_record_commit", params: ["ptr"], results: [] }, + { module: "env", name: "__wpk_fork_module_state_record_find", params: ["i32", "i32", "i32", "i32"], results: ["ptr"] }, + { module: "env", name: "__wpk_fork_module_state_record_reserve", params: ["i32", "i32", "i32", "ptr"], results: ["ptr"] }, + { module: "env", name: "__wpk_fork_module_state_table_dirty_count", params: ["i32"], results: ["i32"] }, + { module: "env", name: "__wpk_fork_module_state_table_dirty_mark", params: ["i32", "i64", "i64"], results: [] }, + { module: "env", name: "__wpk_fork_module_state_table_dirty_page", params: ["i32", "i32"], results: ["i64"] }, + { module: "env", name: "__wpk_fork_module_state_table_mutation_abort", params: [], results: [] }, + { module: "env", name: "__wpk_fork_module_state_table_mutation_begin", params: [], results: ["i64"] }, + { module: "env", name: "__wpk_fork_module_state_table_mutation_commit", params: ["i32", "i64", "i64"], results: [] }, + { module: "env", name: "__wpk_fork_module_state_table_reconcile", params: [], results: ["i64"] }, + { module: "env", name: "__wpk_fork_module_state_table_state_owned", params: ["i32"], results: ["i32"] }, + { module: "env", name: "__wpk_fork_ref_decode_funcref", params: ["i32"], results: ["funcref"] }, + { module: "env", name: "__wpk_fork_ref_encode_funcref", params: ["funcref"], results: ["i32"] }, + { module: "env", name: "__wpk_fork_ref_exn_broker_encode", params: ["i32"], results: ["i32"] }, + { module: "env", name: "__wpk_fork_ref_exn_broker_throw_recipe", params: ["i32"], results: [] }, + { module: "env", name: "__wpk_fork_ref_exn_cache_index", params: ["i32"], results: ["i32"] }, + { module: "env", name: "__wpk_fork_ref_exn_claim", params: ["i32"], results: ["i32"] }, + { module: "env", name: "__wpk_fork_ref_exn_define", params: ["i32", "i32", "i32", "i32", "ptr", "i32", "ptr", "i32"], results: [] }, + { module: "env", name: "__wpk_fork_ref_exn_ingress_throw", params: ["i32"], results: [] }, + { module: "env", name: "__wpk_fork_ref_exn_load", params: ["i32", "i32", "i32", "i32", "ptr", "i32", "ptr", "i32"], results: ["i32"] }, + { module: "env", name: "__wpk_fork_ref_exn_lookup", params: ["i32"], results: ["i32"] }, + { module: "env", name: "__wpk_fork_ref_exn_route", params: ["i32", "i32"], results: ["i32"] }, + { module: "env", name: "__wpk_fork_ref_gc_broker_encode", params: ["i32"], results: ["i32"] }, + { module: "env", name: "__wpk_fork_ref_gc_capture_layout", params: ["i32", "i32", "i32"], results: ["i32"] }, + { module: "env", name: "__wpk_fork_ref_gc_claim", params: ["i32"], results: ["i32"] }, + { module: "env", name: "__wpk_fork_ref_gc_define", params: ["i32", "i32", "i32", "i32", "i32", "ptr", "i32", "i32"], results: [] }, + { module: "env", name: "__wpk_fork_ref_gc_i31", params: ["i32"], results: ["i32"] }, + { module: "env", name: "__wpk_fork_ref_gc_load", params: ["i32", "i32", "i32", "i32", "i32", "ptr", "i32"], results: ["i32"] }, + { module: "env", name: "__wpk_fork_ref_gc_lookup", params: ["i32"], results: ["i32"] }, + { module: "env", name: "__wpk_fork_ref_gc_payload_len", params: ["i32", "i32", "i32"], results: ["i32"] }, + { module: "env", name: "__wpk_fork_ref_gc_provenance_begin", params: ["i32", "i32", "i32", "i32", "i64", "i64", "i32"], results: ["i32"] }, + { module: "env", name: "__wpk_fork_ref_gc_provenance_end", params: ["i32"], results: [] }, + { module: "env", name: "__wpk_fork_ref_gc_provenance_ref", params: ["i32", "i32", "i32"], results: [] }, + { module: "env", name: "__wpk_fork_ref_gc_route", params: ["i32", "i32"], results: ["i32"] }, + { module: "env", name: "__wpk_fork_ref_scratch_release", params: ["ptr", "ptr"], results: [] }, + { module: "env", name: "__wpk_fork_ref_scratch_reserve", params: ["ptr"], results: ["ptr"] }, + { module: "env", name: "__wpk_fork_ref_vector_append", params: ["i32", "i32"], results: [] }, + { module: "env", name: "__wpk_fork_ref_vector_begin", params: ["i32"], results: ["i32"] }, + { module: "env", name: "__wpk_fork_ref_vector_finish", params: ["i32"], results: ["i32"] }, + { module: "env", name: "__wpk_fork_ref_vector_get", params: ["i32", "i32"], results: ["i32"] }, + { module: "env", name: "__wpk_fork_resume_peek", params: ["i32"], results: ["i32"] }, +] as const; +export const WPK_FORK_REQUIRED_TABLE_IMPORTS = [ + { module: "env", name: "__wpk_fork_ref_gc_transit", table64: false, element: "anyref", minimum: 1, maximum: null }, + { module: "env", name: "__wpk_fork_resume_table", table64: false, element: "funcref", minimum: 1, maximum: null }, ] as const; export const WPK_FORK_REQUIRED_EXPORTS = [ + { name: "__wpk_fork_exception_materialize", params: ["i32"], results: [] }, + { name: "__wpk_fork_ref_decode_exnref", params: ["i32"], results: ["exnref"] }, + { name: "__wpk_fork_ref_encode_exnref", params: ["exnref"], results: ["i32"] }, + { name: "__wpk_fork_ref_exn_abort", params: [], results: [] }, + { name: "__wpk_fork_ref_exn_clear", params: [], results: [] }, + { name: "__wpk_fork_ref_exn_encode_ingress", params: ["i32"], results: ["i32"] }, + { name: "__wpk_fork_ref_exn_throw_recipe", params: ["i32"], results: [] }, + { name: "__wpk_fork_ref_exn_throw_slot", params: ["i32"], results: [] }, + { name: "__wpk_fork_ref_gc_allocate", params: ["i32"], results: [] }, + { name: "__wpk_fork_ref_gc_encode_slot", params: ["i32"], results: ["i32"] }, + { name: "__wpk_fork_ref_gc_fill", params: ["i32"], results: [] }, + { name: "__wpk_fork_ref_gc_probe", params: ["i32"], results: ["i64"] }, + { name: "__wpk_fork_ref_gc_publish_externref", params: ["i32", "externref"], results: [] }, + { name: "__wpk_fork_static_root_harvest", params: [], results: [] }, { name: "wpk_fork_abort_begin", params: ["ptr"], results: [] }, { name: "wpk_fork_abort_end", params: [], results: [] }, + { name: "wpk_fork_module_bootstrap", params: [], results: [] }, + { name: "wpk_fork_module_state_finish_restore", params: ["i32"], results: [] }, + { name: "wpk_fork_module_state_restore", params: ["i32"], results: [] }, + { name: "wpk_fork_module_state_save", params: ["i32"], results: [] }, + { name: "wpk_fork_module_table_state_restore", params: ["i32"], results: [] }, + { name: "wpk_fork_module_table_state_save", params: ["i32"], results: [] }, + { name: "wpk_fork_module_thread_bootstrap", params: [], results: [] }, { name: "wpk_fork_rewind_begin", params: ["ptr"], results: [] }, { name: "wpk_fork_rewind_end", params: [], results: [] }, { name: "wpk_fork_state", params: [], results: ["i32"] }, @@ -120,6 +431,8 @@ export const CH_ARGS_COUNT = 6 as const; export const CH_ARG_SIZE = 8 as const; export const CH_RETURN = 56 as const; export const CH_ERRNO = 64 as const; +export const CH_REQUEST_FLAGS = 68 as const; +export const CH_REQUEST_FLAG_DEFER_SIGNAL_DELIVERY = 1 as const; export const CH_DATA = 72 as const; export const CH_DATA_SIZE = 65536 as const; export const CH_HEADER_SIZE = 72 as const; diff --git a/host/src/kernel-worker.ts b/host/src/kernel-worker.ts index af81b64fc2..ed0a2597f4 100644 --- a/host/src/kernel-worker.ts +++ b/host/src/kernel-worker.ts @@ -42,6 +42,8 @@ import { CH_DATA, CH_DATA_SIZE, CH_ERRNO, + CH_REQUEST_FLAGS, + CH_REQUEST_FLAG_DEFER_SIGNAL_DELIVERY, CH_RETURN, CH_SIG_BASE, CH_SIG_FLAGS, @@ -4739,6 +4741,21 @@ export class CentralizedKernelWorker { * handler signal number, or zero when no caught handler was dequeued. */ private dequeueSignalForDelivery(channel: ChannelInfo): number { + const requestFlags = new DataView( + channel.memory.buffer, + channel.channelOffset, + ).getUint32(CH_REQUEST_FLAGS, true); + if ( + (requestFlags & CH_REQUEST_FLAG_DEFER_SIGNAL_DELIVERY) !== 0 + ) { + // WHY: process-worker JavaScript consumes this completion outside + // libc's post-syscall signal trampoline. Dequeuing here would consume + // the kernel signal and block it for a handler that this completion can + // never invoke. Leave it pending for the explicit guest checkpoint after + // the owning fork, clone, or staged-loader transition. + return 0; + } + const preparedSignals = this.resumePreparedSignals; if (preparedSignals?.has(channel)) { const existingSignal = new DataView( diff --git a/host/src/node-kernel-worker-entry.ts b/host/src/node-kernel-worker-entry.ts index fe5e7b5d6a..f049ea6355 100644 --- a/host/src/node-kernel-worker-entry.ts +++ b/host/src/node-kernel-worker-entry.ts @@ -53,7 +53,13 @@ import { ThreadPageAllocator } from "./thread-allocator"; import { patchWasmForThread } from "./worker-main"; import { ThreadExitCoordinator } from "./thread-exit-coordinator"; import { readForkContinuationAnchor } from "./fork-continuation"; -import { detectPtrWidth, extractAbiVersion, extractHeapBase, isWasmModuleBytes } from "./constants"; +import { + describeWasmArtifactPolicyFailures, + detectPtrWidth, + extractAbiVersion, + extractHeapBase, + isWasmModuleBytes, +} from "./constants"; import { CH_TOTAL_SIZE, DEFAULT_MAX_PAGES, PAGES_PER_THREAD, WASM_PAGE_SIZE } from "./constants"; import { classifiedSignalOrFallback, @@ -67,6 +73,16 @@ import { } from "./thread-worker-disposition"; import { VmInterruptTimerManager } from "./vm-interrupt-timer"; import { RootfsSnapshotGate } from "./rootfs-snapshot-gate"; +import { + ForkReplayGateCoordinator, + observeForkReplayWorker, +} from "./fork-replay-gate"; +import { ForkExternrefProcessOwner } from "./fork-externref-process-owner"; +import type { ForkExternrefGeneration } from "./fork-reference-broker"; +import { + ForkHostImportOwnerRuntime, + type ForkHostImportOwnerWorker, +} from "./fork-host-import-runtime"; import { computeProcessMemoryLayout, createProcessMemory, @@ -130,10 +146,17 @@ interface ProcessInfo { ptrWidth: 4 | 8; layout: ProcessMemoryLayout; threadAllocator: ThreadPageAllocator; + /** Exact broker authority for this PID's current Wasm image. */ + externrefGeneration: ForkExternrefGeneration; /** Non-_start continuation root inherited from a pthread fork until exec. */ forkReplayContext?: ForkReplayContext; } const processes = new Map(); +const externrefProcessOwner = new ForkExternrefProcessOwner(); +const forkHostImportOwnerRuntime = + new ForkHostImportOwnerRuntime(externrefProcessOwner); +const forkHostImportsByWorker = + new WeakMap(); const processTeardowns = new Map>(); const vmInterruptTimers = new VmInterruptTimerManager( (pid) => processes.get(pid), @@ -196,9 +219,33 @@ async function terminateTrackedWorker( worker: ReturnType, ): Promise { intentionallyTerminated.add(worker as object); + forkHostImportsByWorker.get(worker as object)?.close(); await worker.terminate().catch(() => {}); } +function bindForkHostImports( + worker: ReturnType, + owner: ForkHostImportOwnerWorker, +): void { + forkHostImportsByWorker.set(worker as object, owner); +} + +function dispatchForkHostImport( + worker: ReturnType, + message: Extract, +): void { + const owner = forkHostImportsByWorker.get(worker as object); + if (!owner || !owner.dispatch(message.wake)) { + reportHostDiagnostic({ + pid: message.wake.pid, + source: "fork host-import protocol", + message: + `[kernel-worker] ignored stale or unbound fork host-import wake ` + + `pid=${message.wake.pid} sender=${message.wake.senderId}`, + }, "warn"); + } +} + async function terminateThreadWorkers(pid: number): Promise { const threads = threadWorkers.get(pid); if (!threads) return; @@ -297,6 +344,7 @@ async function finalizeProcessWorker( reportProcessExit(pid, exitStatus); await terminateThreadWorkers(pid); await terminateTrackedWorker(worker); + externrefProcessOwner.releaseGeneration(cur.externrefGeneration); } function processWorkerErrorDisposition(reason: string | undefined): { @@ -525,6 +573,10 @@ async function resolveExecutableForLaunch( const shebang = parseShebang(bytes); if (!shebang) { if (!isWasmModuleBytes(bytes)) return { errno: ENOEXEC }; + const artifactFailures = describeWasmArtifactPolicyFailures(bytes, { + expectedAbi: kernelWorker.getKernelAbiVersion(), + }); + if (artifactFailures.length > 0) return { errno: ENOEXEC }; let programModule: WebAssembly.Module; try { programModule = await WebAssembly.compile(bytes); @@ -719,6 +771,8 @@ async function handleInit(msg: InitMessage) { async function handleSpawn(msg: SpawnMessage) { let releaseMutation: (() => void) | undefined; let createdPid: number | undefined; + let createdExternrefGeneration: ForkExternrefGeneration | undefined; + let createdForkHostImports: ForkHostImportOwnerWorker | undefined; try { releaseMutation = rootfsSnapshotGate.beginMutation("spawn a process"); const hasProgramBytes = msg.programBytes !== undefined; @@ -792,6 +846,24 @@ async function handleSpawn(msg: SpawnMessage) { } } + const externrefGeneration = externrefProcessOwner.startGeneration(pid); + createdExternrefGeneration = externrefGeneration; + let worker: ReturnType; + const forkHostImports = forkHostImportOwnerRuntime.createWorker({ + pid, + generationId: externrefGeneration.id, + authorizeSender: () => { + const current = processes.get(pid); + if ( + !current + || current.worker !== worker + || current.externrefGeneration !== externrefGeneration + ) { + throw new Error(`stale fork host-import sender for pid=${pid}`); + } + }, + }); + createdForkHostImports = forkHostImports; const initData: CentralizedWorkerInitMessage = { type: "centralized_init", pid, @@ -799,13 +871,16 @@ async function handleSpawn(msg: SpawnMessage) { programModule, memory, channelOffset, + externrefGenerationId: externrefGeneration.id, + forkHostImports: forkHostImports.init, env: msg.env, argv: msg.argv, ptrWidth, kernelAbiVersion: kernelWorker.getKernelAbiVersion(), }; - const worker = workerAdapter.createWorker(initData); + worker = workerAdapter.createWorker(initData); + bindForkHostImports(worker, forkHostImports); processes.set(pid, { memory, programBytes, @@ -815,6 +890,7 @@ async function handleSpawn(msg: SpawnMessage) { ptrWidth, layout, threadAllocator, + externrefGeneration, }); worker.on("error", (err: Error) => finalizeUnexpectedWorkerError(pid, worker, "worker error", err)); @@ -837,14 +913,22 @@ async function handleSpawn(msg: SpawnMessage) { } else if (m.type === "vm_interrupt_timer" && m.pid === pid) { const process = processes.get(pid); if (process?.worker === worker) handleVmInterruptTimer(m, pid, process); + } else if (m.type === "fork_host_import") { + dispatchForkHostImport(worker, m); } }); installCrashSafetyNet(worker, pid); createdPid = undefined; + createdExternrefGeneration = undefined; + createdForkHostImports = undefined; respond(msg.requestId, pid); } catch (e) { + createdForkHostImports?.close(); + if (createdExternrefGeneration) { + externrefProcessOwner.releaseGeneration(createdExternrefGeneration); + } if (createdPid !== undefined) { kernelWorker.unregisterProcess(createdPid); kernelWorker.removeProcessFromKernelTable(createdPid); @@ -911,73 +995,139 @@ async function handleFork( ? { ...parentInfo.forkReplayContext, forkBufAddr: activeForkBufAddr } : undefined; const forkBufAddr = activeForkBufAddr; - - const childInitData: CentralizedWorkerInitMessage = { - type: "centralized_init", - pid: childPid, - programBytes: parentProgram, - programModule: parentInfo.programModule, - memory: childMemory, - channelOffset: childChannelOffset, - isForkChild: true, - forkBufAddr, - forkChildThreadFnPtr: forkReplayContext?.fnPtr, - forkChildThreadArgPtr: forkReplayContext?.argPtr, - ptrWidth, - kernelAbiVersion: kernelWorker.getKernelAbiVersion(), - }; - - const childWorker = new DeferredWorkerHandle( - () => workerAdapter.createWorker(childInitData), + const forkReplay = new ForkReplayGateCoordinator( + `fork child pid=${childPid}`, ); - processes.set(childPid, { - memory: childMemory, - programBytes: parentProgram, - programModule: parentInfo.programModule, - worker: childWorker, - channelOffset: childChannelOffset, + const externrefGrant = externrefProcessOwner.forkGenerationFromContinuation( + parentInfo.externrefGeneration, + childPid, + parentMemory, ptrWidth, - layout: childLayout, - threadAllocator: threadAllocatorForLayout(childLayout, ptrWidth, childPid), - forkReplayContext, - }); + forkBufAddr, + ); + let childWorker: DeferredWorkerHandle | undefined; + let childForkHostImports: ForkHostImportOwnerWorker | undefined; + try { + let launchedWorker: DeferredWorkerHandle; + const forkHostImports = forkHostImportOwnerRuntime.createWorker({ + pid: childPid, + generationId: externrefGrant.generation.id, + authorizeSender: () => { + const current = processes.get(childPid); + if ( + !current + || current.worker !== launchedWorker + || current.externrefGeneration !== externrefGrant.generation + ) { + throw new Error( + `stale fork host-import sender for child pid=${childPid}`, + ); + } + }, + }); + childForkHostImports = forkHostImports; + const childInitData: CentralizedWorkerInitMessage = { + type: "centralized_init", + pid: childPid, + programBytes: parentProgram, + programModule: parentInfo.programModule, + memory: childMemory, + channelOffset: childChannelOffset, + externrefGenerationId: externrefGrant.generation.id, + forkHostImports: forkHostImports.init, + isForkChild: true, + forkBufAddr, + forkReplayGate: forkReplay.gate, + forkChildThreadFnPtr: forkReplayContext?.fnPtr, + forkChildThreadArgPtr: forkReplayContext?.argPtr, + ptrWidth, + kernelAbiVersion: kernelWorker.getKernelAbiVersion(), + }; - childWorker.on("error", (err: Error) => finalizeUnexpectedWorkerError(childPid, childWorker, "worker error", err)); + childWorker = new DeferredWorkerHandle( + () => workerAdapter.createWorker(childInitData), + ); + launchedWorker = childWorker; + bindForkHostImports(launchedWorker, forkHostImports); + processes.set(childPid, { + memory: childMemory, + programBytes: parentProgram, + programModule: parentInfo.programModule, + worker: launchedWorker, + channelOffset: childChannelOffset, + ptrWidth, + layout: childLayout, + threadAllocator: threadAllocatorForLayout(childLayout, ptrWidth, childPid), + forkReplayContext, + externrefGeneration: externrefGrant.generation, + }); - childWorker.on("message", (raw: unknown) => { - const m = raw as WorkerToHostMessage; - if (m.type === "error" && m.pid === childPid) { - finalizeProcessWorkerError(childPid, childWorker, m.message); - } else if (m.type === "exit" && m.pid === childPid) { - void finalizeProcessWorker(childPid, childWorker, m.status ?? 0); - } else if (m.type === "vm_interrupt_timer" && m.pid === childPid) { - const process = processes.get(childPid); - if (process?.worker === childWorker) { - handleVmInterruptTimer(m, childPid, process); + observeForkReplayWorker( + forkReplay, + launchedWorker, + childPid, + () => processes.get(childPid)?.worker === launchedWorker, + ); + launchedWorker.on("error", (err: Error) => + finalizeUnexpectedWorkerError(childPid, launchedWorker, "worker error", err)); + launchedWorker.on("message", (raw: unknown) => { + const m = raw as WorkerToHostMessage; + if (m.type === "error" && m.pid === childPid) { + finalizeProcessWorkerError(childPid, launchedWorker, m.message); + } else if (m.type === "exit" && m.pid === childPid) { + void finalizeProcessWorker(childPid, launchedWorker, m.status ?? 0); + } else if (m.type === "vm_interrupt_timer" && m.pid === childPid) { + const process = processes.get(childPid); + if (process?.worker === launchedWorker) { + handleVmInterruptTimer(m, childPid, process); + } + } else if (m.type === "fork_host_import") { + dispatchForkHostImport(launchedWorker, m); } - } - }); - - installCrashSafetyNet(childWorker, childPid); + }); + installCrashSafetyNet(launchedWorker, childPid); - try { const startDisposition = kernelWorker.startProcessWorkerWhenRunnable( childPid, childMemory, - () => { childWorker.start(); }, - () => { void childWorker.terminate(); }, + () => { launchedWorker.start(); }, + () => { + forkReplay.cancel( + new Error( + `Fork child ${childPid} launch was cancelled before replay readiness`, + ), + ); + forkHostImports.close(); + void launchedWorker.terminate(); + }, ); if (startDisposition === "stale") { throw new Error(`Fork child ${childPid} changed generation before Worker launch`); } + await forkReplay.waitUntilReady(); + if (processes.get(childPid)?.worker !== launchedWorker) { + throw new Error( + `Fork child ${childPid} changed generation before replay commit`, + ); + } + if (!kernelWorker.shouldLaunchPendingChild(childPid)) { + throw new Error(`Fork child ${childPid} exited before replay commit`); + } + // WHY: only this commit wakes the child inside the inherited fork import. + // Resolve onFork afterward so the parent cannot observe a child whose + // continuation has not proved it reached the copied activation. + forkReplay.commit(); } catch (error) { - if (processes.get(childPid)?.worker === childWorker) { + forkReplay.cancel(error); + childForkHostImports?.close(); + externrefProcessOwner.releaseGeneration(externrefGrant.generation); + if (childWorker && processes.get(childPid)?.worker === childWorker) { processes.delete(childPid); threadModuleCache.delete(childPid); ptyByPid.delete(childPid); vmInterruptTimers.clear(childPid); } - void childWorker.terminate(); + if (childWorker) void childWorker.terminate(); throw error; } @@ -1024,6 +1174,8 @@ async function handleExec( const addressSpaceResult = kernelWorker.prepareAddressSpaceForExec(pid); if (addressSpaceResult < 0) return addressSpaceResult; let replacementWorker: ReturnType | undefined; + let replacementExternrefGeneration: ForkExternrefGeneration | undefined; + let replacementForkHostImports: ForkHostImportOwnerWorker | undefined; try { const setupResult = kernelWorker.kernelExecSetup(pid, callerTid); if (setupResult < 0) return setupResult; @@ -1036,6 +1188,9 @@ async function handleExec( intentionallyTerminated.add(initiatingInfo.worker as object); } kernelWorker.prepareProcessForExec(pid); + replacementExternrefGeneration = externrefProcessOwner.replaceGeneration( + initiatingInfo.externrefGeneration, + ); const finalizeResult = kernelWorker.finalizeAddressSpaceForExec(pid); if (finalizeResult < 0) { @@ -1044,9 +1199,14 @@ async function handleExec( await terminateThreadWorkers(pid); if (initiatingInfo.worker) { + forkHostImportsByWorker.get(initiatingInfo.worker as object)?.close(); await initiatingInfo.worker.terminate().catch(() => {}); } - if (kernelWorker.finalizeExecHandoffTermination(pid) > 0) return 0; + if (kernelWorker.finalizeExecHandoffTermination(pid) > 0) { + externrefProcessOwner.releaseGeneration(replacementExternrefGeneration); + replacementExternrefGeneration = undefined; + return 0; + } const { memory: newMemory, @@ -1054,6 +1214,21 @@ async function handleExec( threadAllocator: newThreadAllocator, } = prepared; const newChannelOffset = newLayout.channelOffset; + replacementForkHostImports = forkHostImportOwnerRuntime.createWorker({ + pid, + generationId: replacementExternrefGeneration.id, + authorizeSender: () => { + const current = processes.get(pid); + if ( + !replacementWorker + || !current + || current.worker !== replacementWorker + || current.externrefGeneration !== replacementExternrefGeneration + ) { + throw new Error(`stale fork host-import sender for exec pid=${pid}`); + } + }, + }); const initData: CentralizedWorkerInitMessage = { type: "centralized_init", @@ -1062,6 +1237,8 @@ async function handleExec( programModule, memory: newMemory, channelOffset: newChannelOffset, + externrefGenerationId: replacementExternrefGeneration.id, + forkHostImports: replacementForkHostImports.init, argv: launchArgv, env: envp, ptrWidth: newPtrWidth, @@ -1083,6 +1260,7 @@ async function handleExec( replacementWorker = new DeferredWorkerHandle( () => workerAdapter.createWorker(initData), ); + bindForkHostImports(replacementWorker, replacementForkHostImports); // Clear thread module cache — new program binary is different threadModuleCache.delete(pid); @@ -1096,6 +1274,7 @@ async function handleExec( ptrWidth: newPtrWidth, layout: newLayout, threadAllocator: newThreadAllocator, + externrefGeneration: replacementExternrefGeneration, }); replacementWorker.on("error", (err: Error) => @@ -1115,6 +1294,8 @@ async function handleExec( if (process && process.worker === replacementWorker) { handleVmInterruptTimer(m, pid, process); } + } else if (m.type === "fork_host_import") { + dispatchForkHostImport(replacementWorker!, m); } }); @@ -1123,12 +1304,16 @@ async function handleExec( pid, newMemory, () => { (replacementWorker as DeferredWorkerHandle).start(); }, - () => { void replacementWorker?.terminate(); }, + () => { + replacementForkHostImports?.close(); + void replacementWorker?.terminate(); + }, ); if (startDisposition === "stale") { throw new Error(`Exec pid ${pid} changed generation before Worker launch`); } if (startDisposition === "dead") { + replacementForkHostImports.close(); kernelWorker.finishProcessExecHandoff(pid); kernelWorker.finalizeExecHandoffTermination(pid); return 0; @@ -1136,6 +1321,11 @@ async function handleExec( kernelWorker.finishProcessExecHandoff(pid); return 0; } catch (err) { + replacementForkHostImports?.close(); + if (replacementExternrefGeneration) { + externrefProcessOwner.releaseGeneration(replacementExternrefGeneration); + replacementExternrefGeneration = undefined; + } // A kernel trap can leave the commit point uncertain. We cannot safely // return to the caller, so invalidate the old generation before yielding // and report a truthful signal death. @@ -1241,6 +1431,24 @@ async function handlePosixSpawn( maxAddr: layout.maxAddr, }); + const externrefGeneration = externrefProcessOwner.startGeneration(childPid); + let newWorker: DeferredWorkerHandle; + const forkHostImports = forkHostImportOwnerRuntime.createWorker({ + pid: childPid, + generationId: externrefGeneration.id, + authorizeSender: () => { + const current = processes.get(childPid); + if ( + !current + || current.worker !== newWorker + || current.externrefGeneration !== externrefGeneration + ) { + throw new Error( + `stale fork host-import sender for spawn pid=${childPid}`, + ); + } + }, + }); const initData: CentralizedWorkerInitMessage = { type: "centralized_init", pid: childPid, @@ -1248,15 +1456,18 @@ async function handlePosixSpawn( programModule, memory, channelOffset, + externrefGenerationId: externrefGeneration.id, + forkHostImports: forkHostImports.init, argv, env: envp, ptrWidth, kernelAbiVersion: kernelWorker.getKernelAbiVersion(), }; - const newWorker = new DeferredWorkerHandle( + newWorker = new DeferredWorkerHandle( () => workerAdapter.createWorker(initData), ); + bindForkHostImports(newWorker, forkHostImports); processes.set(childPid, { memory, programBytes, @@ -1266,6 +1477,7 @@ async function handlePosixSpawn( ptrWidth, layout, threadAllocator, + externrefGeneration, }); newWorker.on("error", (err: Error) => finalizeUnexpectedWorkerError(childPid, newWorker, "spawn worker error", err)); @@ -1281,6 +1493,8 @@ async function handlePosixSpawn( if (process?.worker === newWorker) { handleVmInterruptTimer(m, childPid, process); } + } else if (m.type === "fork_host_import") { + dispatchForkHostImport(newWorker, m); } }); @@ -1291,12 +1505,17 @@ async function handlePosixSpawn( childPid, memory, () => { newWorker.start(); }, - () => { void newWorker.terminate(); }, + () => { + forkHostImports.close(); + void newWorker.terminate(); + }, ); if (startDisposition === "stale") { throw new Error(`Spawn child ${childPid} changed generation before Worker launch`); } } catch (error) { + forkHostImports.close(); + externrefProcessOwner.releaseGeneration(externrefGeneration); if (processes.get(childPid)?.worker === newWorker) { processes.delete(childPid); threadModuleCache.delete(childPid); @@ -1363,6 +1582,25 @@ async function handleClone( throw err; } + let threadWorker: DeferredWorkerHandle; + let threadEntry: ThreadWorkerInfo; + const forkHostImports = forkHostImportOwnerRuntime.createWorker({ + pid, + generationId: processInfo.externrefGeneration.id, + authorizeSender: () => { + const entries = threadWorkers.get(pid); + if ( + !belongsToCurrentProcessImage() + || !threadEntry + || threadEntry.worker !== threadWorker + || !entries?.includes(threadEntry) + ) { + throw new Error( + `stale fork host-import sender for pid=${pid} tid=${tid}`, + ); + } + }, + }); const threadInitData: CentralizedThreadInitMessage = { type: "centralized_thread_init", pid, @@ -1372,6 +1610,8 @@ async function handleClone( memory, processChannelOffset: processInfo.channelOffset, channelOffset: alloc.channelOffset, + externrefGenerationId: processInfo.externrefGeneration.id, + forkHostImports: forkHostImports.init, fnPtr, argPtr, stackPtr, @@ -1383,11 +1623,12 @@ async function handleClone( kernelAbiVersion: kernelWorker.getKernelAbiVersion(), }; - const threadWorker = new DeferredWorkerHandle( + threadWorker = new DeferredWorkerHandle( () => workerAdapter.createWorker(threadInitData), ); + bindForkHostImports(threadWorker, forkHostImports); if (!threadWorkers.has(pid)) threadWorkers.set(pid, []); - const threadEntry: ThreadWorkerInfo = { + threadEntry = { worker: threadWorker, channelOffset: alloc.channelOffset, tid, @@ -1459,6 +1700,8 @@ async function handleClone( if (isCurrentThreadGeneration() && m.pid === pid) { handleVmInterruptTimer(m, pid, processInfo); } + } else if (m.type === "fork_host_import") { + dispatchForkHostImport(threadWorker, m); } }); threadWorker.on("error", (err: Error) => failThread(`worker error: ${err.message ?? err}`)); @@ -1471,7 +1714,10 @@ async function handleClone( pid, memory, () => { threadWorker.start(); }, - () => { void threadWorker.terminate(); }, + () => { + forkHostImports.close(); + void threadWorker.terminate(); + }, () => { kernelWorker.finalizeThreadExit(pid, tid, alloc.channelOffset); const failedClone = kernelWorker.failDeferredCloneLaunch(pid, tid, 12); @@ -1532,6 +1778,7 @@ async function finishProcessExit( // termination so no further guest syscalls can arrive on its channel. kernelWorker.deactivateProcess(pid); + externrefProcessOwner.releaseGeneration(info.externrefGeneration); processes.delete(pid); threadModuleCache.delete(pid); ptyByPid.delete(pid); @@ -1562,6 +1809,7 @@ async function handleTerminate(msg: TerminateProcessMessage) { if (threads) { for (const t of threads) { intentionallyTerminated.add(t.worker as object); + forkHostImportsByWorker.get(t.worker as object)?.close(); await t.worker.terminate().catch(() => {}); try { kernelWorker.notifyThreadExit(pid, t.tid); @@ -1576,6 +1824,9 @@ async function handleTerminate(msg: TerminateProcessMessage) { if (info?.worker) { await terminateTrackedWorker(info.worker); } + if (info) { + externrefProcessOwner.releaseGeneration(info.externrefGeneration); + } try { kernelWorker.unregisterProcess(pid); @@ -1627,6 +1878,7 @@ async function handleDestroy(msg: { requestId: number }) { vmInterruptTimers.clear(pid); await terminateThreadWorkers(pid); await terminateTrackedWorker(info.worker); + externrefProcessOwner.releaseGeneration(info.externrefGeneration); try { kernelWorker.unregisterProcess(pid); } catch {} } await Promise.allSettled([...processTeardowns.values()]); @@ -1636,6 +1888,7 @@ async function handleDestroy(msg: { requestId: number }) { for (const threads of threadWorkers.values()) { for (const t of threads) { intentionallyTerminated.add(t.worker as object); + forkHostImportsByWorker.get(t.worker as object)?.close(); t.worker.terminate().catch(() => {}); } } diff --git a/host/src/worker-main.ts b/host/src/worker-main.ts index 56b9a8e3aa..a80c6885a4 100644 --- a/host/src/worker-main.ts +++ b/host/src/worker-main.ts @@ -19,10 +19,20 @@ import { readForkInstrumentCapabilityClaim, requireCppExceptionTag, requireLongjmpTag, + type DylinkForkActivationOwner, + type DylinkForkState, type LoadedSharedLibrary, - type SideModuleForkState, } from "./dylink"; -import { extractAbiVersion, WASM_PAGE_SIZE } from "./constants"; +import { + DylinkForkArchive, + DylinkForkTableReplica, + type DylinkForkArchiveSnapshot, +} from "./dylink-fork-archive"; +import { + describeWasmArtifactPolicyFailures, + extractAbiVersion, + WASM_PAGE_SIZE, +} from "./constants"; import { ABI_SYSCALLS, CHANNEL_STATUS_IDLE, @@ -31,13 +41,20 @@ import { CH_ARGS, CH_DATA, CH_ERRNO, + CH_REQUEST_FLAGS, + CH_REQUEST_FLAG_DEFER_SIGNAL_DELIVERY, CH_RETURN, CH_STATUS, CH_SYSCALL, CH_TOTAL_SIZE, HOST_INTERCEPTED_SYSCALLS, + WPK_FORK_EXPORT_MODULE_THREAD_BOOTSTRAP, + WPK_FORK_MODULE_STATE_IMPORT_RECORD_COMMIT, + WPK_FORK_MODULE_STATE_IMPORT_RECORD_FIND, + WPK_FORK_MODULE_STATE_IMPORT_RECORD_RESERVE, WPK_FORK_REQUIRED_EXPORTS, WPK_FORK_REQUIRED_IMPORTS, + WPK_FORK_CAP_ACTIVATION_STATE_SAFE, } from "./generated/abi"; import { FORK_SAVE_BUFFER_SIZE, @@ -50,6 +67,61 @@ import { readLinkedFrameFormat, writeForkContinuationAnchor, } from "./fork-continuation"; +import { + createForkUnwindTag, + FORK_UNWIND_TAG_IMPORT_MODULE, + FORK_UNWIND_TAG_IMPORT_NAME, + isForkUnwindException, + requireForkUnwindTag, +} from "./fork-unwind-transport"; +import { waitForForkReplayCommit } from "./fork-replay-gate"; +import { + computeForkModuleTemplateId, + computeForkModuleTemplateIdSync, + ForkModuleStateArena, + readForkModuleStateDescriptor, + readForkModuleStateRoot, +} from "./fork-module-state"; +import { + buildForkActivationStateImports, + ForkActivationRegistry, + forkActivationRegistrationFromInstance, + type ForkActivationTableReplication, + type ForkActivationReferenceReplayImports, + type ForkActivationRegistration, +} from "./fork-activation-registry"; +import { + buildForkExceptionImports, + ForkExceptionBroker, + forkExceptionProviderFromInstance, + readForkExceptionCodecDescriptor, + type ForkExceptionReferenceReplayImports, + type ForkExceptionProvider, +} from "./fork-exception-provider"; +import { ForkEarlyChildReferenceProvider } from "./fork-early-reference-provider"; +import { + decodeSegmentedForkReferenceTransaction, + type DecodedSegmentedForkReferenceTransaction, +} from "./fork-reference-segments"; +import { FORK_REFERENCE_TRANSACTION_OWNER_ID } from "./fork-reference-transaction"; +import { + forkGcCodecProviderFromInstance, + readForkGcCodecDescriptor, + type ForkGcCodecProvider, +} from "./fork-gc-codec"; +import { ForkProcessContinuationCoordinator } from "./fork-process-continuation"; +import { forkResumeTargetsFromInstance } from "./fork-resume-catalog"; +import { + ForkExternrefTokenCache, + ForkExternrefTokenRecipeProvider, +} from "./fork-reference-broker"; +import { ForkHostImportWorkerRuntime } from "./fork-host-import-runtime"; +import { + ForkImportedGlobalCapture, + ForkImportedGlobalPlanner, + type ForkWasmImports, + type PreparedForkParentActivation, +} from "./fork-imported-globals"; import { checkedWasmGuestPointerOffset, type WasmGuestPointer, @@ -74,6 +146,24 @@ const SYS_MMAP_NR = ABI_SYSCALLS.Mmap; const PROT_READ_WRITE = 3; const MAP_PRIVATE_ANONYMOUS = 0x22; +function markDeferredSignalDelivery( + view: DataView, + channelOffset: number, +): void { + view.setUint32( + channelOffset + CH_REQUEST_FLAGS, + CH_REQUEST_FLAG_DEFER_SIGNAL_DELIVERY, + true, + ); +} + +function clearDeferredSignalDelivery( + view: DataView, + channelOffset: number, +): void { + view.setUint32(channelOffset + CH_REQUEST_FLAGS, 0, true); +} + function continuationMmap( memory: WebAssembly.Memory, channelOffset: number, @@ -85,19 +175,33 @@ function continuationMmap( view.setInt32(base + CH_SYSCALL, SYS_MMAP_NR, true); view.setBigInt64(base + CH_ARGS + 0 * CH_ARG_SIZE, 0n, true); view.setBigInt64(base + CH_ARGS + 1 * CH_ARG_SIZE, BigInt(size), true); - view.setBigInt64(base + CH_ARGS + 2 * CH_ARG_SIZE, BigInt(PROT_READ_WRITE), true); - view.setBigInt64(base + CH_ARGS + 3 * CH_ARG_SIZE, BigInt(MAP_PRIVATE_ANONYMOUS), true); + view.setBigInt64( + base + CH_ARGS + 2 * CH_ARG_SIZE, + BigInt(PROT_READ_WRITE), + true, + ); + view.setBigInt64( + base + CH_ARGS + 3 * CH_ARG_SIZE, + BigInt(MAP_PRIVATE_ANONYMOUS), + true, + ); view.setBigInt64(base + CH_ARGS + 4 * CH_ARG_SIZE, -1n, true); view.setBigInt64(base + CH_ARGS + 5 * CH_ARG_SIZE, 0n, true); + markDeferredSignalDelivery(view, base); let i32 = new Int32Array(memory.buffer); Atomics.store(i32, (base + CH_STATUS) / 4, CHANNEL_STATUS_PENDING); Atomics.notify(i32, (base + CH_STATUS) / 4, 1); - while (Atomics.wait(i32, (base + CH_STATUS) / 4, CHANNEL_STATUS_PENDING) === "ok") { /* */ } + while ( + Atomics.wait(i32, (base + CH_STATUS) / 4, CHANNEL_STATUS_PENDING) === "ok" + ) { + /* */ + } view = new DataView(memory.buffer); i32 = new Int32Array(memory.buffer); const result = Number(view.getBigInt64(base + CH_RETURN, true)); const err = view.getUint32(base + CH_ERRNO, true); + clearDeferredSignalDelivery(view, base); Atomics.store(i32, (base + CH_STATUS) / 4, CHANNEL_STATUS_IDLE); if (err || result < 0) { const errno = err || -result; @@ -125,17 +229,25 @@ function continuationMunmap( for (let i = 2; i < 6; i++) { view.setBigInt64(base + CH_ARGS + i * CH_ARG_SIZE, 0n, true); } + markDeferredSignalDelivery(view, base); const i32 = new Int32Array(memory.buffer); Atomics.store(i32, (base + CH_STATUS) / 4, CHANNEL_STATUS_PENDING); Atomics.notify(i32, (base + CH_STATUS) / 4, 1); - while (Atomics.wait(i32, (base + CH_STATUS) / 4, CHANNEL_STATUS_PENDING) === "ok") { /* */ } + while ( + Atomics.wait(i32, (base + CH_STATUS) / 4, CHANNEL_STATUS_PENDING) === "ok" + ) { + /* */ + } const resultView = new DataView(memory.buffer); const resultI32 = new Int32Array(memory.buffer); const result = Number(resultView.getBigInt64(base + CH_RETURN, true)); const err = resultView.getUint32(base + CH_ERRNO, true); + clearDeferredSignalDelivery(resultView, base); Atomics.store(resultI32, (base + CH_STATUS) / 4, CHANNEL_STATUS_IDLE); if (err || result < 0) { - throw new Error(`${label}: munmap(0x${addr.toString(16)}, ${size}) failed errno=${err || -result}`); + throw new Error( + `${label}: munmap(0x${addr.toString(16)}, ${size}) failed errno=${err || -result}`, + ); } } @@ -163,35 +275,53 @@ function buildKernelImports( const _envVars = envVars || []; const encoder = new TextEncoder(); /** Convert wasm64 BigInt pointer to number (safe since addresses < 4GB) */ - const n = (v: number | bigint): number => typeof v === "bigint" ? Number(v) : v; + const n = (v: number | bigint): number => + typeof v === "bigint" ? Number(v) : v; return { // CRT argv support kernel_get_argc: (): number => _argv.length, - kernel_argv_read: (index: number, bufPtr: number | bigint, bufMax: number): number => { + kernel_argv_read: ( + index: number, + bufPtr: number | bigint, + bufMax: number, + ): number => { if (index >= _argv.length) return 0; const encoded = encoder.encode(_argv[index]); const len = Math.min(encoded.length, bufMax); - new Uint8Array(memory.buffer, n(bufPtr), len).set(encoded.subarray(0, len)); + new Uint8Array(memory.buffer, n(bufPtr), len).set( + encoded.subarray(0, len), + ); return len; }, // CRT environ support kernel_environ_count: (): number => _envVars.length, - kernel_environ_get: (index: number, bufPtr: number | bigint, bufMax: number): number => { + kernel_environ_get: ( + index: number, + bufPtr: number | bigint, + bufMax: number, + ): number => { if (index >= _envVars.length) return -1; const encoded = encoder.encode(_envVars[index]); const len = Math.min(encoded.length, bufMax); - new Uint8Array(memory.buffer, n(bufPtr), len).set(encoded.subarray(0, len)); + new Uint8Array(memory.buffer, n(bufPtr), len).set( + encoded.subarray(0, len), + ); return len; }, // Fork/exec state — not a fork child. kernel_is_fork_child: (): number => 0, kernel_apply_fork_fd_actions: (): number => 0, - kernel_get_fork_exec_path: (_buf: number | bigint, _max: number): number => 0, + kernel_get_fork_exec_path: (_buf: number | bigint, _max: number): number => + 0, kernel_get_fork_exec_argc: (): number => 0, - kernel_get_fork_exec_argv: (_index: number, _buf: number | bigint, _max: number): number => 0, + kernel_get_fork_exec_argv: ( + _index: number, + _buf: number | bigint, + _max: number, + ): number => 0, kernel_push_argv: (_ptr: number | bigint, _len: number): void => {}, kernel_clear_fork_exec: (): number => 0, @@ -208,7 +338,12 @@ function buildKernelImports( Atomics.store(i32, (base + CH_STATUS) / 4, CHANNEL_STATUS_PENDING); Atomics.notify(i32, (base + CH_STATUS) / 4, 1); // Wait for complete, then trap - while (Atomics.wait(i32, (base + CH_STATUS) / 4, CHANNEL_STATUS_PENDING) === "ok") { /* */ } + while ( + Atomics.wait(i32, (base + CH_STATUS) / 4, CHANNEL_STATUS_PENDING) === + "ok" + ) { + /* */ + } onKernelExit?.(status); // Per-thread exit is non-returning. Throwing here prevents libc's // mandated SYS_exit retry loop from parking a second time on a channel @@ -217,14 +352,25 @@ function buildKernelImports( }, // Clone dispatches through channel (SYS_CLONE) - kernel_clone: (fnPtr: number | bigint, stackPtr: number | bigint, flags: number, - arg: number | bigint, ptidPtr: number | bigint, tlsPtr: number | bigint, ctidPtr: number | bigint): number => { + kernel_clone: ( + fnPtr: number | bigint, + stackPtr: number | bigint, + flags: number, + arg: number | bigint, + ptidPtr: number | bigint, + tlsPtr: number | bigint, + ctidPtr: number | bigint, + ): number => { const SYS_CLONE_NR = ABI_SYSCALLS.Clone; const view = new DataView(memory.buffer); const base = channelOffset; view.setInt32(base + CH_SYSCALL, SYS_CLONE_NR, true); view.setBigInt64(base + CH_ARGS + 0 * CH_ARG_SIZE, BigInt(flags), true); - view.setBigInt64(base + CH_ARGS + 1 * CH_ARG_SIZE, BigInt(stackPtr), true); + view.setBigInt64( + base + CH_ARGS + 1 * CH_ARG_SIZE, + BigInt(stackPtr), + true, + ); view.setBigInt64(base + CH_ARGS + 2 * CH_ARG_SIZE, BigInt(ptidPtr), true); view.setBigInt64(base + CH_ARGS + 3 * CH_ARG_SIZE, BigInt(tlsPtr), true); view.setBigInt64(base + CH_ARGS + 4 * CH_ARG_SIZE, BigInt(ctidPtr), true); @@ -233,13 +379,20 @@ function buildKernelImports( view.setUint32(base + CH_DATA, n(fnPtr), true); view.setUint32(base + CH_DATA + 4, n(arg), true); + markDeferredSignalDelivery(view, base); const i32 = new Int32Array(memory.buffer); Atomics.store(i32, (base + CH_STATUS) / 4, CHANNEL_STATUS_PENDING); Atomics.notify(i32, (base + CH_STATUS) / 4, 1); - while (Atomics.wait(i32, (base + CH_STATUS) / 4, CHANNEL_STATUS_PENDING) === "ok") { /* */ } + while ( + Atomics.wait(i32, (base + CH_STATUS) / 4, CHANNEL_STATUS_PENDING) === + "ok" + ) { + /* */ + } const result = Number(view.getBigInt64(base + CH_RETURN, true)); const err = view.getUint32(base + CH_ERRNO, true); + clearDeferredSignalDelivery(view, base); Atomics.store(i32, (base + CH_STATUS) / 4, CHANNEL_STATUS_IDLE); if (err) return -err; @@ -250,16 +403,28 @@ function buildKernelImports( kernel_fork: (): number => { const view = new DataView(memory.buffer); const base = channelOffset; - view.setInt32(base + CH_SYSCALL, HOST_INTERCEPTED_SYSCALLS.SYS_FORK, true); - for (let i = 0; i < 6; i++) view.setBigInt64(base + CH_ARGS + i * CH_ARG_SIZE, 0n, true); + view.setInt32( + base + CH_SYSCALL, + HOST_INTERCEPTED_SYSCALLS.SYS_FORK, + true, + ); + for (let i = 0; i < 6; i++) + view.setBigInt64(base + CH_ARGS + i * CH_ARG_SIZE, 0n, true); + markDeferredSignalDelivery(view, base); const i32 = new Int32Array(memory.buffer); Atomics.store(i32, (base + CH_STATUS) / 4, CHANNEL_STATUS_PENDING); Atomics.notify(i32, (base + CH_STATUS) / 4, 1); - while (Atomics.wait(i32, (base + CH_STATUS) / 4, CHANNEL_STATUS_PENDING) === "ok") { /* */ } + while ( + Atomics.wait(i32, (base + CH_STATUS) / 4, CHANNEL_STATUS_PENDING) === + "ok" + ) { + /* */ + } const result = Number(view.getBigInt64(base + CH_RETURN, true)); const err = view.getUint32(base + CH_ERRNO, true); + clearDeferredSignalDelivery(view, base); Atomics.store(i32, (base + CH_STATUS) / 4, CHANNEL_STATUS_IDLE); if (err) return -err; @@ -270,32 +435,421 @@ function buildKernelImports( export interface DlopenSupport { imports: Record; - /** Replay the parent's dlopen list (read from the archive in linear - * memory). No-op if the archive head pointer is 0. Call this in the - * fork-child path AFTER setupChannelBase and BEFORE the wpk_fork - * rewind into _start. */ - replayDlopens: () => void; - /** Finish the one active side-module unwind after the main image unwinds. */ - completeSideModuleForkUnwind: () => void; - /** Begin the active side-module rewind after fork-child dlopen replay. */ - beginSideModuleForkRewind: () => void; - /** Replay and discard active side-module frames after main allocation failure. */ - beginSideModuleForkAbort: (errno: number) => void; - /** Reject a leaked active side-module identity on a normal main return. */ - assertNoActiveSideModuleFork: () => void; + /** Validate and return the compact copied live-module closure. */ + readForkState: () => DylinkForkState; + /** Recreate the parent's live module and handle state from linear memory. */ + replayDlopens: (validatedState?: DylinkForkState) => void; /** Clear a fork parent's copied archive lock in the child's private memory. */ resetForkChildLock: () => void; + readonly archive: DylinkForkArchive; + /** Acquire one reentrant process-archive writer depth, blocking if needed. */ + acquireArchiveWriter(): void; + /** Release exactly one writer depth acquired by this Worker. */ + releaseArchiveWriter(): void; + /** Acquire one process-archive reader token, blocking behind a writer. */ + acquireArchiveReader(): void; + /** Release one reader token acquired by this Worker. */ + releaseArchiveReader(): void; + withArchiveWriter(operation: () => T): T; + withArchiveReader(operation: () => T): T; + writerOwned(): boolean; + /** Run after a fresh writer acquisition and before the protected operation. */ + setWriterAcquireObserver(observer: () => void): void; + /** Clean up state owned by a failed linker operation before releasing it. */ + setOperationAbortObserver(observer: () => void): void; + setCommitObserver( + observer: ( + linkerPublication: DylinkForkArchiveSnapshot | undefined, + tableMutationCommitted: boolean, + ) => void, + ): void; +} + +interface ProcessDylinkActivationOwnerOptions { + readonly memory: WebAssembly.Memory; + readonly ptrWidth: 4 | 8; + readonly channelOffset: number; + readonly forkUnwindTag: WebAssembly.Tag | undefined; + readonly coordinator: ForkProcessContinuationCoordinator; + readonly registry: ForkActivationRegistry; + readonly exceptionBroker: ForkExceptionBroker; + readonly importedStateCapture?: ForkImportedGlobalCapture; + readonly tableReplication?: ForkActivationTableReplication; + /** + * The child planner needs the copied dlopen archive, while the archive + * reader needs the activation owner installed first. Resolve it lazily at + * the actual side-module instantiation boundary to break that construction + * cycle without permitting a side activation to instantiate unplanned. + */ + readonly importedStatePlanner?: () => ForkImportedGlobalPlanner | null; + readonly referenceReplay?: () => ProcessReferenceReplayImports; + readonly registerChildReferenceActivation?: ( + activationId: number, + module: WebAssembly.Module, + registration: ForkActivationRegistration, + typedReferenceProvider: ForkGcCodecProvider, + ) => void; + readonly isForkChild: boolean; + /** + * A pthread owns a separate instance graph but adopts the process archive's + * stable activation coordinates. Unlike a fork child it captures live state + * and therefore uses the parent imported-state owner and bootstrap path. + */ + readonly isPthreadReplica?: boolean; + readonly invokeProcessFork: () => number; + readonly label: string; +} + +interface ProcessReferenceReplayImports + extends + ForkActivationReferenceReplayImports, + ForkExceptionReferenceReplayImports {} + +/** + * Bind every instrumented side-module instance to the one process + * continuation transaction. + * + * Activation IDs are monotonic in a parent and copied verbatim through the + * dlopen replay archive. They are coordinates in KFMS recipes and replay + * events, not reusable loader handles. + */ +function createProcessDylinkActivationOwner( + options: ProcessDylinkActivationOwnerOptions, +): DylinkForkActivationOwner { + let nextActivationId = 1; + const claimed = new Set(); + + const claimActivationId = ( + replayActivationId: number | undefined, + ): number => { + const activationId = replayActivationId ?? nextActivationId; + if ( + !Number.isInteger(activationId) || + activationId <= 0 || + activationId > 0xffff_ffff + ) { + throw new RangeError( + `${options.label}: side-module activation id ${String(activationId)} is invalid`, + ); + } + if (claimed.has(activationId)) { + throw new Error( + `${options.label}: side-module activation id ${activationId} was claimed twice`, + ); + } + claimed.add(activationId); + if (activationId >= nextActivationId) { + if (activationId === 0xffff_ffff) { + nextActivationId = 0x1_0000_0000; + } else { + nextActivationId = activationId + 1; + } + } + return activationId; + }; + + return { + prepare(request) { + if (options.isForkChild && request.replayActivationId === undefined) { + throw new Error( + `${request.name}: fresh-child replay is missing its activation id`, + ); + } + if ( + !options.isForkChild && + !options.tableReplication && + request.replayActivationId !== undefined + ) { + throw new Error( + `${request.name}: a parent load supplied a replay activation id`, + ); + } + // WHY: any live process Worker may reconcile an activation published by + // a peer or originate dlopen while holding the process archive writer. + // Its writer-acquire hook first adopts every published activation, so + // the same monotonic allocator safely claims the next process-wide ID. + const activationId = claimActivationId(request.replayActivationId); + let prepared = false; + let registered = false; + let released = false; + let exceptionProvider: ForkExceptionProvider | null = null; + let importedStatePreparation: PreparedForkParentActivation | null = null; + let childImportedStatePlanner: ForkImportedGlobalPlanner | null = null; + let importedStateRegistered = false; + let importsWrapped = false; + const continuation = new LinkedForkContinuation( + options.memory, + readLinkedFrameFormat(request.module), + (size) => + continuationMmap( + options.memory, + options.channelOffset, + size, + `${options.label}: ${request.name} continuation`, + ), + (addr, size) => + continuationMunmap( + options.memory, + options.channelOffset, + addr, + size, + `${options.label}: ${request.name} continuation`, + ), + `${options.label}: ${request.name}`, + ); + if (continuation.format.ptrWidth !== options.ptrWidth) { + throw new Error( + `${request.name}: linked continuation pointer width ` + + `${continuation.format.ptrWidth} does not match the process ` + + `pointer width ${options.ptrWidth}`, + ); + } + const moduleState = readForkModuleStateDescriptor(request.module); + if (moduleState.ptrWidth !== options.ptrWidth) { + throw new Error( + `${request.name}: module-state pointer width ${moduleState.ptrWidth} ` + + `does not match the process pointer width ${options.ptrWidth}`, + ); + } + const templateId = computeForkModuleTemplateIdSync(request.moduleBytes); + + try { + options.coordinator.prepareActivation({ + activationId, + continuation, + }); + prepared = true; + } catch (error) { + claimed.delete(activationId); + throw error; + } + + const env: Record = { + fork: (): number => options.invokeProcessFork(), + [FORK_UNWIND_TAG_IMPORT_NAME]: requireForkUnwindTag( + options.forkUnwindTag, + `${request.name}: fork activation`, + ) as unknown as WebAssembly.ImportValue, + ...options.coordinator.continuationImports(activationId, (errno) => + options.coordinator.beginCaptureAbort(errno), + ), + ...buildForkActivationStateImports( + activationId, + options.registry, + options.referenceReplay, + options.tableReplication, + ), + ...buildForkExceptionImports({ + activationId, + ptrWidth: options.ptrWidth, + registry: options.registry, + broker: options.exceptionBroker, + provider: () => { + if (!exceptionProvider) { + throw new Error( + `${request.name}: exception codec called before activation registration`, + ); + } + return exceptionProvider; + }, + referenceReplay: options.referenceReplay, + }), + }; + + return { + activationId, + env, + wrapImports: (imports) => { + if (importsWrapped) { + throw new Error( + `${request.name}: activation ${activationId} wrapped its imports twice`, + ); + } + importsWrapped = true; + childImportedStatePlanner = options.importedStatePlanner?.() ?? null; + if (options.isForkChild && !childImportedStatePlanner) { + throw new Error( + `${request.name}: child activation ${activationId} has no ` + + "pre-instantiation imported-state plan", + ); + } + let resolvedImports = imports; + if (childImportedStatePlanner) { + resolvedImports = childImportedStatePlanner.importsForActivation( + activationId, + imports as unknown as ForkWasmImports, + ) as unknown as WebAssembly.Imports; + } + if (!options.importedStateCapture) return resolvedImports; + // WHY: a fresh child must become a parent-capable owner after replay. + // Plan the copied identities first, then observe the exact Global and + // Table objects WebAssembly binds so a later fork can publish fresh + // provenance instead of depending on its parent's consumed arena. + importedStatePreparation = + options.importedStateCapture.prepareActivation( + activationId, + request.module, + resolvedImports, + ); + return importedStatePreparation.imports as unknown as WebAssembly.Imports; + }, + register(instance) { + if (released || registered || !prepared) { + throw new Error( + `${request.name}: side-module activation ${activationId} ` + + "cannot be registered in its current state", + ); + } + if (options.importedStateCapture) { + if (!importedStatePreparation) { + throw new Error( + `${request.name}: activation ${activationId} did not wrap its final imports`, + ); + } + importedStatePreparation.complete(instance); + importedStateRegistered = true; + } + if (options.isForkChild && !childImportedStatePlanner) { + throw new Error( + `${request.name}: child activation ${activationId} did not wrap its final imports`, + ); + } + exceptionProvider = forkExceptionProviderFromInstance( + activationId, + instance, + ); + const typedReferenceProvider = forkGcCodecProviderFromInstance( + activationId, + request.module, + instance, + ); + const registration = forkActivationRegistrationFromInstance({ + activationId, + module: request.module, + instance, + templateId, + exceptionProvider, + typedReferenceProvider, + }); + options.coordinator.registerActivation( + registration, + forkResumeTargetsFromInstance(request.module, instance), + ); + registered = true; + prepared = false; + childImportedStatePlanner?.registerInstance(activationId, instance); + options.registerChildReferenceActivation?.( + activationId, + request.module, + registration, + typedReferenceProvider, + ); + options.importedStateCapture?.bindTableDirtyTrackers( + new Map( + options.registry + .activations() + .map((activation) => [ + activation.activationId, + activation.tableDirty, + ]), + ), + ); + if ( + options.isPthreadReplica && + request.replayActivationId !== undefined + ) { + const threadBootstrap = + instance.exports[WPK_FORK_EXPORT_MODULE_THREAD_BOOTSTRAP]; + if (typeof threadBootstrap !== "function") { + throw new Error( + `${request.name}: pthread replica is missing its table bootstrap`, + ); + } + // WHY: this Worker needs fresh instance-local element functions, + // but process linear memory and constructors are already live. + // The thread helper initializes only tables and drops data + // segments; the parent bootstrap would re-run side effects. + threadBootstrap(); + } + }, + unregister() { + if (released) { + throw new Error( + `${request.name}: side-module activation ${activationId} was released twice`, + ); + } + released = true; + let failure: unknown; + try { + if (registered) { + try { + options.coordinator.unregisterActivation(activationId); + } catch (error) { + failure = error; + } + } else if (prepared) { + try { + options.coordinator.discardPreparedActivation(activationId); + exceptionProvider?.abort(); + } catch (error) { + failure = error; + } + } + if (importedStateRegistered) { + try { + options.importedStateCapture!.unregisterActivation( + activationId, + ); + } catch (error) { + failure ??= error; + } + } else if (importedStatePreparation) { + try { + importedStatePreparation.abort(); + } catch (error) { + failure ??= error; + } + } + } finally { + registered = false; + prepared = false; + exceptionProvider = null; + importedStatePreparation = null; + childImportedStatePlanner = null; + importedStateRegistered = false; + importsWrapped = false; + } + if (failure !== undefined) throw failure; + }, + }; + }, + }; } +/** + * Wasm-owned codecs for reference hierarchies that cannot appear in a + * JavaScript function signature. + * + * These are intentionally dependencies, not optional fallbacks. The + * activation provider registry must resolve them before instantiating a module + * that imports the corresponding ABI hook. + */ const MAX_SAFE_BIGINT = BigInt(Number.MAX_SAFE_INTEGER); -function checkedWasmByteLength(value: number | bigint, context: string): number { +function checkedWasmByteLength( + value: number | bigint, + context: string, +): number { if (typeof value === "number" && !Number.isSafeInteger(value)) { - throw new RangeError(`${context}: length is not an exact non-negative JavaScript integer`); + throw new RangeError( + `${context}: length is not an exact non-negative JavaScript integer`, + ); } const exact = typeof value === "bigint" ? value : BigInt(value); if (exact < 0n || exact > MAX_SAFE_BIGINT) { - throw new RangeError(`${context}: length is not an exact non-negative JavaScript integer`); + throw new RangeError( + `${context}: length is not an exact non-negative JavaScript integer`, + ); } return Number(exact); } @@ -318,38 +872,6 @@ function checkedWasmMemoryRange( return { offset, length }; } -/** - * Thread workers instantiate a separate Wasm module/table/tag graph, so they - * cannot safely load or invoke process side modules. Keep dlopen's ordinary C - * failure contract (NULL plus dlerror text) instead of letting a generic - * unresolved-import stub trap the pthread. - */ -function buildUnsupportedThreadDlopenImports( - memory: WebAssembly.Memory, -): Record { - const message = new TextEncoder().encode( - "dlopen is unsupported from pthread workers; load side modules on the process main worker", - ); - const n = (value: number | bigint): number => - typeof value === "bigint" ? Number(value) : value; - return { - __wasm_dlopen: (): number => 0, - __wasm_dlsym: (): number => 0, - __wasm_dlclose: (): number => -1, - __wasm_dlerror: (bufPtr: number | bigint, bufMax: number | bigint): number => { - const ptr = n(bufPtr); - const max = n(bufMax); - if (!Number.isSafeInteger(ptr) || !Number.isSafeInteger(max) || ptr < 0 || max <= 0) { - return 0; - } - const len = Math.min(message.length, max, memory.buffer.byteLength - ptr); - if (len <= 0) return 0; - new Uint8Array(memory.buffer, ptr, len).set(message.subarray(0, len)); - return len; - }, - }; -} - /** * Build dlopen host imports for a process. These are called directly from * the user program's dlopen/dlsym/dlclose C stubs (libc/glue/dlopen.c). @@ -375,66 +897,185 @@ export function buildDlopenImports( ptrWidth: 4 | 8, longjmpTag: WebAssembly.Tag | undefined, cppExceptionTag: WebAssembly.Tag | undefined, - mainHasDylinkForkRole: boolean, - beginMainForkAbort?: (errno: number) => void, + forkActivationOwner?: DylinkForkActivationOwner, + forkActivationOwnerUnavailableReason?: string, + forkUnwindTag?: WebAssembly.Tag, + onTableMutation?: ( + table: WebAssembly.Table, + firstIndex: number, + length: number, + ) => void, + hostImportRuntime?: ForkHostImportWorkerRuntime, + workerIdentity = 1, ): DlopenSupport { + if ( + !Number.isInteger(workerIdentity) || + workerIdentity <= 0 || + workerIdentity > 0x7fff_ffff + ) { + throw new RangeError( + `invalid dynamic-loader Worker identity ${String(workerIdentity)}`, + ); + } let linker: DynamicLinker | null = null; const loadedLibraries = new Map(); - let activeSideFork: SideModuleForkState | null = null; const decoder = new TextDecoder(); const encoder = new TextEncoder(); - const n = (v: number | bigint): number => typeof v === "bigint" ? Number(v) : v; - - const headOffset = ptrWidth === 8 ? DLOPEN_HEAD_OFFSET_WASM64 : DLOPEN_HEAD_OFFSET_WASM32; - const sideForkOffset = ptrWidth === 8 - ? DLOPEN_ACTIVE_SIDE_FORK_OFFSET_WASM64 - : DLOPEN_ACTIVE_SIDE_FORK_OFFSET_WASM32; - const lockOffset = ptrWidth === 8 - ? DLOPEN_LOCK_OFFSET_WASM64 - : DLOPEN_LOCK_OFFSET_WASM32; + const n = (v: number | bigint): number => + typeof v === "bigint" ? Number(v) : v; + const resolvedLibraryPaths = new Map(); + + const headOffset = + ptrWidth === 8 ? DLOPEN_HEAD_OFFSET_WASM64 : DLOPEN_HEAD_OFFSET_WASM32; + const lockOffset = + ptrWidth === 8 ? DLOPEN_LOCK_OFFSET_WASM64 : DLOPEN_LOCK_OFFSET_WASM32; + const generationOffset = + ptrWidth === 8 + ? DLOPEN_GENERATION_OFFSET_WASM64 + : DLOPEN_GENERATION_OFFSET_WASM32; + const ownerOffset = + ptrWidth === 8 ? DLOPEN_OWNER_OFFSET_WASM64 : DLOPEN_OWNER_OFFSET_WASM32; const headSlot = archiveControlAddr - headOffset; - const activeSideForkSlot = archiveControlAddr - sideForkOffset; - const archiveLock = new Int32Array(memory.buffer, archiveControlAddr - lockOffset, 1); - const entrySize = ptrWidth === 8 ? DLOPEN_ENTRY_SIZE_WASM64 : DLOPEN_ENTRY_SIZE_WASM32; - - const readPtr = (view: DataView, addr: number): number => - ptrWidth === 8 ? Number(view.getBigUint64(addr, true)) : view.getUint32(addr, true); - const writePtr = (view: DataView, addr: number, value: number): void => { - if (ptrWidth === 8) view.setBigUint64(addr, BigInt(value), true); - else view.setUint32(addr, value, true); + const archiveLock = new Int32Array( + memory.buffer, + archiveControlAddr - lockOffset, + 1, + ); + const loaderOwner = new Int32Array( + memory.buffer, + archiveControlAddr - ownerOffset, + 1, + ); + const generationSlot = archiveControlAddr - generationOffset; + const readGenerationFence = (): number => { + const value = + typeof SharedArrayBuffer !== "undefined" && + memory.buffer instanceof SharedArrayBuffer + ? Atomics.load(new BigUint64Array(memory.buffer, generationSlot, 1), 0) + : new DataView(memory.buffer).getBigUint64(generationSlot, true); + if (value > BigInt(Number.MAX_SAFE_INTEGER)) { + throw new RangeError( + "dlopen process generation exceeds exact host integers", + ); + } + return Number(value); + }; + const writeGenerationFence = (generation: number): void => { + if (!Number.isSafeInteger(generation) || generation <= 0) { + throw new RangeError( + `invalid dlopen process generation ${String(generation)}`, + ); + } + if ( + typeof SharedArrayBuffer !== "undefined" && + memory.buffer instanceof SharedArrayBuffer + ) { + Atomics.store( + new BigUint64Array(memory.buffer, generationSlot, 1), + 0, + BigInt(generation), + ); + } else { + new DataView(memory.buffer).setBigUint64( + generationSlot, + BigInt(generation), + true, + ); + } }; - const readArchiveHead = (): number => ptrWidth === 8 - ? Number(Atomics.load(new BigUint64Array(memory.buffer, headSlot, 1), 0)) - : Atomics.load(new Uint32Array(memory.buffer, headSlot, 1), 0); + const readArchiveHead = (): number => + ptrWidth === 8 + ? Number(Atomics.load(new BigUint64Array(memory.buffer, headSlot, 1), 0)) + : Atomics.load(new Uint32Array(memory.buffer, headSlot, 1), 0); const writeArchiveHead = (value: number): void => { if (ptrWidth === 8) { - Atomics.store(new BigUint64Array(memory.buffer, headSlot, 1), 0, BigInt(value)); + Atomics.store( + new BigUint64Array(memory.buffer, headSlot, 1), + 0, + BigInt(value), + ); } else { Atomics.store(new Uint32Array(memory.buffer, headSlot, 1), 0, value); } }; - const linkerAllocations = new Map(); - const archiveEntries = new Map(); + const linkerAllocations = new Map< + number, + { rawAddr: number; length: number } + >(); let hostDlopenError: string | null = null; let mainDlopenDepth = 0; - const acquireMainDlopenLock = (): boolean => { - if (mainDlopenDepth > 0) { - mainDlopenDepth++; - return true; + let mainArchiveReaderDepth = 0; + const ownedDlopenTransactions = new Set(); + let tableMutationPending = false; + let commitObserver: + | (( + linkerPublication: DylinkForkArchiveSnapshot | undefined, + tableMutationCommitted: boolean, + ) => void) + | null = null; + let writerAcquireObserver: (() => void) | null = null; + let operationAbortObserver: (() => void) | null = null; + const finishFreshWriterAcquisition = (): void => { + mainDlopenDepth = 1; + try { + writerAcquireObserver?.(); + } catch (error) { + releaseMainDlopenLock(); + throw error; } + }; + const foreignLoaderOwner = (): number => { + const owner = Atomics.load(loaderOwner, 0); + return owner !== DLOPEN_OWNER_IDLE && owner !== workerIdentity + ? owner + : DLOPEN_OWNER_IDLE; + }; + const releaseRawWriterLock = (): void => { const owner = Atomics.compareExchange( archiveLock, 0, - DLOPEN_LOCK_IDLE, DLOPEN_LOCK_WRITER, + DLOPEN_LOCK_IDLE, ); - if (owner !== 0) { - hostDlopenError = owner > 0 - ? "dlopen is temporarily unavailable while pthreads are forking" - : "dlopen is temporarily unavailable while another dlopen operation owns the process lock"; - return false; + if (owner !== DLOPEN_LOCK_WRITER) { + throw new Error( + `dlopen process lock lost writer ownership (state=${owner})`, + ); } - mainDlopenDepth = 1; + Atomics.notify(archiveLock, 0); + }; + const claimLoaderOwnership = (): void => { + if (mainDlopenDepth <= 0) { + throw new Error("dynamic-loader ownership requires the archive writer"); + } + const owner = Atomics.compareExchange( + loaderOwner, + 0, + DLOPEN_OWNER_IDLE, + workerIdentity, + ); + if (owner !== DLOPEN_OWNER_IDLE && owner !== workerIdentity) { + throw new Error(`dynamic-loader ownership belongs to Worker ${owner}`); + } + }; + const releaseLoaderOwnershipIfIdle = (): void => { + if (ownedDlopenTransactions.size !== 0) return; + const owner = Atomics.compareExchange( + loaderOwner, + 0, + workerIdentity, + DLOPEN_OWNER_IDLE, + ); + if (owner !== workerIdentity && owner !== DLOPEN_OWNER_IDLE) { + throw new Error(`dynamic-loader ownership changed to Worker ${owner}`); + } + Atomics.notify(loaderOwner, 0); + }; + const acquireMainDlopenLock = (): boolean => { + // POSIX loader serialization is blocking. Imports run in process Workers, + // so Atomics.wait can suspend only the contending pthread while the owner + // continues its staged guest initializer in a different Worker. + acquireArchiveWriter(); return true; }; const releaseMainDlopenLock = (): void => { @@ -443,85 +1084,491 @@ export function buildDlopenImports( } mainDlopenDepth--; if (mainDlopenDepth === 0) { + releaseRawWriterLock(); + } + }; + const withArchiveWriter = (operation: () => T): T => { + acquireArchiveWriter(); + try { + return operation(); + } finally { + releaseMainDlopenLock(); + } + }; + const acquireArchiveWriter = (): void => { + if (mainArchiveReaderDepth > 0) { + throw new Error( + "cannot acquire the process archive writer while owning a reader", + ); + } + if (mainDlopenDepth > 0) { + mainDlopenDepth++; + return; + } + for (;;) { + const transactionOwner = foreignLoaderOwner(); + if (transactionOwner !== DLOPEN_OWNER_IDLE) { + Atomics.wait(loaderOwner, 0, transactionOwner); + continue; + } const owner = Atomics.compareExchange( archiveLock, 0, - DLOPEN_LOCK_WRITER, DLOPEN_LOCK_IDLE, + DLOPEN_LOCK_WRITER, ); - if (owner !== DLOPEN_LOCK_WRITER) { + if (owner === DLOPEN_LOCK_IDLE) { + const racedTransactionOwner = foreignLoaderOwner(); + if (racedTransactionOwner === DLOPEN_OWNER_IDLE) break; + releaseRawWriterLock(); + Atomics.wait(loaderOwner, 0, racedTransactionOwner); + continue; + } + Atomics.wait(archiveLock, 0, owner); + } + finishFreshWriterAcquisition(); + }; + const acquireArchiveReader = (): void => { + if (mainDlopenDepth > 0) { + throw new Error( + "cannot acquire a process archive reader while owning its writer", + ); + } + for (;;) { + const transactionOwner = foreignLoaderOwner(); + if (transactionOwner !== DLOPEN_OWNER_IDLE) { + // POSIX fork preserves only its calling thread. Waiting here prevents + // a child from inheriting another thread's half-executed constructor, + // whose Wasm continuation cannot exist in the child. + Atomics.wait(loaderOwner, 0, transactionOwner); + continue; + } + const owner = Atomics.load(archiveLock, 0); + if (owner < 0) { + Atomics.wait(archiveLock, 0, owner); + continue; + } + if (owner >= DLOPEN_LOCK_MAX_READERS) { + throw new RangeError("dlopen process archive reader count exhausted"); + } + if (Atomics.compareExchange(archiveLock, 0, owner, owner + 1) !== owner) { + continue; + } + mainArchiveReaderDepth++; + return; + } + }; + const releaseArchiveReader = (): void => { + if (mainArchiveReaderDepth <= 0) { + throw new Error( + "dlopen process archive reader released without ownership", + ); + } + for (;;) { + const owner = Atomics.load(archiveLock, 0); + if (owner <= DLOPEN_LOCK_IDLE) { throw new Error( - `dlopen process lock lost writer ownership (state=${owner})`, + `dlopen process archive reader lost ownership (state=${owner})`, ); } - Atomics.notify(archiveLock, 0); + if (Atomics.compareExchange(archiveLock, 0, owner, owner - 1) !== owner) { + continue; + } + mainArchiveReaderDepth--; + if (owner === 1) Atomics.notify(archiveLock, 0); + return; + } + }; + const withArchiveReader = (operation: () => T): T => { + acquireArchiveReader(); + try { + return operation(); + } finally { + releaseArchiveReader(); } }; + const notifyCommit = ( + publication: DylinkForkArchiveSnapshot | undefined, + ): void => { + const mutated = tableMutationPending; + tableMutationPending = false; + commitObserver?.(publication, mutated); + }; + const abortLinkerOperation = (): void => { + tableMutationPending = false; + operationAbortObserver?.(); + }; - // The kernel mmap allocator. Shared with the linker, but also used - // directly by persistArchiveEntry to obtain blocks for the archive. - const allocateMemory = (size: number, align: number): number => { - const requested = size + Math.max(align, 1) - 1; + const invokeChannelSyscall = ( + syscall: number, + args: readonly (number | bigint)[], + ): { result: number; errno: number } => { + const view = new DataView(memory.buffer); + const base = channelOffset; + view.setInt32(base + CH_SYSCALL, syscall, true); + for (let i = 0; i < 6; i++) { + view.setBigInt64( + base + CH_ARGS + i * CH_ARG_SIZE, + BigInt(args[i] ?? 0), + true, + ); + } + markDeferredSignalDelivery(view, base); + const i32 = new Int32Array(memory.buffer); + Atomics.store(i32, (base + CH_STATUS) / 4, CHANNEL_STATUS_PENDING); + Atomics.notify(i32, (base + CH_STATUS) / 4, 1); + while ( + Atomics.wait(i32, (base + CH_STATUS) / 4, CHANNEL_STATUS_PENDING) === "ok" + ) { + /* wait for the kernel Worker */ + } + const result = Number(view.getBigInt64(base + CH_RETURN, true)); + const errno = view.getUint32(base + CH_ERRNO, true); + clearDeferredSignalDelivery(view, base); + Atomics.store(i32, (base + CH_STATUS) / 4, CHANNEL_STATUS_IDLE); + return { result, errno }; + }; + + // The kernel mmap allocator. Shared with the linker, but also used + // directly by persistArchiveEntry to obtain blocks for the archive. + const allocateMemory = (size: number, align: number): number => { + const requested = size + Math.max(align, 1) - 1; const view = new DataView(memory.buffer); const base = channelOffset; view.setInt32(base + CH_SYSCALL, SYS_MMAP_NR, true); view.setBigInt64(base + CH_ARGS + 0 * CH_ARG_SIZE, 0n, true); view.setBigInt64(base + CH_ARGS + 1 * CH_ARG_SIZE, BigInt(requested), true); - view.setBigInt64(base + CH_ARGS + 2 * CH_ARG_SIZE, BigInt(PROT_READ_WRITE), true); - view.setBigInt64(base + CH_ARGS + 3 * CH_ARG_SIZE, BigInt(MAP_PRIVATE_ANONYMOUS), true); + view.setBigInt64( + base + CH_ARGS + 2 * CH_ARG_SIZE, + BigInt(PROT_READ_WRITE), + true, + ); + view.setBigInt64( + base + CH_ARGS + 3 * CH_ARG_SIZE, + BigInt(MAP_PRIVATE_ANONYMOUS), + true, + ); view.setBigInt64(base + CH_ARGS + 4 * CH_ARG_SIZE, -1n, true); view.setBigInt64(base + CH_ARGS + 5 * CH_ARG_SIZE, 0n, true); + markDeferredSignalDelivery(view, base); const i32 = new Int32Array(memory.buffer); Atomics.store(i32, (base + CH_STATUS) / 4, CHANNEL_STATUS_PENDING); Atomics.notify(i32, (base + CH_STATUS) / 4, 1); - while (Atomics.wait(i32, (base + CH_STATUS) / 4, CHANNEL_STATUS_PENDING) === "ok") { /* wait for mmap */ } + while ( + Atomics.wait(i32, (base + CH_STATUS) / 4, CHANNEL_STATUS_PENDING) === "ok" + ) { + /* wait for mmap */ + } const result = Number(view.getBigInt64(base + CH_RETURN, true)); const err = view.getUint32(base + CH_ERRNO, true); + clearDeferredSignalDelivery(view, base); Atomics.store(i32, (base + CH_STATUS) / 4, CHANNEL_STATUS_IDLE); if (err || result < 0) { - throw new Error(`dlopen: mmap(${requested}) failed errno=${err || -result}`); + throw new Error( + `dlopen: mmap(${requested}) failed errno=${err || -result}`, + ); } const aligned = alignUp(n(result), Math.max(align, 1)); linkerAllocations.set(aligned, { rawAddr: n(result), length: requested }); return aligned; }; - const deallocateMemory = (addr: number, _size: number): void => { + const deallocateMemory = ( + addr: number, + size: number, + allowCopiedArchiveAllocation = false, + ): void => { const allocation = linkerAllocations.get(addr); - if (!allocation) { - throw new Error(`dlopen rollback: unknown allocation 0x${addr.toString(16)}`); + if (!allocation && !allowCopiedArchiveAllocation) { + throw new Error( + `dlopen rollback: unknown allocation 0x${addr.toString(16)}`, + ); + } + const rawAddr = allocation?.rawAddr ?? addr; + const length = allocation?.length ?? size; + if ( + !Number.isSafeInteger(rawAddr) || + rawAddr <= 0 || + !Number.isSafeInteger(length) || + length <= 0 || + rawAddr > memory.buffer.byteLength - length + ) { + throw new Error("dlopen archive release names an invalid copied mapping"); } const view = new DataView(memory.buffer); const base = channelOffset; view.setInt32(base + CH_SYSCALL, ABI_SYSCALLS.Munmap, true); - view.setBigInt64(base + CH_ARGS + 0 * CH_ARG_SIZE, BigInt(allocation.rawAddr), true); - view.setBigInt64(base + CH_ARGS + 1 * CH_ARG_SIZE, BigInt(allocation.length), true); + view.setBigInt64(base + CH_ARGS + 0 * CH_ARG_SIZE, BigInt(rawAddr), true); + view.setBigInt64(base + CH_ARGS + 1 * CH_ARG_SIZE, BigInt(length), true); for (let i = 2; i < 6; i++) { view.setBigInt64(base + CH_ARGS + i * CH_ARG_SIZE, 0n, true); } + markDeferredSignalDelivery(view, base); const i32 = new Int32Array(memory.buffer); Atomics.store(i32, (base + CH_STATUS) / 4, CHANNEL_STATUS_PENDING); Atomics.notify(i32, (base + CH_STATUS) / 4, 1); - while (Atomics.wait(i32, (base + CH_STATUS) / 4, CHANNEL_STATUS_PENDING) === "ok") { /* wait */ } + while ( + Atomics.wait(i32, (base + CH_STATUS) / 4, CHANNEL_STATUS_PENDING) === "ok" + ) { + /* wait */ + } const result = Number(view.getBigInt64(base + CH_RETURN, true)); const err = view.getUint32(base + CH_ERRNO, true); + clearDeferredSignalDelivery(view, base); Atomics.store(i32, (base + CH_STATUS) / 4, CHANNEL_STATUS_IDLE); if (err || result < 0) { throw new Error(`dlopen rollback: munmap failed errno=${err || -result}`); } - linkerAllocations.delete(addr); + if (allocation) linkerAllocations.delete(addr); + }; + + const describeMemoryAllocation = ( + address: number, + size: number, + ): Readonly<{ mappingAddress: number; mappingSize: number }> => { + const allocation = linkerAllocations.get(address); + if (!allocation) { + throw new Error( + `dlopen: allocation 0x${address.toString(16)} has no mmap owner`, + ); + } + if ( + !Number.isSafeInteger(size) || + size <= 0 || + address > allocation.rawAddr + allocation.length - size + ) { + throw new RangeError("dlopen: logical allocation escapes its mmap owner"); + } + return { + mappingAddress: allocation.rawAddr, + mappingSize: allocation.length, + }; + }; + + const adoptMemoryAllocation = ( + allocation: Readonly<{ + address: number; + size: number; + mappingAddress: number; + mappingSize: number; + }>, + ): void => { + const existing = linkerAllocations.get(allocation.address); + if (existing) { + if ( + existing.rawAddr === allocation.mappingAddress && + existing.length === allocation.mappingSize + ) + return; + throw new Error( + `dlopen replay: allocation 0x${allocation.address.toString(16)} ` + + "has conflicting mmap ownership", + ); + } + if ( + !Number.isSafeInteger(allocation.mappingAddress) || + allocation.mappingAddress <= 0 || + !Number.isSafeInteger(allocation.mappingSize) || + allocation.mappingSize <= 0 || + allocation.address < allocation.mappingAddress || + allocation.address + allocation.size > + allocation.mappingAddress + allocation.mappingSize || + allocation.mappingAddress > + memory.buffer.byteLength - allocation.mappingSize + ) { + throw new RangeError("dlopen replay: invalid copied mmap ownership"); + } + linkerAllocations.set(allocation.address, { + rawAddr: allocation.mappingAddress, + length: allocation.mappingSize, + }); + }; + + const forgetMemoryAllocation = ( + allocation: Readonly<{ + address: number; + mappingAddress: number; + mappingSize: number; + }>, + ): void => { + const existing = linkerAllocations.get(allocation.address); + if (!existing) return; + if ( + existing.rawAddr !== allocation.mappingAddress || + existing.length !== allocation.mappingSize + ) { + throw new Error( + `dlopen replay: allocation 0x${allocation.address.toString(16)} ` + + "changed before peer unload", + ); + } + linkerAllocations.delete(allocation.address); + }; + + const readDependencyFile = (path: string): Uint8Array | null => { + if (path.includes("\0")) { + throw new Error("dlopen dependency path contains NUL"); + } + const pathBytes = encoder.encode(`${path}\0`); + const pathAddr = allocateMemory(pathBytes.length, 1); + let openResult: { result: number; errno: number } | undefined; + let pathFailure: unknown; + try { + new Uint8Array(memory.buffer, pathAddr, pathBytes.length).set(pathBytes); + openResult = invokeChannelSyscall(ABI_SYSCALLS.Openat, [ + -100, + pathAddr, + 0, + 0, + ]); + } catch (error) { + pathFailure = error; + } finally { + try { + deallocateMemory(pathAddr, pathBytes.length); + } catch (error) { + pathFailure ??= error; + } + } + if (pathFailure !== undefined) { + if (openResult && openResult.errno === 0 && openResult.result >= 0) { + try { + invokeChannelSyscall(ABI_SYSCALLS.Close, [openResult.result]); + } catch { + // Preserve the path-allocation failure. + } + } + throw pathFailure; + } + if (!openResult) { + throw new Error(`dlopen dependency open(${path}) returned no result`); + } + if (openResult.errno === 2 || openResult.errno === 20) return null; + if (openResult.errno || openResult.result < 0) { + throw new Error( + `dlopen dependency open(${path}) failed errno=` + + `${openResult.errno || -openResult.result}`, + ); + } + + const fd = openResult.result; + const chunkSize = 64 * 1024; + const maxBytes = 64 * 1024 * 1024; + let chunkAddr: number | undefined; + const chunks: Uint8Array[] = []; + let total = 0; + let failure: unknown; + try { + chunkAddr = allocateMemory(chunkSize, 16); + for (;;) { + const read = invokeChannelSyscall(ABI_SYSCALLS.Read, [ + fd, + chunkAddr, + chunkSize, + ]); + if (read.errno || read.result < 0) { + throw new Error( + `dlopen dependency read(${path}) failed errno=` + + `${read.errno || -read.result}`, + ); + } + if (read.result === 0) break; + if (read.result > chunkSize) { + throw new Error( + `dlopen dependency read(${path}) returned ${read.result} bytes`, + ); + } + total += read.result; + if (total > maxBytes) { + throw new Error( + `dlopen dependency ${path} exceeds ${maxBytes} bytes`, + ); + } + chunks.push( + new Uint8Array(new Uint8Array(memory.buffer, chunkAddr, read.result)), + ); + } + } catch (error) { + failure = error; + } finally { + try { + if (chunkAddr !== undefined) { + deallocateMemory(chunkAddr, chunkSize); + } + } catch (error) { + failure ??= error; + } + try { + const close = invokeChannelSyscall(ABI_SYSCALLS.Close, [fd]); + if ((close.errno || close.result < 0) && failure === undefined) { + failure = new Error( + `dlopen dependency close(${path}) failed errno=` + + `${close.errno || -close.result}`, + ); + } + } catch (error) { + failure ??= error; + } + } + if (failure !== undefined) throw failure; + + const bytes = new Uint8Array(total); + let offset = 0; + for (const chunk of chunks) { + bytes.set(chunk, offset); + offset += chunk.length; + } + return bytes; + }; + + const resolveLibrarySync = ( + dependency: string, + requester?: string, + ): Uint8Array | null => { + const candidates: string[] = []; + const addCandidate = (candidate: string): void => { + if (!candidates.includes(candidate)) candidates.push(candidate); + }; + if (dependency.startsWith("/")) { + addCandidate(dependency); + } else { + const requesterPath = + requester === undefined + ? undefined + : (resolvedLibraryPaths.get(requester) ?? requester); + const slash = requesterPath?.lastIndexOf("/") ?? -1; + if (requesterPath && slash >= 0) { + const directory = slash === 0 ? "/" : requesterPath.slice(0, slash); + addCandidate( + directory === "/" ? `/${dependency}` : `${directory}/${dependency}`, + ); + } + addCandidate(dependency); + addCandidate(`/lib/${dependency}`); + addCandidate(`/usr/lib/${dependency}`); + addCandidate(`/usr/local/lib/${dependency}`); + } + + for (const candidate of candidates) { + const bytes = readDependencyFile(candidate); + if (bytes === null) continue; + resolvedLibraryPaths.set(dependency, candidate); + return bytes; + } + return null; }; const getLinker = (): DynamicLinker => { if (linker) return linker; const table = getTable(); const sp = getStackPointer(); - if (!table || !sp) throw new Error("dlopen: program has no table or stack pointer"); + if (!table || !sp) + throw new Error("dlopen: program has no table or stack pointer"); // Register main program's exported functions and data globals as global // symbols so shared libraries can resolve references to libc, libphp, etc. @@ -530,97 +1577,43 @@ export function buildDlopenImports( // RESERVED names are handled per-module by the dylink env Proxy and must // not be shadowed by main exports. const RESERVED = new Set([ - "memory", "__indirect_function_table", - "__memory_base", "__table_base", "__stack_pointer", "__c_longjmp", + "memory", + "__indirect_function_table", + "__memory_base", + "__table_base", + "__stack_pointer", + "__c_longjmp", "__cpp_exception", + FORK_UNWIND_TAG_IMPORT_NAME, ]); const globalSymbols = new Map(); + const globalSymbolOwners = new Map(); const inst = getInstance(); if (inst) { for (const [name, exp] of Object.entries(inst.exports)) { if (RESERVED.has(name)) continue; if (typeof exp === "function" || exp instanceof WebAssembly.Global) { globalSymbols.set(name, exp); + globalSymbolOwners.set(name, undefined); } } } - const mainModuleSymbols = new Set(globalSymbols.keys()); // A main-defined/exported tag is the process ABI authority. If the main // image instead imports and re-exports the host tag, the identity is the // same; if it has no export, retain the process-owned fallback created // before main instantiation. Every side module must receive this one // canonical identity for cross-module exception propagation. const exportedLongjmpTag = inst?.exports.__c_longjmp; - const canonicalLongjmpTag = exportedLongjmpTag === undefined - ? longjmpTag - : requireLongjmpTag(exportedLongjmpTag, "main module export"); + const canonicalLongjmpTag = + exportedLongjmpTag === undefined + ? longjmpTag + : requireLongjmpTag(exportedLongjmpTag, "main module export"); const exportedCppExceptionTag = inst?.exports.__cpp_exception; - const canonicalCppExceptionTag = exportedCppExceptionTag === undefined - ? cppExceptionTag - : requireCppExceptionTag(exportedCppExceptionTag, "main module export"); - const mainFork = inst?.exports.fork; - const mainForkState = inst?.exports.wpk_fork_state; - const sideModuleFork = mainHasDylinkForkRole - && typeof mainFork === "function" - && typeof mainForkState === "function" - ? { - setActiveFork: (state: SideModuleForkState) => { - const persisted = readPtr(new DataView(memory.buffer), activeSideForkSlot); - if (activeSideFork || persisted !== 0) { - throw new Error( - `${state.name}: nested or concurrent side-module fork is unsupported`, - ); - } - activeSideFork = state; - const loaded = loadedLibraries.get(state.name); - if (!loaded || loaded.forkContinuation !== state.continuation) { - throw new Error(`${state.name}: linked continuation owner mismatch`); - } - loaded.forkBufAddr = state.forkBufAddr; - updateArchiveForkBuffer(state.name, state.forkBufAddr); - writePtr(new DataView(memory.buffer), activeSideForkSlot, state.forkBufAddr); - }, - clearActiveFork: (state: SideModuleForkState) => { - const view = new DataView(memory.buffer); - const persisted = readPtr(view, activeSideForkSlot); - if ( - !activeSideFork - || activeSideFork.name !== state.name - || activeSideFork.instance !== state.instance - || activeSideFork.forkBufAddr !== state.forkBufAddr - || persisted !== state.forkBufAddr - ) { - throw new Error(`${state.name}: stale side-module fork identity during rewind`); - } - activeSideFork = null; - const loaded = loadedLibraries.get(state.name); - if (loaded) loaded.forkBufAddr = undefined; - updateArchiveForkBuffer(state.name, 0); - writePtr(view, activeSideForkSlot, 0); - }, - invokeMainFork: (expectedStateAfter: 0 | 1 | readonly (0 | 1)[]): number => { - const result = Number((mainFork as () => number)()); - const actualState = Number((mainForkState as () => number)()); - const expectedStates = Array.isArray(expectedStateAfter) - ? expectedStateAfter - : [expectedStateAfter]; - if (!expectedStates.includes(actualState as 0 | 1)) { - throw new Error( - `main-module fork transition ended in state ${actualState}; ` + - `expected ${expectedStates.join(" or ")}`, - ); - } - return result; - }, - beginMainAbort: (errno: number): void => { - if (!beginMainForkAbort) { - throw new Error("main-module continuation abort coordinator is unavailable"); - } - beginMainForkAbort(errno); - }, - } - : undefined; + const canonicalCppExceptionTag = + exportedCppExceptionTag === undefined + ? cppExceptionTag + : requireCppExceptionTag(exportedCppExceptionTag, "main module export"); linker = new DynamicLinker({ memory, @@ -628,299 +1621,295 @@ export function buildDlopenImports( stackPointer: sp, allocateMemory, deallocateMemory, - allocateContinuation: (size) => continuationMmap( - memory, - channelOffset, - size, - "side-module continuation", - ), - deallocateContinuation: (addr, size) => continuationMunmap( - memory, - channelOffset, - addr, - size, - "side-module continuation", - ), + describeMemoryAllocation, + adoptMemoryAllocation, + forgetMemoryAllocation, globalSymbols, + globalSymbolOwners, got: new Map(), loadedLibraries, + resolveLibrarySync, longjmpTag: canonicalLongjmpTag, cppExceptionTag: canonicalCppExceptionTag, + forkUnwindTag, ptrWidth, - mainModuleSymbols, - sideModuleFork, - sideModuleForkUnavailableReason: !mainHasDylinkForkRole - ? "main module lacks the versioned dlopen-main fork capability; rebuild it with the current wasm-fork-instrument" - : sideModuleFork - ? undefined - : "main module does not export the fork trampoline and wpk_fork_state required for side-module fork", + forkActivationOwner, + forkActivationOwnerUnavailableReason, + onTableMutation: (table, firstIndex, length) => { + onTableMutation?.(table, firstIndex, length); + tableMutationPending = true; + }, + routeFunctionImport: hostImportRuntime + ? (imported, implementation) => + hostImportRuntime.routeFunction(imported, implementation) + : undefined, }); return linker; }; - // Append an entry to the linked-list archive in linear memory. Each - // entry is one mmap block: struct, then name UTF-8 (padded to 8-byte - // alignment), then the side-module wasm bytes. Pointers are absolute - // — fork's memcpy preserves the parent's address space. - const persistArchiveEntry = ( - name: string, - bytes: Uint8Array, - memoryBase: number, - tableBase: number, - sideForkBufAddr: number, - tlsBase: number, - ): void => { - const nameBytes = encoder.encode(name); - const nameLen = nameBytes.length; - const nameAligned = (nameLen + 7) & ~7; - const totalSize = entrySize + nameAligned + bytes.length; + const forkArchive = new DylinkForkArchive( + memory, + ptrWidth, + readArchiveHead, + writeArchiveHead, + (size) => ({ + address: allocateMemory(size, 1), + size, + }), + ({ address, size }) => { + deallocateMemory(address, size, true); + }, + "process dylink archive", + { + read: readGenerationFence, + write: writeGenerationFence, + }, + ); - const entry = allocateMemory(totalSize, 8); - archiveEntries.set(name, entry); - const namePtr = entry + entrySize; - const bytesPtr = namePtr + nameAligned; + const readForkState = (): DylinkForkState => forkArchive.read(); - const view = new DataView(memory.buffer); - if (ptrWidth === 8) { - view.setBigUint64(entry + 0, 0n, true); - view.setBigUint64(entry + 8, BigInt(namePtr), true); - view.setBigUint64(entry + 16, BigInt(nameLen), true); - view.setBigUint64(entry + 24, BigInt(bytesPtr), true); - view.setBigUint64(entry + 32, BigInt(bytes.length), true); - view.setBigUint64(entry + 40, BigInt(memoryBase), true); - view.setBigUint64(entry + 48, BigInt(tableBase), true); - view.setBigUint64(entry + 56, BigInt(sideForkBufAddr), true); - view.setBigUint64(entry + 64, BigInt(tlsBase), true); - } else { - view.setUint32(entry + 0, 0, true); - view.setUint32(entry + 4, namePtr, true); - view.setUint32(entry + 8, nameLen, true); - view.setUint32(entry + 12, bytesPtr, true); - view.setUint32(entry + 16, bytes.length, true); - view.setUint32(entry + 20, memoryBase, true); - view.setUint32(entry + 24, tableBase, true); - view.setUint32(entry + 28, sideForkBufAddr, true); - view.setUint32(entry + 32, tlsBase, true); - } - - new Uint8Array(memory.buffer, namePtr, nameLen).set(nameBytes); - new Uint8Array(memory.buffer, bytesPtr, bytes.length).set(bytes); - - // Append to tail (preserves insertion order). - const head = readArchiveHead(); - if (head === 0) { - // Publish only after the complete entry and payload are visible. A - // pthread fork acquire-loads this word before deciding whether it can - // safely fork without access to the process side-module graph. - writeArchiveHead(entry); + const replayDlopens = (validatedState?: DylinkForkState): void => { + const state = validatedState ?? readForkState(); + if ( + state.nextHandle === 2 && + state.libraries.length === 0 && + linker === null + ) return; - } - let cursor = head; - for (;;) { - const next = readPtr(view, cursor); - if (next === 0) { - writePtr(view, cursor, entry); - return; - } - cursor = next; - } - }; - - const updateArchiveForkBuffer = (name: string, forkBufAddr: number): void => { - const entry = archiveEntries.get(name); - if (entry === undefined) { - throw new Error(`${name}: missing dlopen archive entry for fork continuation`); - } - const view = new DataView(memory.buffer); - if (ptrWidth === 8) view.setBigUint64(entry + 56, BigInt(forkBufAddr), true); - else view.setUint32(entry + 28, forkBufAddr, true); - }; - - const replayDlopens = (): void => { - const view = new DataView(memory.buffer); - let cursor = readArchiveHead(); - if (cursor === 0) return; - // Force linker creation: it's lazily built on the first C-side - // __wasm_dlopen call, which the fork child hasn't made yet. We need - // it now to drive replay before _start resumes. + // Materialize only missing modules, then replace the Worker-local handle + // view. Pthread Workers can call this for every process generation. const lk = getLinker(); - - while (cursor !== 0) { - let next: number; - let namePtr: number; - let nameLen: number; - let bytesPtr: number; - let bytesLen: number; - let memoryBase: number; - let tableBase: number; - let sideForkBufAddr: number; - let tlsBase: number; - if (ptrWidth === 8) { - next = Number(view.getBigUint64(cursor + 0, true)); - namePtr = Number(view.getBigUint64(cursor + 8, true)); - nameLen = Number(view.getBigUint64(cursor + 16, true)); - bytesPtr = Number(view.getBigUint64(cursor + 24, true)); - bytesLen = Number(view.getBigUint64(cursor + 32, true)); - memoryBase = Number(view.getBigUint64(cursor + 40, true)); - tableBase = Number(view.getBigUint64(cursor + 48, true)); - sideForkBufAddr = Number(view.getBigUint64(cursor + 56, true)); - tlsBase = Number(view.getBigUint64(cursor + 64, true)); - } else { - next = view.getUint32(cursor + 0, true); - namePtr = view.getUint32(cursor + 4, true); - nameLen = view.getUint32(cursor + 8, true); - bytesPtr = view.getUint32(cursor + 12, true); - bytesLen = view.getUint32(cursor + 16, true); - memoryBase = view.getUint32(cursor + 20, true); - tableBase = view.getUint32(cursor + 24, true); - sideForkBufAddr = view.getUint32(cursor + 28, true); - tlsBase = view.getUint32(cursor + 32, true); - } - - // Copy name + bytes out of shared memory before passing to - // WebAssembly / TextDecoder — some engines reject SAB-backed - // views, and we already pay the bytes copy cost on the parent's - // initial dlopen path. - const name = decoder.decode( - new Uint8Array(new Uint8Array(memory.buffer, namePtr, nameLen)), - ); - archiveEntries.set(name, cursor); - const bytesCopy = new Uint8Array(new Uint8Array(memory.buffer, bytesPtr, bytesLen)); - - // DynamicLinker.dlopenSync returns 0 on error, >0 on success. - const handle = lk.dlopenSync(name, bytesCopy, { - memoryBase, - tableBase, - forkBufAddr: sideForkBufAddr || undefined, - tlsBase: tlsBase === 0 ? undefined : tlsBase, - }); - if (handle === 0) { - throw new Error(`dlopen(${name}): ${lk.dlerror() || "unknown"}`); - } - if (sideForkBufAddr !== 0) { - const loaded = loadedLibraries.get(name); - if (!loaded || loaded.forkBufAddr !== sideForkBufAddr) { - throw new Error(`${name}: fork replay restored a mismatched save buffer`); - } - } - if (tlsBase !== 0) { - const loaded = loadedLibraries.get(name); - if (!loaded || loaded.tlsBase !== tlsBase) { - throw new Error(`${name}: fork replay restored a mismatched TLS base`); - } - } - - cursor = next; + try { + lk.reconcileForkModules(state); + lk.reconcileForkHandleState(state); + } catch (error) { + abortLinkerOperation(); + throw error; + } finally { + // Replay materializes a publication already owned by the archive; its + // local table writes are not a new source mutation to republish. + tableMutationPending = false; } }; - const findActiveSideFork = (): SideModuleForkState | null => { - const persisted = readPtr(new DataView(memory.buffer), activeSideForkSlot); - if (persisted === 0) { - if (activeSideFork) { - throw new Error(`${activeSideFork.name}: active side fork lost its persisted identity`); - } - return null; - } - if (activeSideFork) { - if (activeSideFork.forkBufAddr !== persisted) { - throw new Error(`${activeSideFork.name}: active side fork buffer identity changed`); - } - return activeSideFork; - } - - const matches = Array.from(loadedLibraries.values()).filter( - (loaded) => loaded.forkBufAddr === persisted, - ); - if (matches.length !== 1) { - throw new Error( - `fork replay could not resolve active side-module buffer 0x${persisted.toString(16)}`, - ); + const resetForkChildLock = (): void => { + Atomics.store(archiveLock, 0, 0); + Atomics.notify(archiveLock, 0); + const copiedOwner = Atomics.load(loaderOwner, 0); + if (copiedOwner !== DLOPEN_OWNER_IDLE) { + // The loader continuation belongs to the one thread that survived fork. + // Rebind its copied process lease to the child's new Worker coordinate. + Atomics.store(loaderOwner, 0, workerIdentity); } - const loaded = matches[0]!; - activeSideFork = { - name: loaded.name, - instance: loaded.instance, - forkBufAddr: persisted, - continuation: loaded.forkContinuation!, - }; - return activeSideFork; + Atomics.notify(loaderOwner, 0); }; - const sideForkState = (state: SideModuleForkState): number => - Number((state.instance.exports.wpk_fork_state as () => number)()); - - const completeSideModuleForkUnwind = (): void => { - const state = findActiveSideFork(); - if (!state) return; - finalizeSideModuleForkUnwind(memory, state, ptrWidth); - }; - - const beginSideModuleForkRewind = (): void => { - const state = findActiveSideFork(); - if (!state) return; - if (sideForkState(state) !== 0) { - throw new Error(`${state.name}: expected NORMAL before side-module rewind`); - } - if (state.continuation.hasActiveContinuation()) { - state.continuation.beginReplay(); - } else { - // WHY: attachment validates the copied pointer through the same guest - // ABI boundary as frame callbacks, where memory64 i64 values are BigInt. - state.continuation.attachForReplay( - ptrWidth === 8 ? BigInt(state.forkBufAddr) : state.forkBufAddr, - ); - } - invokeForkContinuationBegin( - state.instance.exports.wpk_fork_rewind_begin, - state.forkBufAddr, + const readDlopenRequest = ( + bytesPtr: WasmGuestPointer, + bytesLen: number | bigint, + namePtr: WasmGuestPointer, + nameLen: number | bigint, + ): { readonly name: string; readonly bytes: Uint8Array } => { + const bytesRange = checkedWasmMemoryRange( + memory, + bytesPtr, + bytesLen, ptrWidth, - `${state.name}: side-module linked fork rewind`, + "__wasm_dlopen_prepare bytes", ); - if (sideForkState(state) !== 2) { - throw new Error(`${state.name}: side-module rewind did not enter REWINDING`); - } - }; - - const beginSideModuleForkAbort = (errno: number): void => { - const state = findActiveSideFork(); - if (!state) return; - if (sideForkState(state) !== 1) { - throw new Error(`${state.name}: expected UNWINDING before side-module abort replay`); - } - state.continuation.beginAbortReplay(errno); - invokeForkContinuationBegin( - state.instance.exports.wpk_fork_abort_begin, - state.forkBufAddr, + const nameRange = checkedWasmMemoryRange( + memory, + namePtr, + nameLen, ptrWidth, - `${state.name}: side-module linked fork abort`, + "__wasm_dlopen_prepare name", ); - if (sideForkState(state) !== 3) { - throw new Error(`${state.name}: side-module abort did not enter ABORT_UNWINDING`); - } + const bytes = new Uint8Array( + memory.buffer, + bytesRange.offset, + bytesRange.length, + ); + const nameBytes = new Uint8Array( + memory.buffer, + nameRange.offset, + nameRange.length, + ); + // WHY: compilation may grow/detach memory, and Firefox/Chrome reject + // TextDecoder views backed directly by SharedArrayBuffer. + return { + // The first Kandelo dlopen import carried only (bytes, length) and + // historically keyed the module as `dlopen::`. ABI 43's + // lowering supplies an empty name range for that exact form. + name: nameRange.length === 0 && bytesRange.length !== 0 + ? `dlopen:${bytesRange.offset}:${bytesRange.length}` + : decoder.decode(new Uint8Array(nameBytes)), + bytes: new Uint8Array(bytes), + }; }; - const assertNoActiveSideModuleFork = (): void => { - const persisted = readPtr(new DataView(memory.buffer), activeSideForkSlot); - if (activeSideFork || persisted !== 0) { - throw new Error( - `${activeSideFork?.name ?? "unknown side module"}: main image returned with an active side-module fork`, - ); - } - }; + const imports: Record = { + __wasm_dlopen_main: (): number => { + if (!acquireMainDlopenLock()) return 0; + hostDlopenError = null; + try { + return getLinker().dlopenMain(); + } finally { + releaseMainDlopenLock(); + } + }, - const resetForkChildLock = (): void => { - Atomics.store(archiveLock, 0, 0); - Atomics.notify(archiveLock, 0); - }; + __wasm_dlopen_prepare: ( + bytesPtr: WasmGuestPointer, + bytesLen: number | bigint, + namePtr: WasmGuestPointer, + nameLen: number | bigint, + flags: number, + ): number => { + if (!acquireMainDlopenLock()) return 0; + hostDlopenError = null; + let claimedLoader = false; + try { + if (!Number.isInteger(flags)) { + throw new Error( + "__wasm_dlopen_prepare requires ABI 43 dlopen flags; rebuild the process", + ); + } + if (ownedDlopenTransactions.size === 0) { + claimLoaderOwnership(); + claimedLoader = true; + } + const request = readDlopenRequest(bytesPtr, bytesLen, namePtr, nameLen); + const transaction = getLinker().beginDlopenSync( + request.name, + request.bytes, + (flags & RTLD_GLOBAL) !== 0, + ); + if (transaction > 0) { + ownedDlopenTransactions.add(transaction); + } else if (claimedLoader) { + releaseLoaderOwnershipIfIdle(); + } + return transaction; + } catch (error) { + if (claimedLoader) releaseLoaderOwnershipIfIdle(); + abortLinkerOperation(); + throw error; + } finally { + releaseMainDlopenLock(); + } + }, - const imports: Record = { - __wasm_dlopen: (bytesPtr: WasmGuestPointer, bytesLen: number | bigint, - namePtr: WasmGuestPointer, nameLen: number | bigint): number => { + __wasm_dlopen_next: ( + transaction: number, + handlePtr?: WasmGuestPointer, + ): number => { + if (!acquireMainDlopenLock()) return -1; + hostDlopenError = null; + try { + const linker = getLinker(); + if ( + !ownedDlopenTransactions.has(transaction) && + Atomics.load(loaderOwner, 0) === workerIdentity && + linker.hasPendingDlopen(transaction) + ) { + // A fresh fork child reconstructed this token from the copied + // archive. Its loader lease was rebound before module replay. + ownedDlopenTransactions.add(transaction); + } + if (handlePtr === undefined) { + // Transitional standalone callers still use the explicit commit + // import. ABI-43 libc always supplies the output pointer and takes + // the atomic finish path below. + const entry = linker.nextDlopenInitialization(transaction); + if (entry !== 0) { + notifyCommit(forkArchive.sync(linker.forkState())); + } + if (entry < 0) { + ownedDlopenTransactions.delete(transaction); + releaseLoaderOwnershipIfIdle(); + } + return entry; + } + const handleRange = checkedWasmMemoryRange( + memory, + handlePtr, + 4, + ptrWidth, + "__wasm_dlopen_next handle", + ); + const { entry, handle } = linker.advanceDlopenSync(transaction); + new DataView(memory.buffer).setInt32(handleRange.offset, handle, true); + if (entry > 0) { + // Publish the exact provisional activation/stage before libc can + // enter it. A fork from that table call can therefore reconstruct + // both the fresh side instance and the stopped loader generator. + notifyCommit(forkArchive.sync(linker.forkState())); + } else { + // Completion opens the public handle and removes the private + // transaction in this same host transition. Rollback likewise + // removes the issued entry before control returns to Wasm. + notifyCommit(forkArchive.sync(linker.forkState())); + ownedDlopenTransactions.delete(transaction); + releaseLoaderOwnershipIfIdle(); + } + return entry; + } catch (error) { + getLinker().abortDlopenTransaction(transaction, error); + ownedDlopenTransactions.delete(transaction); + releaseLoaderOwnershipIfIdle(); + abortLinkerOperation(); + throw error; + } finally { + releaseMainDlopenLock(); + } + }, + + __wasm_dlopen_commit: (transaction: number): number => { + if (!acquireMainDlopenLock()) return 0; + hostDlopenError = null; + try { + const linker = getLinker(); + const handle = linker.commitDlopenSync(transaction); + notifyCommit(forkArchive.sync(linker.forkState())); + // WHY: commit deliberately returns zero without destroying a + // transaction whose initializer is still outstanding. Retaining the + // process lease keeps another pthread from interleaving loader state + // if arbitrary Wasm calls this transitional import too early. + if (!linker.hasPendingDlopen(transaction)) { + ownedDlopenTransactions.delete(transaction); + releaseLoaderOwnershipIfIdle(); + } + return handle; + } catch (error) { + getLinker().abortDlopenTransaction(transaction, error); + ownedDlopenTransactions.delete(transaction); + releaseLoaderOwnershipIfIdle(); + abortLinkerOperation(); + throw error; + } finally { + releaseMainDlopenLock(); + } + }, + + __wasm_dlopen: ( + bytesPtr: WasmGuestPointer, + bytesLen: number | bigint, + namePtr: WasmGuestPointer, + nameLen: number | bigint, + flags = RTLD_GLOBAL, + ): number => { if (!acquireMainDlopenLock()) return 0; hostDlopenError = null; + let claimedLoader = false; try { + if (!Number.isInteger(flags)) { + throw new Error("__wasm_dlopen received invalid dlopen flags"); + } + if (ownedDlopenTransactions.size === 0) { + claimLoaderOwnership(); + claimedLoader = true; + } const bytesRange = checkedWasmMemoryRange( memory, bytesPtr, @@ -942,7 +1931,11 @@ export function buildDlopenImports( return getLinker().dlopenMain(); } - const bytes = new Uint8Array(memory.buffer, bytesRange.offset, bytesRange.length); + const bytes = new Uint8Array( + memory.buffer, + bytesRange.offset, + bytesRange.length, + ); // Copy bytes since memory.buffer may detach during Wasm instantiation const bytesCopy = new Uint8Array(bytes); // TextDecoder.decode() rejects views backed by SharedArrayBuffer @@ -956,27 +1949,24 @@ export function buildDlopenImports( ); const nameBytesCopy = new Uint8Array(nameBytesView); const name = decoder.decode(nameBytesCopy); - const handle = getLinker().dlopenSync(name, bytesCopy); + const lk = getLinker(); + const handle = lk.dlopenSync( + name, + bytesCopy, + undefined, + (flags & RTLD_GLOBAL) !== 0, + ); if (handle > 0) { - // The linker just instantiated this — the map MUST contain it. - // A miss means the shared-map ref got rewired and replay would - // silently see an empty archive after fork; fail loudly here - // instead of corrupting the fork child later. - const loaded = loadedLibraries.get(name); - if (!loaded) { - throw new Error(`__wasm_dlopen(${name}): handle=${handle} but loadedLibraries lookup failed`); - } - persistArchiveEntry( - name, - bytesCopy, - loaded.memoryBase, - loaded.tableBase, - loaded.forkBufAddr ?? 0, - loaded.tlsBase ?? 0, - ); + notifyCommit(forkArchive.sync(lk.forkState())); + } else { + abortLinkerOperation(); } return handle; + } catch (error) { + abortLinkerOperation(); + throw error; } finally { + if (claimedLoader) releaseLoaderOwnershipIfIdle(); releaseMainDlopenLock(); } }, @@ -986,31 +1976,60 @@ export function buildDlopenImports( namePtr: WasmGuestPointer, nameLen: number | bigint, ): number => { - // See __wasm_dlopen above: copy off the shared buffer before - // TextDecoder.decode() touches it. - const nameRange = checkedWasmMemoryRange( - memory, - namePtr, - nameLen, - ptrWidth, - "__wasm_dlsym name", - ); - const nameBytesView = new Uint8Array( - memory.buffer, - nameRange.offset, - nameRange.length, - ); - const nameBytesCopy = new Uint8Array(nameBytesView); - const name = decoder.decode(nameBytesCopy); - const result = getLinker().dlsym(handle, name); - return result === null ? 0 : (result as number); + if (!acquireMainDlopenLock()) return 0; + hostDlopenError = null; + try { + // See __wasm_dlopen above: copy off the shared buffer before + // TextDecoder.decode() touches it. + const nameRange = checkedWasmMemoryRange( + memory, + namePtr, + nameLen, + ptrWidth, + "__wasm_dlsym name", + ); + const nameBytesView = new Uint8Array( + memory.buffer, + nameRange.offset, + nameRange.length, + ); + const nameBytesCopy = new Uint8Array(nameBytesView); + const name = decoder.decode(nameBytesCopy); + const result = getLinker().dlsym(handle, name); + notifyCommit(undefined); + return result === null ? 0 : (result as number); + } catch (error) { + abortLinkerOperation(); + throw error; + } finally { + releaseMainDlopenLock(); + } }, __wasm_dlclose: (handle: number): number => { - return getLinker().dlclose(handle); + if (!acquireMainDlopenLock()) return -1; + hostDlopenError = null; + try { + const lk = getLinker(); + const result = lk.dlclose(handle); + if (result === 0) { + notifyCommit(forkArchive.sync(lk.forkState())); + } else { + abortLinkerOperation(); + } + return result; + } catch (error) { + abortLinkerOperation(); + throw error; + } finally { + releaseMainDlopenLock(); + } }, - __wasm_dlerror: (bufPtr: WasmGuestPointer, bufMax: number | bigint): number => { + __wasm_dlerror: ( + bufPtr: WasmGuestPointer, + bufMax: number | bigint, + ): number => { const err = hostDlopenError ?? getLinker().dlerror(); hostDlopenError = null; if (!err) return 0; @@ -1023,20 +2042,35 @@ export function buildDlopenImports( ptrWidth, "__wasm_dlerror buffer", ); - new Uint8Array(memory.buffer, range.offset, range.length) - .set(encoded.subarray(0, range.length)); + new Uint8Array(memory.buffer, range.offset, range.length).set( + encoded.subarray(0, range.length), + ); return range.length; }, }; return { imports, + readForkState, replayDlopens, - completeSideModuleForkUnwind, - beginSideModuleForkRewind, - beginSideModuleForkAbort, - assertNoActiveSideModuleFork, resetForkChildLock, + archive: forkArchive, + acquireArchiveWriter, + releaseArchiveWriter: releaseMainDlopenLock, + acquireArchiveReader, + releaseArchiveReader, + withArchiveWriter, + withArchiveReader, + writerOwned: () => mainDlopenDepth > 0, + setWriterAcquireObserver: (observer) => { + writerAcquireObserver = observer; + }, + setOperationAbortObserver: (observer) => { + operationAbortObserver = observer; + }, + setCommitObserver: (observer) => { + commitObserver = observer; + }, }; } @@ -1053,74 +2087,142 @@ function buildImportObject( ptrWidth: 4 | 8 = 4, longjmpTag?: WebAssembly.Tag, cppExceptionTag?: WebAssembly.Tag, + forkUnwindTag?: WebAssembly.Tag, postVmInterruptTimer?: ( timedOutPtr: number, vmInterruptPtr: number, seconds: number, ) => void, - forkContinuation?: LinkedForkContinuation, - onContinuationAbort?: () => void, + forkEnvImports?: Record, ): WebAssembly.Imports { const envImports: Record = { memory }; /** Convert wasm64 BigInt pointer to number (safe since addresses < 4GB) */ - const n = (v: number | bigint): number => typeof v === "bigint" ? Number(v) : v; + const n = (v: number | bigint): number => + typeof v === "bigint" ? Number(v) : v; /** Wrap a number as the correct return type for pointer-returning imports */ - const retPtr = (v: number): number | bigint => ptrWidth === 8 ? BigInt(v) : v; + const retPtr = (v: number): number | bigint => + ptrWidth === 8 ? BigInt(v) : v; // Provide __channel_base as a mutable wasm global if the module imports it. // Each instance gets its own global, immune to cross-thread shared memory corruption. // On wasm64, __channel_base is i64 (BigInt); on wasm32 it's i32 (number). const moduleImports = WebAssembly.Module.imports(module); - const importsFunction = (name: string): boolean => moduleImports.some( - (i) => i.module === "env" && i.name === name && i.kind === "function", - ); + const importsFunction = (name: string): boolean => + moduleImports.some( + (i) => i.module === "env" && i.name === name && i.kind === "function", + ); const linkedFrameImports = WPK_FORK_REQUIRED_IMPORTS.filter( ({ module }) => module === "env", ); - const linkedFrameImportCount = linkedFrameImports.filter( - ({ name }) => importsFunction(name), + const linkedFrameImportCount = linkedFrameImports.filter(({ name }) => + importsFunction(name), ).length; - if (linkedFrameImportCount !== 0 && linkedFrameImportCount !== linkedFrameImports.length) { - throw new Error("incomplete linked fork instrumentation imports; rebuild the program"); + if ( + linkedFrameImportCount !== 0 && + linkedFrameImportCount !== linkedFrameImports.length + ) { + throw new Error( + "incomplete linked fork instrumentation imports; rebuild the program", + ); } if (linkedFrameImportCount !== 0) { - if (!forkContinuation) { - throw new Error("linked fork instrumentation requested without continuation storage"); + if (!forkEnvImports) { + throw new Error( + "linked fork instrumentation requested without continuation and activation-state owners", + ); + } + for (const imported of moduleImports) { + if ( + imported.module !== "env" || + !imported.name.startsWith("__wpk_fork_") || + (imported.name === FORK_UNWIND_TAG_IMPORT_NAME && + (imported.kind as string) === "tag") + ) { + continue; + } + const value = forkEnvImports[imported.name]; + if (value === undefined) { + throw new Error( + `linked fork activation owner is missing env.${imported.name}`, + ); + } + if ( + imported.kind !== "function" && + imported.kind !== "global" && + imported.kind !== "table" + ) { + throw new Error( + `linked fork activation import env.${imported.name} has invalid kind ` + + `${imported.kind}`, + ); + } + envImports[imported.name] = value as WebAssembly.ExportValue; } - envImports.__wpk_fork_frame_reserve = (size: number | bigint) => { - const frame = forkContinuation.reserveFrame(size); - if (frame === 0 || frame === 0n) onContinuationAbort?.(); - return frame; - }; - envImports.__wpk_fork_frame_commit = (payload: number | bigint) => - forkContinuation.commitFrame(payload); - envImports.__wpk_fork_frame_next = (size: number | bigint) => - forkContinuation.nextFrame(size); } - if (moduleImports.some(i => i.module === "env" && i.name === "__channel_base" && i.kind === "global")) { + if ( + moduleImports.some( + (i) => + i.module === "env" && + i.name === "__channel_base" && + i.kind === "global", + ) + ) { if (ptrWidth === 8) { - envImports.__channel_base = new WebAssembly.Global({ value: "i64", mutable: true }, BigInt(channelOffset)); + envImports.__channel_base = new WebAssembly.Global( + { value: "i64", mutable: true }, + BigInt(channelOffset), + ); } else { - envImports.__channel_base = new WebAssembly.Global({ value: "i32", mutable: true }, channelOffset); + envImports.__channel_base = new WebAssembly.Global( + { value: "i32", mutable: true }, + channelOffset, + ); } } // LLVM/lld >= 22 import this tag for setjmp users. The process owns its // identity so a longjmp thrown through a side module can be caught by the // main image (and vice versa). - if (moduleImports.some(i => i.module === "env" && i.name === "__c_longjmp" && (i.kind as string) === "tag")) { + if ( + moduleImports.some( + (i) => + i.module === "env" && + i.name === "__c_longjmp" && + (i.kind as string) === "tag", + ) + ) { envImports.__c_longjmp = requireLongjmpTag( longjmpTag, "process module", ) as unknown as WebAssembly.ExportValue; } - if (moduleImports.some(i => i.module === "env" && i.name === "__cpp_exception" && (i.kind as string) === "tag")) { + if ( + moduleImports.some( + (i) => + i.module === "env" && + i.name === "__cpp_exception" && + (i.kind as string) === "tag", + ) + ) { envImports.__cpp_exception = requireCppExceptionTag( cppExceptionTag, "process module", ) as unknown as WebAssembly.ExportValue; } + if ( + moduleImports.some( + (i) => + i.module === FORK_UNWIND_TAG_IMPORT_MODULE && + i.name === FORK_UNWIND_TAG_IMPORT_NAME && + (i.kind as string) === "tag", + ) + ) { + envImports[FORK_UNWIND_TAG_IMPORT_NAME] = requireForkUnwindTag( + forkUnwindTag, + "process module", + ) as unknown as WebAssembly.ExportValue; + } // Add dlopen imports if provided if (dlopenImports) { @@ -1136,7 +2238,9 @@ function buildImportObject( ) ) { if (!postVmInterruptTimer) { - throw new Error("VM interrupt timer import requested without a host timer route"); + throw new Error( + "VM interrupt timer import requested without a host timer route", + ); } envImports.__wasm_posix_vm_interrupt_after = ( timedOutPtr: number | bigint, @@ -1152,21 +2256,23 @@ function buildImportObject( if (getInstance) { const cppMalloc = (size: number | bigint): number | bigint => { const inst = getInstance(); - const malloc = inst?.exports.malloc as ((n: number | bigint) => number | bigint) | undefined; + const malloc = inst?.exports.malloc as + ((n: number | bigint) => number | bigint) | undefined; if (!malloc) return ptrWidth === 8 ? 0n : 0; return malloc(size || (ptrWidth === 8 ? 1n : 1)); }; const cppFree = (ptr: number | bigint): void => { const inst = getInstance(); - const free = inst?.exports.free as ((p: number | bigint) => void) | undefined; + const free = inst?.exports.free as + ((p: number | bigint) => void) | undefined; if (free) free(ptr); }; - envImports._Znwm = cppMalloc; // operator new(size_t) - envImports._Znam = cppMalloc; // operator new[](size_t) - envImports._ZdlPv = cppFree; // operator delete(void*) - envImports._ZdlPvm = cppFree; // operator delete(void*, size_t) - envImports._ZdaPv = cppFree; // operator delete[](void*) - envImports._ZdaPvm = cppFree; // operator delete[](void*, size_t) + envImports._Znwm = cppMalloc; // operator new(size_t) + envImports._Znam = cppMalloc; // operator new[](size_t) + envImports._ZdlPv = cppFree; // operator delete(void*) + envImports._ZdlPvm = cppFree; // operator delete(void*, size_t) + envImports._ZdaPv = cppFree; // operator delete[](void*) + envImports._ZdaPvm = cppFree; // operator delete[](void*, size_t) envImports._ZnwmRKSt9nothrow_t = cppMalloc; // operator new(size_t, nothrow) envImports._ZnamRKSt9nothrow_t = cppMalloc; // operator new[](size_t, nothrow) } @@ -1184,7 +2290,9 @@ function buildImportObject( const view = new Uint8Array(memory.buffer); view[n(guardPtr)] = 1; // mark initialized }; - envImports.__cxa_guard_abort = (_guardPtr: number | bigint): void => { /* no-op */ }; + envImports.__cxa_guard_abort = (_guardPtr: number | bigint): void => { + /* no-op */ + }; envImports.__cxa_pure_virtual = (): void => { throw new Error("pure virtual method called"); }; @@ -1192,13 +2300,20 @@ function buildImportObject( envImports.__cxa_thread_atexit = (): number => 0; // no-op, return success // libc++ verbose abort — called on internal library errors - envImports._ZNSt3__122__libcpp_verbose_abortEPKcz = (_fmt: number | bigint, _args: number | bigint): void => { + envImports._ZNSt3__122__libcpp_verbose_abortEPKcz = ( + _fmt: number | bigint, + _args: number | bigint, + ): void => { throw new Error("libc++ verbose abort"); }; // libc++ sort — MariaDB doesn't actually call this at runtime // (linked from empty stub libc++.a). Signature: sort, ull*>(first, last, comp) - envImports["_ZNSt3__16__sortIRNS_6__lessIyyEEPyEEvT0_S5_T_"] = (_first: number | bigint, _last: number | bigint, _comp: number | bigint): void => { + envImports["_ZNSt3__16__sortIRNS_6__lessIyyEEPyEEvT0_S5_T_"] = ( + _first: number | bigint, + _last: number | bigint, + _comp: number | bigint, + ): void => { throw new Error("libc++ sort called unexpectedly"); }; const dcTiClassCache = new Map(); // typeinfo addr → metaclass (0=leaf, 1=SI, 2=VMI) @@ -1206,7 +2321,12 @@ function buildImportObject( // Reads RTTI from the object's vtable and walks the type hierarchy to // check if dst_type is reachable from the object's runtime type. // Args: (src_ptr, src_typeinfo*, dst_typeinfo*, src2dst_hint) - envImports.__dynamic_cast = (srcPtr_: number | bigint, _srcType: number | bigint, dstType_: number | bigint, _src2dst: number | bigint): number | bigint => { + envImports.__dynamic_cast = ( + srcPtr_: number | bigint, + _srcType: number | bigint, + dstType_: number | bigint, + _src2dst: number | bigint, + ): number | bigint => { const srcPtr = n(srcPtr_); const dstType = n(dstType_); if (srcPtr === 0) return retPtr(0); @@ -1214,9 +2334,13 @@ function buildImportObject( const memSize = memory.buffer.byteLength; const PS = ptrWidth; // pointer size in bytes const readPtr = (addr: number): number => - PS === 8 ? Number(view.getBigUint64(addr, true)) : view.getUint32(addr, true); + PS === 8 + ? Number(view.getBigUint64(addr, true)) + : view.getUint32(addr, true); const readSPtr = (addr: number): number => - PS === 8 ? Number(view.getBigInt64(addr, true)) : view.getInt32(addr, true); + PS === 8 + ? Number(view.getBigInt64(addr, true)) + : view.getInt32(addr, true); // Read vtable pointer from object (Itanium ABI: first word is vtable ptr) const vtablePtr = readPtr(srcPtr); @@ -1250,7 +2374,11 @@ function buildImportObject( const tiClassCache = dcTiClassCache; - const isTypeAncestor = (ti: number, target: number, visited: Set): boolean => { + const isTypeAncestor = ( + ti: number, + target: number, + visited: Set, + ): boolean => { if (ti === target) return true; if (ti === 0 || ti >= memSize || visited.has(ti)) return false; visited.add(ti); @@ -1269,7 +2397,8 @@ function buildImportObject( const baseCount = view.getUint32(ti + TI_FIELD2 + 4, true); for (let i = 0; i < baseCount; i++) { const baseType = readPtr(ti + TI_FIELD2 + 8 + i * BASE_INFO_STRIDE); - if (baseType > 0 && isTypeAncestor(baseType, target, visited)) return true; + if (baseType > 0 && isTypeAncestor(baseType, target, visited)) + return true; } return false; } @@ -1291,11 +2420,16 @@ function buildImportObject( const flags32 = view.getUint32(ti + TI_FIELD2, true); if (flags32 <= 3 && ti + TI_FIELD2 + 8 <= memSize) { const baseCount = view.getUint32(ti + TI_FIELD2 + 4, true); - if (baseCount > 0 && baseCount < 100 && ti + TI_FIELD2 + 8 + baseCount * BASE_INFO_STRIDE <= memSize) { + if ( + baseCount > 0 && + baseCount < 100 && + ti + TI_FIELD2 + 8 + baseCount * BASE_INFO_STRIDE <= memSize + ) { tiClassCache.set(ti, 2); for (let i = 0; i < baseCount; i++) { const baseType = readPtr(ti + TI_FIELD2 + 8 + i * BASE_INFO_STRIDE); - if (baseType > 0 && isTypeAncestor(baseType, target, visited)) return true; + if (baseType > 0 && isTypeAncestor(baseType, target, visited)) + return true; } return false; } @@ -1312,16 +2446,20 @@ function buildImportObject( }; // libc++ sort specialization — sort uint64 array in-place - envImports['_ZNSt3__16__sortIRNS_6__lessIyyEEPyEEvT0_S5_T_'] = ( - begin_: number | bigint, end_: number | bigint, + envImports["_ZNSt3__16__sortIRNS_6__lessIyyEEPyEEvT0_S5_T_"] = ( + begin_: number | bigint, + end_: number | bigint, ): void => { - const begin = n(begin_), end = n(end_); + const begin = n(begin_), + end = n(end_); const view = new DataView(memory.buffer); const count = (end - begin) / 8; const arr: bigint[] = []; - for (let i = 0; i < count; i++) arr.push(view.getBigUint64(begin + i * 8, true)); + for (let i = 0; i < count; i++) + arr.push(view.getBigUint64(begin + i * 8, true)); arr.sort((a, b) => (a < b ? -1 : a > b ? 1 : 0)); - for (let i = 0; i < count; i++) view.setBigUint64(begin + i * 8, arr[i], true); + for (let i = 0; i < count; i++) + view.setBigUint64(begin + i * 8, arr[i], true); }; // Stub any remaining unresolved function imports @@ -1353,7 +2491,7 @@ const FORK_BUF_SIZE = FORK_SAVE_BUFFER_SIZE; /** * Detect a legacy contiguous fork-save-buffer overrun after unwind. * - * ABI 42 linked continuations do not use this check. It remains exported for + * Linked continuations do not use this check. It remains exported for * stale-buffer regression coverage. Legacy instrumentation keeps * `current_pos` — the pointer-width integer at the * base of the save buffer (`forkBufAddr + 0`) — seeded to the absolute address @@ -1379,45 +2517,20 @@ export function forkSaveBufferOverrun( forkBufSize: number, ): number { const view = new DataView(memory.buffer); - const currentPos = ptrWidth === 8 - ? Number(view.getBigUint64(forkBufAddr, true)) - : view.getUint32(forkBufAddr, true); + const currentPos = + ptrWidth === 8 + ? Number(view.getBigUint64(forkBufAddr, true)) + : view.getUint32(forkBufAddr, true); const bufferEnd = forkBufAddr + forkBufSize; return currentPos > bufferEnd ? currentPos - bufferEnd : 0; } -/** - * Finish the active side-module unwind and reject an overrun before the main - * worker is allowed to send SYS_FORK. Side modules own a save-buffer - * allocation separate from the main process channel, so checking only the - * main buffer cannot protect this continuation. - */ -export function finalizeSideModuleForkUnwind( - _memory: WebAssembly.Memory, - state: SideModuleForkState, - _ptrWidth: 4 | 8, -): void { - const sideForkState = (): number => - Number((state.instance.exports.wpk_fork_state as () => number)()); - if (sideForkState() !== 1) { - throw new Error(`${state.name}: expected UNWINDING before side-module unwind completion`); - } - (state.instance.exports.wpk_fork_unwind_end as () => void)(); - if (sideForkState() !== 0) { - throw new Error(`${state.name}: side-module unwind did not return to NORMAL`); - } - - state.continuation.finishUnwind(); -} - // Host-private control slots below the process main channel's fork buffer. // Fork's memcpy carries the parent's dlopen archive into the child intact; // the child walks it to replay each module before wpk_fork rewind. These are // intentionally not relative to a pthread's rewind buffer. const DLOPEN_HEAD_OFFSET_WASM32 = 12; const DLOPEN_HEAD_OFFSET_WASM64 = 24; -const DLOPEN_ACTIVE_SIDE_FORK_OFFSET_WASM32 = 16; -const DLOPEN_ACTIVE_SIDE_FORK_OFFSET_WASM64 = 32; // Atomic host-private reader/writer arbitration between process-main dlopen // and pthread fork. A negative value is the exclusive main-worker dlopen // writer; a positive value counts concurrent pthread forks from their @@ -1427,44 +2540,277 @@ const DLOPEN_ACTIVE_SIDE_FORK_OFFSET_WASM64 = 32; // clears its copied value before replay because its memory is independent. const DLOPEN_LOCK_OFFSET_WASM32 = 20; const DLOPEN_LOCK_OFFSET_WASM64 = 40; +// Positive identity of the Worker whose ordinary Wasm stack owns every live +// staged loader transaction. Unlike the short archive writer, this lease spans +// guest bootstrap/relocation/constructor calls. Same-owner fork is legal; +// another pthread must wait because its child cannot inherit the owner's stack. +const DLOPEN_OWNER_OFFSET_WASM32 = 24; +const DLOPEN_OWNER_OFFSET_WASM64 = 36; +// One fixed, naturally aligned u64 fence lets instrumented Wasm detect a +// newer process table snapshot without crossing into JavaScript on the steady +// state path. The archive header remains the authoritative validated value. +const DLOPEN_GENERATION_OFFSET_WASM32 = 32; +const DLOPEN_GENERATION_OFFSET_WASM64 = 48; const DLOPEN_MAX_CONTROL_OFFSET = Math.max( DLOPEN_HEAD_OFFSET_WASM32, DLOPEN_HEAD_OFFSET_WASM64, - DLOPEN_ACTIVE_SIDE_FORK_OFFSET_WASM32, - DLOPEN_ACTIVE_SIDE_FORK_OFFSET_WASM64, DLOPEN_LOCK_OFFSET_WASM32, DLOPEN_LOCK_OFFSET_WASM64, + DLOPEN_OWNER_OFFSET_WASM32, + DLOPEN_OWNER_OFFSET_WASM64, + DLOPEN_GENERATION_OFFSET_WASM32, + DLOPEN_GENERATION_OFFSET_WASM64, ); if ( - FORK_BUF_SIZE % 16 !== 0 - || FORK_SAVE_CONTROL_PREFIX_SIZE + FORK_BUF_SIZE !== WASM_PAGE_SIZE - || DLOPEN_MAX_CONTROL_OFFSET > FORK_SAVE_CONTROL_PREFIX_SIZE + FORK_BUF_SIZE % 16 !== 0 || + FORK_SAVE_CONTROL_PREFIX_SIZE + FORK_BUF_SIZE !== WASM_PAGE_SIZE || + DLOPEN_MAX_CONTROL_OFFSET > FORK_SAVE_CONTROL_PREFIX_SIZE ) { throw new Error("invalid fork-save scratch-page geometry"); } -const DLOPEN_LOCK_IDLE = 0; -const DLOPEN_LOCK_WRITER = -1; -const DLOPEN_LOCK_MAX_READERS = 0x7fff_ffff; -// Each entry also carries the side module's instance-local TLS base. Fork -// copies the TLS bytes in memory, but a new replay instance's mutable global -// must be restored explicitly. Zero is the explicit no-TLS sentinel; TLS -// allocations are required to have a positive base. -// -// This is a host-private, transient replay record: the same host build writes -// and reads it around one fork, and neither guest code nor persisted package -// artifacts interpret the layout. Enlarging it therefore does not alter the -// guest/kernel ABI. The ABI classifier/check still guards the public contract. -const DLOPEN_ENTRY_SIZE_WASM32 = 40; -const DLOPEN_ENTRY_SIZE_WASM64 = 72; +const DLOPEN_LOCK_IDLE = 0; +const DLOPEN_LOCK_WRITER = -1; +const DLOPEN_LOCK_MAX_READERS = 0x7fff_ffff; +const DLOPEN_OWNER_IDLE = 0; +const RTLD_GLOBAL = 0x100; +const WPK_FORK_EXPORTS = WPK_FORK_REQUIRED_EXPORTS.map(({ name }) => name); + +interface ProcessTableReplicationOwner extends ForkActivationTableReplication { + /** Bring this Worker to the latest complete process generation. */ + reconcileNow(): number; + /** Check the archive fence while the caller already excludes writers. */ + isCurrentUnderLock(): boolean; + /** Release any mutation writer depths unwound by a Wasm trap. */ + abortActiveMutations(): void; +} + +function createProcessTableReplicationOwner(options: { + readonly generationAddress: number; + readonly registry: ForkActivationRegistry; + readonly dlopen: DlopenSupport; + readonly newArena: () => ForkModuleStateArena; + readonly materializeModules: (snapshot: DylinkForkArchiveSnapshot) => void; + readonly restoreSnapshots: boolean; + readonly label: string; +}): ProcessTableReplicationOwner { + const generationAddress = new WebAssembly.Global( + { value: "i64", mutable: false }, + BigInt(options.generationAddress), + ); + let deferredPublication = false; + let replicaMaterializing = false; + let suppressInitialSnapshotRestore = !options.restoreSnapshots; + const mutationContexts: Array<{ readonly deferPublication: boolean }> = []; + + const releaseArena = (root: number): void => { + if (root === 0) return; + const arena = options.newArena(); + arena.attach(root); + arena.release(); + }; + const replica = new DylinkForkTableReplica( + options.dlopen.archive, + (snapshot, previousGeneration) => { + options.materializeModules(snapshot); + if (suppressInitialSnapshotRestore) { + // WHY: a fork child restores the exact capture-time table graph from + // its normal KFMS arena after all activations exist. The archive + // generation is still adopted now; only this first redundant restore + // is suppressed. Later pthread/process mutations must be applied. + suppressInitialSnapshotRestore = false; + return; + } + let patchFloor = previousGeneration; + if ( + snapshot.tableStateRoot !== 0 && + snapshot.tableCheckpointGeneration > previousGeneration + ) { + const arena = options.newArena(); + arena.attach(snapshot.tableStateRoot); + options.registry.restoreTableState(arena); + // The archive, not this temporary validated view, owns the mappings. + patchFloor = snapshot.tableCheckpointGeneration; + } + for (const patch of snapshot.tablePatches) { + if (patch.generation! > patchFloor) { + options.registry.applyFuncrefTablePatch(patch); + } + } + }, + `${options.label}: table replica`, + ); + + const reconcileLocked = (): number => { + replicaMaterializing = true; + try { + const suppressingInitialRestore = suppressInitialSnapshotRestore; + const changed = replica.reconcile(); + if (suppressingInitialRestore && !changed) { + // A child whose copied process has never published a dylink/table + // archive still completed its one startup reconciliation. Do not + // suppress the first real peer mutation published later. + suppressInitialSnapshotRestore = false; + } + return replica.generation(); + } finally { + replicaMaterializing = false; + // Module constructors and loader table writes performed while applying + // a validated archive snapshot are effects of that publication, not a + // new mutation authored by this Worker. + deferredPublication = false; + } + }; + + const publishLocked = (): DylinkForkArchiveSnapshot => { + const arena = options.newArena(); + arena.begin(); + let root: number; + try { + root = options.registry.captureTableState(arena); + } catch (error) { + if (arena.hasActiveArena()) arena.release(); + throw error; + } + let publication; + try { + publication = options.dlopen.archive.publishTableState(root); + } catch (error) { + arena.release(); + throw error; + } + replica.adoptPublishedGeneration(publication.snapshot.generation); + if ( + publication.previousTableStateRoot !== 0 && + publication.previousTableStateRoot !== root + ) { + releaseArena(publication.previousTableStateRoot); + } + deferredPublication = false; + return publication.snapshot; + }; + + options.dlopen.setCommitObserver( + (linkerPublication, tableMutationCommitted) => { + if (linkerPublication) { + replica.adoptPublishedGeneration(linkerPublication.generation); + } + const hasPriorGuestOverlay = + linkerPublication?.tableStateRoot !== undefined && + (linkerPublication.tableStateRoot !== 0 || + linkerPublication.tablePatches.length !== 0); + // Loader-owned table entries are already deterministic module recipes in + // the dylink archive. Until a guest/dlsym overlay exists, publishing the + // same entries again as a typed KFMS snapshot would turn ordinary dlopen + // into an O(table closure) operation. Once an overlay exists, a module-set + // change still needs a fresh exact activation manifest. + if ( + deferredPublication || + (!linkerPublication && tableMutationCommitted) || + hasPriorGuestOverlay + ) { + publishLocked(); + } + }, + ); + options.dlopen.setWriterAcquireObserver(() => { + // Called with exclusive process ownership. Reconcile before any linker or + // guest mutation reads this Worker's instance-local table so the protected + // operation cannot overwrite a newer peer publication. + reconcileLocked(); + }); + + const reconcileNow = (): number => + options.dlopen.withArchiveWriter(reconcileLocked); + const abortActiveMutations = (): void => { + while (mutationContexts.length > 0) { + mutationContexts.pop(); + options.dlopen.releaseArchiveWriter(); + } + deferredPublication = false; + }; + options.dlopen.setOperationAbortObserver(abortActiveMutations); + + return { + generationAddress, + beginMutation: (): bigint => { + const deferPublication = options.dlopen.writerOwned(); + options.dlopen.acquireArchiveWriter(); + mutationContexts.push({ deferPublication }); + return BigInt(replica.generation()); + }, + reconcile: (): bigint => BigInt(reconcileNow()), + commit: (activationId, ownerId, firstIndex, length): void => { + const context = mutationContexts.pop(); + if (!context) { + throw new Error( + `${options.label}: table mutation committed without ownership`, + ); + } + try { + // Dlopen/start mutations occur before the module manifest and handle + // graph are publishable. The enclosing linker transaction snapshots + // once at its commit. Replica materialization is already represented + // by the generation being applied and must not echo a publication. + if (context.deferPublication) { + if (!replicaMaterializing) deferredPublication = true; + } else { + const patch = options.registry.captureFuncrefTablePatch( + activationId, + ownerId, + firstIndex, + length, + ); + if ( + patch !== null && + options.dlopen.archive.canPublishTablePatch(patch) + ) { + const publication = options.dlopen.archive.publishTablePatch(patch); + replica.adoptPublishedGeneration(publication.snapshot.generation); + } else { + // Typed/opaque entries stay on the Wasm codec path. The same full + // checkpoint transparently compacts a bounded patch journal; no + // table shape or mutation is rejected at either threshold. + publishLocked(); + } + } + } finally { + options.dlopen.releaseArchiveWriter(); + } + }, + abort: (): void => { + const context = mutationContexts.pop(); + if (!context) { + throw new Error( + `${options.label}: table mutation aborted without ownership`, + ); + } + options.dlopen.releaseArchiveWriter(); + }, + reconcileNow, + isCurrentUnderLock: () => + replica.generation() === options.dlopen.archive.generation(), + abortActiveMutations, + }; +} -const WPK_FORK_EXPORTS = WPK_FORK_REQUIRED_EXPORTS.map(({ name }) => name); +/** @internal Exact publication-lifecycle seam; not re-exported by the host API. */ +export function __testCreateProcessTableReplicationOwner( + options: unknown, +): unknown { + return createProcessTableReplicationOwner( + options as Parameters[0], + ); +} function hasCompleteForkInstrumentation( - moduleExports: WebAssembly.ModuleExportDescriptor[], + module: WebAssembly.Module, pid: number, ): boolean { + const moduleExports = WebAssembly.Module.exports(module); const exportNames = new Set(moduleExports.map((e) => e.name)); - const legacyAsyncifyExports = [...exportNames].filter((name) => name.startsWith("asyncify_")); + const legacyAsyncifyExports = [...exportNames].filter((name) => + name.startsWith("asyncify_"), + ); if (legacyAsyncifyExports.length > 0) { throw new Error( `pid=${pid}: user program exports legacy Asyncify instrumentation ` + @@ -1473,8 +2819,13 @@ function hasCompleteForkInstrumentation( ); } - const presentWpkExports = WPK_FORK_EXPORTS.filter((name) => exportNames.has(name)); - if (presentWpkExports.length > 0 && presentWpkExports.length !== WPK_FORK_EXPORTS.length) { + const presentWpkExports = WPK_FORK_EXPORTS.filter((name) => + exportNames.has(name), + ); + if ( + presentWpkExports.length > 0 && + presentWpkExports.length !== WPK_FORK_EXPORTS.length + ) { const missing = WPK_FORK_EXPORTS.filter((name) => !exportNames.has(name)); throw new Error( `pid=${pid}: incomplete wasm-fork-instrument exports; missing ${missing.join(", ")}. ` + @@ -1482,7 +2833,20 @@ function hasCompleteForkInstrumentation( ); } - return presentWpkExports.length === WPK_FORK_EXPORTS.length; + const complete = presentWpkExports.length === WPK_FORK_EXPORTS.length; + if (complete) { + const claim = readForkInstrumentCapabilityClaim(module); + if ( + !claim.present || + (claim.flags & WPK_FORK_CAP_ACTIVATION_STATE_SAFE) === 0 + ) { + throw new Error( + `pid=${pid}: wasm-fork-instrument artifact lacks the required ` + + "activation-state-safe capability; rebuild it for ABI 43.", + ); + } + } + return complete; } /** @@ -1548,9 +2912,19 @@ export async function centralizedWorkerMain( port: MessagePort, initData: CentralizedWorkerInitMessage, ): Promise { + let processHostImportRuntime: ForkHostImportWorkerRuntime | null = null; try { const { memory, programBytes, channelOffset, pid } = initData; const ptrWidth = initData.ptrWidth ?? 4; + const artifactFailures = describeWasmArtifactPolicyFailures(programBytes, { + expectedAbi: initData.kernelAbiVersion, + }); + if (artifactFailures.length > 0) { + throw new Error( + `pid=${pid}: refusing unsafe program artifact before execution: ` + + artifactFailures.join("; "), + ); + } // Use pre-compiled module if provided (avoids recompilation in workers) const module = initData.programModule ? initData.programModule @@ -1560,7 +2934,7 @@ export async function centralizedWorkerMain( if (wasiModuleDefinesMemory(module)) { throw new Error( "WASI module defines its own memory. Only modules that import memory " + - "(compiled with --import-memory) are supported.", + "(compiled with --import-memory) are supported.", ); } @@ -1570,13 +2944,19 @@ export async function centralizedWorkerMain( const { WasiShim, WasiExit } = await import("./wasi-shim"); const wasiShim = new WasiShim( - memory, channelOffset, initData.argv || [], initData.env || [], + memory, + channelOffset, + initData.argv || [], + initData.env || [], ); const wasiImports = wasiShim.getImports(); // Build import object: provide wasi_snapshot_preview1 namespace + env.memory const importObject: WebAssembly.Imports = { - wasi_snapshot_preview1: wasiImports as Record, + wasi_snapshot_preview1: wasiImports as Record< + string, + WebAssembly.ExportValue + >, env: { memory }, }; @@ -1587,7 +2967,11 @@ export async function centralizedWorkerMain( if (!(importObject.env as Record)[imp.name]) { (importObject.env as Record)[imp.name] = imp.kind === "function" - ? (..._args: unknown[]) => { throw new Error(`Unimplemented WASI env import: ${imp.name}`); } + ? (..._args: unknown[]) => { + throw new Error( + `Unimplemented WASI env import: ${imp.name}`, + ); + } : undefined; } } @@ -1614,26 +2998,32 @@ export async function centralizedWorkerMain( } } - port.postMessage({ type: "exit", pid, status: exitCode } satisfies WorkerToHostMessage); + port.postMessage({ + type: "exit", + pid, + status: exitCode, + } satisfies WorkerToHostMessage); return; } // --- SDK module path (existing) --- const processLongjmpTag = createLongjmpTag(ptrWidth); const processCppExceptionTag = createCppExceptionTag(ptrWidth); + const processForkUnwindTag = createForkUnwindTag(); let kernelExitStatus: number | null = null; const kernelImports = buildKernelImports( memory, channelOffset, initData.argv || [], initData.env || [], - (status) => { kernelExitStatus = status; }, + (status) => { + kernelExitStatus = status; + }, ); // Check if the module has complete wpk_fork_* instrumentation exports, // and reject stale legacy fork artifacts before they can run. - const moduleExports = WebAssembly.Module.exports(module); - const hasForkInstrumentation = hasCompleteForkInstrumentation(moduleExports, pid); + const hasForkInstrumentation = hasCompleteForkInstrumentation(module, pid); const forkCapabilityClaim = readForkInstrumentCapabilityClaim(module); const hasDylinkForkRole = forkInstrumentRoleAvailable( forkCapabilityClaim, @@ -1646,90 +3036,578 @@ export async function centralizedWorkerMain( if (hasForkInstrumentation) { const linkedFrameFormat = readLinkedFrameFormat(module); + const moduleStateFormat = readForkModuleStateDescriptor(module); + if (moduleStateFormat.ptrWidth !== linkedFrameFormat.ptrWidth) { + throw new Error( + `pid=${pid}: module-state pointer width ${moduleStateFormat.ptrWidth} ` + + `does not match linked frames ${linkedFrameFormat.ptrWidth}`, + ); + } + const mainTemplateId = await computeForkModuleTemplateId(programBytes); const forkContinuation = new LinkedForkContinuation( memory, linkedFrameFormat, (size) => continuationMmap(memory, channelOffset, size, `pid=${pid}`), - (addr, size) => continuationMunmap(memory, channelOffset, addr, size, `pid=${pid}`), + (addr, size) => + continuationMunmap(memory, channelOffset, addr, size, `pid=${pid}`), `pid=${pid}`, ); - // Override kernel_fork with fork-instrumentation-aware version. - // Late-bound: processInstance is set after instantiation. let processInstance: WebAssembly.Instance | null = null; + const newModuleStateArena = (): ForkModuleStateArena => + new ForkModuleStateArena( + memory, + ptrWidth, + (size) => + continuationMmap( + memory, + channelOffset, + size, + `pid=${pid}: module state`, + ), + (addr, size) => + continuationMunmap( + memory, + channelOffset, + addr, + size, + `pid=${pid}: module state`, + ), + `pid=${pid}`, + ); + + if ( + initData.forkHostImports === undefined || + initData.externrefGenerationId === undefined + ) { + throw new Error( + `pid=${pid}: ABI 43 fork artifact requires its process owner ` + + "host-import mailbox and externref generation", + ); + } + const externrefTokens = new ForkExternrefTokenCache( + initData.externrefGenerationId, + ); + processHostImportRuntime = new ForkHostImportWorkerRuntime( + initData.forkHostImports, + pid, + initData.externrefGenerationId, + externrefTokens, + (wake) => { + port.postMessage({ + type: "fork_host_import", + wake, + } satisfies WorkerToHostMessage); + }, + ); + const externrefRecipes = new ForkExternrefTokenRecipeProvider( + externrefTokens, + (value) => + processHostImportRuntime!.localExceptions.normalizeUnclaimedForkValue( + value, + ), + ); + const activationRegistry = new ForkActivationRegistry( + memory, + externrefRecipes, + `pid=${pid}: fork activations`, + (size) => + continuationMmap( + memory, + channelOffset, + size, + `pid=${pid}: reference scratch`, + ), + (addr, size) => + continuationMunmap( + memory, + channelOffset, + addr, + size, + `pid=${pid}: reference scratch`, + ), + ); + // Every process instance, including a freshly reconstructed child, owns + // the provenance manifest for any fork it may issue later. + const importedStateCapture = new ForkImportedGlobalCapture( + `pid=${pid}: imported activation state`, + ); + let importedStatePlanner: ForkImportedGlobalPlanner | null = null; + let earlyChildReferences: ForkEarlyChildReferenceProvider | null = null; + let decodedChildReferences: DecodedSegmentedForkReferenceTransaction | null = + null; + let childDylinkState: DylinkForkState | null = null; + const referenceReplay = (): ProcessReferenceReplayImports => + earlyChildReferences ?? activationRegistry.currentReferences(); + const processContinuation = new ForkProcessContinuationCoordinator( + memory, + activationRegistry, + `pid=${pid}: process continuation`, + ); + let processDlopenSupport: DlopenSupport | null = null; + let processForkArchiveReaderHeld = false; + const tableGenerationOffset = + ptrWidth === 8 + ? DLOPEN_GENERATION_OFFSET_WASM64 + : DLOPEN_GENERATION_OFFSET_WASM32; + const tableGenerationAddress = + dlopenArchiveControlAddr - tableGenerationOffset; + let processTableReplication: ProcessTableReplicationOwner | null = null; + const tableReplicationImports: ForkActivationTableReplication = { + generationAddress: new WebAssembly.Global( + { value: "i64", mutable: false }, + BigInt(tableGenerationAddress), + ), + reconcile: (): bigint => processTableReplication?.reconcile() ?? 0n, + beginMutation: (): bigint => + processTableReplication?.beginMutation() ?? 0n, + commit: (activationId, ownerId, firstIndex, length): void => { + processTableReplication?.commit( + activationId, + ownerId, + firstIndex, + length, + ); + }, + abort: (): void => { + processTableReplication?.abort(); + }, + }; + const exceptionBroker = new ForkExceptionBroker( + activationRegistry, + `pid=${pid}: exception broker`, + () => earlyChildReferences ?? activationRegistry.currentReferences(), + (value) => + processHostImportRuntime!.localExceptions.normalizeUnclaimedForkException( + value, + ), + ); + let mainExceptionProvider: ForkExceptionProvider | null = null; + const registerChildReferenceActivation = ( + activationId: number, + activationModule: WebAssembly.Module, + registration: ForkActivationRegistration, + typedReferenceProvider: ForkGcCodecProvider, + ): void => { + const early = earlyChildReferences; + if (!early) { + throw new Error( + `pid=${pid}: child activation ${activationId} registered ` + + "outside early reference reconstruction", + ); + } + if ( + registration.activationId !== activationId || + typedReferenceProvider.activationId !== activationId + ) { + throw new Error( + `pid=${pid}: child activation ${activationId} provider coordinate mismatch`, + ); + } + // Parse against this exact compiled module before publishing any + // provider. ForkEarlyChildReferenceProvider compares the resulting + // descriptor to the pre-instantiation declaration. + readForkGcCodecDescriptor(activationModule); + // WHY: these catalogs contain fresh-instance identities. Register the + // activation only after the full registry has harvested static roots, + // but before a later module's immutable import getter can request one. + early.registerActivation({ + activationId, + functions: { + decode(ordinal) { + if ( + !Number.isInteger(ordinal) || + ordinal < 0 || + ordinal >= registration.functionCatalog.length + ) { + throw new RangeError( + `pid=${pid}: function recipe ${activationId}:` + + `${String(ordinal)} is out of bounds`, + ); + } + const value = registration.functionCatalog.get(ordinal); + if (typeof value !== "function") { + throw new TypeError( + `pid=${pid}: function recipe ${activationId}:${ordinal} ` + + "did not resolve to a function", + ); + } + return value as CallableFunction; + }, + }, + staticRoots: { + decode: (ordinal) => + activationRegistry.decodeStaticRoot(activationId, ordinal), + }, + typed: typedReferenceProvider, + exceptions: registration.exceptionProvider, + }); + }; + + const readProcessLaunchRoot = (): number => { + const view = new DataView(memory.buffer); + return ptrWidth === 8 + ? Number(view.getBigUint64(dlopenArchiveControlAddr, true)) + : view.getUint32(dlopenArchiveControlAddr, true); + }; + let copiedLaunchRoot = 0; + let childArena: ForkModuleStateArena | null = null; + if (initData.isForkChild) { + if ( + initData.forkChildThreadFnPtr !== undefined && + initData.forkBufAddr !== undefined + ) { + // A pthread continuation is rooted in the caller's channel page, + // not the process-main anchor copied into the child. Publish the + // kernel-validated launch root under activation zero before any + // child reconstruction recipe is inspected. + writeForkContinuationAnchor( + memory, + dlopenArchiveControlAddr, + ptrWidth, + initData.forkBufAddr, + ); + } + copiedLaunchRoot = readProcessLaunchRoot(); + if ( + initData.forkBufAddr !== undefined && + copiedLaunchRoot !== initData.forkBufAddr + ) { + throw new Error( + `pid=${pid}: copied process launch root ${copiedLaunchRoot} ` + + `does not match launch root ${initData.forkBufAddr}`, + ); + } + if (!Number.isSafeInteger(copiedLaunchRoot) || copiedLaunchRoot <= 0) { + throw new Error( + `pid=${pid}: fork child has no copied process launch root`, + ); + } + const moduleStateRoot = readForkModuleStateRoot( + memory, + copiedLaunchRoot, + ptrWidth, + ); + childArena = newModuleStateArena(); + childArena.attach( + ptrWidth === 8 ? BigInt(moduleStateRoot) : moduleStateRoot, + ); + } + processContinuation.prepareActivation({ + activationId: 0, + continuation: forkContinuation, + publishProcessLaunchRoot: (address) => { + // WHY: this copied control-page word is the fresh child's route to + // the main activation. No JavaScript closure survives fork. + writeForkContinuationAnchor( + memory, + dlopenArchiveControlAddr, + ptrWidth, + address, + ); + forkBufAddr = address; + }, + readProcessLaunchRoot, + }); + + const releaseProcessForkArchiveReader = (): void => { + if (!processForkArchiveReaderHeld) return; + processForkArchiveReaderHeld = false; + processDlopenSupport?.releaseArchiveReader(); + }; + const acquireCurrentProcessForkArchiveReader = (): void => { + if (!processDlopenSupport || !processTableReplication) { + throw new Error(`pid=${pid}: fork archive owner is not initialized`); + } + for (;;) { + processTableReplication.reconcileNow(); + processDlopenSupport.acquireArchiveReader(); + processForkArchiveReaderHeld = true; + if (processTableReplication.isCurrentUnderLock()) return; + releaseProcessForkArchiveReader(); + } + }; + kernelImports.kernel_fork = (): number => { if (!processInstance) return -38; // ENOSYS - const getState = processInstance.exports.wpk_fork_state as () => number; - const state = getState(); - if (state === 2) { - // Rewinding: end rewind and return the stored fork result - (processInstance.exports.wpk_fork_rewind_end as () => void)(); - forkContinuation.finishReplayAndRelease(); - writeForkContinuationAnchor(memory, dlopenArchiveControlAddr, ptrWidth, 0); - forkBufAddr = 0; + const phase = processContinuation.phaseName(); + if (phase === "parent-replay" || phase === "child-replay") { + try { + processContinuation.finishReplay(); + } finally { + releaseProcessForkArchiveReader(); + } + if (initData.isForkChild) { + const gate = initData.forkReplayGate; + if (!gate) { + throw new Error( + `pid=${pid}: fork child is missing its replay commit gate`, + ); + } + // Every outer activation has already restored its frame before + // descending to this import. Reaching here is therefore the exact + // point at which the host may commit the fresh child. + port.postMessage({ + type: "fork_replay_ready", + pid, + } satisfies WorkerToHostMessage); + waitForForkReplayCommit(gate, `pid=${pid}`); + } return forkResult; } - if (state === 3) { + if (phase === "abort-replay") { const errno = forkContinuation.abortErrno(); - (processInstance.exports.wpk_fork_abort_end as () => void)(); - forkContinuation.finishAbortReplayAndRelease(); - writeForkContinuationAnchor(memory, dlopenArchiveControlAddr, ptrWidth, 0); - forkBufAddr = 0; + try { + processContinuation.finishAbortReplay(); + } finally { + releaseProcessForkArchiveReader(); + } return -errno; } + if (phase !== "idle") { + throw new Error( + `pid=${pid}: fork import reached while process continuation is ${phase}`, + ); + } - // Normal call: start unwind to save the call stack. - // SYS_FORK is sent after _start returns (unwind complete). - // wpk_fork_unwind_begin self-initializes current_pos and snapshots - // saved_globals (including __tls_base and __stack_pointer) into the - // buffer — the host no longer pre-seeds the header. + // The arena and every activation prefix are allocated before any user + // frame commits. If this fails, fork returns errno with no partially + // published activation graph. + acquireCurrentProcessForkArchiveReader(); + const arena = newModuleStateArena(); try { - forkBufAddr = Number(forkContinuation.beginUnwind()); + arena.begin(); + processContinuation.beginCapture(arena); + importedStateCapture?.appendTo(arena); } catch (error) { + if (processContinuation.phaseName() !== "idle") { + try { + processContinuation.abort(); + } catch { + // Preserve the capture failure; abort has already made the + // transaction unreachable before attempting cleanup. + } + } else if (arena.hasActiveArena()) { + arena.release(); + } + releaseProcessForkArchiveReader(); + forkBufAddr = 0; if (error instanceof ContinuationAllocationError) return -error.errno; throw error; } - writeForkContinuationAnchor( - memory, - dlopenArchiveControlAddr, - ptrWidth, - forkBufAddr, - ); - invokeForkContinuationBegin( - processInstance.exports.wpk_fork_unwind_begin, - forkBufAddr, - ptrWidth, - `pid=${pid}: linked fork unwind`, - ); return 0; // ignored during unwind }; + const dylinkForkActivationOwner = hasDylinkForkRole + ? createProcessDylinkActivationOwner({ + memory, + ptrWidth, + channelOffset, + forkUnwindTag: processForkUnwindTag, + coordinator: processContinuation, + registry: activationRegistry, + exceptionBroker, + importedStateCapture, + tableReplication: tableReplicationImports, + importedStatePlanner: initData.isForkChild + ? () => importedStatePlanner + : undefined, + referenceReplay, + registerChildReferenceActivation: initData.isForkChild + ? registerChildReferenceActivation + : undefined, + isForkChild: Boolean(initData.isForkChild), + invokeProcessFork: () => { + const fork = processInstance?.exports.fork; + if (typeof fork !== "function") { + throw new Error( + `pid=${pid}: dylink fork role is missing the main libc fork export`, + ); + } + return Number((fork as () => number)()); + }, + label: `pid=${pid}: dylink activations`, + }) + : undefined; + // Build import object and instantiate const dlopenSupport = buildDlopenImports( memory, channelOffset, dlopenArchiveControlAddr, - () => processInstance?.exports.__indirect_function_table as WebAssembly.Table | undefined, - () => processInstance?.exports.__stack_pointer as WebAssembly.Global | undefined, + () => + processInstance?.exports.__indirect_function_table as + WebAssembly.Table | undefined, + () => + processInstance?.exports.__stack_pointer as + WebAssembly.Global | undefined, () => processInstance ?? undefined, ptrWidth, processLongjmpTag, processCppExceptionTag, - hasDylinkForkRole, - (errno) => { - if (!processInstance) throw new Error(`pid=${pid}: side abort before main instantiation`); - forkContinuation.beginAbortReplay(errno); - invokeForkContinuationBegin( - processInstance.exports.wpk_fork_abort_begin, - forkBufAddr, - ptrWidth, - `pid=${pid}: linked fork abort`, - ); + dylinkForkActivationOwner, + hasDylinkForkRole + ? undefined + : `pid=${pid}: main artifact lacks the dylink fork role capability`, + processForkUnwindTag, + (table, firstIndex, length) => { + activationRegistry.markTableMutation(table, firstIndex, length); }, + processHostImportRuntime, + pid, ); - const importObject = buildImportObject(module, memory, kernelImports, channelOffset, dlopenSupport.imports, - () => processInstance ?? undefined, ptrWidth, processLongjmpTag, processCppExceptionTag, + processDlopenSupport = dlopenSupport; + processTableReplication = createProcessTableReplicationOwner({ + generationAddress: tableGenerationAddress, + registry: activationRegistry, + dlopen: dlopenSupport, + newArena: newModuleStateArena, + materializeModules: (snapshot) => { + dlopenSupport.replayDlopens(snapshot); + }, + // The copied fork arena restores a child process's complete + // global/table/reference graph and preserves aliases with live frames. + // The process table journal is for separately instantiated pthread + // Workers and later generations, not a second initial child restore. + restoreSnapshots: !initData.isForkChild, + label: `pid=${pid}`, + }); + if (initData.isForkChild) { + if (!childArena) { + throw new Error( + `pid=${pid}: fork child lost its validated module-state arena`, + ); + } + // A parent can be copied while the archive mutex word names its + // now-nonexistent Worker. The validated archive bytes are immutable + // for this child launch, so clear that private lock before creating + // any loader state. + dlopenSupport.resetForkChildLock(); + childDylinkState = dlopenSupport.readForkState(); + const records = childArena.recordViews(); + decodedChildReferences = decodeSegmentedForkReferenceTransaction( + records, + FORK_REFERENCE_TRANSACTION_OWNER_ID, + ); + const modules = new Map([[0, module]]); + for (const library of childDylinkState.libraries) { + if (library.activationId === undefined) continue; + if (modules.has(library.activationId)) { + throw new Error( + `pid=${pid}: archived activation ${library.activationId} ` + + "is duplicated or aliases the main activation", + ); + } + modules.set( + library.activationId, + new WebAssembly.Module( + library.moduleBytes as unknown as BufferSource, + ), + ); + } + const declarations = [...modules] + .sort(([left], [right]) => left - right) + .map(([activationId, activationModule]) => ({ + activationId, + gcDescriptor: readForkGcCodecDescriptor(activationModule), + exceptionDescriptor: + readForkExceptionCodecDescriptor(activationModule), + })); + earlyChildReferences = new ForkEarlyChildReferenceProvider({ + records, + transaction: decodedChildReferences, + declarations, + externrefs: externrefRecipes, + transit: { + prepare: (maxRecipeId) => + activationRegistry.prepareEarlyGcTransit(maxRecipeId), + publish: (recipeId, value) => + activationRegistry.publishEarlyGcTransit(recipeId, value), + read: (recipeId) => activationRegistry.readEarlyGcTransit(recipeId), + abort: () => activationRegistry.abortEarlyGcTransit(), + }, + memory, + allocateScratch: (size) => + continuationMmap( + memory, + channelOffset, + size, + `pid=${pid}: early reference scratch`, + ), + deallocateScratch: (addr, size) => + continuationMunmap( + memory, + channelOffset, + addr, + size, + `pid=${pid}: early reference scratch`, + ), + label: `pid=${pid}: early child references`, + }); + importedStatePlanner = new ForkImportedGlobalPlanner( + records, + modules, + earlyChildReferences, + `pid=${pid}: child imported activation state`, + ); + const archivedOrder = [ + 0, + ...childDylinkState.libraries.flatMap(({ activationId }) => + activationId === undefined ? [] : [activationId], + ), + ]; + const plannedOrder = importedStatePlanner.instantiationOrder(); + if ( + archivedOrder.length !== plannedOrder.length || + archivedOrder.some( + (activationId, index) => activationId !== plannedOrder[index], + ) + ) { + throw new Error( + `pid=${pid}: copied activation import dependencies require order ` + + `${plannedOrder.join(",")}, but the replay archive provides ` + + archivedOrder.join(","), + ); + } + } + const forkEnvImports: Record = { + ...processContinuation.continuationImports(0, (errno) => { + processContinuation.beginCaptureAbort(errno); + }), + ...buildForkActivationStateImports( + 0, + activationRegistry, + referenceReplay, + tableReplicationImports, + ), + ...buildForkExceptionImports({ + activationId: 0, + ptrWidth, + registry: activationRegistry, + broker: exceptionBroker, + provider: () => { + if (!mainExceptionProvider) { + throw new Error( + `pid=${pid}: exception codec called before registration`, + ); + } + return mainExceptionProvider; + }, + referenceReplay, + }), + }; + const importObject = buildImportObject( + module, + memory, + kernelImports, + channelOffset, + dlopenSupport.imports, + () => processInstance ?? undefined, + ptrWidth, + processLongjmpTag, + processCppExceptionTag, + processForkUnwindTag, (timedOutPtr, vmInterruptPtr, seconds) => { port.postMessage({ type: "vm_interrupt_timer", @@ -1739,35 +3617,172 @@ export async function centralizedWorkerMain( seconds, } satisfies WorkerToHostMessage); }, - forkContinuation, - () => { - if (!processInstance) throw new Error(`pid=${pid}: continuation abort before instantiation`); - const errno = forkContinuation.abortErrno(); - dlopenSupport.beginSideModuleForkAbort(errno); - invokeForkContinuationBegin( - processInstance.exports.wpk_fork_abort_begin, - forkBufAddr, - ptrWidth, - `pid=${pid}: linked fork abort`, - ); - }); - const instance = await WebAssembly.instantiate(module, importObject); + forkEnvImports, + ); + const routedImportObject = processHostImportRuntime.routeImportObject( + programBytes, + importObject, + ); + const reconstructedMainImports = importedStatePlanner + ? importedStatePlanner.importsForActivation( + 0, + routedImportObject as unknown as ForkWasmImports, + ) + : routedImportObject; + // WHY: reconstruction supplies copied identities; capture wraps that + // resolved view so this child can safely become the parent of another + // fresh instance without retaining the previous fork arena. + const mainImportedStatePreparation = + importedStateCapture.prepareActivation( + 0, + module, + reconstructedMainImports, + ); + const mainInstantiationImports = mainImportedStatePreparation.imports; + let instance: WebAssembly.Instance; + try { + instance = await WebAssembly.instantiate( + module, + mainInstantiationImports as unknown as WebAssembly.Imports, + ); + } catch (error) { + mainImportedStatePreparation?.abort(); + if (earlyChildReferences) { + try { + earlyChildReferences.abort(); + } catch { + // Preserve the instantiation failure. No replay can proceed, and + // the outer process teardown still clears registered providers. + } + earlyChildReferences = null; + } + importedStatePlanner?.clear(); + throw error; + } processInstance = instance; + mainImportedStatePreparation?.complete(instance); + mainExceptionProvider = forkExceptionProviderFromInstance(0, instance); + const mainTypedReferenceProvider = forkGcCodecProviderFromInstance( + 0, + module, + instance, + ); + const mainRegistration = forkActivationRegistrationFromInstance({ + activationId: 0, + module, + instance, + templateId: mainTemplateId, + exceptionProvider: mainExceptionProvider, + typedReferenceProvider: mainTypedReferenceProvider, + }); + processContinuation.registerActivation( + mainRegistration, + forkResumeTargetsFromInstance(module, instance), + ); + importedStatePlanner?.registerInstance(0, instance); if (initData.isForkChild) { - dlopenSupport.resetForkChildLock(); + registerChildReferenceActivation( + 0, + module, + mainRegistration, + mainTypedReferenceProvider, + ); + } + importedStateCapture?.bindTableDirtyTrackers( + new Map( + activationRegistry + .activations() + .map((activation) => [ + activation.activationId, + activation.tableDirty, + ]), + ), + ); + if (!initData.isForkChild) { + try { + // Registration harvests static roots before bootstrap consumes the + // converted active segments, and installs the dirty-table owner + // before the original start can mutate a table. + activationRegistry.bootstrapActivation(0); + } catch (error) { + processTableReplication.abortActiveMutations(); + processContinuation.unregisterActivation(0); + mainExceptionProvider = null; + throw error; + } } verifyProgramAbi(programBytes, initData.kernelAbiVersion, pid); - // For the fork-parent case (initial launch, not a fork child), install - // __channel_base now — the parent's __tls_base is already correctly - // populated by instantiation, so setupChannelBase can read it. - // - // For fork children: defer until AFTER wpk_fork_rewind_begin runs - // (inside the loop below), because rewind_begin is what restores - // the child's __tls_base from the fork save buffer; setupChannelBase - // would otherwise see a zeroed __tls_base. if (!initData.isForkChild) { - setupChannelBase(instance, module, memory, channelOffset, programBytes as ArrayBuffer, ptrWidth); + setupChannelBase( + instance, + module, + memory, + channelOffset, + programBytes as ArrayBuffer, + ptrWidth, + ); + } else { + // Every side activation must exist before the process transaction is + // attached: module/reference recipes name activation coordinates, not + // whichever instance happens to load first in the child. + try { + if (!childDylinkState) { + throw new Error("copied dynamic-linker state was not prepared"); + } + dlopenSupport.replayDlopens(childDylinkState); + processTableReplication.reconcileNow(); + } catch (error) { + throw new Error( + `fork-replay-dlopen failed: ${ + error instanceof Error ? error.message : String(error) + }`, + ); + } + if (!childArena) { + throw new Error( + `pid=${pid}: fork child lost its validated module-state arena`, + ); + } + if (!importedStatePlanner || !earlyChildReferences) { + throw new Error( + `pid=${pid}: fork child lost its pre-instantiation reference plan`, + ); + } + importedStatePlanner.bindTableDirtyTrackers( + new Map( + activationRegistry + .activations() + .map((activation) => [ + activation.activationId, + activation.tableDirty, + ]), + ), + ); + const early = earlyChildReferences; + processContinuation.attachChild( + childArena, + () => { + early.adoptInto(activationRegistry.currentReferences()); + earlyChildReferences = null; + }, + decodedChildReferences ?? undefined, + ); + decodedChildReferences = null; + importedStatePlanner.clear(); + importedStatePlanner = null; + forkResult = 0; + + // attachChild restores __tls_base/__stack_pointer for every + // activation before any continuation frame can execute. + setupChannelBase( + instance, + module, + memory, + channelOffset, + programBytes as ArrayBuffer, + ptrWidth, + ); } // Signal ready @@ -1777,121 +3792,99 @@ export async function centralizedWorkerMain( let exitCode = 0; try { const start = instance.exports._start as () => void; - const getState = instance.exports.wpk_fork_state as () => number; - const unwindEnd = instance.exports.wpk_fork_unwind_end as () => void; - - // For fork children: start with rewind to resume from fork point - let needsRewind = !!initData.isForkChild; - if (needsRewind) { - forkResult = 0; // fork() returns 0 in child + const resumeStart = instance.exports.wpk_fork_resume_start as + (() => void) | undefined; + if (typeof resumeStart !== "function") { + throw new Error( + `pid=${pid}: fork-capable program is missing wpk_fork_resume_start`, + ); } - let replayedForkChildDlopens = false; - let attachedForkChildContinuation = false; - // Choose entry: normal _start, or — for a fork-from-non-main-thread // child — call the parent thread's thread function directly. _start // is not in the thread's fork-path call chain, so rewinding through // it would never reach the saved fork() call site. The thread // function's instrumented body sees state==REWINDING on entry and // replays the saved frames back to fork(). - let entry: () => void; + let lexicalEntry: () => void; + let replayEntry: () => void; if (initData.isForkChild && initData.forkChildThreadFnPtr != null) { - const table = instance.exports.__indirect_function_table as WebAssembly.Table | undefined; - if (!table) { - throw new Error("Fork-from-thread child: no __indirect_function_table export"); - } const fnIdx = initData.forkChildThreadFnPtr; - const tableIdx = ptrWidth === 8 ? (BigInt(fnIdx) as unknown as number) : fnIdx; - const threadFn = table.get(tableIdx) as ((arg: number | bigint) => unknown) | null; - if (!threadFn) { - throw new Error(`Fork-from-thread child: thread function at index ${fnIdx} is null`); - } - const childArgPtr = initData.forkChildThreadArgPtr ?? 0; - const threadArg = ptrWidth === 8 ? BigInt(childArgPtr) : childArgPtr; - entry = () => { threadFn(threadArg); }; - } else { - entry = start; - } - - for (;;) { - if (needsRewind) { - const rewindAddr = initData.isForkChild - && !attachedForkChildContinuation - && initData.forkBufAddr != null - ? initData.forkBufAddr - : forkBufAddr; - if (initData.isForkChild && !attachedForkChildContinuation) { - // A fork child has copied chunks but a fresh JS owner. - // Preserve the guest ABI type when handing that copied pointer - // to the continuation validator: memory64 i64 requires BigInt. - forkContinuation.attachForReplay( - ptrWidth === 8 ? BigInt(rewindAddr) : rewindAddr, - ); - attachedForkChildContinuation = true; - } else { - forkContinuation.beginReplay(); - } - // wpk_fork_rewind_begin restores all saved mutable globals - // (including __tls_base and __stack_pointer) from the fork - // buffer. Must run before setupChannelBase, which reads - // __tls_base to locate the channel-base TLS slot. - invokeForkContinuationBegin( - instance.exports.wpk_fork_rewind_begin, - rewindAddr, - ptrWidth, - `pid=${pid}: linked fork rewind`, + const childArgPtr = initData.forkChildThreadArgPtr ?? 0; + const threadArg = ptrWidth === 8 ? BigInt(childArgPtr) : childArgPtr; + const resumeThread = instance.exports.wpk_fork_resume_thread as + | ((tableIndex: number, arg: number | bigint) => number | bigint) + | undefined; + if (typeof resumeThread !== "function") { + throw new Error( + "Fork-from-thread child: missing wpk_fork_resume_thread", ); - // Now that rewind_begin has restored __tls_base, install - // __channel_base for this (child) instance. - setupChannelBase(instance, module, memory, channelOffset, programBytes as ArrayBuffer, ptrWidth); - if (initData.isForkChild && !replayedForkChildDlopens) { - try { - dlopenSupport.replayDlopens(); - } catch (e) { - throw new Error(`fork-replay-dlopen failed: ${e instanceof Error ? e.message : String(e)}`); - } - replayedForkChildDlopens = true; - } - dlopenSupport.beginSideModuleForkRewind(); - needsRewind = false; } + // A fork child never executes the lexical pthread entry. Keep the + // two closures structurally complete so the loop can select solely + // from coordinator phase below. + lexicalEntry = () => { + throw new Error( + "Fork-from-thread child entered lexical thread path", + ); + }; + replayEntry = () => { + resumeThread(fnIdx, threadArg); + }; + } else { + lexicalEntry = start; + replayEntry = resumeStart; + } + for (;;) { + let transportedForkUnwind = false; try { + const phaseBeforeEntry = processContinuation.phaseName(); + const entry = + phaseBeforeEntry === "idle" ? lexicalEntry : replayEntry; entry(); } catch (e) { - if (e instanceof Error && e.message.includes("unreachable")) { + if (isForkUnwindException(e, processForkUnwindTag)) { + transportedForkUnwind = true; + } else if ( + e instanceof Error && + e.message.includes("unreachable") + ) { if (kernelExitStatus !== null) { exitCode = kernelExitStatus; break; // Normal exit via kernel_exit -> unreachable trap } + throw e; + } else { + throw e; } - throw e; } - const forkState = getState(); - if (forkState === 1) { - // Unwind completed (fork) — finalize and send SYS_FORK. - unwindEnd(); - forkContinuation.finishUnwind(); - - dlopenSupport.completeSideModuleForkUnwind(); - - // Send SYS_FORK through the channel now that memory has the - // fork save buffer populated (saved_globals + frames). + const phase = processContinuation.phaseName(); + if (transportedForkUnwind && phase !== "capture") { + throw new Error( + `pid=${pid}: private fork-unwind exception escaped while ` + + `process continuation is ${phase}`, + ); + } + if (phase === "capture") { + processContinuation.sealCapture(); const childPid = sendForkSyscall(memory, channelOffset); + forkResult = childPid; if (childPid < 0) { - forkResult = childPid; - needsRewind = true; - continue; + processContinuation.beginAbortReplay(-childPid); + } else { + processContinuation.beginParentReplay(); } - forkResult = childPid; - needsRewind = true; continue; } + if (phase !== "idle") { + throw new Error( + `pid=${pid}: process entry returned while continuation is ${phase}`, + ); + } // Normal return — program finished - dlopenSupport.assertNoActiveSideModuleFork(); if (kernelExitStatus === null) { kernelImports.kernel_exit(0); exitCode = kernelExitStatus ?? 0; @@ -1899,14 +3892,37 @@ export async function centralizedWorkerMain( break; } } catch (e) { - if (e instanceof Error && e.message.includes("unreachable") && kernelExitStatus !== null) { + processTableReplication.abortActiveMutations(); + releaseProcessForkArchiveReader(); + if ( + e instanceof Error && + e.message.includes("unreachable") && + kernelExitStatus !== null + ) { exitCode = kernelExitStatus; } else { + if (processContinuation.phaseName() !== "idle") { + try { + processContinuation.abort(); + } catch { + // Preserve the execution failure; abort already made its + // transaction state unreachable before attempting deallocation. + } + } throw e; } } - port.postMessage({ type: "exit", pid, status: exitCode } satisfies WorkerToHostMessage); + processContinuation.clear(); + releaseProcessForkArchiveReader(); + importedStateCapture?.clear(); + externrefTokens.clear(); + processHostImportRuntime.clear(); + port.postMessage({ + type: "exit", + pid, + status: exitCode, + } satisfies WorkerToHostMessage); } else { // No fork instrumentation: fork cannot be represented safely because // the child cannot resume at the fork call site. Fail loudly if the @@ -1923,16 +3939,34 @@ export async function centralizedWorkerMain( memory, channelOffset, dlopenArchiveControlAddr, - () => processInstance?.exports.__indirect_function_table as WebAssembly.Table | undefined, - () => processInstance?.exports.__stack_pointer as WebAssembly.Global | undefined, + () => + processInstance?.exports.__indirect_function_table as + WebAssembly.Table | undefined, + () => + processInstance?.exports.__stack_pointer as + WebAssembly.Global | undefined, () => processInstance ?? undefined, ptrWidth, processLongjmpTag, processCppExceptionTag, - false, + undefined, + `pid=${pid}: main artifact has no fork activation coordinator`, + processForkUnwindTag, + undefined, + undefined, + pid, ); - const importObject = buildImportObject(module, memory, kernelImports, channelOffset, dlopenSupport.imports, - () => processInstance ?? undefined, ptrWidth, processLongjmpTag, processCppExceptionTag, + const importObject = buildImportObject( + module, + memory, + kernelImports, + channelOffset, + dlopenSupport.imports, + () => processInstance ?? undefined, + ptrWidth, + processLongjmpTag, + processCppExceptionTag, + processForkUnwindTag, (timedOutPtr, vmInterruptPtr, seconds) => { port.postMessage({ type: "vm_interrupt_timer", @@ -1941,12 +3975,20 @@ export async function centralizedWorkerMain( vmInterruptPtr, seconds, } satisfies WorkerToHostMessage); - }); + }, + ); const instance = await WebAssembly.instantiate(module, importObject); processInstance = instance; verifyProgramAbi(programBytes, initData.kernelAbiVersion, pid); - setupChannelBase(instance, module, memory, channelOffset, programBytes as ArrayBuffer, ptrWidth); + setupChannelBase( + instance, + module, + memory, + channelOffset, + programBytes as ArrayBuffer, + ptrWidth, + ); port.postMessage({ type: "ready", pid } satisfies WorkerToHostMessage); @@ -1973,13 +4015,21 @@ export async function centralizedWorkerMain( exitCode = kernelExitStatus ?? exitCode; } - port.postMessage({ type: "exit", pid, status: exitCode } satisfies WorkerToHostMessage); + port.postMessage({ + type: "exit", + pid, + status: exitCode, + } satisfies WorkerToHostMessage); } } catch (err) { + processHostImportRuntime?.clear(); let errMsg: string; if (err instanceof Error) { errMsg = `${err.message}\n${err.stack}`; - } else if ((WebAssembly as any).Exception && err instanceof (WebAssembly as any).Exception) { + } else if ( + (WebAssembly as any).Exception && + err instanceof (WebAssembly as any).Exception + ) { // WebAssembly.Exception isn't an Error subclass in V8, so String(err) // produces the useless "[object WebAssembly.Exception]". Surface // anything we can read off it for build-time debugging. @@ -2016,7 +4066,9 @@ function detectChannelBaseTlsOffset(programBytes: ArrayBuffer): number { if (src.length < 8) return -1; function readLEB128(buf: Uint8Array, off: number): [number, number] { - let result = 0, shift = 0, pos = off; + let result = 0, + shift = 0, + pos = off; for (;;) { const byte = buf[pos++]; result |= (byte & 0x7f) << shift; @@ -2027,7 +4079,11 @@ function detectChannelBaseTlsOffset(programBytes: ArrayBuffer): number { } // Parse sections to find Export and Code sections - interface Section { id: number; contentOffset: number; contentSize: number; } + interface Section { + id: number; + contentOffset: number; + contentSize: number; + } const sections: Section[] = []; let numFuncImports = 0; let offset = 8; @@ -2035,7 +4091,11 @@ function detectChannelBaseTlsOffset(programBytes: ArrayBuffer): number { while (offset < src.length) { const sectionId = src[offset]; const [sectionSize, sizeBytes] = readLEB128(src, offset + 1); - sections.push({ id: sectionId, contentOffset: offset + 1 + sizeBytes, contentSize: sectionSize }); + sections.push({ + id: sectionId, + contentOffset: offset + 1 + sizeBytes, + contentSize: sectionSize, + }); offset += 1 + sizeBytes + sectionSize; } @@ -2046,13 +4106,35 @@ function detectChannelBaseTlsOffset(programBytes: ArrayBuffer): number { const [importCount, countBytes] = readLEB128(src, pos); pos += countBytes; for (let i = 0; i < importCount; i++) { - const [modLen, modLenBytes] = readLEB128(src, pos); pos += modLenBytes + modLen; - const [fieldLen, fieldLenBytes] = readLEB128(src, pos); pos += fieldLenBytes + fieldLen; + const [modLen, modLenBytes] = readLEB128(src, pos); + pos += modLenBytes + modLen; + const [fieldLen, fieldLenBytes] = readLEB128(src, pos); + pos += fieldLenBytes + fieldLen; const kind = src[pos++]; - if (kind === 0) { numFuncImports++; const [, n] = readLEB128(src, pos); pos += n; } - else if (kind === 1) { pos++; const f = src[pos++]; const [, n] = readLEB128(src, pos); pos += n; if (f & 1) { const [, n2] = readLEB128(src, pos); pos += n2; } } - else if (kind === 2) { const f = src[pos++]; const [, n] = readLEB128(src, pos); pos += n; if (f & 1) { const [, n2] = readLEB128(src, pos); pos += n2; } } - else if (kind === 3) { pos += 2; } + if (kind === 0) { + numFuncImports++; + const [, n] = readLEB128(src, pos); + pos += n; + } else if (kind === 1) { + pos++; + const f = src[pos++]; + const [, n] = readLEB128(src, pos); + pos += n; + if (f & 1) { + const [, n2] = readLEB128(src, pos); + pos += n2; + } + } else if (kind === 2) { + const f = src[pos++]; + const [, n] = readLEB128(src, pos); + pos += n; + if (f & 1) { + const [, n2] = readLEB128(src, pos); + pos += n2; + } + } else if (kind === 3) { + pos += 2; + } } break; } @@ -2063,12 +4145,16 @@ function detectChannelBaseTlsOffset(programBytes: ArrayBuffer): number { for (const sec of sections) { if (sec.id === 7) { let pos = sec.contentOffset; - const [exportCount, countBytes] = readLEB128(src, pos); pos += countBytes; + const [exportCount, countBytes] = readLEB128(src, pos); + pos += countBytes; for (let i = 0; i < exportCount; i++) { - const [nameLen, nameLenBytes] = readLEB128(src, pos); pos += nameLenBytes; - const name = new TextDecoder().decode(src.subarray(pos, pos + nameLen)); pos += nameLen; + const [nameLen, nameLenBytes] = readLEB128(src, pos); + pos += nameLenBytes; + const name = new TextDecoder().decode(src.subarray(pos, pos + nameLen)); + pos += nameLen; const kind = src[pos++]; - const [idx, idxBytes] = readLEB128(src, pos); pos += idxBytes; + const [idx, idxBytes] = readLEB128(src, pos); + pos += idxBytes; if (kind === 0 && name === "__get_channel_base_addr") { channelBaseExportFuncIdx = idx; break; @@ -2089,17 +4175,24 @@ function detectChannelBaseTlsOffset(programBytes: ArrayBuffer): number { for (const sec of sections) { if (sec.id !== 10) continue; let pos = sec.contentOffset; - const [, funcCountBytes] = readLEB128(src, pos); pos += funcCountBytes; + const [, funcCountBytes] = readLEB128(src, pos); + pos += funcCountBytes; // Skip to the exported function's body for (let i = 0; i < exportCodeEntry; i++) { const [bodySize, bodySizeBytes] = readLEB128(src, pos); pos += bodySizeBytes + bodySize; } - const [, bodySizeBytes] = readLEB128(src, pos); pos += bodySizeBytes; + const [, bodySizeBytes] = readLEB128(src, pos); + pos += bodySizeBytes; // Skip locals - const [localCount, lcBytes] = readLEB128(src, pos); pos += lcBytes; - for (let i = 0; i < localCount; i++) { const [, n] = readLEB128(src, pos); pos += n; pos++; } + const [localCount, lcBytes] = readLEB128(src, pos); + pos += lcBytes; + for (let i = 0; i < localCount; i++) { + const [, n] = readLEB128(src, pos); + pos += n; + pos++; + } // i32.const = 0x41, i64.const = 0x42 (wasm64 uses i64 for addresses) const I32_CONST = 0x41; @@ -2115,7 +4208,8 @@ function detectChannelBaseTlsOffset(programBytes: ArrayBuffer): number { // Pattern 3: instrumented/optimized — global.get ; i32/i64.const ; i32/i64.add if (src[pos] === 0x23) { let p3 = pos + 1; - const [, globalIdxBytes] = readLEB128(src, p3); p3 += globalIdxBytes; + const [, globalIdxBytes] = readLEB128(src, p3); + p3 += globalIdxBytes; if (src[p3] === I32_CONST || src[p3] === I64_CONST) { p3++; const [tlsOffset] = readLEB128(src, p3); @@ -2126,7 +4220,8 @@ function detectChannelBaseTlsOffset(programBytes: ArrayBuffer): number { // Pattern 2: wrapper — call ; call ; end if (src[pos] !== 0x10) return -1; pos++; - const [, ctorIdxBytes] = readLEB128(src, pos); pos += ctorIdxBytes; + const [, ctorIdxBytes] = readLEB128(src, pos); + pos += ctorIdxBytes; if (src[pos] !== 0x10) return -1; pos++; const [actualFuncIdx] = readLEB128(src, pos); @@ -2135,14 +4230,21 @@ function detectChannelBaseTlsOffset(programBytes: ArrayBuffer): number { if (actualCodeEntry < 0) return -1; let pos2 = sec.contentOffset; - const [, fcb2] = readLEB128(src, pos2); pos2 += fcb2; + const [, fcb2] = readLEB128(src, pos2); + pos2 += fcb2; for (let i = 0; i < actualCodeEntry; i++) { const [bs, bsb] = readLEB128(src, pos2); pos2 += bsb + bs; } - const [, bsb2] = readLEB128(src, pos2); pos2 += bsb2; - const [lc2, lcb2] = readLEB128(src, pos2); pos2 += lcb2; - for (let i = 0; i < lc2; i++) { const [, n] = readLEB128(src, pos2); pos2 += n; pos2++; } + const [, bsb2] = readLEB128(src, pos2); + pos2 += bsb2; + const [lc2, lcb2] = readLEB128(src, pos2); + pos2 += lcb2; + for (let i = 0; i < lc2; i++) { + const [, n] = readLEB128(src, pos2); + pos2 += n; + pos2++; + } if (src[pos2] !== I32_CONST && src[pos2] !== I64_CONST) return -1; pos2++; @@ -2164,7 +4266,14 @@ function setupChannelBase( // If the module imports env.__channel_base as a global, the channel offset was // already set at instantiation via WebAssembly.Global in buildImportObject. const moduleImports = WebAssembly.Module.imports(module); - if (moduleImports.some(i => i.module === "env" && i.name === "__channel_base" && i.kind === "global")) { + if ( + moduleImports.some( + (i) => + i.module === "env" && + i.name === "__channel_base" && + i.kind === "global", + ) + ) { return; } @@ -2191,20 +4300,37 @@ function setupChannelBase( * Send SYS_FORK through the channel and wait for the result. * Returns child pid on success, or -errno on failure. */ -function sendForkSyscall(memory: WebAssembly.Memory, channelOffset: number): number { +function sendForkSyscall( + memory: WebAssembly.Memory, + channelOffset: number, +): number { const view = new DataView(memory.buffer); - view.setInt32(channelOffset + CH_SYSCALL, HOST_INTERCEPTED_SYSCALLS.SYS_FORK, true); + view.setInt32( + channelOffset + CH_SYSCALL, + HOST_INTERCEPTED_SYSCALLS.SYS_FORK, + true, + ); for (let i = 0; i < 6; i++) { view.setBigInt64(channelOffset + CH_ARGS + i * CH_ARG_SIZE, 0n, true); } + markDeferredSignalDelivery(view, channelOffset); const i32 = new Int32Array(memory.buffer); Atomics.store(i32, (channelOffset + CH_STATUS) / 4, CHANNEL_STATUS_PENDING); Atomics.notify(i32, (channelOffset + CH_STATUS) / 4, 1); - while (Atomics.wait(i32, (channelOffset + CH_STATUS) / 4, CHANNEL_STATUS_PENDING) === "ok") { /* */ } + while ( + Atomics.wait( + i32, + (channelOffset + CH_STATUS) / 4, + CHANNEL_STATUS_PENDING, + ) === "ok" + ) { + /* */ + } const result = Number(view.getBigInt64(channelOffset + CH_RETURN, true)); const err = view.getUint32(channelOffset + CH_ERRNO, true); + clearDeferredSignalDelivery(view, channelOffset); Atomics.store(i32, (channelOffset + CH_STATUS) / 4, CHANNEL_STATUS_IDLE); if (err) return -err; @@ -2257,7 +4383,13 @@ export function patchWasmForThread(bytes: ArrayBuffer): ArrayBuffer { } // Parse all sections - interface Section { id: number; offset: number; totalSize: number; contentOffset: number; contentSize: number; } + interface Section { + id: number; + offset: number; + totalSize: number; + contentOffset: number; + contentSize: number; + } const sections: Section[] = []; let numFuncImports = 0; let hasStartSection = false; @@ -2268,46 +4400,27 @@ export function patchWasmForThread(bytes: ArrayBuffer): ArrayBuffer { const [sectionSize, sizeBytes] = readLEB128(src, offset + 1); const contentOffset = offset + 1 + sizeBytes; const totalSize = 1 + sizeBytes + sectionSize; - sections.push({ id: sectionId, offset, totalSize, contentOffset, contentSize: sectionSize }); + sections.push({ + id: sectionId, + offset, + totalSize, + contentOffset, + contentSize: sectionSize, + }); if (sectionId === 8) hasStartSection = true; offset += totalSize; } if (!hasStartSection) return bytes; - // Count function imports from Import section (id=2) - for (const sec of sections) { - if (sec.id === 2) { - let pos = sec.contentOffset; - const [importCount, countBytes] = readLEB128(src, pos); - pos += countBytes; - for (let i = 0; i < importCount; i++) { - const [modLen, modLenBytes] = readLEB128(src, pos); - pos += modLenBytes + modLen; - const [fieldLen, fieldLenBytes] = readLEB128(src, pos); - pos += fieldLenBytes + fieldLen; - const kind = src[pos++]; - if (kind === 0) { // function import - numFuncImports++; - const [, typeIdxBytes] = readLEB128(src, pos); - pos += typeIdxBytes; - } else if (kind === 1) { // table - pos++; // reftype - const flags = src[pos++]; - const [, minBytes] = readLEB128(src, pos); pos += minBytes; - if (flags & 1) { const [, maxBytes] = readLEB128(src, pos); pos += maxBytes; } - } else if (kind === 2) { // memory - const flags = src[pos++]; - const [, minBytes] = readLEB128(src, pos); pos += minBytes; - if (flags & 1) { const [, maxBytes] = readLEB128(src, pos); pos += maxBytes; } - } else if (kind === 3) { // global - pos++; // valtype - pos++; // mutability - } - } - break; - } - } + // WHY: import descriptors can contain recursive GC types, multi-byte + // concrete references, table64 limits, tags, and future standardized + // imports. The engine has already validated and decoded that grammar; using + // its reflection avoids a second partial parser shifting every function + // index when a non-function import is not one byte wide. + numFuncImports = WebAssembly.Module.imports( + new WebAssembly.Module(bytes), + ).filter((entry) => entry.kind === "function").length; // Find the constructor function by looking at the exported helper wrappers. // Plain lld output puts `call $__wasm_call_ctors` first. After @@ -2332,7 +4445,8 @@ export function patchWasmForThread(bytes: ArrayBuffer): ArrayBuffer { const kind = src[pos++]; const [idx, idxBytes] = readLEB128(src, pos); pos += idxBytes; - if (kind === 0) { // function export + if (kind === 0) { + // function export exportedFuncIndices.push(idx); exportFuncIndicesByName.set(name, idx); } @@ -2351,6 +4465,28 @@ export function patchWasmForThread(bytes: ArrayBuffer): ArrayBuffer { return skipLEB(pos); // offset } + function skipValueType(pos: number): number { + const kind = src[pos++]; + // `(ref null )` and `(ref )` carry a signed + // heap-type/type-index LEB. Abstract shorthand references and all numeric + // value types are single-byte encodings. + return kind === 0x63 || kind === 0x64 ? skipLEB(pos) : pos; + } + + function skipBlockType(pos: number): number { + const kind = src[pos]; + if (kind === 0x40) return pos + 1; // empty + if ( + (kind >= 0x7b && kind <= 0x7f) || + (kind >= 0x65 && kind <= 0x70) || + kind === 0x63 || + kind === 0x64 + ) { + return skipValueType(pos); + } + return skipLEB(pos); // signed type index + } + function getInstructionStartAndEnd( codeSection: Section, funcIndex: number, @@ -2376,7 +4512,7 @@ export function patchWasmForThread(bytes: ArrayBuffer): ArrayBuffer { pos += localCountBytes; for (let i = 0; i < localCount; i++) { pos = skipLEB(pos); // count - pos++; // valtype + pos = skipValueType(pos); } return { start: pos, end: bodyEnd }; @@ -2390,21 +4526,29 @@ export function patchWasmForThread(bytes: ArrayBuffer): ArrayBuffer { let pos = bounds.start; while (pos < bounds.end) { const op = src[pos++]; - if (op === 0x10) { // call + if (op === 0x10) { + // call const [target, n] = readLEB128(src, pos); pos += n; calls.push(target); - } else if (op === 0x11 || op === 0x13) { // call_indirect / return_call_indirect + } else if (op === 0x11 || op === 0x13) { + // call_indirect / return_call_indirect pos = skipLEB(pos); pos = skipLEB(pos); } else if (op === 0x12 || op === 0x14 || op === 0x15) { pos = skipLEB(pos); } else if (op === 0x02 || op === 0x03 || op === 0x04) { - // blocktype: empty marker, valtype, or signed type index. - pos = src[pos] === 0x40 || src[pos] >= 0x70 ? pos + 1 : skipLEB(pos); - } else if (op === 0x0c || op === 0x0d || (op >= 0x20 && op <= 0x26) || op === 0xd0 || op === 0xd2) { + pos = skipBlockType(pos); + } else if ( + op === 0x0c || + op === 0x0d || + (op >= 0x20 && op <= 0x26) || + op === 0xd0 || + op === 0xd2 + ) { pos = skipLEB(pos); - } else if (op === 0x0e) { // br_table + } else if (op === 0x0e) { + // br_table const [count, n] = readLEB128(src, pos); pos += n; for (let i = 0; i <= count; i++) pos = skipLEB(pos); @@ -2455,7 +4599,11 @@ export function patchWasmForThread(bytes: ArrayBuffer): ArrayBuffer { for (const name of helperNames) { const funcIndex = exportFuncIndicesByName.get(name); if (funcIndex === undefined) continue; - const perFunction = new Set(scanCallTargets(sec, funcIndex).filter(target => target >= numFuncImports)); + const perFunction = new Set( + scanCallTargets(sec, funcIndex).filter( + (target) => target >= numFuncImports, + ), + ); for (const target of perFunction) { const entry = counts.get(target); if (entry) { @@ -2466,11 +4614,13 @@ export function patchWasmForThread(bytes: ArrayBuffer): ArrayBuffer { } } - let best: { target: number; count: number; firstOrder: number } | null = null; + let best: { target: number; count: number; firstOrder: number } | null = + null; for (const [target, value] of counts) { if ( value.count >= 2 && - (!best || value.count > best.count || + (!best || + value.count > best.count || (value.count === best.count && value.firstOrder < best.firstOrder)) ) { best = { target, count: value.count, firstOrder: value.firstOrder }; @@ -2496,7 +4646,8 @@ export function patchWasmForThread(bytes: ArrayBuffer): ArrayBuffer { } } - const ctorCodeEntry = ctorFuncIndex >= 0 ? ctorFuncIndex - numFuncImports : -1; + const ctorCodeEntry = + ctorFuncIndex >= 0 ? ctorFuncIndex - numFuncImports : -1; if (ctorFuncIndex < 0) { // No ctor found — still strip start section but can't neuter the ctor body } @@ -2522,7 +4673,10 @@ export function patchWasmForThread(bytes: ArrayBuffer): ArrayBuffer { const [bodySize, bodySizeBytes] = readLEB128(src, targetBodyStart); targetBodyStart += bodySizeBytes + bodySize; } - const [origBodySize, origBodySizeBytes] = readLEB128(src, targetBodyStart); + const [origBodySize, origBodySizeBytes] = readLEB128( + src, + targetBodyStart, + ); const origBodyEnd = targetBodyStart + origBodySizeBytes + origBodySize; // New body: size=2, content = 0x00 (0 locals) + 0x0B (end) @@ -2530,7 +4684,7 @@ export function patchWasmForThread(bytes: ArrayBuffer): ArrayBuffer { // Compute new section content size const beforeTarget = targetBodyStart - sec.contentOffset; - const afterTarget = (sec.contentOffset + sec.contentSize) - origBodyEnd; + const afterTarget = sec.contentOffset + sec.contentSize - origBodyEnd; const newContentSize = beforeTarget + newBody.length + afterTarget; const newSectionSizeBytes = encodeLEB128(newContentSize); @@ -2538,7 +4692,9 @@ export function patchWasmForThread(bytes: ArrayBuffer): ArrayBuffer { chunks.push(new Uint8Array(newSectionSizeBytes)); chunks.push(src.subarray(sec.contentOffset, targetBodyStart)); // func count + bodies before target chunks.push(newBody); // patched function body - chunks.push(src.subarray(origBodyEnd, sec.contentOffset + sec.contentSize)); // bodies after target + chunks.push( + src.subarray(origBodyEnd, sec.contentOffset + sec.contentSize), + ); // bodies after target } else { // Copy section as-is chunks.push(src.subarray(sec.offset, sec.offset + sec.totalSize)); @@ -2591,26 +4747,46 @@ export async function centralizedThreadWorkerMain( const ptrWidth = initData.ptrWidth ?? 4; let threadInstance: WebAssembly.Instance | undefined; + let threadProcessContinuation: ForkProcessContinuationCoordinator | null = + null; + let threadTableReplication: ProcessTableReplicationOwner | null = null; + let threadHostImportRuntime: ForkHostImportWorkerRuntime | null = null; + let threadExternrefTokens: ForkExternrefTokenCache | null = null; let processDlopenLock: Int32Array | undefined; + let processDlopenOwner: Int32Array | undefined; let pthreadForkLockHeld = false; const acquirePthreadForkLock = (): boolean => { - if (!processDlopenLock) { - throw new Error(`pid=${pid} tid=${tid}: missing process dlopen lock`); + if (!processDlopenLock || !processDlopenOwner) { + throw new Error( + `pid=${pid} tid=${tid}: missing process dlopen ownership`, + ); } if (pthreadForkLockHeld) { throw new Error(`pid=${pid} tid=${tid}: pthread fork lock already held`); } for (;;) { + const transactionOwner = Atomics.load(processDlopenOwner, 0); + if (transactionOwner !== DLOPEN_OWNER_IDLE && transactionOwner !== tid) { + Atomics.wait(processDlopenOwner, 0, transactionOwner); + continue; + } const owner = Atomics.load(processDlopenLock, 0); - if (owner < DLOPEN_LOCK_IDLE) return false; + if (owner < DLOPEN_LOCK_IDLE) { + // dlopen is finite and publishes the archive generation before + // releasing this writer token. Waiting preserves ordinary pthread + // fork/dlopen semantics instead of exposing a scheduler race as + // ENOTSUP. + Atomics.wait(processDlopenLock, 0, owner); + continue; + } if (owner >= DLOPEN_LOCK_MAX_READERS) { throw new Error( `pid=${pid} tid=${tid}: process dlopen lock reader overflow`, ); } if ( - Atomics.compareExchange(processDlopenLock, 0, owner, owner + 1) - === owner + Atomics.compareExchange(processDlopenLock, 0, owner, owner + 1) === + owner ) { pthreadForkLockHeld = true; return true; @@ -2629,8 +4805,8 @@ export async function centralizedThreadWorkerMain( ); } if ( - Atomics.compareExchange(processDlopenLock, 0, owner, owner - 1) - === owner + Atomics.compareExchange(processDlopenLock, 0, owner, owner - 1) === + owner ) { pthreadForkLockHeld = false; if (owner === 1) Atomics.notify(processDlopenLock, 0); @@ -2651,55 +4827,223 @@ export async function centralizedThreadWorkerMain( ? initData.programModule : new WebAssembly.Module(programBytes!); - const moduleExports = WebAssembly.Module.exports(module); - const hasForkInstrumentation = hasCompleteForkInstrumentation(moduleExports, pid); + const hasForkInstrumentation = hasCompleteForkInstrumentation(module, pid); + if (hasForkInstrumentation) { + if ( + initData.forkHostImports === undefined || + initData.externrefGenerationId === undefined + ) { + throw new Error( + `pid=${pid} tid=${tid}: ABI 43 fork artifact requires its process ` + + "owner host-import mailbox and externref generation", + ); + } + threadExternrefTokens = new ForkExternrefTokenCache( + initData.externrefGenerationId, + ); + threadHostImportRuntime = new ForkHostImportWorkerRuntime( + initData.forkHostImports, + pid, + initData.externrefGenerationId, + threadExternrefTokens, + (wake) => { + port.postMessage({ + type: "fork_host_import", + wake, + } satisfies WorkerToHostMessage); + }, + ); + } + const threadForkCapabilityClaim = readForkInstrumentCapabilityClaim(module); + const hasDylinkForkRole = forkInstrumentRoleAvailable( + threadForkCapabilityClaim, + FORK_CAP_DYLINK_MAIN, + ); let forkBufAddr = 0; const forkAnchorAddr = channelOffset - FORK_BUF_SIZE; const threadForkContinuation = hasForkInstrumentation ? new LinkedForkContinuation( memory, readLinkedFrameFormat(module), - (size) => continuationMmap(memory, channelOffset, size, `pid=${pid} tid=${tid}`), - (addr, size) => continuationMunmap( + (size) => + continuationMmap( + memory, + channelOffset, + size, + `pid=${pid} tid=${tid}`, + ), + (addr, size) => + continuationMunmap( + memory, + channelOffset, + addr, + size, + `pid=${pid} tid=${tid}`, + ), + `pid=${pid} tid=${tid}`, + ) + : null; + const threadTemplateId = hasForkInstrumentation + ? await computeForkModuleTemplateId(initData.programBytes) + : null; + const newThreadModuleStateArena = (): ForkModuleStateArena => + new ForkModuleStateArena( + memory, + ptrWidth, + (size) => + continuationMmap( + memory, + channelOffset, + size, + `pid=${pid} tid=${tid}: module state`, + ), + (addr, size) => + continuationMunmap( memory, channelOffset, addr, size, - `pid=${pid} tid=${tid}`, + `pid=${pid} tid=${tid}: module state`, ), - `pid=${pid} tid=${tid}`, + `pid=${pid} tid=${tid}`, + ); + const threadActivationRegistry = hasForkInstrumentation + ? new ForkActivationRegistry( + memory, + new ForkExternrefTokenRecipeProvider( + threadExternrefTokens!, + (value) => + threadHostImportRuntime!.localExceptions.normalizeUnclaimedForkValue( + value, + ), + ), + `pid=${pid} tid=${tid}: fork activations`, + (size) => + continuationMmap( + memory, + channelOffset, + size, + `pid=${pid} tid=${tid}: reference scratch`, + ), + (addr, size) => + continuationMunmap( + memory, + channelOffset, + addr, + size, + `pid=${pid} tid=${tid}: reference scratch`, + ), + ) + : null; + const threadImportedStateCapture = threadActivationRegistry + ? new ForkImportedGlobalCapture( + `pid=${pid} tid=${tid}: imported activation state`, + ) + : null; + threadProcessContinuation = threadActivationRegistry + ? new ForkProcessContinuationCoordinator( + memory, + threadActivationRegistry, + `pid=${pid} tid=${tid}: process continuation`, + ) + : null; + const threadExceptionBroker = threadActivationRegistry + ? new ForkExceptionBroker( + threadActivationRegistry, + `pid=${pid} tid=${tid}: exception broker`, + undefined, + (value) => + threadHostImportRuntime!.localExceptions.normalizeUnclaimedForkException( + value, + ), ) : null; - const processArchiveHeadOffset = ptrWidth === 8 - ? DLOPEN_HEAD_OFFSET_WASM64 - : DLOPEN_HEAD_OFFSET_WASM32; - const processArchiveHeadAddr = processChannelOffset - - FORK_BUF_SIZE - - processArchiveHeadOffset; - const processArchiveLockOffset = ptrWidth === 8 - ? DLOPEN_LOCK_OFFSET_WASM64 - : DLOPEN_LOCK_OFFSET_WASM32; - const processArchiveLockAddr = processChannelOffset - - FORK_BUF_SIZE - - processArchiveLockOffset; + let threadExceptionProvider: ForkExceptionProvider | null = null; + if (threadProcessContinuation && threadForkContinuation) { + threadProcessContinuation.prepareActivation({ + activationId: 0, + continuation: threadForkContinuation, + publishProcessLaunchRoot: (address) => { + writeForkContinuationAnchor( + memory, + forkAnchorAddr, + ptrWidth, + address, + ); + forkBufAddr = address; + }, + readProcessLaunchRoot: () => { + const view = new DataView(memory.buffer); + return ptrWidth === 8 + ? Number(view.getBigUint64(forkAnchorAddr, true)) + : view.getUint32(forkAnchorAddr, true); + }, + }); + } + const processArchiveHeadOffset = + ptrWidth === 8 ? DLOPEN_HEAD_OFFSET_WASM64 : DLOPEN_HEAD_OFFSET_WASM32; + const processArchiveHeadAddr = + processChannelOffset - FORK_BUF_SIZE - processArchiveHeadOffset; + const processArchiveLockOffset = + ptrWidth === 8 ? DLOPEN_LOCK_OFFSET_WASM64 : DLOPEN_LOCK_OFFSET_WASM32; + const processArchiveLockAddr = + processChannelOffset - FORK_BUF_SIZE - processArchiveLockOffset; + const processArchiveOwnerOffset = + ptrWidth === 8 ? DLOPEN_OWNER_OFFSET_WASM64 : DLOPEN_OWNER_OFFSET_WASM32; + const processArchiveOwnerAddr = + processChannelOffset - FORK_BUF_SIZE - processArchiveOwnerOffset; if ( - !Number.isSafeInteger(processArchiveHeadAddr) - || processArchiveHeadAddr <= 0 - || processArchiveHeadAddr + ptrWidth > memory.buffer.byteLength - || !Number.isSafeInteger(processArchiveLockAddr) - || processArchiveLockAddr <= 0 - || processArchiveLockAddr + 4 > memory.buffer.byteLength + !Number.isSafeInteger(processArchiveHeadAddr) || + processArchiveHeadAddr <= 0 || + processArchiveHeadAddr + ptrWidth > memory.buffer.byteLength || + !Number.isSafeInteger(processArchiveLockAddr) || + processArchiveLockAddr <= 0 || + processArchiveLockAddr + 4 > memory.buffer.byteLength || + !Number.isSafeInteger(processArchiveOwnerAddr) || + processArchiveOwnerAddr <= 0 || + processArchiveOwnerAddr + 4 > memory.buffer.byteLength ) { throw new Error( `pid=${pid} tid=${tid}: invalid process dlopen archive anchor ` + `${String(processArchiveHeadAddr)}`, ); } - processDlopenLock = new Int32Array(memory.buffer, processArchiveLockAddr, 1); - const processHasDlopenArchive = (): boolean => { - return ptrWidth === 8 - ? Atomics.load(new BigUint64Array(memory.buffer, processArchiveHeadAddr, 1), 0) !== 0n - : Atomics.load(new Uint32Array(memory.buffer, processArchiveHeadAddr, 1), 0) !== 0; + processDlopenLock = new Int32Array( + memory.buffer, + processArchiveLockAddr, + 1, + ); + processDlopenOwner = new Int32Array( + memory.buffer, + processArchiveOwnerAddr, + 1, + ); + const processArchiveControlAddr = processChannelOffset - FORK_BUF_SIZE; + const processGenerationOffset = + ptrWidth === 8 + ? DLOPEN_GENERATION_OFFSET_WASM64 + : DLOPEN_GENERATION_OFFSET_WASM32; + const processGenerationAddress = + processArchiveControlAddr - processGenerationOffset; + const threadTableReplicationImports: ForkActivationTableReplication = { + generationAddress: new WebAssembly.Global( + { value: "i64", mutable: false }, + BigInt(processGenerationAddress), + ), + reconcile: (): bigint => threadTableReplication?.reconcile() ?? 0n, + beginMutation: (): bigint => + threadTableReplication?.beginMutation() ?? 0n, + commit: (activationId, ownerId, firstIndex, length): void => { + threadTableReplication?.commit( + activationId, + ownerId, + firstIndex, + length, + ); + }, + abort: (): void => { + threadTableReplication?.abort(); + }, }; let forkResult = 0; @@ -2715,62 +5059,61 @@ export async function centralizedThreadWorkerMain( ); if (hasForkInstrumentation) { kernelImports.kernel_fork = (): number => { - if (!threadInstance) return -38; // ENOSYS + if (!threadInstance || !threadProcessContinuation) return -38; // ENOSYS - const getState = threadInstance.exports.wpk_fork_state as () => number; - const state = getState(); - if (state === 2) { + const phase = threadProcessContinuation.phaseName(); + if (phase === "parent-replay") { try { - (threadInstance.exports.wpk_fork_rewind_end as () => void)(); - threadForkContinuation!.finishReplayAndRelease(); - writeForkContinuationAnchor(memory, forkAnchorAddr, ptrWidth, 0); - forkBufAddr = 0; + threadProcessContinuation.finishReplay(); } finally { releasePthreadForkLock(); } return forkResult; } - if (state === 3) { + if (phase === "abort-replay") { const errno = threadForkContinuation!.abortErrno(); try { - (threadInstance.exports.wpk_fork_abort_end as () => void)(); - threadForkContinuation!.finishAbortReplayAndRelease(); - writeForkContinuationAnchor(memory, forkAnchorAddr, ptrWidth, 0); - forkBufAddr = 0; + threadProcessContinuation.finishAbortReplay(); } finally { releasePthreadForkLock(); } return -errno; } - - // Side modules live in the process main worker's module/table/tag - // graph. A pthread worker cannot replay that graph into its own - // instance, so fork must fail before unwind once the process has ever - // loaded a side module. The head is read live from shared memory so a - // dlopen after pthread creation is still observed. - if (!acquirePthreadForkLock()) { - return -95; // ENOTSUP: process-main dlopen is active + if (phase !== "idle") { + throw new Error( + `pid=${pid} tid=${tid}: fork import reached while process ` + + `continuation is ${phase}`, + ); } - if (processHasDlopenArchive()) { + + try { + // Reconciliation may instantiate a missing side module and execute + // its start function, so it requires writer ownership. Afterward, + // acquire the long-lived fork reader and verify no publication won + // the handoff race before capturing activation state. + for (;;) { + threadTableReplication?.reconcileNow(); + acquirePthreadForkLock(); + if ( + !threadTableReplication || + threadTableReplication.isCurrentUnderLock() + ) { + break; + } + releasePthreadForkLock(); + } + } catch (error) { releasePthreadForkLock(); - return -95; // ENOTSUP: pthreads cannot replay process side modules + throw error; } + const arena = newThreadModuleStateArena(); try { - forkBufAddr = Number(threadForkContinuation!.beginUnwind()); - writeForkContinuationAnchor( - memory, - forkAnchorAddr, - ptrWidth, - forkBufAddr, - ); - invokeForkContinuationBegin( - threadInstance.exports.wpk_fork_unwind_begin, - forkBufAddr, - ptrWidth, - `pid=${pid} tid=${tid}: linked fork unwind`, - ); + arena.begin(); + threadProcessContinuation.beginCapture(arena); + threadImportedStateCapture?.appendTo(arena); } catch (error) { + if (arena.hasActiveArena()) arena.release(); releasePthreadForkLock(); if (error instanceof ContinuationAllocationError) return -error.errno; throw error; @@ -2779,7 +5122,6 @@ export async function centralizedThreadWorkerMain( }; } else { kernelImports.kernel_fork = (): number => { - if (processHasDlopenArchive()) return -95; // ENOTSUP throw new Error( `pid=${pid} tid=${tid}: kernel_fork reached without complete ` + "wasm-fork-instrument exports. Rebuild the program with " + @@ -2789,9 +5131,115 @@ export async function centralizedThreadWorkerMain( } const threadLongjmpTag = createLongjmpTag(ptrWidth); const threadCppExceptionTag = createCppExceptionTag(ptrWidth); - const threadDlopenImports = buildUnsupportedThreadDlopenImports(memory); - const importObject = buildImportObject(module, memory, kernelImports, channelOffset, threadDlopenImports, - () => threadInstance, ptrWidth, threadLongjmpTag, threadCppExceptionTag, + const threadForkUnwindTag = createForkUnwindTag(); + const replicaActivationOwner = + hasDylinkForkRole && + threadProcessContinuation && + threadActivationRegistry && + threadExceptionBroker + ? createProcessDylinkActivationOwner({ + memory, + ptrWidth, + channelOffset, + forkUnwindTag: threadForkUnwindTag, + coordinator: threadProcessContinuation, + registry: threadActivationRegistry, + exceptionBroker: threadExceptionBroker, + importedStateCapture: threadImportedStateCapture ?? undefined, + tableReplication: threadTableReplicationImports, + isForkChild: false, + isPthreadReplica: true, + invokeProcessFork: () => { + const fork = threadInstance?.exports.fork; + if (typeof fork !== "function") { + throw new Error( + `pid=${pid} tid=${tid}: dylink fork role is missing ` + + "the main libc fork export", + ); + } + return Number((fork as () => number)()); + }, + label: `pid=${pid} tid=${tid}: dylink table activations`, + }) + : undefined; + const threadDlopenSupport = buildDlopenImports( + memory, + channelOffset, + processArchiveControlAddr, + () => + threadInstance?.exports.__indirect_function_table as + WebAssembly.Table | undefined, + () => + threadInstance?.exports.__stack_pointer as + WebAssembly.Global | undefined, + () => threadInstance, + ptrWidth, + threadLongjmpTag, + threadCppExceptionTag, + replicaActivationOwner, + hasDylinkForkRole + ? undefined + : `pid=${pid} tid=${tid}: main artifact lacks the dylink fork role capability`, + threadForkUnwindTag, + (table, firstIndex, length) => { + threadActivationRegistry?.markTableMutation(table, firstIndex, length); + }, + threadHostImportRuntime ?? undefined, + tid, + ); + if (threadActivationRegistry) { + threadTableReplication = createProcessTableReplicationOwner({ + generationAddress: processGenerationAddress, + registry: threadActivationRegistry, + dlopen: threadDlopenSupport, + newArena: newThreadModuleStateArena, + materializeModules: (snapshot) => { + threadDlopenSupport.replayDlopens(snapshot); + }, + restoreSnapshots: true, + label: `pid=${pid} tid=${tid}`, + }); + } + const threadCoordinator = threadProcessContinuation; + const threadForkEnvImports = + threadCoordinator && threadActivationRegistry && threadExceptionBroker + ? { + ...threadCoordinator.continuationImports(0, (errno) => { + threadCoordinator.beginCaptureAbort(errno); + }), + ...buildForkActivationStateImports( + 0, + threadActivationRegistry, + undefined, + threadTableReplicationImports, + ), + ...buildForkExceptionImports({ + activationId: 0, + ptrWidth, + registry: threadActivationRegistry, + broker: threadExceptionBroker, + provider: () => { + if (!threadExceptionProvider) { + throw new Error( + `pid=${pid} tid=${tid}: exception codec called before registration`, + ); + } + return threadExceptionProvider; + }, + }), + } + : undefined; + const importObject = buildImportObject( + module, + memory, + kernelImports, + channelOffset, + threadDlopenSupport.imports, + () => threadInstance, + ptrWidth, + threadLongjmpTag, + threadCppExceptionTag, + threadForkUnwindTag, (timedOutPtr, vmInterruptPtr, seconds) => { port.postMessage({ type: "vm_interrupt_timer", @@ -2801,23 +5249,90 @@ export async function centralizedThreadWorkerMain( seconds, } satisfies WorkerToHostMessage); }, - threadForkContinuation ?? undefined, - () => { - if (!threadInstance) { - throw new Error(`pid=${pid} tid=${tid}: continuation abort before instantiation`); - } - invokeForkContinuationBegin( - threadInstance.exports.wpk_fork_abort_begin, - forkBufAddr, - ptrWidth, - `pid=${pid} tid=${tid}: linked fork abort`, - ); - }); - const instance = new WebAssembly.Instance(module, importObject); + threadForkEnvImports, + ); + const routedThreadImportObject = threadHostImportRuntime + ? threadHostImportRuntime.routeImportObject( + initData.programBytes, + importObject, + ) + : importObject; + const threadMainImportedState = + threadImportedStateCapture?.prepareActivation( + 0, + module, + routedThreadImportObject, + ); + const threadInstanceImports = (threadMainImportedState?.imports ?? + routedThreadImportObject) as WebAssembly.Imports; + const instance = new WebAssembly.Instance(module, threadInstanceImports); threadInstance = instance; + threadMainImportedState?.complete(instance); + if ( + hasForkInstrumentation && + threadProcessContinuation && + threadActivationRegistry && + threadTemplateId + ) { + const threadBootstrap = instance.exports + .wpk_fork_module_thread_bootstrap as (() => void) | undefined; + if (!threadBootstrap) { + throw new Error( + `pid=${pid} tid=${tid}: fork module is missing thread bootstrap`, + ); + } + threadExceptionProvider = forkExceptionProviderFromInstance(0, instance); + threadProcessContinuation.registerActivation( + forkActivationRegistrationFromInstance({ + activationId: 0, + module, + instance, + templateId: threadTemplateId, + exceptionProvider: threadExceptionProvider, + }), + forkResumeTargetsFromInstance(module, instance), + ); + try { + // The pthread bootstrap consumes passive element segments, so static + // root harvesting and table-dirty registration must precede it just as + // they do for the process-main bootstrap. + threadBootstrap(); + threadImportedStateCapture?.bindTableDirtyTrackers( + new Map( + threadActivationRegistry + .activations() + .map((activation) => [ + activation.activationId, + activation.tableDirty, + ]), + ), + ); + } catch (error) { + threadTableReplication?.abortActiveMutations(); + threadProcessContinuation.unregisterActivation(0); + threadExceptionProvider = null; + throw error; + } + } + + const threadTable = instance.exports.__indirect_function_table as + WebAssembly.Table | undefined; + const threadStackPointer = instance.exports.__stack_pointer as + WebAssembly.Global | undefined; + if ( + (!threadTable || !threadStackPointer) && + threadDlopenSupport.archive.generation() !== 0 + ) { + throw new Error( + `pid=${pid} tid=${tid}: process has dlopen table recipes but ` + + "the pthread instance has no shared table/stack binding", + ); + } + threadTableReplication?.reconcileNow(); // Initialize Wasm TLS for this thread in the slot's explicit TLS/control page. - const wasmInitTls = instance.exports.__wasm_init_tls as ((addr: number | bigint) => void) | undefined; + const wasmInitTls = instance.exports.__wasm_init_tls as + ((addr: number | bigint) => void) | undefined; const tlsBlock = tlsOffset; if (wasmInitTls && tlsBlock > 0) { @@ -2825,13 +5340,15 @@ export async function centralizedThreadWorkerMain( } // Set __stack_pointer - const stackPointer = instance.exports.__stack_pointer as WebAssembly.Global | undefined; + const stackPointer = instance.exports.__stack_pointer as + WebAssembly.Global | undefined; if (stackPointer) { stackPointer.value = ptrWidth === 8 ? BigInt(stackPtr) : stackPtr; } // Initialize musl thread pointer if available - const wasmThreadInit = instance.exports.__wasm_thread_init as ((tp: number | bigint) => void) | undefined; + const wasmThreadInit = instance.exports.__wasm_thread_init as + ((tp: number | bigint) => void) | undefined; if (wasmThreadInit && tlsPtr > 0) { wasmThreadInit(ptrWidth === 8 ? BigInt(tlsPtr) : tlsPtr); } @@ -2839,77 +5356,92 @@ export async function centralizedThreadWorkerMain( // Set __channel_base without calling the exported helper. lld can prefix // exported functions with __wasm_call_ctors, and thread workers must not // re-run constructors in shared process memory. - setupChannelBase(instance, module, memory, channelOffset, initData.programBytes, ptrWidth); + setupChannelBase( + instance, + module, + memory, + channelOffset, + initData.programBytes, + ptrWidth, + ); // Call the thread function via indirect function table - const table = instance.exports.__indirect_function_table as WebAssembly.Table | undefined; + const table = threadTable; if (!table) { - throw new Error("No __indirect_function_table export — cannot call thread function"); + throw new Error( + "No __indirect_function_table export — cannot call thread function", + ); } // On wasm64, table indices may require BigInt (table64 extension) const tableIdx = ptrWidth === 8 ? BigInt(fnPtr) : fnPtr; - const threadFn = table.get(tableIdx as number) as ((...args: (number | bigint)[]) => number | bigint) | null; + const threadFn = table.get(tableIdx as number) as + ((...args: (number | bigint)[]) => number | bigint) | null; if (!threadFn) { throw new Error(`Thread function at table index ${fnPtr} is null`); } const threadArg = ptrWidth === 8 ? BigInt(argPtr) : argPtr; + const resumeThread = hasForkInstrumentation + ? (instance.exports.wpk_fork_resume_thread as + | ((tableIndex: number, arg: number | bigint) => number | bigint) + | undefined) + : undefined; + if (hasForkInstrumentation && typeof resumeThread !== "function") { + throw new Error( + `pid=${pid} tid=${tid}: fork-capable program is missing ` + + "wpk_fork_resume_thread", + ); + } let result = 0; - if (hasForkInstrumentation) { - const getState = instance.exports.wpk_fork_state as () => number; - const unwindEnd = instance.exports.wpk_fork_unwind_end as () => void; - let needsRewind = false; - + if (hasForkInstrumentation && threadProcessContinuation) { for (;;) { - if (needsRewind) { - threadForkContinuation!.beginReplay(); - invokeForkContinuationBegin( - instance.exports.wpk_fork_rewind_begin, - forkBufAddr, - ptrWidth, - `pid=${pid} tid=${tid}: linked fork rewind`, - ); - needsRewind = false; - } - + let transportedForkUnwind = false; try { - const raw = threadFn(threadArg); + const raw = + threadProcessContinuation.phaseName() === "idle" + ? threadFn(threadArg) + : resumeThread!(fnPtr, threadArg); result = Number(raw); } catch (e) { - if ( + if (isForkUnwindException(e, threadForkUnwindTag)) { + transportedForkUnwind = true; + } else if ( e instanceof Error && e.message.includes("unreachable") && kernelThreadExitStatus !== null ) { result = kernelThreadExitStatus; break; + } else { + throw e; } - throw e; } - const forkState = getState(); - if (forkState === 1) { - unwindEnd(); - threadForkContinuation!.finishUnwind(); - // Close the race where the process main worker dlopens after this - // pthread began unwinding but before it completed. Rewind locally - // with ENOTSUP and do not create a child. - if (processHasDlopenArchive()) { - forkResult = -95; - needsRewind = true; - continue; - } + const phase = threadProcessContinuation.phaseName(); + if (transportedForkUnwind && phase !== "capture") { + throw new Error( + `pid=${pid} tid=${tid}: private fork-unwind exception escaped ` + + `while process continuation is ${phase}`, + ); + } + if (phase === "capture") { + threadProcessContinuation.sealCapture(); const childPid = sendForkSyscall(memory, channelOffset); + forkResult = childPid; if (childPid < 0) { - forkResult = childPid; - needsRewind = true; - continue; + threadProcessContinuation.beginAbortReplay(-childPid); + } else { + threadProcessContinuation.beginParentReplay(); } - forkResult = childPid; - needsRewind = true; continue; } + if (phase !== "idle") { + throw new Error( + `pid=${pid} tid=${tid}: pthread entry returned while process ` + + `continuation is ${phase}`, + ); + } break; } } else { @@ -2929,10 +5461,13 @@ export async function centralizedThreadWorkerMain( } } - // A well-formed fork releases its reader token from the state=2 import - // above. Keep normal-return cleanup defensive so an unexpected - // instrumenter state cannot strand the process-wide writer lock. + // A well-formed replay releases its reader token from the inherited fork + // import above. Keep normal-return cleanup defensive so an unexpected + // execution exit cannot strand the process-wide writer lock. releasePthreadForkLock(); + threadProcessContinuation?.clear(); + threadExternrefTokens?.clear(); + threadHostImportRuntime?.clear(); // A normal return has not passed through libc's noreturn kernel_exit // import, so publish SYS_EXIT here. When kernel_exit already ran it sent @@ -2950,7 +5485,12 @@ export async function centralizedThreadWorkerMain( Atomics.notify(i32, (base + CH_STATUS) / 4, 1); // Wait for kernel to process the exit. The kernel completes the channel // (CH_STATUS -> COMPLETE), which returns this Atomics.wait. - while (Atomics.wait(i32, (base + CH_STATUS) / 4, CHANNEL_STATUS_PENDING) === "ok") { /* */ } + while ( + Atomics.wait(i32, (base + CH_STATUS) / 4, CHANNEL_STATUS_PENDING) === + "ok" + ) { + /* */ + } // Intentionally do NOT reset CH_STATUS back to IDLE here. A normal syscall // resets to IDLE so the next syscall can set PENDING, but an exiting thread // issues no further syscalls — the channel is torn down and the slot is @@ -2965,10 +5505,18 @@ export async function centralizedThreadWorkerMain( tid, } satisfies WorkerToHostMessage); } catch (err) { + threadTableReplication?.abortActiveMutations(); releasePthreadForkLock(); - const message = err instanceof Error - ? `${err.message}\n${err.stack ?? ""}` - : String(err); + try { + threadProcessContinuation?.clear(); + } catch { + // Preserve the original worker failure after making transaction roots + // unreachable as far as the coordinator can. + } + threadExternrefTokens?.clear(); + threadHostImportRuntime?.clear(); + const message = + err instanceof Error ? `${err.message}\n${err.stack ?? ""}` : String(err); port.postMessage({ type: "error", pid, diff --git a/host/src/worker-protocol.ts b/host/src/worker-protocol.ts index d72da141d1..8bace89473 100644 --- a/host/src/worker-protocol.ts +++ b/host/src/worker-protocol.ts @@ -1,3 +1,10 @@ +import type { + ForkHostImportWorkerInit, +} from "./fork-host-import-runtime"; +import type { + ForkExternrefImportWake, +} from "./fork-externref-import-mailbox"; + // --- Host → Worker messages --- export type HostToWorkerMessage = @@ -22,6 +29,20 @@ export interface CentralizedWorkerInitMessage { memory: WebAssembly.Memory; /** Channel offset within the shared Memory for this thread's syscall channel */ channelOffset: number; + /** + * Exact process-image generation issued by the kernel-side externref owner. + * Workers use this scalar only when routing token-bearing host imports; the + * broker capability and real JavaScript values never cross the Worker edge. + * Optional only for direct non-fork harnesses; an instrumented artifact must + * reject launch unless this and `forkHostImports` are both present. + */ + externrefGenerationId?: number; + /** + * One fixed owner-import mailbox for this Worker. Side modules reuse it. + * Optional only for direct non-fork test harnesses that do not create a + * durable process owner; production Node/browser launch paths always set it. + */ + forkHostImports?: ForkHostImportWorkerInit; /** Optional env vars to set up in the program */ env?: string[]; /** Optional argv */ @@ -32,6 +53,12 @@ export interface CentralizedWorkerInitMessage { isForkChild?: boolean; /** Address of the fork save-buffer in memory (used for fork child rewind) */ forkBufAddr?: number; + /** + * Two-phase launch gate for a fork child. The child announces that all + * reconstruction and activation frames reached the inherited fork import, + * then waits here until the kernel host commits the launch. + */ + forkReplayGate?: SharedArrayBuffer; /** * Entry-point override for fork children created by a non-main thread. * @@ -67,6 +94,13 @@ export interface CentralizedThreadInitMessage { * archive head relative to this live shared-memory anchor before fork. */ processChannelOffset: number; channelOffset: number; + /** + * Same process-image externref generation as the process's main Worker. + * Optional only for direct non-fork harnesses. + */ + externrefGenerationId?: number; + /** Distinct pthread mailbox; side modules in this pthread reuse it. */ + forkHostImports?: ForkHostImportWorkerInit; fnPtr: number; argPtr: number; stackPtr: number; @@ -90,19 +124,26 @@ export interface WorkerTerminateMessage { export type WorkerToHostMessage = | WorkerReadyMessage + | ForkReplayReadyMessage | WorkerExitMessage | ThreadExitMessage | WorkerErrorMessage | ExecRequestMessage | ExecCompleteMessage | AlarmSetMessage - | VmInterruptTimerMessage; + | VmInterruptTimerMessage + | ForkHostImportWakeMessage; export interface WorkerReadyMessage { type: "ready"; pid: number; } +export interface ForkReplayReadyMessage { + type: "fork_replay_ready"; + pid: number; +} + export interface WorkerExitMessage { type: "exit"; pid: number; @@ -146,6 +187,12 @@ export interface VmInterruptTimerMessage { seconds: number; } +export interface ForkHostImportWakeMessage { + type: "fork_host_import"; + /** Contains scalar identity/sequence fields only; the SAB moved at init. */ + wake: ForkExternrefImportWake; +} + export interface ExecReplyMessage { type: "exec_reply"; wasmBytes: ArrayBuffer; diff --git a/host/test/abi-version.test.ts b/host/test/abi-version.test.ts index 3169078314..135d08be81 100644 --- a/host/test/abi-version.test.ts +++ b/host/test/abi-version.test.ts @@ -2,6 +2,19 @@ import { describe, it, expect } from "vitest"; import { readFileSync } from "node:fs"; import { resolveBinary } from "../src/binary-resolver"; import { detectPtrWidth } from "../src/constants"; +import { + FORK_ANYREF_TRANSIT_IMPORT, + ForkAnyrefTransitTable, +} from "../src/fork-anyref-transit"; +import { FORK_MODULE_TABLE_GENERATION_ADDR_IMPORT } from "../src/fork-activation-registry"; +import { + createForkUnwindTag, + FORK_UNWIND_TAG_IMPORT_NAME, +} from "../src/fork-unwind-transport"; +import { + WPK_FORK_EXCEPTION_IMPORT_ACTIVATION, + WPK_FORK_RESUME_IMPORT_TABLE, +} from "../src/generated/abi"; /** * Coverage for the ABI version surface: @@ -20,7 +33,10 @@ describe("ABI version marker", () => { const kernelWasm = readFileSync(resolveBinary("kernel.wasm")); function toArrayBuffer(bytes: Uint8Array): ArrayBuffer { - return bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength); + return bytes.buffer.slice( + bytes.byteOffset, + bytes.byteOffset + bytes.byteLength, + ); } async function instantiateKernelOnly( @@ -29,18 +45,19 @@ describe("ABI version marker", () => { const ptrWidth = detectPtrWidth(toArrayBuffer(bytes)); // Match host/src/kernel.ts. Keep headroom above the kernel Wasm's // linker-derived minimum without re-tuning this test per change. - const memory = ptrWidth === 8 - ? new WebAssembly.Memory({ - initial: 24n, - maximum: 16384n, - shared: true, - address: "i64", - } as unknown as WebAssembly.MemoryDescriptor) - : new WebAssembly.Memory({ - initial: 24, - maximum: 16384, - shared: true, - }); + const memory = + ptrWidth === 8 + ? new WebAssembly.Memory({ + initial: 24n, + maximum: 16384n, + shared: true, + address: "i64", + } as unknown as WebAssembly.MemoryDescriptor) + : new WebAssembly.Memory({ + initial: 24, + maximum: 16384, + shared: true, + }); const module = await WebAssembly.compile(bytes as BufferSource); // The kernel imports many host functions. We only need to inspect // the exports, so provide minimal stubs for every import. @@ -60,9 +77,7 @@ describe("ABI version marker", () => { it("kernel exports __abi_version as a function returning u32", async () => { const instance = await instantiateKernelOnly(kernelWasm); - const fn = instance.exports.__abi_version as - | (() => number) - | undefined; + const fn = instance.exports.__abi_version as (() => number) | undefined; expect(typeof fn).toBe("function"); const value = fn!(); expect(typeof value).toBe("number"); @@ -96,18 +111,52 @@ describe("ABI version marker", () => { }); const importObject: WebAssembly.Imports = { env: { memory } }; const envImports = importObject.env as Record; + const gcTransit = new ForkAnyrefTransitTable(); + const resumeTable = new WebAssembly.Table({ + initial: 1, + element: "anyfunc", + }); + const unwindTag = createForkUnwindTag(); for (const imp of WebAssembly.Module.imports(module)) { if (imp.module === "env" && imp.name === "memory") continue; const target = (importObject[imp.module] ??= {}) as Record< string, unknown >; - target[imp.name] ??= - imp.kind === "function" - ? (..._args: unknown[]) => 0 - : imp.kind === "global" - ? new WebAssembly.Global({ value: "i32", mutable: true }, 0) - : undefined; + if (target[imp.name] !== undefined) continue; + if (imp.kind === "function") { + target[imp.name] = (..._args: unknown[]) => 0; + } else if (imp.kind === "global") { + target[imp.name] = + imp.name === FORK_MODULE_TABLE_GENERATION_ADDR_IMPORT + ? new WebAssembly.Global({ value: "i64", mutable: false }, 0n) + : new WebAssembly.Global( + { + value: "i32", + mutable: imp.name !== WPK_FORK_EXCEPTION_IMPORT_ACTIVATION, + }, + 0, + ); + } else if (imp.kind === "table") { + // WHY: ABI 43's GC transit table has `(ref null any)` element type and + // cannot be replaced by the legacy `anyfunc` resume table. + if (imp.name === FORK_ANYREF_TRANSIT_IMPORT) { + target[imp.name] = gcTransit.table; + } else if (imp.name === WPK_FORK_RESUME_IMPORT_TABLE) { + target[imp.name] = resumeTable; + } else { + throw new Error(`unhandled table import ${imp.module}.${imp.name}`); + } + } else if ( + (imp.kind as string) === "tag" && + imp.name === FORK_UNWIND_TAG_IMPORT_NAME + ) { + target[imp.name] = unwindTag; + } else { + throw new Error( + `unhandled ${imp.kind} import ${imp.module}.${imp.name}`, + ); + } void envImports; } const instance = await WebAssembly.instantiate(module, importObject); diff --git a/host/test/audio-integration.test.ts b/host/test/audio-integration.test.ts index b226df5f9f..e91808ea17 100644 --- a/host/test/audio-integration.test.ts +++ b/host/test/audio-integration.test.ts @@ -25,7 +25,11 @@ import { CAPTURED_STDIO, CentralizedKernelWorker } from "../src/kernel-worker"; import { NodePlatformIO } from "../src/platform/node"; import { NodeWorkerAdapter } from "../src/worker-adapter"; import { detectPtrWidth } from "../src/constants"; -import type { CentralizedWorkerInitMessage } from "../src/worker-protocol"; +import type { + CentralizedWorkerInitMessage, + WorkerToHostMessage, +} from "../src/worker-protocol"; +import { TestProcessReferenceOwners } from "./process-reference-owner-helper"; const __dirname = dirname(fileURLToPath(import.meta.url)); @@ -57,22 +61,34 @@ describe.skipIf(!existsSync(audiotestBinary))("audio integration", () => { const io = new NodePlatformIO(); const workerAdapter = new NodeWorkerAdapter(); - const workers = new Map>(); + const referenceOwners = new TestProcessReferenceOwners(); + const workers = new Map< + number, + ReturnType + >(); let pid = 0; let stdout = ""; let resolveExit: (status: number) => void; - const exitPromise = new Promise((resolve) => { + let rejectExit: (reason: Error) => void; + const exitPromise = new Promise((resolve, reject) => { resolveExit = resolve; + rejectExit = reject; }); const kernel = new CentralizedKernelWorker( - { maxWorkers: 4, dataBufferSize: 65536, useSharedMemory: true, enableSyscallLog: false }, + { + maxWorkers: 4, + dataBufferSize: 65536, + useSharedMemory: true, + enableSyscallLog: false, + }, io, { onExit: (exitPid, exitStatus) => { if (exitPid === pid) { + referenceOwners.release(exitPid); kernel.unregisterProcess(exitPid); const w = workers.get(exitPid); if (w) { @@ -104,6 +120,7 @@ describe.skipIf(!existsSync(audiotestBinary))("audio integration", () => { new Uint8Array(memory.buffer, channelOffset, CH_TOTAL_SIZE).fill(0); kernel.registerProcess(pid, memory, [channelOffset], { ptrWidth }); + const referenceInit = referenceOwners.start(pid); const initData: CentralizedWorkerInitMessage = { type: "centralized_init", @@ -114,16 +131,34 @@ describe.skipIf(!existsSync(audiotestBinary))("audio integration", () => { argv: ["audiotest"], env: [], ptrWidth, + ...referenceInit, }; const mainWorker = workerAdapter.createWorker(initData); + referenceOwners.attach(pid, mainWorker); + mainWorker.on("error", rejectExit); + mainWorker.on("message", (raw: unknown) => { + const message = raw as WorkerToHostMessage; + if (message.type === "error" && message.pid === pid) { + rejectExit(new Error(message.message)); + } + }); workers.set(pid, mainWorker); try { const exitCode = await Promise.race([ exitPromise, new Promise((_, reject) => - setTimeout(() => reject(new Error("audiotest didn't exit in 10s")), 10_000), + setTimeout( + () => + reject( + new Error( + "audiotest didn't exit in 10s" + + (stderr ? `: ${stderr}` : ""), + ), + ), + 10_000, + ), ), ]); expect(exitCode).toBe(0); @@ -169,6 +204,7 @@ describe.skipIf(!existsSync(audiotestBinary))("audio integration", () => { expect(kernel.drainAudio(after)).toBe(0); } finally { for (const [, w] of workers) await w.terminate().catch(() => {}); + referenceOwners.close(); void exitPromise.catch(() => {}); } }, 30_000); diff --git a/host/test/catch-ref-fresh-worker.test.ts b/host/test/catch-ref-fresh-worker.test.ts new file mode 100644 index 0000000000..19610d5d72 --- /dev/null +++ b/host/test/catch-ref-fresh-worker.test.ts @@ -0,0 +1,102 @@ +import { execFileSync } from "node:child_process"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { runCentralizedProgram } from "./centralized-test-helper"; + +const testDir = dirname(fileURLToPath(import.meta.url)); +const fixtureSource = resolve( + testDir, + "fixtures/catch-ref-fresh-worker.wat", +); +const referencePayloadFixtureSource = resolve( + testDir, + "fixtures/reference-catch-payload-fresh-worker.wat", +); +const instrumenter = resolve( + testDir, + "../../tools/bin/wasm-fork-instrument", +); + +describe("CatchRef fresh process worker replay", () => { + let workDir = ""; + let programPath = ""; + let referencePayloadProgramPath = ""; + + beforeAll(() => { + workDir = mkdtempSync(join(tmpdir(), "kandelo-catch-ref-worker-")); + const rawPath = join(workDir, "catch-ref-fresh-worker.raw.wasm"); + programPath = join(workDir, "catch-ref-fresh-worker.wasm"); + execFileSync("wat2wasm", [ + "--enable-exceptions", + "--enable-threads", + fixtureSource, + "-o", + rawPath, + ]); + execFileSync(instrumenter, [rawPath, "-o", programPath]); + + const referencePayloadRawPath = join( + workDir, + "reference-catch-payload-fresh-worker.raw.wasm", + ); + referencePayloadProgramPath = join( + workDir, + "reference-catch-payload-fresh-worker.wasm", + ); + execFileSync("wat2wasm", [ + "--enable-exceptions", + "--enable-threads", + referencePayloadFixtureSource, + "-o", + referencePayloadRawPath, + ]); + execFileSync(instrumenter, [ + referencePayloadRawPath, + "-o", + referencePayloadProgramPath, + ]); + }); + + afterAll(() => { + if (workDir) rmSync(workDir, { recursive: true, force: true }); + }); + + it("reconstructs the caught exception in a fresh Node child worker", async () => { + // The fixture's parent waits for the fork child. The child exits 91 if + // CatchRef replay did not restore payload 42; the parent converts any + // failed wait status into exit 92. + const result = await runCentralizedProgram({ + programPath, + argv: ["catch-ref-fresh-worker"], + timeout: 30_000, + useDefaultRootfs: false, + }); + + expect( + result.exitCode, + `stdout:\n${result.stdout}\nstderr:\n${result.stderr}`, + ).toBe(0); + expect(result.stderr).toBe(""); + }); + + it("reconstructs reference-bearing catches in fresh Node child workers", async () => { + // The first child calls a non-null funcref reconstructed from the child's + // static function catalog. The second verifies a nullable externref + // payload; both values originated in a caught exception recipe. + const result = await runCentralizedProgram({ + programPath: referencePayloadProgramPath, + argv: ["reference-catch-payload-fresh-worker"], + timeout: 30_000, + useDefaultRootfs: false, + }); + + expect( + result.exitCode, + `stdout:\n${result.stdout}\nstderr:\n${result.stderr}`, + ).toBe(0); + expect(result.stderr).toBe(""); + }); +}); diff --git a/host/test/centralized-test-helper.ts b/host/test/centralized-test-helper.ts index 587c32c6a4..b222c8c9af 100644 --- a/host/test/centralized-test-helper.ts +++ b/host/test/centralized-test-helper.ts @@ -22,6 +22,16 @@ import { } from "../src/process-memory"; import { NodeKernelHost } from "../src/node-kernel-host"; import { readForkContinuationAnchor } from "../src/fork-continuation"; +import { + ForkHostImportOwnerRuntime, + type ForkHostImportOwnerWorker, +} from "../src/fork-host-import-runtime"; +import { ForkExternrefProcessOwner } from "../src/fork-externref-process-owner"; +import { + ForkReplayGateCoordinator, + observeForkReplayWorker, +} from "../src/fork-replay-gate"; +import type { ForkExternrefGeneration } from "../src/fork-reference-broker"; import type { HostDiagnostic } from "../src/host-diagnostic"; import type { CentralizedWorkerInitMessage, CentralizedThreadInitMessage, WorkerToHostMessage } from "../src/worker-protocol"; import type { PlatformIO } from "../src/types"; @@ -333,10 +343,25 @@ async function runOnMainThread(options: RunProgramOptions): Promise(); const processPtrWidths = new Map(); const forkReplayContexts = new Map(); + const externrefProcessOwner = new ForkExternrefProcessOwner(); + const forkHostImportOwnerRuntime = + new ForkHostImportOwnerRuntime(externrefProcessOwner); + const externrefGenerations = new Map(); + const processForkHostImports = new Map(); let mainThreadForkCount: bigint | undefined; let pid = 0; + const releaseProcessReferenceOwner = (releasePid: number): void => { + processForkHostImports.get(releasePid)?.close(); + processForkHostImports.delete(releasePid); + const generation = externrefGenerations.get(releasePid); + if (generation) { + externrefProcessOwner.releaseGeneration(generation); + externrefGenerations.delete(releasePid); + } + }; + const kernelWorker = new CentralizedKernelWorker( { maxWorkers: 4, dataBufferSize: 65536, useSharedMemory: true, enableSyscallLog: !!process.env.KERNEL_SYSCALL_LOG }, io, @@ -381,8 +406,42 @@ async function runOnMainThread(options: RunProgramOptions): Promise; + const childForkHostImports = forkHostImportOwnerRuntime.createWorker({ + pid: childPid, + generationId: childGeneration.id, + authorizeSender: () => { + if ( + workers.get(childPid) !== childWorker + || externrefGenerations.get(childPid) !== childGeneration + ) { + throw new Error( + `stale centralized-test host-import sender for pid=${childPid}`, + ); + } + }, + }); const childInitData: CentralizedWorkerInitMessage = { type: "centralized_init", @@ -392,13 +451,24 @@ async function runOnMainThread(options: RunProgramOptions): Promise {}); }; childWorker.on("error", finalizeChildWorkerError); @@ -435,10 +506,46 @@ async function runOnMainThread(options: RunProgramOptions): Promise workers.get(childPid) === childWorker, + ); - return [childChannelOffset]; + try { + await forkReplay.waitUntilReady(); + if (workers.get(childPid) !== childWorker) { + throw new Error( + `Fork child ${childPid} changed generation before replay commit`, + ); + } + if (!kernelWorker.shouldLaunchPendingChild(childPid)) { + throw new Error(`Fork child ${childPid} exited before replay commit`); + } + // Match the real Node/browser host: the parent cannot observe the + // child until replay has reached the inherited fork import and this + // separate commit wakes that exact Worker generation. + forkReplay.commit(); + return [childChannelOffset]; + } catch (error) { + forkReplay.cancel(error); + if (workers.get(childPid) === childWorker) { + workers.delete(childPid); + processProgramBytes.delete(childPid); + processLayouts.delete(childPid); + threadAllocators.delete(childPid); + processPtrWidths.delete(childPid); + forkReplayContexts.delete(childPid); + releaseProcessReferenceOwner(childPid); + } + childWorker.terminate().catch(() => {}); + throw error; + } }, onExec: async (execPid, path, argv, envp, callerTid) => { const wasmPath = options.execPrograms?.get(path); @@ -471,10 +578,21 @@ async function runOnMainThread(options: RunProgramOptions): Promise | undefined; + let replacementGeneration: ForkExternrefGeneration | undefined; + let replacementForkHostImports: ForkHostImportOwnerWorker | undefined; try { const setupResult = kernelWorker.kernelExecSetup(execPid, callerTid); if (setupResult < 0) return setupResult; kernelWorker.prepareProcessForExec(execPid); + const previousGeneration = externrefGenerations.get(execPid); + if (!previousGeneration) { + throw new Error( + `Unknown externref generation for exec pid ${execPid}`, + ); + } + replacementGeneration = + externrefProcessOwner.replaceGeneration(previousGeneration); + externrefGenerations.set(execPid, replacementGeneration); const finalizeResult = kernelWorker.finalizeAddressSpaceForExec(execPid); if (finalizeResult < 0) { @@ -482,11 +600,18 @@ async function runOnMainThread(options: RunProgramOptions): Promise {}); workers.delete(execPid); } - if (kernelWorker.finalizeExecHandoffTermination(execPid) > 0) return 0; + if (kernelWorker.finalizeExecHandoffTermination(execPid) > 0) { + externrefProcessOwner.releaseGeneration(replacementGeneration); + externrefGenerations.delete(execPid); + replacementGeneration = undefined; + return 0; + } kernelWorker.registerProcess(execPid, newMemory, [newChannelOffset], { preserveProcessState: true, @@ -504,6 +629,23 @@ async function runOnMainThread(options: RunProgramOptions): Promise { + if ( + !replacementWorker + || workers.get(execPid) !== replacementWorker + || externrefGenerations.get(execPid) + !== replacementGeneration + ) { + throw new Error( + `stale centralized-test host-import sender for exec pid=${execPid}`, + ); + } + }, + }); const initData: CentralizedWorkerInitMessage = { type: "centralized_init", pid: execPid, @@ -513,16 +655,26 @@ async function runOnMainThread(options: RunProgramOptions): Promise { console.error(`[exec] worker error for pid ${execPid}:`, err); }); + replacementWorker.on("message", (msg: unknown) => { + const m = msg as WorkerToHostMessage; + if (m.type === "fork_host_import") { + replacementForkHostImports?.dispatch(m.wake); + } + }); kernelWorker.finishProcessExecHandoff(execPid); return 0; } catch (err) { + replacementForkHostImports?.close(); try { kernelWorker.prepareProcessForExec(execPid); } catch { /* best-effort */ } if (replacementWorker && workers.get(execPid) !== replacementWorker) { await replacementWorker.terminate().catch(() => {}); @@ -539,6 +691,7 @@ async function runOnMainThread(options: RunProgramOptions): Promise; + let threadWorkerLive = true; + const threadForkHostImports = forkHostImportOwnerRuntime.createWorker({ + pid: clonePid, + generationId: processGeneration.id, + authorizeSender: () => { + if ( + !threadWorkerLive + || externrefGenerations.get(clonePid) !== processGeneration + ) { + throw new Error( + `stale centralized-test pthread host-import sender ` + + `for pid=${clonePid} tid=${tid}`, + ); + } + }, + }); const threadInitData: CentralizedThreadInitMessage = { type: "centralized_thread_init", @@ -587,17 +764,32 @@ async function runOnMainThread(options: RunProgramOptions): Promise { const m = msg as WorkerToHostMessage; if (m.type === "thread_exit") { + threadWorkerLive = false; + threadForkHostImports.close(); threadAllocator.free(alloc.basePage); threadWorker.terminate().catch(() => {}); + } else if (m.type === "fork_host_import") { + threadForkHostImports.dispatch(m.wake); } }); threadWorker.on("error", () => { + threadWorkerLive = false; + threadForkHostImports.close(); kernelWorker.notifyThreadExit(clonePid, tid); kernelWorker.removeChannel(clonePid, alloc.channelOffset); threadAllocator.free(alloc.basePage); @@ -615,6 +807,7 @@ async function runOnMainThread(options: RunProgramOptions): Promise {}); @@ -628,6 +821,7 @@ async function runOnMainThread(options: RunProgramOptions): Promise {}); @@ -683,6 +877,23 @@ async function runOnMainThread(options: RunProgramOptions): Promise; + const mainForkHostImports = forkHostImportOwnerRuntime.createWorker({ + pid, + generationId: mainGeneration.id, + authorizeSender: () => { + if ( + workers.get(pid) !== mainWorker + || externrefGenerations.get(pid) !== mainGeneration + ) { + throw new Error( + `stale centralized-test host-import sender for pid=${pid}`, + ); + } + }, + }); const initData: CentralizedWorkerInitMessage = { type: "centralized_init", pid, @@ -692,10 +903,20 @@ async function runOnMainThread(options: RunProgramOptions): Promise { for (const [, w] of workers) w.terminate().catch(() => {}); + for (const livePid of [...externrefGenerations.keys()]) { + releaseProcessReferenceOwner(livePid); + } rejectExit(new Error(`Program timed out after ${timeout}ms`)); }, timeout); mainWorker.on("error", (err: Error) => { clearTimeout(timer); + releaseProcessReferenceOwner(pid); rejectExit(err); }); @@ -721,7 +946,10 @@ async function runOnMainThread(options: RunProgramOptions): Promise {}); + releaseProcessReferenceOwner(pid); rejectExit(new Error(m.message)); + } else if (m.type === "fork_host_import") { + mainForkHostImports.dispatch(m.wake); } }); diff --git a/host/test/dlopen-host-imports.test.ts b/host/test/dlopen-host-imports.test.ts index 4d640bd780..0ec62486ec 100644 --- a/host/test/dlopen-host-imports.test.ts +++ b/host/test/dlopen-host-imports.test.ts @@ -1,4 +1,13 @@ -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; +import { + ABI_SYSCALLS, + CHANNEL_STATUS_COMPLETE, + CHANNEL_STATUS_PENDING, + CH_ERRNO, + CH_RETURN, + CH_STATUS, + CH_SYSCALL, +} from "../src/generated/abi"; import { buildDlopenImports } from "../src/worker-main"; type WasmPointer = number | bigint; @@ -18,6 +27,29 @@ type DlsymImport = ( type DlerrorImport = (bufPtr: WasmPointer, bufMax: number) => number; +type DlopenPrepareImport = ( + bytesPtr: WasmPointer, + bytesLen: number, + namePtr: WasmPointer, + nameLen: number, + flags: number, +) => number; + +type DlopenNextImport = (transaction: number) => number; +type DlopenCommitImport = (transaction: number) => number; + +const STAGED_CTOR_SIDE_MODULE = Uint8Array.from( + Buffer.from( + "0061736d01000000" + + "000f0864796c696e6b2e30010400000000" + + "010401600000" + + "03020100" + + "071501115f5f7761736d5f63616c6c5f63746f72730000" + + "0a040102000b", + "hex", + ), +); + function createImports(ptrWidth: 4 | 8): { memory: WebAssembly.Memory; pointer: (value: number) => WasmPointer; @@ -125,4 +157,101 @@ describe("dlopen host import pointer widths", () => { .toThrow(/__wasm_dlopen bytes: memory range .* exceeds 65536 bytes/); expect(dlopen(pointer(0), 0, pointer(0), 0)).toBe(1); }); + + it("retains the loader lease after a premature explicit staged commit", () => { + const channelOffset = 4_096; + const archiveControlAddr = 128; + const memory = new WebAssembly.Memory({ + initial: 2, + maximum: 2, + shared: true, + }); + const table = new WebAssembly.Table({ initial: 1, element: "anyfunc" }); + const stackPointer = new WebAssembly.Global( + { value: "i32", mutable: true }, + 32_768, + ); + const support = buildDlopenImports( + memory, + channelOffset, + archiveControlAddr, + () => table, + () => stackPointer, + () => undefined, + 4, + undefined, + undefined, + false, + ); + const prepare = support.imports + .__wasm_dlopen_prepare as DlopenPrepareImport; + const next = support.imports.__wasm_dlopen_next as DlopenNextImport; + const commit = support.imports.__wasm_dlopen_commit as DlopenCommitImport; + const moduleOffset = 8_192; + const nameOffset = 8_512; + const name = new TextEncoder().encode("libstaged-lease.so"); + const bytes = new Uint8Array(memory.buffer); + bytes.set(STAGED_CTOR_SIDE_MODULE, moduleOffset); + bytes.set(name, nameOffset); + + let nextMapping = 16_384; + const wait = vi.spyOn(Atomics, "wait").mockImplementation( + (array, index, expected) => { + if (Atomics.load(array, index) !== expected) return "not-equal"; + if ( + array.buffer !== memory.buffer + || index !== (channelOffset + CH_STATUS) / 4 + || expected !== CHANNEL_STATUS_PENDING + ) { + throw new Error("unexpected Atomics.wait in staged-loader test"); + } + const view = new DataView(memory.buffer); + const syscall = view.getInt32(channelOffset + CH_SYSCALL, true); + const result = syscall === ABI_SYSCALLS.Mmap + ? nextMapping + : syscall === ABI_SYSCALLS.Munmap + ? 0 + : -1; + if (syscall === ABI_SYSCALLS.Mmap) nextMapping += 8_192; + view.setBigInt64(channelOffset + CH_RETURN, BigInt(result), true); + view.setUint32( + channelOffset + CH_ERRNO, + result < 0 ? 38 : 0, + true, + ); + Atomics.store(array, index, CHANNEL_STATUS_COMPLETE); + return "ok"; + }, + ); + + try { + const transaction = prepare( + moduleOffset, + STAGED_CTOR_SIDE_MODULE.length, + nameOffset, + name.length, + 0x100, + ); + expect(transaction).toBeGreaterThan(0); + + const entry = next(transaction); + expect(entry).toBeGreaterThan(0); + // Once `next` has issued an entry, commit must wait for ordinary Wasm + // to execute it. This is a truthful misuse failure, not cancellation. + expect(commit(transaction)).toBe(0); + const loaderOwner = new Int32Array( + memory.buffer, + archiveControlAddr - 24, + 1, + ); + expect(loaderOwner[0]).toBe(1); + + (table.get(entry) as () => void)(); + expect(next(transaction)).toBe(0); + expect(commit(transaction)).toBeGreaterThan(0); + expect(loaderOwner[0]).toBe(0); + } finally { + wait.mockRestore(); + } + }); }); diff --git a/host/test/dri-cube-pyramid.test.ts b/host/test/dri-cube-pyramid.test.ts index fdd6a5c966..ea8c71c4a8 100644 --- a/host/test/dri-cube-pyramid.test.ts +++ b/host/test/dri-cube-pyramid.test.ts @@ -13,12 +13,17 @@ import { NodeWorkerAdapter } from "../src/worker-adapter"; import { detectPtrWidth, extractHeapBase } from "../src/constants"; import { tryResolveBinary } from "../src/binary-resolver"; import { readForkContinuationAnchor } from "../src/fork-continuation"; +import { + ForkReplayGateCoordinator, + observeForkReplayWorker, +} from "../src/fork-replay-gate"; import { GlMuxer } from "../src/webgl/muxer"; import type { GlBinding } from "../src/webgl/registry"; import type { CentralizedWorkerInitMessage, WorkerToHostMessage, } from "../src/worker-protocol"; +import { TestProcessReferenceOwners } from "./process-reference-owner-helper"; const programBinary = tryResolveBinary("programs/cube_pyramid.wasm") ?? ""; const kernelBinary = tryResolveBinary("kernel.wasm") ?? ""; @@ -41,7 +46,10 @@ function createProcessMemory(pages: number): WebAssembly.Memory { /** Proxy WebGL2 context: every call lands in `log`; `create*` returns * a fresh object so the bridge's per-name maps stay distinct. */ -function makeFakeGl(): { log: Array<[string, unknown[]]>; gl: WebGL2RenderingContext } { +function makeFakeGl(): { + log: Array<[string, unknown[]]>; + gl: WebGL2RenderingContext; +} { const log: Array<[string, unknown[]]> = []; const handler: ProxyHandler = { get(_t, prop) { @@ -54,7 +62,10 @@ function makeFakeGl(): { log: Array<[string, unknown[]]>; gl: WebGL2RenderingCon }; }, }; - return { log, gl: new Proxy({}, handler) as unknown as WebGL2RenderingContext }; + return { + log, + gl: new Proxy({}, handler) as unknown as WebGL2RenderingContext, + }; } function makeFakeCanvas(gl: WebGL2RenderingContext) { @@ -86,7 +97,11 @@ describe.skipIf(!existsSync(programBinary) || !existsSync(kernelBinary))( const io = new NodePlatformIO(); const workerAdapter = new NodeWorkerAdapter(); - const workers = new Map>(); + const referenceOwners = new TestProcessReferenceOwners(); + const workers = new Map< + number, + ReturnType + >(); let parentPid = 0; let stdout = ""; @@ -116,11 +131,20 @@ describe.skipIf(!existsSync(programBinary) || !existsSync(kernelBinary))( new Uint8Array(childMemory.buffer).set(parentBuf); const childChannelOffset = (MAX_PAGES - 2) * 65536; - new Uint8Array(childMemory.buffer, childChannelOffset, CH_TOTAL_SIZE).fill(0); + new Uint8Array( + childMemory.buffer, + childChannelOffset, + CH_TOTAL_SIZE, + ).fill(0); - kernel.registerProcess(childPid, childMemory, [childChannelOffset], { - ptrWidth, - }); + kernel.registerProcess( + childPid, + childMemory, + [childChannelOffset], + { + ptrWidth, + }, + ); kernel.inheritProcessSharedMappings(parentForkPid, childPid); // Same canvas → same WebGL2 context → same muxer instance @@ -132,6 +156,16 @@ describe.skipIf(!existsSync(programBinary) || !existsSync(kernelBinary))( childChannelOffset - FORK_SAVE_BUFFER_SIZE, ptrWidth, ); + const childReferenceInit = referenceOwners.fork( + parentForkPid, + childPid, + parentMemory, + ptrWidth, + forkBufAddr, + ); + const forkReplay = new ForkReplayGateCoordinator( + `DRI cube fork child pid=${childPid}`, + ); const childInit: CentralizedWorkerInitMessage = { type: "centralized_init", @@ -141,25 +175,60 @@ describe.skipIf(!existsSync(programBinary) || !existsSync(kernelBinary))( channelOffset: childChannelOffset, isForkChild: true, forkBufAddr, + forkReplayGate: forkReplay.gate, ptrWidth, + ...childReferenceInit, }; const childWorker = workerAdapter.createWorker(childInit); + referenceOwners.attach(childPid, childWorker); workers.set(childPid, childWorker); childWorker.on("error", () => { + referenceOwners.release(childPid); kernel.unregisterProcess(childPid); workers.delete(childPid); }); childWorker.on("message", (m: unknown) => { const msg = m as WorkerToHostMessage; if (msg.type !== "error") return; + referenceOwners.release(childPid); kernel.unregisterProcess(childPid); workers.delete(childPid); }); + observeForkReplayWorker( + forkReplay, + childWorker, + childPid, + () => workers.get(childPid) === childWorker, + ); - return [childChannelOffset]; + try { + await forkReplay.waitUntilReady(); + if (workers.get(childPid) !== childWorker) { + throw new Error( + `fork child ${childPid} changed generation before commit`, + ); + } + if (!kernel.shouldLaunchPendingChild(childPid)) { + throw new Error( + `fork child ${childPid} exited before replay commit`, + ); + } + forkReplay.commit(); + return [childChannelOffset]; + } catch (error) { + forkReplay.cancel(error); + if (workers.get(childPid) === childWorker) { + workers.delete(childPid); + referenceOwners.release(childPid); + kernel.unregisterProcess(childPid); + } + childWorker.terminate().catch(() => {}); + throw error; + } }, onExit: (exitPid, exitStatus) => { + referenceOwners.release(exitPid); const w = workers.get(exitPid); if (w) { w.terminate().catch(() => {}); @@ -195,6 +264,7 @@ describe.skipIf(!existsSync(programBinary) || !existsSync(kernelBinary))( kernel.registerProcess(parentPid, memory, [channelOffset], { ptrWidth }); const heapBase = extractHeapBase(programBytes); if (heapBase !== null) kernel.setBrkBase(parentPid, heapBase); + const parentReferenceInit = referenceOwners.start(parentPid); // Attach before the worker starts so eglInitialize → host_gl_bind // sees the canvas. @@ -209,14 +279,20 @@ describe.skipIf(!existsSync(programBinary) || !existsSync(kernelBinary))( argv: ["cube_pyramid", "200"], env: [], ptrWidth, + ...parentReferenceInit, }; const mainWorker = workerAdapter.createWorker(initData); + referenceOwners.attach(parentPid, mainWorker); workers.set(parentPid, mainWorker); const timer = setTimeout(() => { for (const [, w] of workers) w.terminate().catch(() => {}); - rejectExit(new Error(`cube_pyramid timed out. stdout=${stdout} stderr=${stderr}`)); + rejectExit( + new Error( + `cube_pyramid timed out. stdout=${stdout} stderr=${stderr}`, + ), + ); }, 30_000); mainWorker.on("error", (err: Error) => { clearTimeout(timer); @@ -235,17 +311,26 @@ describe.skipIf(!existsSync(programBinary) || !existsSync(kernelBinary))( exitCode = await exitPromise; } finally { clearTimeout(timer); + for (const [, worker] of workers) { + await worker.terminate().catch(() => {}); + } + referenceOwners.close(); } expect(exitCode, `stdout=${stdout}\nstderr=${stderr}`).toBe(0); - expect(stdout).toMatch(/cube_pyramid: parent pid=\d+ rc=0, child pid=\d+ rc=0/); + expect(stdout).toMatch( + /cube_pyramid: parent pid=\d+ rc=0, child pid=\d+ rc=0/, + ); const switchedKeys = new Set(); for (const call of switchSpy.mock.calls) { const b = call[0] as Pick; switchedKeys.add(`${b.pid}/${b.contextId ?? "-"}`); } - expect(switchedKeys.size, `switched keys = ${[...switchedKeys].join(",")}`).toBeGreaterThanOrEqual(2); + expect( + switchedKeys.size, + `switched keys = ${[...switchedKeys].join(",")}`, + ).toBeGreaterThanOrEqual(2); }, 60_000); }, ); diff --git a/host/test/dri-smoke.test.ts b/host/test/dri-smoke.test.ts index 2dc8eda021..8f502240d8 100644 --- a/host/test/dri-smoke.test.ts +++ b/host/test/dri-smoke.test.ts @@ -27,7 +27,11 @@ import { NodePlatformIO } from "../src/platform/node"; import { NodeWorkerAdapter } from "../src/worker-adapter"; import { detectPtrWidth } from "../src/constants"; import { tryResolveBinary } from "../src/binary-resolver"; -import type { CentralizedWorkerInitMessage } from "../src/worker-protocol"; +import type { + CentralizedWorkerInitMessage, + WorkerToHostMessage, +} from "../src/worker-protocol"; +import { TestProcessReferenceOwners } from "./process-reference-owner-helper"; const driSmokeBinary = tryResolveBinary("programs/dri-smoke.wasm") ?? ""; const kernelBinary = tryResolveBinary("kernel.wasm") ?? ""; @@ -57,7 +61,11 @@ describe.skipIf(!existsSync(driSmokeBinary))("dri-smoke integration", () => { const io = new NodePlatformIO(); const workerAdapter = new NodeWorkerAdapter(); - const workers = new Map>(); + const referenceOwners = new TestProcessReferenceOwners(); + const workers = new Map< + number, + ReturnType + >(); let pid = 0; @@ -65,8 +73,10 @@ describe.skipIf(!existsSync(driSmokeBinary))("dri-smoke integration", () => { let stderr = ""; let stdoutResolved = false; let resolveOk: () => void; - const okPromise = new Promise((resolve) => { + let rejectOk: (reason: Error) => void; + const okPromise = new Promise((resolve, reject) => { resolveOk = resolve; + rejectOk = reject; }); let resolveExit: (status: number) => void; const exitPromise = new Promise((resolve) => { @@ -74,11 +84,17 @@ describe.skipIf(!existsSync(driSmokeBinary))("dri-smoke integration", () => { }); const kernel = new CentralizedKernelWorker( - { maxWorkers: 4, dataBufferSize: 65536, useSharedMemory: true, enableSyscallLog: false }, + { + maxWorkers: 4, + dataBufferSize: 65536, + useSharedMemory: true, + enableSyscallLog: false, + }, io, { onExit: (exitPid, exitStatus) => { if (exitPid === pid) { + referenceOwners.release(exitPid); kernel.unregisterProcess(exitPid); const w = workers.get(exitPid); if (w) { @@ -113,6 +129,7 @@ describe.skipIf(!existsSync(driSmokeBinary))("dri-smoke integration", () => { new Uint8Array(memory.buffer, channelOffset, CH_TOTAL_SIZE).fill(0); kernel.registerProcess(pid, memory, [channelOffset], { ptrWidth }); + const referenceInit = referenceOwners.start(pid); const initData: CentralizedWorkerInitMessage = { type: "centralized_init", @@ -123,16 +140,33 @@ describe.skipIf(!existsSync(driSmokeBinary))("dri-smoke integration", () => { argv: ["dri-smoke"], env: [], ptrWidth, + ...referenceInit, }; const mainWorker = workerAdapter.createWorker(initData); + referenceOwners.attach(pid, mainWorker); + mainWorker.on("error", rejectOk); + mainWorker.on("message", (raw: unknown) => { + const message = raw as WorkerToHostMessage; + if (message.type === "error" && message.pid === pid) { + rejectOk(new Error(message.message)); + } + }); workers.set(pid, mainWorker); try { await Promise.race([ okPromise, new Promise((_, reject) => - setTimeout(() => reject(new Error(`dri-smoke didn't print 'ok' in 10s. stdout=${stdout!} stderr=${stderr!}`)), 10_000), + setTimeout( + () => + reject( + new Error( + `dri-smoke didn't print 'ok' in 10s. stdout=${stdout!} stderr=${stderr!}`, + ), + ), + 10_000, + ), ), ]); @@ -154,11 +188,15 @@ describe.skipIf(!existsSync(driSmokeBinary))("dri-smoke integration", () => { // Read pixel pattern from the bound region of the process Memory SAB. const procMem = kernel.getProcessMemory(pid); expect(procMem).toBeDefined(); - const view = new DataView(procMem!.buffer, bo.binding!.addr, bo.binding!.len); + const view = new DataView( + procMem!.buffer, + bo.binding!.addr, + bo.binding!.len, + ); const sample = (r: number, c: number) => view.getUint32((r * bo.w + c) * 4, /*littleEndian*/ true); const expected = (r: number, c: number) => - ((0xff000000 | (r << 16) | c) >>> 0); + (0xff000000 | (r << 16) | c) >>> 0; expect(sample(0, 0)).toBe(expected(0, 0)); expect(sample(10, 20)).toBe(expected(10, 20)); expect(sample(255, 255)).toBe(expected(255, 255)); @@ -168,6 +206,7 @@ describe.skipIf(!existsSync(driSmokeBinary))("dri-smoke integration", () => { if (mainW) { await mainW.terminate().catch(() => {}); } + referenceOwners.close(); await Promise.race([ exitPromise, new Promise((resolve) => setTimeout(() => resolve(0), 1_000)), diff --git a/host/test/dylink-fork-archive.test.ts b/host/test/dylink-fork-archive.test.ts new file mode 100644 index 0000000000..b824c7aa44 --- /dev/null +++ b/host/test/dylink-fork-archive.test.ts @@ -0,0 +1,492 @@ +import { describe, expect, it } from "vitest"; +import type { DylinkForkState } from "../src/dylink"; +import { + DylinkForkArchive, + DylinkForkTableReplica, +} from "../src/dylink-fork-archive"; + +function fixture(ptrWidth: 4 | 8 = 4) { + const memory = new WebAssembly.Memory({ initial: 4, maximum: 4 }); + let head = 0; + let next = 4096; + const allocations = new Map(); + const deallocated: Array<{ address: number; size: number }> = []; + const archive = () => new DylinkForkArchive( + memory, + ptrWidth, + () => head, + (value) => { head = value; }, + (size) => { + const address = next; + next += Math.ceil(size / 8) * 8; + allocations.set(address, size); + return { address, size }; + }, + ({ address, size }) => { + expect(allocations.get(address)).toBe(size); + allocations.delete(address); + deallocated.push({ address, size }); + }, + "test dylink archive", + ); + return { + memory, + archive, + allocations, + deallocated, + get head() { + return head; + }, + }; +} + +function state(): DylinkForkState { + return { + nextHandle: 4, + libraries: [ + { + name: "libdependency.so", + moduleBytes: new Uint8Array([0, 97, 115, 109, 1]), + memoryBase: 8192, + tableBase: 3, + activationId: 7, + globalVisibility: true, + allocations: [{ + address: 8192, + size: 64, + mappingAddress: 8176, + mappingSize: 95, + }], + }, + { + name: "libconsumer.so", + moduleBytes: new Uint8Array([0, 97, 115, 109, 2]), + memoryBase: 12288, + tableBase: 9, + activationId: 8, + tlsBase: 16384, + globalVisibility: true, + committedGlobalRoot: true, + allocations: [{ + address: 12288, + size: 128, + mappingAddress: 12272, + mappingSize: 159, + }], + handle: 3, + refCount: 2, + }, + ], + }; +} + +describe("compact dylink fork archive", () => { + it("round-trips dependency-first live state and updates records in place", () => { + const f = fixture(); + const parent = f.archive(); + expect(parent.generation()).toBe(0); + const first = parent.sync(state()); + expect(first.generation).toBe(1); + const allocationCount = f.allocations.size; + expect(parent.read()).toEqual({ + generation: 1, + tableStateRoot: 0, + tableCheckpointGeneration: 0, + tablePatches: [], + ...state(), + }); + + const updated = state(); + updated.libraries[1] = { + ...updated.libraries[1]!, + refCount: 3, + }; + expect(parent.sync(updated).generation).toBe(2); + expect(f.allocations.size).toBe(allocationCount); + expect(parent.read()).toEqual({ + generation: 2, + tableStateRoot: 0, + tableCheckpointGeneration: 0, + tablePatches: [], + ...updated, + }); + + // A separately constructed worker owns no JS cache and must validate the + // complete copied archive before returning any module bytes. + const replica = f.archive(); + expect(replica.generation()).toBe(2); + expect(replica.read()).toEqual({ + generation: 2, + tableStateRoot: 0, + tableCheckpointGeneration: 0, + tablePatches: [], + ...updated, + }); + + const closed: DylinkForkState = { + nextHandle: 4, + libraries: [updated.libraries[0]!], + }; + parent.sync(closed); + expect(parent.read()).toEqual({ + generation: 3, + tableStateRoot: 0, + tableCheckpointGeneration: 0, + tablePatches: [], + ...closed, + }); + // The same replica must invalidate its JavaScript index after another + // Worker publishes a generation. + expect(replica.read()).toEqual({ + generation: 3, + tableStateRoot: 0, + tableCheckpointGeneration: 0, + tablePatches: [], + ...closed, + }); + expect(f.deallocated).toHaveLength(1); + expect(f.allocations.size).toBe(2); // persistent header + dependency + }); + + it("round-trips an issued initialization stage and retires its transaction", () => { + const f = fixture(); + const archive = f.archive(); + const moduleBytes = new Uint8Array([0, 97, 115, 109, 43]); + const pending: DylinkForkState = { + nextHandle: 2, + libraries: [{ + name: "libinitializing.so", + moduleBytes, + memoryBase: 8192, + tableBase: 3, + activationId: 7, + globalVisibility: false, + initialization: { + transactionToken: 11, + stage: "bootstrap", + tableIndex: 19, + }, + }], + transactions: [{ + token: 11, + name: "libinitializing.so", + moduleBytes, + globalVisibility: false, + }], + }; + + archive.sync(pending); + expect(f.archive().read()).toMatchObject({ + generation: 1, + ...pending, + }); + const allocationCount = f.allocations.size; + + const relocated: DylinkForkState = { + ...pending, + libraries: [{ + ...pending.libraries[0]!, + tlsBase: 12288, + initialization: { + transactionToken: 11, + stage: "constructors", + tableIndex: 19, + }, + }], + }; + archive.sync(relocated); + expect(f.allocations.size).toBe(allocationCount); + expect(f.archive().read()).toMatchObject({ + generation: 2, + ...relocated, + }); + + archive.sync({ + nextHandle: 3, + libraries: [{ + ...relocated.libraries[0]!, + initialization: undefined, + handle: 2, + refCount: 1, + }], + }); + expect(f.archive().read()).toMatchObject({ + generation: 3, + nextHandle: 3, + libraries: [{ + name: "libinitializing.so", + handle: 2, + refCount: 1, + }], + }); + expect(f.deallocated).toHaveLength(1); + }); + + it("replaces a live record when constructor binding ownership grows", () => { + const f = fixture(); + const archive = f.archive(); + const initial = state(); + archive.sync(initial); + const allocationCount = f.allocations.size; + + const withRuntimeProvider: DylinkForkState = { + ...initial, + libraries: [ + initial.libraries[0]!, + { + ...initial.libraries[1]!, + providerDependencies: ["libdependency.so"], + }, + ], + }; + archive.sync(withRuntimeProvider); + expect(f.allocations.size).toBe(allocationCount); + expect(f.deallocated).toHaveLength(1); + expect(f.archive().read().libraries[1]).toMatchObject({ + providerDependencies: ["libdependency.so"], + }); + }); + + it("retains an empty header so closed handle gaps survive another fork", () => { + const f = fixture(8); + const archive = f.archive(); + archive.sync({ nextHandle: 19, libraries: [] }); + + expect(f.head).toBeGreaterThan(0); + expect(f.archive().read()).toEqual({ + generation: 1, + tableStateRoot: 0, + tableCheckpointGeneration: 0, + tablePatches: [], + nextHandle: 19, + libraries: [], + }); + }); + + it("publishes generation last and advances it without reallocating records", () => { + const f = fixture(); + const archive = f.archive(); + const initial = archive.sync(state()); + const addresses = [...f.allocations.keys()]; + expect(initial.generation).toBe(1); + expect(archive.generation()).toBe(1); + + const next = archive.sync(state()); + expect(next.generation).toBe(2); + expect(archive.generation()).toBe(2); + expect([...f.allocations.keys()]).toEqual(addresses); + + const view = new DataView(f.memory.buffer); + view.setBigUint64(f.head + 40, 0n, true); + expect(() => f.archive().read()).toThrow(/unpublished/); + }); + + it("publishes one sealed table root and preserves it across linker updates", () => { + const f = fixture(); + const writer = f.archive(); + const reader = f.archive(); + writer.sync(state()); + + const first = writer.publishTableState(2048); + expect(first.previousTableStateRoot).toBe(0); + expect(first.snapshot).toMatchObject({ + generation: 2, + tableStateRoot: 2048, + tableCheckpointGeneration: 2, + tablePatches: [], + }); + expect(reader.read()).toMatchObject({ + generation: 2, + tableStateRoot: 2048, + }); + + const updated = state(); + updated.libraries[1] = { + ...updated.libraries[1]!, + refCount: 4, + }; + expect(writer.sync(updated)).toMatchObject({ + generation: 3, + tableStateRoot: 2048, + tableCheckpointGeneration: 2, + }); + const replacement = writer.publishTableState(3072); + expect(replacement.previousTableStateRoot).toBe(2048); + expect(reader.read()).toMatchObject({ + generation: 4, + tableStateRoot: 3072, + tableCheckpointGeneration: 4, + tablePatches: [], + }); + }); + + it("round-trips bounded stable funcref patches after a checkpoint", () => { + const f = fixture(); + const writer = f.archive(); + const reader = f.archive(); + writer.sync(state()); + writer.publishTableState(2048); + + const patch = { + activationId: 7, + ownerId: 3, + start: 5, + tableLength: 12, + runs: [ + { length: 2, function: null }, + { + length: 3, + function: { activationId: 8, ordinal: 4 }, + }, + ], + } as const; + expect(writer.canPublishTablePatch(patch)).toBe(true); + const publication = writer.publishTablePatch(patch); + expect(publication.snapshot).toMatchObject({ + generation: 3, + tableStateRoot: 2048, + tableCheckpointGeneration: 2, + tablePatches: [{ ...patch, generation: 3 }], + }); + expect(reader.read()).toMatchObject(publication.snapshot); + + // A linker-only generation remains ordered after the patch without + // duplicating or discarding its deterministic replay recipe. + const linked = writer.sync(state()); + expect(linked.generation).toBe(4); + expect(linked.tablePatches).toEqual([{ ...patch, generation: 3 }]); + + const allocationsBeforeCheckpoint = f.allocations.size; + const replacement = writer.publishTableState(3072); + expect(replacement.snapshot).toMatchObject({ + generation: 5, + tableStateRoot: 3072, + tableCheckpointGeneration: 5, + tablePatches: [], + }); + expect(f.allocations.size).toBe(allocationsBeforeCheckpoint - 1); + }); + + it("bounds the patch journal and requires checkpoint compaction", () => { + const f = fixture(); + const writer = f.archive(); + writer.sync({ nextHandle: 2, libraries: [] }); + const patch = { + activationId: 0, + ownerId: 1, + start: 0, + tableLength: 1, + runs: [{ length: 1, function: null }], + } as const; + + for (let index = 0; index < 256; index++) { + expect(writer.canPublishTablePatch(patch)).toBe(true); + writer.publishTablePatch(patch); + } + expect(writer.canPublishTablePatch(patch)).toBe(false); + expect(() => writer.publishTablePatch(patch)).toThrow( + /requires compaction/, + ); + }); + + it("keeps the steady-state Worker table path to one generation read", () => { + const f = fixture(); + const writer = f.archive(); + const reader = f.archive(); + const materialized: Array<[number, number]> = []; + const replica = new DylinkForkTableReplica( + reader, + (snapshot, previousGeneration) => { + materialized.push([snapshot.generation, previousGeneration]); + }, + "pthread table replica", + ); + + expect(replica.reconcile()).toBe(false); + writer.sync(state()); + expect(replica.reconcile()).toBe(true); + expect(replica.generation()).toBe(1); + expect(replica.reconcile()).toBe(false); + + const updated = state(); + updated.libraries[1] = { + ...updated.libraries[1]!, + refCount: 4, + }; + writer.sync(updated); + expect(replica.reconcile()).toBe(true); + expect(replica.reconcile()).toBe(false); + expect(materialized).toEqual([[1, 0], [2, 1]]); + }); + + it("rejects hash corruption and record cycles before exposing bytes", () => { + const hashFixture = fixture(); + hashFixture.archive().sync(state()); + const hashView = new DataView(hashFixture.memory.buffer); + const first = Number(hashView.getBigUint64(hashFixture.head + 32, true)); + const nameLength = hashView.getUint32(first + 60, true); + const moduleOffset = first + 136 + Math.ceil(nameLength / 8) * 8; + new Uint8Array(hashFixture.memory.buffer)[moduleOffset] ^= 0xff; + expect(() => hashFixture.archive().read()).toThrow(/SHA-256 validation/); + + const cycleFixture = fixture(); + cycleFixture.archive().sync(state()); + const cycleView = new DataView(cycleFixture.memory.buffer); + const cycleFirst = Number(cycleView.getBigUint64(cycleFixture.head + 32, true)); + cycleView.setBigUint64(cycleFirst + 8, BigInt(cycleFirst), true); + expect(() => cycleFixture.archive().read()).toThrow(/cyclic or truncated/); + + const countFixture = fixture(); + countFixture.archive().sync({ nextHandle: 2, libraries: [] }); + new DataView(countFixture.memory.buffer).setUint32( + countFixture.head + 24, + 0xffff_ffff, + true, + ); + expect(() => countFixture.archive().read()).toThrow(/memory geometry/); + }); + + it("rejects duplicate identities, impossible handles, and immutable drift", () => { + const f = fixture(); + const archive = f.archive(); + const valid = state(); + expect(() => archive.sync({ + ...valid, + libraries: [ + valid.libraries[0]!, + { ...valid.libraries[0]! }, + ], + })).toThrow(/duplicate live module/); + expect(() => archive.sync({ + nextHandle: 3, + libraries: [valid.libraries[1]!], + })).toThrow(/handle 3 is out of range/); + + archive.sync(valid); + expect(() => archive.sync({ + ...valid, + libraries: [ + { + ...valid.libraries[0]!, + memoryBase: valid.libraries[0]!.memoryBase + 1, + }, + valid.libraries[1]!, + ], + })).toThrow(/changed immutable archive identity/); + }); + + it("binds the copied archive to its pointer-width contract", () => { + const f = fixture(4); + f.archive().sync(state()); + const wrongWidth = new DylinkForkArchive( + f.memory, + 8, + () => f.head, + () => { throw new Error("must not publish"); }, + () => { throw new Error("must not allocate"); }, + () => { throw new Error("must not deallocate"); }, + "wrong width", + ); + expect(() => wrongWidth.read()).toThrow(/pointer-width mismatch/); + }); +}); diff --git a/host/test/dylink.test.ts b/host/test/dylink.test.ts index 4de05a8e5a..7e16d497bd 100644 --- a/host/test/dylink.test.ts +++ b/host/test/dylink.test.ts @@ -10,6 +10,7 @@ import { loadSharedLibrary, loadSharedLibrarySync, DynamicLinker, + FORK_CAP_ACTIVATION_STATE_SAFE, FORK_CAP_DYLINK_MAIN, FORK_CAP_SIDE_ENTRY, FORK_CAPABILITIES_SECTION, @@ -17,14 +18,26 @@ import { forkInstrumentRoleAvailable, readForkInstrumentCapabilityClaim, readForkInstrumentCapabilities, + SIDE_MODULE_FORK_EXPORTS, + type DylinkForkActivationOwner, + type DylinkForkActivationRequest, type LoadSharedLibraryOptions, - type SideModuleForkState, } from "../src/dylink.ts"; import { execFileSync } from "node:child_process"; import { readFileSync, writeFileSync, mkdirSync } from "node:fs"; -import { join } from "node:path"; +import { dirname, join } from "node:path"; import { tmpdir } from "node:os"; +import { fileURLToPath } from "node:url"; import { LINKED_FRAME_FORMAT_SECTION } from "../src/fork-continuation"; +import { ABI_VERSION } from "../src/generated/abi"; +import { ForkAnyrefTransitTable } from "../src/fork-anyref-transit"; +import { + createForkUnwindTag, + FORK_UNWIND_TAG_IMPORT_NAME, +} from "../src/fork-unwind-transport"; + +const repoRoot = join(dirname(fileURLToPath(import.meta.url)), "../.."); +const forkInstrument = join(repoRoot, "scripts", "run-wasm-fork-instrument.sh"); function hasCompiler(compiler = "wasm32posix-cc"): boolean { try { @@ -79,18 +92,27 @@ function buildDylinkWat( memorySize = 0, wat2wasmFlags: string[] = [], tlsExports: string[] = [], + abiVersion: number | null = ABI_VERSION, + neededDynlibs: string[] = [], ): Uint8Array { const dir = join(tmpdir(), "wasm-dylink-wat-test"); mkdirSync(dir, { recursive: true }); const watPath = join(dir, `${name}.wat`); const wasmPath = join(dir, `${name}.wasm`); - const linkedWat = forkCapabilities !== undefined + let linkedWat = forkCapabilities !== undefined && (forkCapabilities & FORK_CAP_SIDE_ENTRY) !== 0 ? wat.replace("(module", `(module (import "env" "__wpk_fork_frame_reserve" (func (param i32) (result i32))) (import "env" "__wpk_fork_frame_commit" (func (param i32))) (import "env" "__wpk_fork_frame_next" (func (param i32) (result i32)))`) : wat; + if (forkCapabilities !== undefined && abiVersion !== null) { + const moduleEnd = linkedWat.lastIndexOf(")"); + if (moduleEnd < 0) throw new Error("test WAT has no module terminator"); + linkedWat = `${linkedWat.slice(0, moduleEnd)} + (func (export "__abi_version") (result i32) i32.const ${abiVersion}) + ${linkedWat.slice(moduleEnd)}`; + } writeFileSync(watPath, linkedWat); execFileSync("wat2wasm", ["--enable-threads", ...wat2wasmFlags, watPath, "-o", wasmPath], { stdio: "pipe", @@ -107,11 +129,25 @@ function buildDylinkWat( const exportInfo = tlsExports.length > 0 ? [3, 1 + exportInfoBody.length, tlsExports.length, ...exportInfoBody] : []; - const payload = new Uint8Array(1 + dylinkName.length + 6 + exportInfo.length); + const neededBody = neededDynlibs.flatMap((libraryName) => { + const bytes = [...new TextEncoder().encode(libraryName)]; + if (bytes.length >= 128) throw new Error("test dependency name is too long"); + return [bytes.length, ...bytes]; + }); + const neededInfo = neededDynlibs.length > 0 + ? [2, 1 + neededBody.length, neededDynlibs.length, ...neededBody] + : []; + const payload = new Uint8Array( + 1 + dylinkName.length + 6 + neededInfo.length + exportInfo.length, + ); + if (payload.length >= 128) { + throw new Error("test dylink section helper only supports one-byte LEB lengths"); + } payload[0] = dylinkName.length; payload.set(dylinkName, 1); payload.set([1, 4, memorySize, 0, tableSize, 0], 1 + dylinkName.length); - payload.set(exportInfo, 1 + dylinkName.length + 6); + payload.set(neededInfo, 1 + dylinkName.length + 6); + payload.set(exportInfo, 1 + dylinkName.length + 6 + neededInfo.length); const section = new Uint8Array(2 + payload.length); section[0] = 0; section[1] = payload.length; @@ -124,7 +160,10 @@ function buildDylinkWat( let marked = appendCustomSection( out, FORK_CAPABILITIES_SECTION, - new Uint8Array([FORK_CAPABILITIES_VERSION, forkCapabilities]), + new Uint8Array([ + FORK_CAPABILITIES_VERSION, + forkCapabilities | FORK_CAP_ACTIVATION_STATE_SAFE, + ]), ); if ((forkCapabilities & FORK_CAP_SIDE_ENTRY) !== 0) { marked = appendCustomSection( @@ -142,6 +181,155 @@ function buildDylinkWat( return marked; } +/** Build a real ABI-43 side artifact instead of hand-maintaining its contract. */ +function buildInstrumentedDylinkWat( + wat: string, + name: string, + neededDynlibs: string[] = [], +): Uint8Array { + const dir = join(tmpdir(), "wasm-dylink-instrumented-test"); + mkdirSync(dir, { recursive: true }); + const inputPath = join(dir, `${name}.input.wasm`); + const outputPath = join(dir, `${name}.instrumented.wasm`); + const moduleEnd = wat.lastIndexOf(")"); + if (moduleEnd < 0) throw new Error("instrumented test WAT has no module terminator"); + const versionedWat = `${wat.slice(0, moduleEnd)} + (func (export "__abi_version") (result i32) i32.const ${ABI_VERSION}) + ${wat.slice(moduleEnd)}`; + writeFileSync( + inputPath, + buildDylinkWat( + versionedWat, + `${name}-raw`, + undefined, + 0, + 0, + [], + [], + null, + neededDynlibs, + ), + ); + execFileSync( + "bash", + [forkInstrument, inputPath, "-o", outputPath, "--entry", "env.fork"], + { cwd: repoRoot, stdio: "pipe" }, + ); + return new Uint8Array(readFileSync(outputPath)); +} + +interface TestForkActivationOwner { + readonly owner: DylinkForkActivationOwner; + readonly prepares: DylinkForkActivationRequest[]; + readonly registered: Array<{ activationId: number; instance: WebAssembly.Instance }>; + readonly unregistered: number[]; + readonly active: ReadonlySet; + readonly forkImport: () => number; + readonly wrappedImports: WebAssembly.Imports[]; +} + +/** + * Minimal process owner for loader contract tests. + * + * The real worker binds state arenas and typed codecs. These fixtures never + * execute a continuation, so inert functions are sufficient; tables and the + * private exception tag still use their exact WebAssembly types. + */ +function createTestForkActivationOwner( + firstActivationId = 1, +): TestForkActivationOwner { + const prepares: DylinkForkActivationRequest[] = []; + const registered: Array<{ + activationId: number; + instance: WebAssembly.Instance; + }> = []; + const unregistered: number[] = []; + const wrappedImports: WebAssembly.Imports[] = []; + const active = new Set(); + const gcTransit = new ForkAnyrefTransitTable(); + const resumeTable = new WebAssembly.Table({ initial: 1, element: "anyfunc" }); + const unwindTag = createForkUnwindTag(); + const forkImport = () => -12; + let nextActivationId = firstActivationId; + + const owner: DylinkForkActivationOwner = { + prepare(request) { + const activationId = request.replayActivationId ?? nextActivationId++; + prepares.push(request); + if (active.has(activationId)) { + throw new Error(`duplicate test activation id ${activationId}`); + } + active.add(activationId); + + const env: Record = {}; + for (const imported of WebAssembly.Module.imports(request.module)) { + if (imported.module !== "env") continue; + if (imported.name === "fork") { + env[imported.name] = forkImport; + } else if (imported.name === "__wpk_fork_ref_gc_transit") { + env[imported.name] = gcTransit.table; + } else if (imported.name === "__wpk_fork_resume_table") { + env[imported.name] = resumeTable; + } else if ( + imported.name === FORK_UNWIND_TAG_IMPORT_NAME + && (imported.kind as string) === "tag" + ) { + env[imported.name] = + unwindTag as unknown as WebAssembly.ImportValue; + } else if ( + imported.name.startsWith("__wpk_fork_") + && imported.kind === "function" + ) { + env[imported.name] = () => 0; + } else if ( + imported.name.startsWith("__wpk_fork_") + && imported.kind === "global" + ) { + env[imported.name] = + imported.name === "__wpk_fork_module_state_table_generation_addr" + ? new WebAssembly.Global( + { value: "i64", mutable: false }, + 0n, + ) + : new WebAssembly.Global( + { value: "i32", mutable: false }, + activationId, + ); + } + } + + let released = false; + return { + activationId, + env, + wrapImports(imports) { + wrappedImports.push(imports); + return imports; + }, + register(instance) { + registered.push({ activationId, instance }); + }, + unregister() { + if (released) throw new Error(`test activation ${activationId} released twice`); + released = true; + active.delete(activationId); + unregistered.push(activationId); + }, + }; + }, + }; + + return { + owner, + prepares, + registered, + unregistered, + active, + forkImport, + wrappedImports, + }; +} + describe.skipIf(typeof WebAssembly.Tag !== "function")("longjmp tag identity", () => { const cases = [ { ptrWidth: 4 as const, wasmType: "i32", value: 37 }, @@ -636,80 +824,567 @@ describe.skipIf(!hasCompiler())("synchronous loading (loadSharedLibrarySync)", ( }); function createSideForkLoadOptions(): LoadSharedLibraryOptions { - const memory = new WebAssembly.Memory({ initial: 1, maximum: 100, shared: true }); - let nextContinuation = 65536; return { - memory, + memory: new WebAssembly.Memory({ initial: 1, maximum: 100, shared: true }), table: new WebAssembly.Table({ initial: 1, element: "anyfunc" }), stackPointer: new WebAssembly.Global({ value: "i32", mutable: true }, 65536), heapPointer: { value: 1024 }, - allocateContinuation: (size) => { - const addr = nextContinuation; - nextContinuation += size; - const requiredPages = Math.ceil(nextContinuation / 65536); - const currentPages = memory.buffer.byteLength / 65536; - if (requiredPages > currentPages) memory.grow(requiredPages - currentPages); - return addr; - }, - deallocateContinuation: () => {}, globalSymbols: new Map(), got: new Map(), loadedLibraries: new Map(), }; } +describe("DynamicLinker deterministic replay events", () => { + it("loads dependencies without handles and replays exact open/close state", () => { + const dependencyBytes = buildDylinkWat(` + (module + (import "env" "memory" (memory 1 100 shared)) + (func (export "dependency_value") (result i32) i32.const 5)) + `, "replay-event-dependency"); + const consumerBytes = buildDylinkWat(` + (module + (import "env" "memory" (memory 1 100 shared)) + (import "env" "dependency_value" (func $dependency_value (result i32))) + (func (export "consumer_value") (result i32) call $dependency_value)) + `, "replay-event-consumer", undefined, 0, 0, [], [], null, [ + "libevent-dependency.so", + ]); + const options = createSideForkLoadOptions(); + options.resolveLibrarySync = (name) => + name === "libevent-dependency.so" ? dependencyBytes : null; + const linker = new DynamicLinker(options); + + const consumer = linker.loadModuleSync( + "libevent-consumer.so", + consumerBytes, + ); + expect(Array.from(options.loadedLibraries.keys())).toEqual([ + "libevent-dependency.so", + "libevent-consumer.so", + ]); + const dependency = options.loadedLibraries.get("libevent-dependency.so")!; + expect(dependency.moduleBytes).toEqual(dependencyBytes); + expect(dependency.moduleBytes).not.toBe(dependencyBytes); + expect(consumer.moduleBytes).toEqual(consumerBytes); + expect(linker.forkState()).toMatchObject({ + nextHandle: 2, + libraries: [ + { name: "libevent-dependency.so" }, + { name: "libevent-consumer.so" }, + ], + }); + expect(linker.forkLibraryState("libevent-dependency.so")).not.toHaveProperty("handle"); + expect(linker.forkLibraryState("libevent-consumer.so")).not.toHaveProperty("handle"); + expect((consumer.exports.consumer_value as () => number)()).toBe(5); + expect(linker.dlsym(2, "consumer_value")).toBeNull(); + expect(linker.dlerror()).toContain("invalid handle"); + + expect(() => linker.replayOpen("libevent-consumer.so", 3)) + .toThrow(/does not match next handle 2/); + expect(linker.replayOpen("libevent-consumer.so", 2)).toBe(2); + expect(linker.replayOpen("libevent-consumer.so", 2)).toBe(2); + expect(linker.forkLibraryState("libevent-consumer.so")).toMatchObject({ + handle: 2, + refCount: 2, + }); + expect(linker.forkState().nextHandle).toBe(3); + expect(() => linker.replayOpen("libevent-dependency.so", 2)) + .toThrow(/does not match next handle 3/); + expect(linker.replayOpen("libevent-dependency.so", 3)).toBe(3); + + linker.replayClose(2); + expect(linker.forkLibraryState("libevent-consumer.so")).toMatchObject({ + handle: 2, + refCount: 1, + }); + expect(options.loadedLibraries.has("libevent-consumer.so")).toBe(true); + linker.replayClose(2); + expect(options.loadedLibraries.has("libevent-consumer.so")).toBe(false); + expect(options.loadedLibraries.has("libevent-dependency.so")).toBe(true); + expect(() => linker.replayClose(2)).toThrow(/invalid dlopen handle 2/); + + expect(linker.forkState()).toMatchObject({ + nextHandle: 4, + libraries: [ + { + name: "libevent-dependency.so", + handle: 3, + refCount: 1, + }, + ], + }); + }); + + it("retains NEEDED providers until both dependency and handle owners release", () => { + const dependencyBytes = buildDylinkWat(` + (module + (import "env" "memory" (memory 1 100 shared)) + (func (export "dependency_value") (result i32) i32.const 5)) + `, "dependency-retain-provider"); + const consumerBytes = buildDylinkWat(` + (module + (import "env" "memory" (memory 1 100 shared)) + (import "env" "dependency_value" (func $dependency_value (result i32))) + (func (export "consumer_value") (result i32) call $dependency_value)) + `, "dependency-retain-consumer", undefined, 0, 0, [], [], null, [ + "libretain-provider.so", + ]); + const options = createSideForkLoadOptions(); + options.resolveLibrarySync = (name) => + name === "libretain-provider.so" ? dependencyBytes : null; + const linker = new DynamicLinker(options); + + const consumerHandle = linker.dlopenSync("libretain-consumer.so", consumerBytes); + const providerHandle = linker.dlopenSync("libretain-provider.so", dependencyBytes); + expect(consumerHandle).toBe(2); + expect(providerHandle).toBe(3); + + expect(linker.dlclose(providerHandle)).toBe(0); + expect(options.loadedLibraries.has("libretain-provider.so")).toBe(true); + expect(linker.forkLibraryState("libretain-provider.so")).not.toHaveProperty("handle"); + expect(linker.dlclose(consumerHandle)).toBe(0); + expect(options.loadedLibraries.size).toBe(0); + expect(linker.forkState()).toMatchObject({ + nextHandle: 4, + libraries: [], + }); + }); + + it("retains a side-module provider captured by a direct relocation", () => { + const providerBytes = buildDylinkWat(` + (module + (import "env" "memory" (memory 1 100 shared)) + (func (export "runtime_provider") (result i32) i32.const 41)) + `, "runtime-provider-retain-provider"); + const consumerBytes = buildDylinkWat(` + (module + (import "env" "memory" (memory 1 100 shared)) + (import "env" "runtime_provider" + (func $runtime_provider (result i32))) + (func (export "runtime_consumer") (result i32) + call $runtime_provider + i32.const 1 + i32.add)) + `, "runtime-provider-retain-consumer"); + const options = createSideForkLoadOptions(); + const linker = new DynamicLinker(options); + const providerHandle = linker.dlopenSync( + "libruntime-provider.so", + providerBytes, + ); + const consumerHandle = linker.dlopenSync( + "libruntime-consumer.so", + consumerBytes, + ); + expect( + linker.forkLibraryState("libruntime-consumer.so") + ?.providerDependencies, + ).toEqual(["libruntime-provider.so"]); + + expect(linker.dlclose(providerHandle)).toBe(0); + expect(options.loadedLibraries.has("libruntime-provider.so")).toBe(true); + const consumer = linker.dlsym(consumerHandle, "runtime_consumer"); + expect((options.table.get(consumer!) as () => number)()).toBe(42); + + expect(linker.dlclose(consumerHandle)).toBe(0); + expect(options.loadedLibraries.size).toBe(0); + }); + + it("keeps RTLD_LOCAL exports private and archives their later promotion", () => { + const wasmBytes = buildDylinkWat(` + (module + (import "env" "memory" (memory 1 100 shared)) + (func (export "local_then_global") (result i32) i32.const 37)) + `, "staged-local-visibility"); + const options = createSideForkLoadOptions(); + const linker = new DynamicLinker(options); + + const localHandle = linker.dlopenSync( + "liblocal-visibility.so", + wasmBytes, + undefined, + false, + ); + const explicit = linker.dlsym(localHandle, "local_then_global"); + expect(explicit).not.toBeNull(); + expect(linker.dlsym(0, "local_then_global")).toBeNull(); + expect(linker.forkState().libraries[0]).toMatchObject({ + globalVisibility: false, + }); + expect(linker.forkState().libraries[0]).not.toHaveProperty( + "committedGlobalRoot", + ); + + const promotedHandle = linker.dlopenSync( + "liblocal-visibility.so", + wasmBytes, + undefined, + true, + ); + expect(promotedHandle).toBe(localHandle); + expect(linker.dlsym(0, "local_then_global")).toBe(explicit); + expect(linker.forkState().libraries[0]).toMatchObject({ + globalVisibility: true, + committedGlobalRoot: true, + }); + }); + + it("binds an RTLD_LOCAL root through its private NEEDED scope", () => { + const dependency = buildDylinkWat(` + (module + (import "env" "memory" (memory 1 100 shared)) + (func (export "dependency_value") (result i32) i32.const 29)) + `, "staged-local-needed-dependency"); + const root = buildDylinkWat(` + (module + (import "env" "memory" (memory 1 100 shared)) + (import "env" "dependency_value" (func $dependency_value (result i32))) + (func (export "root_value") (result i32) + call $dependency_value + i32.const 8 + i32.add)) + `, "staged-local-needed-root", undefined, 0, 0, [], [], null, [ + "libscope-dep.so", + ]); + const options = createSideForkLoadOptions(); + options.resolveLibrarySync = (name) => + name === "libscope-dep.so" ? dependency : null; + const linker = new DynamicLinker(options); + + const localHandle = linker.dlopenSync( + "libscope-root.so", + root, + undefined, + false, + ); + const rootIndex = linker.dlsym(localHandle, "root_value"); + expect(rootIndex).not.toBeNull(); + expect((options.table.get(rootIndex!) as () => number)()).toBe(37); + expect( + options.loadedLibraries.get("libscope-dep.so")?.globalVisibility, + ).toBe(false); + expect(linker.dlsym(0, "root_value")).toBeNull(); + expect(linker.dlsym(0, "dependency_value")).toBeNull(); + + expect(linker.dlopenSync( + "libscope-root.so", + root, + undefined, + true, + )).toBe(localHandle); + expect( + options.loadedLibraries.get("libscope-dep.so")?.globalVisibility, + ).toBe(true); + expect(linker.dlsym(0, "root_value")).toBe(rootIndex); + expect(linker.dlsym(0, "dependency_value")).not.toBeNull(); + }); + + it("restores compact handle/refcount state with closed-handle gaps", () => { + const firstBytes = buildDylinkWat(` + (module + (import "env" "memory" (memory 1 100 shared)) + (func (export "first_value") (result i32) i32.const 1)) + `, "fork-handle-first"); + const secondBytes = buildDylinkWat(` + (module + (import "env" "memory" (memory 1 100 shared)) + (func (export "second_value") (result i32) i32.const 2)) + `, "fork-handle-second"); + const parentOptions = createSideForkLoadOptions(); + const parent = new DynamicLinker(parentOptions); + + expect(parent.dlopenSync("libfirst.so", firstBytes)).toBe(2); + expect(parent.dlopenSync("libsecond.so", secondBytes)).toBe(3); + expect(parent.dlopenSync("libsecond.so", secondBytes)).toBe(3); + expect(parent.dlclose(2)).toBe(0); + const archived = parent.forkState(); + expect(archived).toMatchObject({ + nextHandle: 4, + libraries: [{ + name: "libsecond.so", + handle: 3, + refCount: 2, + }], + }); + + const childOptions = createSideForkLoadOptions(); + const child = new DynamicLinker(childOptions); + for (const library of archived.libraries) { + child.loadModuleSync( + library.name, + new Uint8Array(library.moduleBytes), + { + memoryBase: library.memoryBase, + tableBase: library.tableBase, + activationId: library.activationId, + tlsBase: library.tlsBase, + globalVisibility: library.globalVisibility, + committedGlobalRoot: library.committedGlobalRoot, + }, + ); + } + child.restoreForkHandleState(archived); + expect(child.forkState()).toEqual(archived); + + // The duplicate open keeps the inherited handle and reference count. + expect(child.dlopenSync("libsecond.so", secondBytes)).toBe(3); + expect(child.forkLibraryState("libsecond.so")).toMatchObject({ + handle: 3, + refCount: 3, + }); + // The next new module must not reuse the parent's closed handle 2. + expect(child.dlopenSync("libfirst.so", firstBytes)).toBe(4); + }); + + it("reconciles dlopen function recipes to fresh Worker-local table entries", () => { + const sideBytes = buildDylinkWat(` + (module + (import "env" "memory" (memory 1 100 shared)) + (import "env" "__indirect_function_table" (table 1 funcref)) + (import "env" "__table_base" (global $table_base i32)) + (func $side_value (export "side_value") (result i32) i32.const 73) + (elem (global.get $table_base) func $side_value)) + `, "fork-table-replica", undefined, 1); + const parentOptions = createSideForkLoadOptions(); + const parentMutations: Array<[number, number]> = []; + parentOptions.onTableMutation = (_table, firstIndex, length) => { + parentMutations.push([firstIndex, length]); + }; + const parent = new DynamicLinker(parentOptions); + expect(parent.dlopenSync("libtable-replica.so", sideBytes)).toBe(2); + const archived = parent.forkState(); + const parentLibrary = archived.libraries[0]!; + const parentFunction = parentOptions.table.get(parentLibrary.tableBase); + expect(typeof parentFunction).toBe("function"); + expect((parentFunction as () => number)()).toBe(73); + expect(parentMutations).toContainEqual([ + parentOptions.table.length - 1, + 1, + ]); + + const replicaOptions = createSideForkLoadOptions(); + const replicaMutations: Array<[number, number]> = []; + replicaOptions.onTableMutation = (_table, firstIndex, length) => { + replicaMutations.push([firstIndex, length]); + }; + const replica = new DynamicLinker(replicaOptions); + replica.reconcileForkModules(archived); + const freshFunction = replicaOptions.table.get(parentLibrary.tableBase); + expect(typeof freshFunction).toBe("function"); + expect(freshFunction).not.toBe(parentFunction); + expect((freshFunction as () => number)()).toBe(73); + expect(replicaMutations).toContainEqual([ + replicaOptions.table.length - 1, + 1, + ]); + + const length = replicaOptions.table.length; + replica.reconcileForkModules(archived); + expect(replicaOptions.table.length).toBe(length); + expect(replicaOptions.table.get(parentLibrary.tableBase)).toBe(freshFunction); + + const parentOwnedEntries = [ + ...parentOptions.loadedLibraries.get("libtable-replica.so")! + .ownedTableEntries, + ]; + const replicaOwnedEntries = [ + ...replicaOptions.loadedLibraries.get("libtable-replica.so")! + .ownedTableEntries, + ]; + expect(parent.dlclose(2)).toBe(0); + for (const index of parentOwnedEntries) { + expect(parentOptions.table.get(index)).toBeNull(); + } + expect(parentOptions.globalSymbols.has("side_value")).toBe(false); + expect(parent.dlsym(0, "side_value")).toBeNull(); + const closed = parent.forkState(); + expect(closed.libraries).toEqual([]); + + replica.reconcileForkModules(closed); + expect(replicaOptions.loadedLibraries.size).toBe(0); + for (const index of replicaOwnedEntries) { + expect(replicaOptions.table.get(index)).toBeNull(); + } + }); + + it("rejects non-pristine or inconsistent compact handle snapshots", () => { + const wasmBytes = buildDylinkWat(` + (module + (import "env" "memory" (memory 1 100 shared)) + (func (export "value") (result i32) i32.const 1)) + `, "fork-handle-validation"); + const options = createSideForkLoadOptions(); + const linker = new DynamicLinker(options); + const live = linker.loadModuleSync("libvalidation.so", wasmBytes); + const baseState = { + nextHandle: 4, + libraries: [{ + name: live.name, + moduleBytes: live.moduleBytes, + memoryBase: live.memoryBase, + tableBase: live.tableBase, + globalVisibility: live.globalVisibility, + handle: 3, + refCount: 1, + }], + }; + + expect(() => linker.restoreForkHandleState({ + ...baseState, + libraries: [ + ...baseState.libraries, + { ...baseState.libraries[0]!, handle: 2 }, + ], + })).toThrow(/exact live module closure/); + expect(() => linker.restoreForkHandleState({ + ...baseState, + libraries: [{ ...baseState.libraries[0]!, handle: 4 }], + })).toThrow(/fork handle 4 is invalid/); + linker.restoreForkHandleState(baseState); + expect(() => linker.restoreForkHandleState(baseState)) + .toThrow(/requires a pristine child handle index/); + }); + + it("rejects replay of the same module-load record twice", () => { + const wasmBytes = buildDylinkWat(` + (module + (import "env" "memory" (memory 1 100 shared)) + (func (export "value") (result i32) i32.const 1)) + `, "duplicate-replay-load"); + const options = createSideForkLoadOptions(); + const linker = new DynamicLinker(options); + const loaded = linker.loadModuleSync("libduplicate-replay.so", wasmBytes, { + memoryBase: 0, + tableBase: 1, + }); + expect(loaded.name).toBe("libduplicate-replay.so"); + + expect(() => linker.loadModuleSync("libduplicate-replay.so", wasmBytes, { + memoryBase: 0, + tableBase: 1, + })).toThrow(/archive entries must be unique/); + }); +}); + describe("side-module fork contract", () => { - it("rejects an uninstrumented side module that imports fork", () => { + it("keeps raw side modules legal when the process has no fork activation owner", () => { const wasmBytes = buildDylinkWat(` (module (import "env" "memory" (memory 1 100 shared)) - (import "env" "fork" (func $fork (result i32))) - (func (export "side_fork") (result i32) call $fork)) - `, "side-fork-uninstrumented"); + (func (export "raw_value") (result i32) i32.const 17)) + `, "raw-side-without-fork-owner"); + const options = createSideForkLoadOptions(); + + const loaded = loadSharedLibrarySync( + "libraw-nonfork.so", + wasmBytes, + options, + ); + + expect((loaded.exports.raw_value as () => number)()).toBe(17); + expect(loaded.activationId).toBeUndefined(); + }); + + it("rejects a raw side module before instantiation in a fork-capable process", () => { + const wasmBytes = buildDylinkWat(` + (module + (import "env" "memory" (memory 1 100 shared)) + (func (export "raw_value") (result i32) i32.const 17)) + `, "raw-side-with-fork-owner"); + const testOwner = createTestForkActivationOwner(); const options = createSideForkLoadOptions(); - options.sideModuleFork = { - setActiveFork: () => {}, - clearActiveFork: () => {}, - invokeMainFork: () => 0, - beginMainAbort: () => {}, + options.forkActivationOwner = testOwner.owner; + + expect(() => loadSharedLibrarySync( + "libraw-fork-process.so", + wasmBytes, + options, + )).toThrow(/requires complete ABI 43 side-boundary instrumentation/); + expect(testOwner.prepares).toEqual([]); + expect(options.loadedLibraries.size).toBe(0); + }); + + it("accepts a complete side-boundary artifact without an env.fork import", () => { + const wasmBytes = buildInstrumentedDylinkWat(` + (module + (import "env" "memory" (memory 1 100 shared)) + (import "env" "host_value" (func $host_value (result i32))) + (func (export "side_value") (result i32) + call $host_value + i32.const 1 + i32.add)) + `, "side-boundary-without-fork-import"); + const module = new WebAssembly.Module( + wasmBytes as unknown as BufferSource, + ); + expect(WebAssembly.Module.imports(module).some( + (entry) => + entry.module === "env" + && entry.name === "fork" + && entry.kind === "function", + )).toBe(false); + expect(readForkInstrumentCapabilities(module) & FORK_CAP_SIDE_ENTRY) + .toBe(FORK_CAP_SIDE_ENTRY); + + const testOwner = createTestForkActivationOwner(21); + const options = createSideForkLoadOptions(); + options.globalSymbols.set("host_value", () => 16); + options.forkActivationOwner = testOwner.owner; + + const loaded = loadSharedLibrarySync( + "libside-boundary.so", + wasmBytes, + options, + ); + + expect(loaded.activationId).toBe(21); + expect((loaded.exports.side_value as () => number)()).toBe(17); + expect(testOwner.registered).toEqual([ + { activationId: 21, instance: loaded.instance }, + ]); + }); + + it("validates ABI 43 reconstruction metadata before side-module instantiation", () => { + // Export every reserved function name so the loader takes the complete + // ABI-43 path. Deliberately give the stubs the wrong signatures and omit + // reconstruction descriptors/imports: none of this module may execute. + const reservedStubs = SIDE_MODULE_FORK_EXPORTS + .map((name) => `(func (export "${name}"))`) + .join("\n"); + const wasmBytes = buildDylinkWat(` + (module + (import "env" "memory" (memory 1 100 shared)) + ${reservedStubs}) + `, "side-fork-invalid-reconstruction-contract", 0); + const options = createSideForkLoadOptions(); + let prepareCalls = 0; + options.forkActivationOwner = { + prepare() { + prepareCalls++; + throw new Error("must not prepare an invalid artifact"); + }, }; - expect(() => loadSharedLibrarySync("libbadfork.so", wasmBytes, options)) - .toThrow(/requires complete side-module instrumentation/); + expect(() => loadSharedLibrarySync("libinvalidfork.so", wasmBytes, options)) + .toThrow( + /invalid ABI 43 fork reconstruction contract: .*exception_codec.*imported_globals/, + ); + expect(prepareCalls).toBe(0); + expect(options.loadedLibraries.has("libinvalidfork.so")).toBe(false); }); - it("applies the generated ABI transition to a legacy five-export side artifact", () => { + it("rejects an uninstrumented side module that imports fork", () => { const wasmBytes = buildDylinkWat(` (module (import "env" "memory" (memory 1 100 shared)) (import "env" "fork" (func $fork (result i32))) - (func (export "wpk_fork_unwind_begin") (param i32)) - (func (export "wpk_fork_unwind_end")) - (func (export "wpk_fork_rewind_begin") (param i32)) - (func (export "wpk_fork_rewind_end")) - (func (export "wpk_fork_abort_begin") (param i32)) - (func (export "wpk_fork_abort_end")) - (func (export "wpk_fork_state") (result i32) i32.const 0) (func (export "side_fork") (result i32) call $fork)) - `, "side-fork-generic"); + `, "side-fork-uninstrumented"); const options = createSideForkLoadOptions(); - options.sideModuleFork = { - setActiveFork: () => {}, - clearActiveFork: () => {}, - invokeMainFork: () => 0, - beginMainAbort: () => {}, - }; - const load = () => loadSharedLibrarySync("liblegacyfork.so", wasmBytes, options); - const legacyAllowed = forkInstrumentRoleAvailable( - { present: false, flags: 0 }, - FORK_CAP_SIDE_ENTRY, - ); - if (legacyAllowed) { - expect(load).not.toThrow(); - } else { - expect(load).toThrow(/versioned side-entry capability/); - } + expect(() => loadSharedLibrarySync("libbadfork.so", wasmBytes, options)) + .toThrow(/requires complete side-module instrumentation/); }); it("makes missing side and main role claims mandatory at ABI 17", () => { @@ -731,30 +1406,49 @@ describe("side-module fork contract", () => { )).toBe(false); }); - it("rejects a marker-present artifact that does not claim side-entry coverage", () => { - const wasmBytes = buildDylinkWat(` + it("binds a side module's activation-safety claim to ABI 43", () => { + const reservedStubs = SIDE_MODULE_FORK_EXPORTS + .map((name) => `(func (export "${name}"))`) + .join("\n"); + const sideWat = ` (module (import "env" "memory" (memory 1 100 shared)) - (import "env" "fork" (func $fork (result i32))) - (func (export "wpk_fork_unwind_begin") (param i32)) - (func (export "wpk_fork_unwind_end")) - (func (export "wpk_fork_rewind_begin") (param i32)) - (func (export "wpk_fork_rewind_end")) - (func (export "wpk_fork_abort_begin") (param i32)) - (func (export "wpk_fork_abort_end")) - (func (export "wpk_fork_state") (result i32) i32.const 0) - (func (export "side_fork") (result i32) call $fork)) - `, "side-fork-wrong-marker", 0); + ${reservedStubs}) + `; const options = createSideForkLoadOptions(); - options.sideModuleFork = { - setActiveFork: () => {}, - clearActiveFork: () => {}, - invokeMainFork: () => 0, - beginMainAbort: () => {}, + let prepareCalls = 0; + options.forkActivationOwner = { + prepare() { + prepareCalls++; + throw new Error("must not prepare an ABI-mismatched artifact"); + }, }; - - expect(() => loadSharedLibrarySync("libwrongmarker.so", wasmBytes, options)) - .toThrow(/versioned side-entry capability/); + const stale = buildDylinkWat( + sideWat, + "side-fork-stale-abi", + 0, + 0, + 0, + [], + [], + ABI_VERSION - 1, + ); + expect(() => loadSharedLibrarySync("libstale.so", stale, options)) + .toThrow(/declares ABI 42, but the host requires ABI 43/); + + const missing = buildDylinkWat( + sideWat, + "side-fork-missing-abi", + 0, + 0, + 0, + [], + [], + null, + ); + expect(() => loadSharedLibrarySync("libmissing.so", missing, options)) + .toThrow(/missing __abi_version/); + expect(prepareCalls).toBe(0); }); it("reads the versioned side-entry capability independently", () => { @@ -764,9 +1458,11 @@ describe("side-module fork contract", () => { const module = new WebAssembly.Module(wasmBytes as unknown as BufferSource); expect(readForkInstrumentCapabilityClaim(module)).toEqual({ present: true, - flags: FORK_CAP_SIDE_ENTRY, + flags: FORK_CAP_SIDE_ENTRY | FORK_CAP_ACTIVATION_STATE_SAFE, }); - expect(readForkInstrumentCapabilities(module)).toBe(FORK_CAP_SIDE_ENTRY); + expect(readForkInstrumentCapabilities(module)).toBe( + FORK_CAP_SIDE_ENTRY | FORK_CAP_ACTIVATION_STATE_SAFE, + ); }); it("rejects a malformed marker even during the ABI-16 compatibility window", () => { @@ -784,183 +1480,292 @@ describe("side-module fork contract", () => { .toThrow(/malformed kandelo\.wpk_fork\.capabilities custom section/); }); - it("reports an explicit stale-main diagnostic for a valid side artifact", () => { - const wasmBytes = buildDylinkWat(` + it("requires a process activation owner for a valid ABI-43 side module", () => { + const wasmBytes = buildInstrumentedDylinkWat(` (module (import "env" "memory" (memory 1 100 shared)) (import "env" "fork" (func $fork (result i32))) - (func (export "wpk_fork_unwind_begin") (param i32)) - (func (export "wpk_fork_unwind_end")) - (func (export "wpk_fork_rewind_begin") (param i32)) - (func (export "wpk_fork_rewind_end")) - (func (export "wpk_fork_abort_begin") (param i32)) - (func (export "wpk_fork_abort_end")) - (func (export "wpk_fork_state") (result i32) i32.const 0) (func (export "side_fork") (result i32) call $fork)) - `, "side-with-stale-main", FORK_CAP_SIDE_ENTRY); + `, "side-fork-owner-required"); const options = createSideForkLoadOptions(); - options.sideModuleForkUnavailableReason = - "main module lacks the versioned dlopen-main fork capability; rebuild it"; + options.forkActivationOwnerUnavailableReason = + "main activation registry is unavailable; rebuild or relaunch the process"; - expect(() => loadSharedLibrarySync("libside.so", wasmBytes, options)) - .toThrow(/main module lacks the versioned dlopen-main fork capability; rebuild it/); + expect(() => loadSharedLibrarySync("libowner-required.so", wasmBytes, options)) + .toThrow(/main activation registry is unavailable/); }); - it("rejects a fork-capable side module without process-mapping storage", () => { - const wasmBytes = buildDylinkWat(` + it("registers multiple linked side activations without module-static fork roots", () => { + const providerBytes = buildInstrumentedDylinkWat(` + (module + (import "env" "memory" (memory 1 100 shared)) + (import "env" "fork" (func $fork (result i32))) + (export "raw_fork_import" (func $fork)) + (func (export "provider_value") (result i32) i32.const 7)) + `, "side-fork-provider"); + const consumerBytes = buildInstrumentedDylinkWat(` (module (import "env" "memory" (memory 1 100 shared)) (import "env" "fork" (func $fork (result i32))) - (func (export "wpk_fork_unwind_begin") (param i32)) - (func (export "wpk_fork_unwind_end")) - (func (export "wpk_fork_rewind_begin") (param i32)) - (func (export "wpk_fork_rewind_end")) - (func (export "wpk_fork_abort_begin") (param i32)) - (func (export "wpk_fork_abort_end")) - (func (export "wpk_fork_state") (result i32) i32.const 0) + (import "env" "provider_value" (func $provider_value (result i32))) + (func (export "nested_value") (result i32) + call $provider_value + i32.const 1 + i32.add) + (func (export "consumer_fork") (result i32) call $fork)) + `, "side-fork-consumer"); + const testOwner = createTestForkActivationOwner(); + const options = createSideForkLoadOptions(); + options.forkActivationOwner = testOwner.owner; + + const provider = loadSharedLibrarySync( + "libfork-provider.so", + providerBytes, + options, + ); + const consumer = loadSharedLibrarySync( + "libfork-consumer.so", + consumerBytes, + options, + ); + + expect(provider.activationId).toBe(1); + expect(consumer.activationId).toBe(2); + expect(testOwner.registered).toEqual([ + { activationId: 1, instance: provider.instance }, + { activationId: 2, instance: consumer.instance }, + ]); + expect((consumer.exports.nested_value as () => number)()).toBe(8); + // Engines expose a Wasm wrapper when an imported JS function is + // re-exported, so behavior—not JS object identity—proves the exact owner + // callback reached the module. + expect((provider.instance.exports.raw_fork_import as () => number)()).toBe(-12); + expect("forkBufAddr" in provider).toBe(false); + expect("forkContinuation" in provider).toBe(false); + expect(typeof provider.activationId).toBe("number"); + }); + + it("wraps the final lazy imports before instantiation without collapsing duplicates", () => { + const wasmBytes = buildInstrumentedDylinkWat(` + (module + (import "env" "memory" (memory 1 100 shared)) + (import "env" "__indirect_function_table" (table 1 funcref)) + (import "env" "shared_counter" (global $first_counter (mut i32))) + (import "env" "shared_counter" (global $second_counter (mut i32))) + (import "env" "fork" (func $fork (result i32))) + (func (export "counter_sum") (result i32) + global.get $first_counter + global.get $second_counter + i32.add) (func (export "side_fork") (result i32) call $fork)) - `, "side-without-continuation-mapping", FORK_CAP_SIDE_ENTRY); + `, "side-fork-lazy-import-capture"); + const testOwner = createTestForkActivationOwner(31); const options = createSideForkLoadOptions(); - options.allocateContinuation = undefined; - options.deallocateContinuation = undefined; - options.sideModuleFork = { - setActiveFork: () => {}, - clearActiveFork: () => {}, - invokeMainFork: () => 0, - beginMainAbort: () => {}, + const sharedCounter = new WebAssembly.Global( + { value: "i32", mutable: true }, + 6, + ); + options.globalSymbols.set("shared_counter", sharedCounter); + const observedGlobals: unknown[] = []; + const observedTables: unknown[] = []; + options.forkActivationOwner = { + prepare(request) { + const prepared = testOwner.owner.prepare(request); + return { + ...prepared, + wrapImports(imports) { + const baseImports = prepared.wrapImports(imports); + return new Proxy(baseImports as object, { + get(target, moduleName, receiver) { + const namespace = Reflect.get(target, moduleName, receiver); + if (moduleName !== "env" || typeof namespace !== "object") { + return namespace; + } + return new Proxy(namespace as object, { + get(namespaceTarget, importName, namespaceReceiver) { + const value = Reflect.get( + namespaceTarget, + importName, + namespaceReceiver, + ); + if (importName === "shared_counter") { + observedGlobals.push(value); + } else if (importName === "__indirect_function_table") { + observedTables.push(value); + } + return value; + }, + }); + }, + }) as WebAssembly.Imports; + }, + }; + }, }; - expect(() => loadSharedLibrarySync("libunmappedfork.so", wasmBytes, options)) - .toThrow(/require process-mapping allocation and cleanup/); + const loaded = loadSharedLibrarySync( + "libfork-lazy-import-capture.so", + wasmBytes, + options, + ); + + expect(observedGlobals).toEqual([sharedCounter, sharedCounter]); + expect(observedTables).toEqual([options.table]); + expect(testOwner.wrappedImports).toHaveLength(1); + expect(testOwner.registered).toEqual([ + { activationId: 31, instance: loaded.instance }, + ]); + expect((loaded.exports.counter_sum as () => number)()).toBe(12); }); - it("drives repeated instrumented side-module forks through exact states", () => { - const wasmBytes = buildDylinkWat(` + it("replays dependency-first with the parent's exact activation ids", () => { + const dependencyBytes = buildInstrumentedDylinkWat(` (module (import "env" "memory" (memory 1 100 shared)) (import "env" "fork" (func $fork (result i32))) - (global $state (mut i32) (i32.const 0)) - (global $buf (mut i32) (i32.const 0)) - (func (export "wpk_fork_unwind_begin") (param $addr i32) - local.get $addr - global.set $buf - i32.const 1 - global.set $state) - (func (export "wpk_fork_unwind_end") - i32.const 0 - global.set $state) - (func (export "wpk_fork_rewind_begin") (param $addr i32) - local.get $addr - global.set $buf - i32.const 2 - global.set $state) - (func (export "wpk_fork_rewind_end") - i32.const 0 - global.set $state) - (func (export "wpk_fork_abort_begin") (param $addr i32) - local.get $addr - global.set $buf - i32.const 3 - global.set $state) - (func (export "wpk_fork_abort_end") - i32.const 0 - global.set $state) - (func (export "wpk_fork_state") (result i32) - global.get $state) - (func (export "side_fork_with_local") (result i32) - i32.const 41 - call $fork - i32.add)) - `, "side-fork-instrumented", FORK_CAP_SIDE_ENTRY); - const options = createSideForkLoadOptions(); - let forkResult = 0; - let active: SideModuleForkState | null = null; - options.sideModuleFork = { - setActiveFork: (state) => { - expect(active).toBeNull(); - active = state; + (func (export "dependency_value") (result i32) i32.const 19) + (func (export "dependency_fork") (result i32) call $fork)) + `, "side-fork-needed-dependency"); + const consumerBytes = buildInstrumentedDylinkWat(` + (module + (import "env" "memory" (memory 1 100 shared)) + (import "env" "fork" (func $fork (result i32))) + (import "env" "dependency_value" (func $dependency_value (result i32))) + (func (export "needed_value") (result i32) call $dependency_value) + (func (export "needed_fork") (result i32) call $fork)) + `, "side-fork-needed-consumer", ["libfork-dependency.so"]); + const parentOwner = createTestForkActivationOwner(41); + const parent = createSideForkLoadOptions(); + parent.forkActivationOwner = parentOwner.owner; + parent.resolveLibrarySync = (name) => + name === "libfork-dependency.so" ? dependencyBytes : null; + + const parentConsumer = loadSharedLibrarySync( + "libfork-consumer.so", + consumerBytes, + parent, + ); + const parentDependency = parent.loadedLibraries.get("libfork-dependency.so")!; + expect(Array.from(parent.loadedLibraries)).toEqual([ + ["libfork-dependency.so", parentDependency], + ["libfork-consumer.so", parentConsumer], + ]); + expect(parentDependency.activationId).toBe(41); + expect(parentConsumer.activationId).toBe(42); + + const childOwner = createTestForkActivationOwner(100); + const child = createSideForkLoadOptions(); + child.forkActivationOwner = childOwner.owner; + expect(() => loadSharedLibrarySync( + "libfork-consumer.so", + consumerBytes, + child, + { + memoryBase: parentConsumer.memoryBase, + tableBase: parentConsumer.tableBase, + activationId: parentConsumer.activationId, }, - clearActiveFork: (state) => { - expect(active).toBe(state); - active = null; + )).toThrow(/archive entries must be replayed in dependency order/); + expect(childOwner.prepares).toHaveLength(0); + + const childDependency = loadSharedLibrarySync( + "libfork-dependency.so", + dependencyBytes, + child, + { + memoryBase: parentDependency.memoryBase, + tableBase: parentDependency.tableBase, + activationId: parentDependency.activationId, }, - invokeMainFork: () => forkResult, - beginMainAbort: () => {}, - }; + ); + const childConsumer = loadSharedLibrarySync( + "libfork-consumer.so", + consumerBytes, + child, + { + memoryBase: parentConsumer.memoryBase, + tableBase: parentConsumer.tableBase, + activationId: parentConsumer.activationId, + }, + ); - const lib = loadSharedLibrarySync("libsidefork.so", wasmBytes, options); - const sideFork = lib.exports.side_fork_with_local as () => number; - const state = lib.instance.exports.wpk_fork_state as () => number; - const unwindEnd = lib.instance.exports.wpk_fork_unwind_end as () => void; - const rewindBegin = lib.instance.exports.wpk_fork_rewind_begin as (addr: number) => void; - - // A main root-allocation failure returns synchronously before either Wasm - // stack has unwound. The side owner must cancel its just-opened root and - // clear the persisted active identity without entering replay. - forkResult = -12; - expect(sideFork()).toBe(29); - expect(state()).toBe(0); - expect(active).toBeNull(); - expect(lib.forkContinuation?.hasActiveContinuation()).toBe(false); - - for (const expectedForkResult of [101, 202]) { - forkResult = 0; - expect(sideFork()).toBe(41); - expect(state()).toBe(1); - expect(active?.forkBufAddr).toBe(lib.forkBufAddr); - expect(active?.continuation).toBe(lib.forkContinuation); - - unwindEnd(); - forkResult = expectedForkResult; - rewindBegin(lib.forkBufAddr!); - expect(sideFork()).toBe(41 + expectedForkResult); - expect(state()).toBe(0); - expect(active).toBeNull(); - } + expect(childDependency.activationId).toBe(41); + expect(childConsumer.activationId).toBe(42); + expect(childOwner.prepares.map((request) => request.replayActivationId)) + .toEqual([41, 42]); + expect((childConsumer.exports.needed_value as () => number)()).toBe(19); }); - it("allows independent extensions but rejects visible side-to-side fork nesting", () => { - const options = createSideForkLoadOptions(); - options.sideModuleFork = { - setActiveFork: () => {}, - clearActiveFork: () => {}, - invokeMainFork: () => 0, - beginMainAbort: () => {}, - }; - const provider = buildDylinkWat(` + it("unregisters exactly once when post-instantiation startup rolls back", () => { + const wasmBytes = buildInstrumentedDylinkWat(` (module (import "env" "memory" (memory 1 100 shared)) - (func (export "provider_value") (result i32) i32.const 7)) - `, "fork-provider"); - loadSharedLibrarySync("libprovider.so", provider, options); + (import "env" "fork" (func $fork (result i32))) + (func (export "__wasm_call_ctors") unreachable) + (func (export "side_fork") (result i32) call $fork)) + `, "side-fork-rollback"); + const testOwner = createTestForkActivationOwner(9); + const options = createSideForkLoadOptions(); + options.forkActivationOwner = testOwner.owner; - const independentFork = buildDylinkWat(` + expect(() => loadSharedLibrarySync("libfork-rollback.so", wasmBytes, options)) + .toThrow(); + expect(testOwner.registered).toHaveLength(1); + expect(testOwner.unregistered).toEqual([9]); + expect(testOwner.active.size).toBe(0); + expect(options.loadedLibraries.size).toBe(0); + }); + + it("does not fall back to process symbols for owner-controlled fork imports", () => { + const wasmBytes = buildInstrumentedDylinkWat(` (module (import "env" "memory" (memory 1 100 shared)) (import "env" "fork" (func $fork (result i32))) - (global $state (mut i32) (i32.const 0)) - (func (export "wpk_fork_unwind_begin") (param i32) - i32.const 1 global.set $state) - (func (export "wpk_fork_unwind_end") i32.const 0 global.set $state) - (func (export "wpk_fork_rewind_begin") (param i32) - i32.const 2 global.set $state) - (func (export "wpk_fork_rewind_end") i32.const 0 global.set $state) - (func (export "wpk_fork_abort_begin") (param i32) - i32.const 3 global.set $state) - (func (export "wpk_fork_abort_end") i32.const 0 global.set $state) - (func (export "wpk_fork_state") (result i32) global.get $state) (func (export "side_fork") (result i32) call $fork)) - `, "independent-fork-side", FORK_CAP_SIDE_ENTRY); - loadSharedLibrarySync("libindependent-fork.so", independentFork, options); + `, "side-fork-owner-import"); + const testOwner = createTestForkActivationOwner(15); + const options = createSideForkLoadOptions(); + options.globalSymbols.set("fork", () => 123); + options.forkActivationOwner = { + prepare(request) { + const prepared = testOwner.owner.prepare(request); + const { fork: _fork, ...envWithoutFork } = prepared.env; + return { ...prepared, env: envWithoutFork }; + }, + }; + + expect(() => loadSharedLibrarySync("libfork-owner-import.so", wasmBytes, options)) + .toThrow(/function import requires a callable/); + expect(testOwner.unregistered).toEqual([15]); + expect(testOwner.active.size).toBe(0); + }); - const visibleConsumer = buildDylinkWat(` + it("keeps a shared activation until the final dlclose reference", () => { + const wasmBytes = buildInstrumentedDylinkWat(` (module (import "env" "memory" (memory 1 100 shared)) - (import "env" "side_fork" (func $side_fork (result i32))) - (func (export "nested") (result i32) call $side_fork)) - `, "visible-side-consumer"); - expect(() => loadSharedLibrarySync("libnested.so", visibleConsumer, options)) - .toThrow(/fork-capable side-module nesting/); + (import "env" "fork" (func $fork (result i32))) + (func (export "side_fork") (result i32) call $fork)) + `, "side-fork-refcount"); + const testOwner = createTestForkActivationOwner(27); + const options = createSideForkLoadOptions(); + options.forkActivationOwner = testOwner.owner; + const linker = new DynamicLinker(options); + + const firstHandle = linker.dlopenSync("libfork-refcount.so", wasmBytes); + const secondHandle = linker.dlopenSync("libfork-refcount.so", wasmBytes); + expect(firstHandle).toBeGreaterThan(0); + expect(secondHandle).toBe(firstHandle); + expect(testOwner.prepares).toHaveLength(1); + + expect(linker.dlclose(firstHandle)).toBe(0); + expect(testOwner.unregistered).toEqual([]); + expect(options.loadedLibraries.has("libfork-refcount.so")).toBe(true); + + expect(linker.dlclose(secondHandle)).toBe(0); + expect(testOwner.unregistered).toEqual([27]); + expect(testOwner.active.size).toBe(0); + expect(options.loadedLibraries.has("libfork-refcount.so")).toBe(false); }); }); @@ -1008,6 +1813,37 @@ describe("dylink symbol interposition", () => { }); describe("dylink replay layout and rollback", () => { + it("does not apply data relocations twice to copied child memory", () => { + const wasmBytes = buildDylinkWat(` + (module + (import "env" "memory" (memory 1 100 shared)) + (func (export "__wasm_apply_data_relocs") + i32.const 32 + i32.const 32 + i32.load + i32.const 1 + i32.add + i32.store)) + `, "replay-does-not-relocate-twice"); + const parent = createSideForkLoadOptions(); + const loaded = loadSharedLibrarySync( + "librelocate-once.so", + wasmBytes, + parent, + ); + expect(new DataView(parent.memory.buffer).getInt32(32, true)).toBe(1); + + const child = createSideForkLoadOptions(); + new Uint8Array(child.memory.buffer).set( + new Uint8Array(parent.memory.buffer), + ); + loadSharedLibrarySync("librelocate-once.so", wasmBytes, child, { + memoryBase: loaded.memoryBase, + tableBase: loaded.tableBase, + }); + expect(new DataView(child.memory.buffer).getInt32(32, true)).toBe(1); + }); + it("restores copied live TLS without re-running side-module TLS initialization", () => { const tlsSide = buildDylinkWat(` (module @@ -1304,6 +2140,532 @@ describe.skipIf(!hasCompiler())("DynamicLinker", () => { expect(linker.dlclose(handle)).toBe(0); }); + it("lets libc drive initialization without a host-to-Wasm callback", () => { + const memory = new WebAssembly.Memory({ initial: 1 }); + const table = new WebAssembly.Table({ initial: 1, element: "anyfunc" }); + const stackPointer = new WebAssembly.Global( + { value: "i32", mutable: true }, + 65536, + ); + const linker = new DynamicLinker({ + memory, + table, + stackPointer, + heapPointer: { value: 1024 }, + globalSymbols: new Map(), + got: new Map(), + loadedLibraries: new Map(), + }); + const wasmBytes = buildDylinkWat(` + (module + (global $state (mut i32) (i32.const 0)) + (func (export "__wasm_apply_data_relocs") + global.get $state + i32.const 1 + i32.add + global.set $state) + (func (export "__wasm_call_ctors") + global.get $state + i32.const 10 + i32.add + global.set $state) + (func (export "initialization_state") (result i32) + global.get $state)) + `, "process-driven-initialization"); + + const token = linker.beginDlopenSync( + "libprocess-driven-initialization.so", + wasmBytes, + false, + ); + expect(token).toBeGreaterThan(0); + + const relocations = linker.nextDlopenInitialization(token); + expect(relocations).toBeGreaterThan(0); + expect(linker.forkState()).toMatchObject({ + libraries: [{ globalVisibility: false }], + transactions: [{ token, globalVisibility: false }], + }); + const relocationEntry = table.get(relocations); + expect(typeof relocationEntry).toBe("function"); + (relocationEntry as () => void)(); + + const constructors = linker.nextDlopenInitialization(token); + expect(constructors).toBe(relocations); + const constructorEntry = table.get(constructors); + expect(typeof constructorEntry).toBe("function"); + (constructorEntry as () => void)(); + + expect(linker.nextDlopenInitialization(token)).toBe(0); + expect(table.get(relocations)).toBeNull(); + const handle = linker.commitDlopenSync(token); + expect(handle).toBeGreaterThan(0); + const stateIndex = linker.dlsym(handle, "initialization_state"); + expect(stateIndex).not.toBeNull(); + expect((table.get(stateIndex!) as () => number)()).toBe(11); + expect(linker.dlsym(0, "initialization_state")).toBeNull(); + }); + + it("reconstructs an issued initialization entry in a fresh instance", () => { + const wasmBytes = buildDylinkWat(` + (module + (import "env" "memory" (memory 1)) + (func (export "__wasm_apply_data_relocs") + i32.const 32 + i32.const 32 + i32.load + i32.const 1 + i32.add + i32.store) + (func (export "__wasm_call_ctors") + i32.const 32 + i32.const 32 + i32.load + i32.const 10 + i32.add + i32.store) + (func (export "initialization_state") (result i32) + i32.const 32 + i32.load)) + `, "fresh-process-driven-initialization"); + const parentMemory = new WebAssembly.Memory({ initial: 1 }); + const parentTable = new WebAssembly.Table({ initial: 1, element: "anyfunc" }); + const parent = new DynamicLinker({ + memory: parentMemory, + table: parentTable, + stackPointer: new WebAssembly.Global( + { value: "i32", mutable: true }, + 65536, + ), + heapPointer: { value: 1024 }, + globalSymbols: new Map(), + got: new Map(), + loadedLibraries: new Map(), + }); + const token = parent.beginDlopenSync( + "libfresh-process-driven.so", + wasmBytes, + false, + ); + const relocations = parent.nextDlopenInitialization(token); + (parentTable.get(relocations) as () => void)(); + const constructors = parent.nextDlopenInitialization(token); + expect(constructors).toBe(relocations); + const archived = parent.forkState(); + expect(archived.transactions).toHaveLength(1); + expect(archived.transactions![0]).toMatchObject({ + globalVisibility: false, + }); + expect(archived.libraries[0]!.initialization).toMatchObject({ + transactionToken: token, + stage: "constructors", + tableIndex: constructors, + }); + + const childMemory = new WebAssembly.Memory({ initial: 1 }); + new Uint8Array(childMemory.buffer).set( + new Uint8Array(parentMemory.buffer), + ); + const childTable = new WebAssembly.Table({ initial: 1, element: "anyfunc" }); + const child = new DynamicLinker({ + memory: childMemory, + table: childTable, + stackPointer: new WebAssembly.Global( + { value: "i32", mutable: true }, + 65536, + ), + heapPointer: { value: 1024 }, + globalSymbols: new Map(), + got: new Map(), + loadedLibraries: new Map(), + }); + child.reconcileForkModules(archived); + child.reconcileForkHandleState(archived); + + const childConstructor = childTable.get(constructors); + expect(typeof childConstructor).toBe("function"); + (childConstructor as () => void)(); + expect(child.nextDlopenInitialization(token)).toBe(0); + const handle = child.commitDlopenSync(token); + expect(handle).toBe(2); + const stateIndex = child.dlsym(handle, "initialization_state"); + expect(stateIndex).not.toBeNull(); + expect((childTable.get(stateIndex!) as () => number)()).toBe(11); + expect(child.dlsym(0, "initialization_state")).toBeNull(); + }); + + it("archives constructor dlsym ownership across a fresh staged replay", () => { + const providerBytes = buildDylinkWat(` + (module + (func (export "runtime_provider") (result i32) i32.const 61)) + `, "fresh-constructor-provider"); + const consumerBytes = buildDylinkWat(` + (module + (import "env" "lookup_provider" + (func $lookup_provider (result i32))) + (func (export "__wasm_call_ctors") + call $lookup_provider + drop) + (func (export "consumer_value") (result i32) i32.const 17)) + `, "fresh-constructor-provider-consumer"); + const parentOptions = createSideForkLoadOptions(); + let parent!: DynamicLinker; + let providerHandle = 0; + parentOptions.globalSymbols.set("lookup_provider", () => { + return parent.dlsym(providerHandle, "runtime_provider") ?? 0; + }); + parent = new DynamicLinker(parentOptions); + providerHandle = parent.dlopenSync( + "libconstructor-provider.so", + providerBytes, + ); + const token = parent.beginDlopenSync( + "libconstructor-consumer.so", + consumerBytes, + ); + const constructor = parent.nextDlopenInitialization(token); + (parentOptions.table.get(constructor) as () => void)(); + const archived = parent.forkState(); + expect( + archived.libraries.find( + (library) => library.name === "libconstructor-consumer.so", + )?.providerDependencies, + ).toEqual(["libconstructor-provider.so"]); + + const childOptions = createSideForkLoadOptions(); + childOptions.globalSymbols.set("lookup_provider", () => 0); + const child = new DynamicLinker(childOptions); + child.reconcileForkModules(archived); + child.reconcileForkHandleState(archived); + expect(child.nextDlopenInitialization(token)).toBe(0); + const consumerHandle = child.commitDlopenSync(token); + expect(consumerHandle).toBe(3); + + expect(child.dlclose(providerHandle)).toBe(0); + expect( + childOptions.loadedLibraries.has("libconstructor-provider.so"), + ).toBe(true); + expect(child.dlclose(consumerHandle)).toBe(0); + expect(Array.from(childOptions.loadedLibraries.keys())).toEqual([]); + }); + + it("reconciles staged initializer generations without repeating guest calls", () => { + const wasmBytes = buildDylinkWat(` + (module + (import "env" "memory" (memory 1)) + (func (export "__wasm_apply_data_relocs") + i32.const 32 + i32.const 32 + i32.load + i32.const 1 + i32.add + i32.store) + (func (export "__wasm_call_ctors") + i32.const 32 + i32.const 32 + i32.load + i32.const 10 + i32.add + i32.store) + (func (export "initialization_state") (result i32) + i32.const 32 + i32.load)) + `, "replicated-process-driven-initialization"); + const memory = new WebAssembly.Memory({ initial: 1 }); + const parentTable = new WebAssembly.Table({ initial: 1, element: "anyfunc" }); + const replicaTable = new WebAssembly.Table({ initial: 1, element: "anyfunc" }); + const makeLinker = (table: WebAssembly.Table): DynamicLinker => + new DynamicLinker({ + memory, + table, + stackPointer: new WebAssembly.Global( + { value: "i32", mutable: true }, + 65536, + ), + heapPointer: { value: 1024 }, + globalSymbols: new Map(), + got: new Map(), + loadedLibraries: new Map(), + }); + const parent = makeLinker(parentTable); + const replica = makeLinker(replicaTable); + + const token = parent.beginDlopenSync("libreplicated-stages.so", wasmBytes); + const relocations = parent.nextDlopenInitialization(token); + const relocationState = parent.forkState(); + replica.reconcileForkModules(relocationState); + replica.reconcileForkHandleState(relocationState); + expect(typeof replicaTable.get(relocations)).toBe("function"); + + (parentTable.get(relocations) as () => void)(); + const constructors = parent.nextDlopenInitialization(token); + expect(constructors).toBe(relocations); + const constructorState = parent.forkState(); + replica.reconcileForkModules(constructorState); + replica.reconcileForkHandleState(constructorState); + expect(typeof replicaTable.get(constructors)).toBe("function"); + // Only the source Worker called the relocation entry. + expect(new DataView(memory.buffer).getInt32(32, true)).toBe(1); + + (parentTable.get(constructors) as () => void)(); + expect(parent.nextDlopenInitialization(token)).toBe(0); + expect(parent.commitDlopenSync(token)).toBe(2); + const committedState = parent.forkState(); + replica.reconcileForkModules(committedState); + replica.reconcileForkHandleState(committedState); + expect(replicaTable.get(constructors)).toBeNull(); + expect(new DataView(memory.buffer).getInt32(32, true)).toBe(11); + const stateIndex = replica.dlsym(2, "initialization_state"); + expect(stateIndex).not.toBeNull(); + expect((replicaTable.get(stateIndex!) as () => number)()).toBe(11); + }); + + it("retires a replicated staged initializer after authoritative rollback", () => { + const wasmBytes = buildDylinkWat(` + (module + (func (export "__wasm_call_ctors")) + ) + `, "replicated-process-driven-rollback"); + const makeLinker = (): { + readonly linker: DynamicLinker; + readonly table: WebAssembly.Table; + } => { + const table = new WebAssembly.Table({ initial: 1, element: "anyfunc" }); + return { + table, + linker: new DynamicLinker({ + memory: new WebAssembly.Memory({ initial: 1 }), + table, + stackPointer: new WebAssembly.Global( + { value: "i32", mutable: true }, + 65536, + ), + heapPointer: { value: 1024 }, + globalSymbols: new Map(), + got: new Map(), + loadedLibraries: new Map(), + }), + }; + }; + const parent = makeLinker(); + const replica = makeLinker(); + const token = parent.linker.beginDlopenSync( + "libreplicated-rollback.so", + wasmBytes, + ); + const constructors = parent.linker.nextDlopenInitialization(token); + const issuedState = parent.linker.forkState(); + replica.linker.reconcileForkModules(issuedState); + replica.linker.reconcileForkHandleState(issuedState); + expect(typeof replica.table.get(constructors)).toBe("function"); + + parent.linker.abortDlopenTransaction(token, new Error("constructor failed")); + const rolledBackState = parent.linker.forkState(); + replica.linker.reconcileForkModules(rolledBackState); + replica.linker.reconcileForkHandleState(rolledBackState); + expect(replica.table.get(constructors)).toBeNull(); + expect(rolledBackState.libraries).toHaveLength(0); + }); + + it("rolls back completed NEEDED objects when the staged root fails", () => { + const dependencyBytes = buildDylinkWat(` + (module + (func (export "__wasm_call_ctors")) + (func (export "dependency_value") (result i32) i32.const 7) + ) + `, "staged-needed-rollback-dependency"); + const rootBytes = buildDylinkWat(` + (module + (import "env" "missing_root_symbol" (func)) + ) + `, "staged-needed-rollback-root", undefined, 0, 0, [], [], null, [ + "libstaged-needed-dependency.so", + ]); + const options = createSideForkLoadOptions(); + options.resolveLibrarySync = (name) => + name === "libstaged-needed-dependency.so" ? dependencyBytes : null; + const linker = new DynamicLinker(options); + const token = linker.beginDlopenSync("libstaged-needed-root.so", rootBytes); + const dependencyConstructor = linker.nextDlopenInitialization(token); + expect(dependencyConstructor).toBeGreaterThan(0); + (options.table.get(dependencyConstructor) as () => void)(); + + expect(linker.nextDlopenInitialization(token)).toBe(-1); + expect(options.loadedLibraries.size).toBe(0); + expect(linker.forkState()).toMatchObject({ + nextHandle: 2, + libraries: [], + }); + expect(options.table.get(dependencyConstructor)).toBeNull(); + }); + + it("preserves an independently committed constructor-nested load", () => { + const nestedBytes = buildDylinkWat(` + (module + (func (export "nested_value") (result i32) i32.const 19) + ) + `, "staged-independent-nested"); + const outerBytes = buildDylinkWat(` + (module + (import "env" "nested_open" (func $nested_open)) + (func (export "__wasm_call_ctors") + call $nested_open + unreachable) + ) + `, "staged-failing-outer-independent"); + const options = createSideForkLoadOptions(); + let linker!: DynamicLinker; + let nestedHandle = 0; + options.globalSymbols.set("nested_open", () => { + const token = linker.beginDlopenSync( + "libindependent-nested.so", + nestedBytes, + ); + const result = linker.advanceDlopenSync(token); + expect(result.entry).toBe(0); + nestedHandle = result.handle; + }); + linker = new DynamicLinker(options); + + const outerToken = linker.beginDlopenSync( + "libfailing-outer-independent.so", + outerBytes, + ); + const constructor = linker.nextDlopenInitialization(outerToken); + let failure: unknown; + try { + (options.table.get(constructor) as () => void)(); + } catch (error) { + failure = error; + } + expect(failure).toBeInstanceOf(WebAssembly.RuntimeError); + linker.abortDlopenTransaction(outerToken, failure); + + expect(nestedHandle).toBe(2); + expect(Array.from(options.loadedLibraries.keys())).toEqual([ + "libindependent-nested.so", + ]); + const nestedValue = linker.dlsym(nestedHandle, "nested_value"); + expect(nestedValue).not.toBeNull(); + expect((options.table.get(nestedValue!) as () => number)()).toBe(19); + }); + + it("preserves an independently committed nested GLOBAL promotion", () => { + const localBytes = buildDylinkWat(` + (module + (func (export "promoted_value") (result i32) i32.const 31) + ) + `, "staged-independent-promotion"); + const outerBytes = buildDylinkWat(` + (module + (import "env" "nested_promote" (func $nested_promote)) + (func (export "__wasm_call_ctors") + call $nested_promote + unreachable) + ) + `, "staged-failing-outer-promotion"); + const options = createSideForkLoadOptions(); + let linker!: DynamicLinker; + let promotedHandle = 0; + options.globalSymbols.set("nested_promote", () => { + const token = linker.beginDlopenSync( + "libpromoted-local.so", + localBytes, + true, + ); + const result = linker.advanceDlopenSync(token); + expect(result.entry).toBe(0); + promotedHandle = result.handle; + }); + linker = new DynamicLinker(options); + const localHandle = linker.dlopenSync( + "libpromoted-local.so", + localBytes, + undefined, + false, + ); + expect(localHandle).toBeGreaterThan(0); + expect(linker.dlsym(0, "promoted_value")).toBeNull(); + + const outerToken = linker.beginDlopenSync( + "libfailing-outer-promotion.so", + outerBytes, + ); + const constructor = linker.nextDlopenInitialization(outerToken); + let failure: unknown; + try { + (options.table.get(constructor) as () => void)(); + } catch (error) { + failure = error; + } + expect(failure).toBeInstanceOf(WebAssembly.RuntimeError); + linker.abortDlopenTransaction(outerToken, failure); + + expect(promotedHandle).toBe(localHandle); + const promoted = options.loadedLibraries.get("libpromoted-local.so")!; + expect(promoted.globalVisibility).toBe(true); + expect(promoted.committedGlobalRoot).toBe(true); + expect(linker.forkLibraryState("libpromoted-local.so")).toMatchObject({ + globalVisibility: true, + committedGlobalRoot: true, + handle: localHandle, + refCount: 2, + }); + const value = linker.dlsym(0, "promoted_value"); + expect(value).not.toBeNull(); + expect((options.table.get(value!) as () => number)()).toBe(31); + }); + + it("cascades rollback into a constructor-nested load bound to the outer", () => { + const nestedBytes = buildDylinkWat(` + (module + (import "env" "outer_value" (func $outer_value (result i32))) + (func (export "nested_value") (result i32) call $outer_value) + ) + `, "staged-dependent-nested"); + const outerBytes = buildDylinkWat(` + (module + (import "env" "nested_open" (func $nested_open)) + (func (export "outer_value") (result i32) i32.const 23) + (func (export "__wasm_call_ctors") + call $nested_open + unreachable) + ) + `, "staged-failing-outer-dependent"); + const options = createSideForkLoadOptions(); + let linker!: DynamicLinker; + let nestedHandle = 0; + options.globalSymbols.set("nested_open", () => { + const token = linker.beginDlopenSync( + "libdependent-nested.so", + nestedBytes, + ); + const result = linker.advanceDlopenSync(token); + expect(result.entry).toBe(0); + nestedHandle = result.handle; + }); + linker = new DynamicLinker(options); + + const outerToken = linker.beginDlopenSync( + "libfailing-outer-dependent.so", + outerBytes, + ); + const constructor = linker.nextDlopenInitialization(outerToken); + let failure: unknown; + try { + (options.table.get(constructor) as () => void)(); + } catch (error) { + failure = error; + } + expect(failure).toBeInstanceOf(WebAssembly.RuntimeError); + linker.abortDlopenTransaction(outerToken, failure); + + expect(nestedHandle).toBe(2); + expect(options.loadedLibraries.size).toBe(0); + expect(linker.dlsym(nestedHandle, "nested_value")).toBeNull(); + expect(linker.dlerror()).toBe("invalid handle"); + }); + it("reserves a stable handle for the main program symbol scope", () => { const memory = new WebAssembly.Memory({ initial: 1, maximum: 100, shared: true }); const table = new WebAssembly.Table({ initial: 1, element: "anyfunc" }); @@ -1367,6 +2729,110 @@ describe.skipIf(!hasCompiler())("DynamicLinker", () => { expect(allocAlign).toBeGreaterThan(0); }); + it("adopts copied mmap ownership and releases it on child dlclose", () => { + const wasmBytes = buildSharedLib( + ` + int value = 7; + int get_value(void) { return value; } + `, + "dl-fork-allocation-owner", + ); + const parentMemory = new WebAssembly.Memory({ + initial: 2, + maximum: 100, + shared: true, + }); + const parentTable = new WebAssembly.Table({ initial: 1, element: "anyfunc" }); + const parentAllocations = new Map< + number, + { mappingAddress: number; mappingSize: number } + >(); + const parent = new DynamicLinker({ + memory: parentMemory, + table: parentTable, + stackPointer: new WebAssembly.Global( + { value: "i32", mutable: true }, + 65536, + ), + allocateMemory: (size) => { + const address = 0x3000; + parentAllocations.set(address, { + mappingAddress: 0x2ff0, + mappingSize: size + 31, + }); + return address; + }, + describeMemoryAllocation: (address) => parentAllocations.get(address)!, + deallocateMemory: () => {}, + globalSymbols: new Map(), + got: new Map(), + loadedLibraries: new Map(), + }); + const handle = parent.dlopenSync("libfork-allocation.so", wasmBytes); + expect(handle).toBeGreaterThan(0); + const forkState = parent.forkState(); + expect(forkState.libraries[0]?.allocations).toEqual([{ + address: 0x3000, + size: expect.any(Number), + mappingAddress: 0x2ff0, + mappingSize: expect.any(Number), + }]); + + const childMemory = new WebAssembly.Memory({ + initial: 2, + maximum: 100, + shared: true, + }); + new Uint8Array(childMemory.buffer).set( + new Uint8Array(parentMemory.buffer), + ); + const adopted = new Map< + number, + { mappingAddress: number; mappingSize: number } + >(); + const released: Array<{ + address: number; + mappingAddress: number; + mappingSize: number; + }> = []; + const child = new DynamicLinker({ + memory: childMemory, + table: new WebAssembly.Table({ initial: 1, element: "anyfunc" }), + stackPointer: new WebAssembly.Global( + { value: "i32", mutable: true }, + 65536, + ), + adoptMemoryAllocation: (allocation) => { + adopted.set(allocation.address, { + mappingAddress: allocation.mappingAddress, + mappingSize: allocation.mappingSize, + }); + }, + deallocateMemory: (address) => { + const mapping = adopted.get(address); + if (!mapping) throw new Error("missing adopted mmap owner"); + released.push({ address, ...mapping }); + adopted.delete(address); + }, + globalSymbols: new Map(), + got: new Map(), + loadedLibraries: new Map(), + }); + child.reconcileForkModules(forkState); + child.reconcileForkHandleState(forkState); + expect(adopted.get(0x3000)).toEqual({ + mappingAddress: 0x2ff0, + mappingSize: forkState.libraries[0]!.allocations![0]!.mappingSize, + }); + expect(child.dlclose(handle)).toBe(0); + expect(released).toEqual([{ + address: 0x3000, + mappingAddress: 0x2ff0, + mappingSize: forkState.libraries[0]!.allocations![0]!.mappingSize, + }]); + expect(adopted.size).toBe(0); + }); + it("dlerror reports failures", () => { const linker = createLinker(); diff --git a/host/test/fixtures/catch-ref-fresh-worker.wat b/host/test/fixtures/catch-ref-fresh-worker.wat new file mode 100644 index 0000000000..bd191cdd27 --- /dev/null +++ b/host/test/fixtures/catch-ref-fresh-worker.wat @@ -0,0 +1,172 @@ +;; ABI 43 integration fixture for real process workers. +;; +;; The parent catches a scalar tag through CatchRef, forks from the handler, +;; and waits for the child. Rewind can succeed only if the copied continuation +;; rethrows the tag in the fresh child instance and the original CatchRef clause +;; creates a new instance-local exnref. The child exits 91 if its scalar payload +;; was not restored; the parent exits 92 if wait4 observes any failure. +(module + (import "env" "memory" (memory 1 16384 shared)) + (import "env" "__channel_base" (global $__channel_base (mut i32))) + (import "kernel" "kernel_exit" (func $kernel_exit (param i32))) + (import "kernel" "kernel_fork" (func $kernel_fork (result i32))) + + (tag $payload (param i32)) + + (global $__stack_pointer (export "__stack_pointer") (mut i32) + (i32.const 65536)) + + (func (export "__abi_version") (result i32) + i32.const 43) + + (func $wait_child (param $pid i32) (result i32) + (local $base i32) + (local $result i32) + + global.get $__channel_base + local.set $base + + ;; SYS_wait4(pid, &status, 0, 0) + local.get $base + i32.const 4 + i32.add + i32.const 139 + i32.store + + local.get $base + i32.const 8 + i32.add + local.get $pid + i64.extend_i32_s + i64.store + + local.get $base + i32.const 16 + i32.add + i64.const 1024 + i64.store + + local.get $base + i32.const 24 + i32.add + i64.const 0 + i64.store + + local.get $base + i32.const 32 + i32.add + i64.const 0 + i64.store + + local.get $base + i32.const 40 + i32.add + i64.const 0 + i64.store + + local.get $base + i32.const 48 + i32.add + i64.const 0 + i64.store + + local.get $base + i32.const 1 + i32.atomic.store + local.get $base + i32.const 1 + memory.atomic.notify + drop + + block $complete + loop $wait + local.get $base + i32.atomic.load + i32.const 1 + i32.ne + br_if $complete + + local.get $base + i32.const 1 + i64.const -1 + memory.atomic.wait32 + drop + br $wait + end + end + + local.get $base + i32.const 64 + i32.add + i32.load + if + i32.const -1 + local.set $result + else + local.get $base + i32.const 56 + i32.add + i64.load + i32.wrap_i64 + local.set $result + end + + local.get $base + i32.const 0 + i32.atomic.store + + local.get $result) + + (func (export "_start") + (local $caught i32) + (local $pid i32) + + (block $handler (result i32 exnref) + (try_table (result i32 exnref) + (catch_ref $payload $handler) + i32.const 42 + throw $payload + unreachable)) + drop + local.set $caught + + call $kernel_fork + local.set $pid + + local.get $pid + i32.eqz + if + local.get $caught + i32.const 42 + i32.ne + if + i32.const 91 + call $kernel_exit + unreachable + end + i32.const 0 + call $kernel_exit + unreachable + end + + local.get $pid + call $wait_child + local.get $pid + i32.ne + if + i32.const 92 + call $kernel_exit + unreachable + end + + i32.const 1024 + i32.load + if + i32.const 92 + call $kernel_exit + unreachable + end + + i32.const 0 + call $kernel_exit + unreachable)) diff --git a/host/test/fixtures/fork-externref-import-worker.ts b/host/test/fixtures/fork-externref-import-worker.ts new file mode 100644 index 0000000000..95275c6c1c --- /dev/null +++ b/host/test/fixtures/fork-externref-import-worker.ts @@ -0,0 +1,68 @@ +import { parentPort, workerData } from "node:worker_threads"; +import { + defineForkExternrefImport, + type ForkExternrefImportBinding, + type ForkExternrefImportWake, + ForkExternrefImportWorkerCaller, +} from "../../src/fork-externref-import-mailbox"; +import { ForkExternrefTokenCache } from "../../src/fork-reference-broker"; + +const { + mailbox, + binding, + inputHandle, +} = workerData as { + mailbox: SharedArrayBuffer; + binding: ForkExternrefImportBinding; + inputHandle: number; +}; + +const aliasDescriptor = defineForkExternrefImport( + 30, + ["externref", "i64"], + ["externref", "i64"], +); +const throwingDescriptor = defineForkExternrefImport( + 31, + ["i32"], + ["i32"], +); +const tokens = new ForkExternrefTokenCache(binding.generationId); +const caller = new ForkExternrefImportWorkerCaller( + mailbox, + binding, + tokens, + (wake: ForkExternrefImportWake) => { + parentPort!.postMessage({ type: "wake", wake }); + }, +); + +try { + const input = tokens.materialize(inputHandle); + const [alias, scalar] = caller.call( + aliasDescriptor, + [input, -9n], + ) as [unknown, bigint]; + + let exception: unknown; + try { + caller.call(throwingDescriptor, [7]); + throw new Error("owner import unexpectedly returned"); + } catch (error) { + exception = error; + } + + parentPort!.postMessage({ + type: "complete", + resultHandle: tokens.encode(alias), + resultScalar: scalar, + exceptionHandle: tokens.encode(exception), + }); +} catch (error) { + parentPort!.postMessage({ + type: "failed", + message: error instanceof Error + ? error.stack ?? error.message + : String(error), + }); +} diff --git a/host/test/fixtures/fork-worker-import-exception-worker.ts b/host/test/fixtures/fork-worker-import-exception-worker.ts new file mode 100644 index 0000000000..24df0908b4 --- /dev/null +++ b/host/test/fixtures/fork-worker-import-exception-worker.ts @@ -0,0 +1,78 @@ +import { parentPort, workerData } from "node:worker_threads"; +import { + defineForkExternrefImport, + type ForkExternrefImportBinding, +} from "../../src/fork-externref-import-mailbox"; +import { + type ForkHostImportWorkerInit, + ForkHostImportWorkerRuntime, +} from "../../src/fork-host-import-runtime"; +import { ForkExternrefTokenCache } from "../../src/fork-reference-broker"; + +if (!parentPort) throw new Error("fork exception fixture requires parentPort"); + +const data = workerData as { + readonly mode: "parent" | "child"; + readonly binding: ForkExternrefImportBinding; + readonly init: ForkHostImportWorkerInit; + readonly inheritedHandle?: number; +}; +const tokens = new ForkExternrefTokenCache(data.binding.generationId); +const runtime = new ForkHostImportWorkerRuntime( + data.init, + data.binding.pid, + data.binding.generationId, + tokens, + (wake) => parentPort.postMessage({ type: "wake", wake }), +); + +const echo = defineForkExternrefImport( + 77, + ["externref"], + ["externref"], +); + +try { + if (data.mode === "parent") { + const localOnly = Object.freeze({ + source: "parent Worker", + callback: () => 42, + }); + const wrapped = runtime.localExceptions.wrap(5, () => { + throw localOnly; + }); + let thrown: unknown; + try { + wrapped(); + throw new Error("expected Worker-local exception"); + } catch (value) { + thrown = value; + } + if (thrown !== localOnly) { + throw new Error("Worker-local import changed exception identity"); + } + const token = + runtime.localExceptions.normalizeUnclaimedForkException(thrown); + const handle = tokens.encode(token); + if (handle === null) { + throw new Error("parent Worker received a noncanonical token"); + } + parentPort.postMessage({ type: "complete", handle }); + } else { + if (data.inheritedHandle === undefined) { + throw new Error("child Worker is missing its inherited handle"); + } + const inherited = tokens.materialize(data.inheritedHandle); + const echoed = runtime.caller.call(echo, [inherited]); + const handle = tokens.encode(echoed); + if (handle === null) { + throw new Error("child Worker received a noncanonical echo"); + } + parentPort.postMessage({ type: "complete", handle }); + } +} catch (error) { + parentPort.postMessage({ + type: "failed", + message: error instanceof Error ? error.stack ?? error.message : String(error), + }); +} diff --git a/host/test/fixtures/gc-reference-state-fresh-worker-bytes.ts b/host/test/fixtures/gc-reference-state-fresh-worker-bytes.ts new file mode 100644 index 0000000000..49cba8ec3d --- /dev/null +++ b/host/test/fixtures/gc-reference-state-fresh-worker-bytes.ts @@ -0,0 +1,15 @@ +// WHY: the dev shell's WABT parser does not accept current typed-reference +// syntax even with --enable-gc. These are the Rust `wat` crate's deterministic +// bytes for the adjacent, reviewed gc-reference-state-fresh-worker.wat source. +// Node and browser integration tests share the exact input artifact. +export const RAW_GC_REFERENCE_STATE_FRESH_WORKER_HEX = [ + "0061736d01000000011e065f027f0163000160017f006000017f60017f017f60016400017f60000002520403656e76066d656d6f727902030180800103656e760e5f5f6368616e6e656c5f62617365037f01066b65726e656c0b6b65726e656c", + "5f657869740001066b65726e656c0b6b65726e656c5f666f726b0002030504020304050406016300010101060e02630001d0000b7f01418080040b072c030f5f5f737461636b5f706f696e74657203020d5f5f6162695f76657273696f6e0002", + "065f737461727400050ad202040400412b0ba60101027f23002101200141046a418b01360200200141086a2000ac370300200141106a428008370300200141186a4200370300200141206a4200370300200141286a4200370300200141306a42", + "0037030020014101fe17020020014101fe0002001a024003402001fe1002004101470d0120014101427ffe0102001a0c000b0b200141c0006a2802000440417f210205200141386a290300a721020b20014100fe17020020020b530201630002", + "7f2000100121022101200245044020012000d323012000d371410025002000d3712000fb02000041cd00467120002000fb020001d4d3712103200345044041db001000000b41001000000b20020b4f02016300017f41cd00d000fb0000210020", + "00d42000fb050001200024014100200026002000d41004210120011003200147044041dc001000000b418008280200044041dc001000000b41001000000b00e401046e616d65014204000b6b65726e656c5f65786974010b6b65726e656c5f66", + "6f726b030a776169745f6368696c640419666f726b5f776974685f7265666572656e63655f7374617465024003030300037069640104626173650206726573756c74040400046e6f64650107636172726965640203706964030576616c696405", + "0200046e6f6465010370696403130103020008636f6d706c65746501047761697404070100046e6f6465050e01000b73617665645f7461626c65072903000e5f5f6368616e6e656c5f6261736501057361766564020f5f5f737461636b5f706f", + "696e746572", +].join(""); diff --git a/host/test/fixtures/gc-reference-state-fresh-worker.wat b/host/test/fixtures/gc-reference-state-fresh-worker.wat new file mode 100644 index 0000000000..1b6028ffaf --- /dev/null +++ b/host/test/fixtures/gc-reference-state-fresh-worker.wat @@ -0,0 +1,230 @@ +;; ABI 43 real-worker integration fixture for activation-owned Wasm GC state. +;; +;; One cyclic object is aliased simultaneously by a reference parameter, an +;; operand-stack carryover across kernel_fork, a mutable reference global, and +;; a mutated reference table. A fresh child must rebuild one canonical local +;; identity for every alias; copying only linear memory cannot make any of +;; these module-instance values survive. +(module + (import "env" "memory" (memory 1 16384 shared)) + (import "env" "__channel_base" (global $__channel_base (mut i32))) + (import "kernel" "kernel_exit" (func $kernel_exit (param i32))) + (import "kernel" "kernel_fork" (func $kernel_fork (result i32))) + + (type $node + (struct + (field (mut i32)) + (field (mut (ref null $node))))) + + (global $saved (mut (ref null $node)) (ref.null $node)) + (table $saved_table 1 1 (ref null $node)) + (global $__stack_pointer (export "__stack_pointer") (mut i32) + (i32.const 65536)) + + (func (export "__abi_version") (result i32) + i32.const 43) + + (func $wait_child (param $pid i32) (result i32) + (local $base i32) + (local $result i32) + + global.get $__channel_base + local.set $base + + ;; SYS_wait4(pid, &status, 0, 0) + local.get $base + i32.const 4 + i32.add + i32.const 139 + i32.store + + local.get $base + i32.const 8 + i32.add + local.get $pid + i64.extend_i32_s + i64.store + + local.get $base + i32.const 16 + i32.add + i64.const 1024 + i64.store + + local.get $base + i32.const 24 + i32.add + i64.const 0 + i64.store + + local.get $base + i32.const 32 + i32.add + i64.const 0 + i64.store + + local.get $base + i32.const 40 + i32.add + i64.const 0 + i64.store + + local.get $base + i32.const 48 + i32.add + i64.const 0 + i64.store + + local.get $base + i32.const 1 + i32.atomic.store + local.get $base + i32.const 1 + memory.atomic.notify + drop + + block $complete + loop $wait + local.get $base + i32.atomic.load + i32.const 1 + i32.ne + br_if $complete + + local.get $base + i32.const 1 + i64.const -1 + memory.atomic.wait32 + drop + br $wait + end + end + + local.get $base + i32.const 64 + i32.add + i32.load + if + i32.const -1 + local.set $result + else + local.get $base + i32.const 56 + i32.add + i64.load + i32.wrap_i64 + local.set $result + end + + local.get $base + i32.const 0 + i32.atomic.store + + local.get $result) + + (func $fork_with_reference_state + (param $node (ref $node)) + (result i32) + (local $carried (ref null $node)) + (local $pid i32) + (local $valid i32) + + ;; Leave the reference below the fork result on the operand stack. This is + ;; a real call carryover, not merely a local that happens to stay live. + local.get $node + call $kernel_fork + local.set $pid + local.set $carried + + local.get $pid + i32.eqz + if + local.get $carried + local.get $node + ref.eq + + global.get $saved + local.get $node + ref.eq + i32.and + + i32.const 0 + table.get $saved_table + local.get $node + ref.eq + i32.and + + local.get $node + struct.get $node 0 + i32.const 77 + i32.eq + i32.and + + local.get $node + local.get $node + struct.get $node 1 + ref.as_non_null + ref.eq + i32.and + local.set $valid + + local.get $valid + i32.eqz + if + i32.const 91 + call $kernel_exit + unreachable + end + i32.const 0 + call $kernel_exit + unreachable + end + + local.get $pid) + + (func (export "_start") + (local $node (ref null $node)) + (local $pid i32) + + i32.const 77 + ref.null $node + struct.new $node + local.set $node + + local.get $node + ref.as_non_null + local.get $node + struct.set $node 1 + + local.get $node + global.set $saved + i32.const 0 + local.get $node + table.set $saved_table + + local.get $node + ref.as_non_null + call $fork_with_reference_state + local.set $pid + + local.get $pid + call $wait_child + local.get $pid + i32.ne + if + i32.const 92 + call $kernel_exit + unreachable + end + + i32.const 1024 + i32.load + if + i32.const 92 + call $kernel_exit + unreachable + end + + i32.const 0 + call $kernel_exit + unreachable)) diff --git a/host/test/fixtures/gc-transit-object.wat b/host/test/fixtures/gc-transit-object.wat new file mode 100644 index 0000000000..08cdd6981e --- /dev/null +++ b/host/test/fixtures/gc-transit-object.wat @@ -0,0 +1,9 @@ +(module + (type $box (struct (field (mut i32)))) + (import "env" "__wpk_fork_ref_gc_transit" + (table $transit 1 (ref null any))) + (func (export "publish") (param $value i32) + i32.const 0 + local.get $value + struct.new $box + table.set $transit)) diff --git a/host/test/fixtures/reference-catch-payload-fresh-worker.wat b/host/test/fixtures/reference-catch-payload-fresh-worker.wat new file mode 100644 index 0000000000..36fca1d75b --- /dev/null +++ b/host/test/fixtures/reference-catch-payload-fresh-worker.wat @@ -0,0 +1,223 @@ +;; ABI 43 integration fixture for reference-bearing exception payloads. +;; +;; Each path catches a reference payload through CatchRef, forks from that +;; handler, and waits for the child. The funcref is non-null; the externref +;; exercises the nullable/null recipe. The child has a fresh Wasm instance, so +;; success requires the complete exception recipe to reconstruct the payload +;; and create a fresh child-local exnref. +(module + (import "env" "memory" (memory 1 16384 shared)) + (import "env" "__channel_base" (global $__channel_base (mut i32))) + (import "kernel" "kernel_exit" (func $kernel_exit (param i32))) + (import "kernel" "kernel_fork" (func $kernel_fork (result i32))) + + (tag $func_payload (param funcref)) + (tag $extern_payload (param externref)) + (type $sentinel_type (func (result i32))) + (table $verify 1 funcref) + + (global $__stack_pointer (export "__stack_pointer") (mut i32) + (i32.const 65536)) + + (func (export "__abi_version") (result i32) + i32.const 43) + + (func $sentinel (type $sentinel_type) (result i32) + i32.const 77) + (elem declare func $sentinel) + + (func $wait_child (param $pid i32) (result i32) + (local $base i32) + (local $result i32) + + global.get $__channel_base + local.set $base + + ;; SYS_wait4(pid, &status, 0, 0) + local.get $base + i32.const 4 + i32.add + i32.const 139 + i32.store + + local.get $base + i32.const 8 + i32.add + local.get $pid + i64.extend_i32_s + i64.store + + local.get $base + i32.const 16 + i32.add + i64.const 1024 + i64.store + + local.get $base + i32.const 24 + i32.add + i64.const 0 + i64.store + + local.get $base + i32.const 32 + i32.add + i64.const 0 + i64.store + + local.get $base + i32.const 40 + i32.add + i64.const 0 + i64.store + + local.get $base + i32.const 48 + i32.add + i64.const 0 + i64.store + + local.get $base + i32.const 1 + i32.atomic.store + local.get $base + i32.const 1 + memory.atomic.notify + drop + + block $complete + loop $wait + local.get $base + i32.atomic.load + i32.const 1 + i32.ne + br_if $complete + + local.get $base + i32.const 1 + i64.const -1 + memory.atomic.wait32 + drop + br $wait + end + end + + local.get $base + i32.const 64 + i32.add + i32.load + if + i32.const -1 + local.set $result + else + local.get $base + i32.const 56 + i32.add + i64.load + i32.wrap_i64 + local.set $result + end + + local.get $base + i32.const 0 + i32.atomic.store + + local.get $result) + + (func $require_child_ok (param $pid i32) + local.get $pid + call $wait_child + local.get $pid + i32.ne + if + i32.const 92 + call $kernel_exit + unreachable + end + + i32.const 1024 + i32.load + if + i32.const 92 + call $kernel_exit + unreachable + end) + + (func $test_funcref_payload + (local $caught funcref) + (local $pid i32) + + (block $handler (result funcref exnref) + (try_table (result funcref exnref) + (catch_ref $func_payload $handler) + ref.func $sentinel + throw $func_payload + unreachable)) + drop + local.set $caught + + call $kernel_fork + local.set $pid + local.get $pid + i32.eqz + if + i32.const 0 + local.get $caught + table.set $verify + i32.const 0 + call_indirect $verify (type $sentinel_type) + i32.const 77 + i32.ne + if + i32.const 91 + call $kernel_exit + unreachable + end + i32.const 0 + call $kernel_exit + unreachable + end + + local.get $pid + call $require_child_ok) + + (func $test_externref_payload + (local $caught externref) + (local $pid i32) + + (block $handler (result externref exnref) + (try_table (result externref exnref) + (catch_ref $extern_payload $handler) + ref.null extern + throw $extern_payload + unreachable)) + drop + local.set $caught + + call $kernel_fork + local.set $pid + local.get $pid + i32.eqz + if + local.get $caught + ref.is_null + i32.eqz + if + i32.const 93 + call $kernel_exit + unreachable + end + i32.const 0 + call $kernel_exit + unreachable + end + + local.get $pid + call $require_child_ok) + + (func (export "_start") + call $test_funcref_payload + call $test_externref_payload + i32.const 0 + call $kernel_exit + unreachable)) diff --git a/host/test/fork-abort-unwind.test.ts b/host/test/fork-abort-unwind.test.ts index a2e12fb36b..fe4a3f3d5b 100644 --- a/host/test/fork-abort-unwind.test.ts +++ b/host/test/fork-abort-unwind.test.ts @@ -9,6 +9,8 @@ import { LinkedForkContinuation, readLinkedFrameFormat, } from "../src/fork-continuation"; +import { ForkModuleStateArena } from "../src/fork-module-state"; +import { SingleActivationForkRuntime } from "./fork-instrument-runtime-harness"; describe("instrumented ABORT_UNWINDING", () => { it("reconstructs committed inner frames and permits a later successful fork", () => { @@ -53,10 +55,10 @@ describe("instrumented ABORT_UNWINDING", () => { const module = new WebAssembly.Module(bytes); const memory = new WebAssembly.Memory({ initial: 8 }); let instance: WebAssembly.Instance; - let moduleBuffer = 0; let forkResult = 0; let failGrowth = true; let nextAddress = 65_536; + let nextArenaAddress = 5 * 65_536; const released: Array<{ addr: number; size: number }> = []; const continuation = new LinkedForkContinuation( memory, @@ -72,41 +74,51 @@ describe("instrumented ABORT_UNWINDING", () => { (addr, size) => released.push({ addr, size }), "abort-e2e", ); + const runtime = new SingleActivationForkRuntime({ + module, + moduleBytes: bytes, + memory, + continuation, + newArena: () => new ForkModuleStateArena( + memory, + 4, + (size) => { + const address = nextArenaAddress; + nextArenaAddress += size; + const missing = nextArenaAddress - memory.buffer.byteLength; + if (missing > 0) memory.grow(Math.ceil(missing / 65_536)); + return address; + }, + () => {}, + "abort-e2e module state", + ), + label: "abort-e2e", + }); const imports = { env: { memory, - __wpk_fork_frame_reserve: (size: number) => { - const frame = continuation.reserveFrame(size); - if (frame === 0) { - (instance.exports.wpk_fork_abort_begin as (addr: number) => void)(moduleBuffer); - } - return frame; - }, - __wpk_fork_frame_commit: (payload: number) => continuation.commitFrame(payload), - __wpk_fork_frame_next: (size: number) => continuation.nextFrame(size), + ...runtime.envImports, }, kernel: { kernel_fork: () => { - const state = (instance.exports.wpk_fork_state as () => number)(); - if (state === 2) { - (instance.exports.wpk_fork_rewind_end as () => void)(); - continuation.finishReplayAndRelease(); + const phase = runtime.coordinator.phaseName(); + if (phase === "parent-replay") { + runtime.coordinator.finishReplay(); return forkResult; } - if (state === 3) { + if (phase === "abort-replay") { const errno = continuation.abortErrno(); - (instance.exports.wpk_fork_abort_end as () => void)(); - continuation.finishAbortReplayAndRelease(); + runtime.coordinator.finishAbortReplay(); return -errno; } - moduleBuffer = Number(continuation.beginUnwind()); - (instance.exports.wpk_fork_unwind_begin as (addr: number) => void)(moduleBuffer); + runtime.beginCapture(); return 0; }, }, }; instance = new WebAssembly.Instance(module, imports); + runtime.register(instance); const run = instance.exports.run as () => number; const state = instance.exports.wpk_fork_state as () => number; @@ -119,25 +131,21 @@ describe("instrumented ABORT_UNWINDING", () => { // SYS_FORK result after a complete unwind must replay to the guest. failGrowth = false; nextAddress = 65_536; - expect(run()).toBe(0); // transformed unwind returns the result-type default + runtime.expectCaptureTransport(run); expect(state()).toBe(1); - (instance.exports.wpk_fork_unwind_end as () => void)(); - continuation.finishUnwind(); + runtime.coordinator.sealCapture(); forkResult = -11; - continuation.beginReplay(); - (instance.exports.wpk_fork_rewind_begin as (addr: number) => void)(moduleBuffer); + runtime.coordinator.beginParentReplay(); expect(run()).toBe(-4); expect(state()).toBe(0); expect(continuation.hasActiveContinuation()).toBe(false); // A later independent fork can still complete successfully. nextAddress = 65_536; - expect(run()).toBe(0); - (instance.exports.wpk_fork_unwind_end as () => void)(); - continuation.finishUnwind(); + runtime.expectCaptureTransport(run); + runtime.coordinator.sealCapture(); forkResult = 123; - continuation.beginReplay(); - (instance.exports.wpk_fork_rewind_begin as (addr: number) => void)(moduleBuffer); + runtime.coordinator.beginParentReplay(); expect(run()).toBe(130); expect(state()).toBe(0); expect(continuation.hasActiveContinuation()).toBe(false); @@ -209,13 +217,15 @@ describe("instrumented ABORT_UNWINDING", () => { instrumentedPath, ]); - const module = new WebAssembly.Module(readFileSync(instrumentedPath)); + const bytes = readFileSync(instrumentedPath); + const module = new WebAssembly.Module(bytes); const memory = new WebAssembly.Memory({ initial: 8 }); const view = new DataView(memory.buffer); let instance: WebAssembly.Instance; let moduleBuffer = 0; let failGrowth = true; let nextAddress = 65_536; + let nextArenaAddress = 5 * 65_536; let abortCommits = 0; let successfulCommits = 0; let lowMemoryUntouched = false; @@ -254,41 +264,52 @@ describe("instrumented ABORT_UNWINDING", () => { (addr, size) => released.push({ addr, size }), "abort-catch-e2e", ); + const runtime = new SingleActivationForkRuntime({ + module, + moduleBytes: bytes, + memory, + continuation, + newArena: () => new ForkModuleStateArena( + memory, + 4, + (size) => { + const address = nextArenaAddress; + nextArenaAddress += size; + const missing = nextArenaAddress - memory.buffer.byteLength; + if (missing > 0) memory.grow(Math.ceil(missing / 65_536)); + return address; + }, + () => {}, + "abort-catch-e2e module state", + ), + label: "abort-catch-e2e", + }); + const coordinatedCommit = runtime.envImports.__wpk_fork_frame_commit as + (payload: number) => void; const imports = { env: { memory, - __wpk_fork_frame_reserve: (size: number) => { - const frame = continuation.reserveFrame(size); - if (frame === 0) { - (instance.exports.wpk_fork_abort_begin as (addr: number) => void)( - moduleBuffer, - ); - } - return frame; - }, + ...runtime.envImports, __wpk_fork_frame_commit: (payload: number) => { - continuation.commitFrame(payload); + coordinatedCommit(payload); if (failGrowth) { abortCommits++; } else { successfulCommits++; } }, - __wpk_fork_frame_next: (size: number) => continuation.nextFrame(size), }, kernel: { kernel_fork: () => { - const state = (instance.exports.wpk_fork_state as () => number)(); - if (state === 2) { - (instance.exports.wpk_fork_rewind_end as () => void)(); - continuation.finishReplayAndRelease(); + const phase = runtime.coordinator.phaseName(); + if (phase === "parent-replay") { + runtime.coordinator.finishReplay(); return 17; } - if (state === 3) { + if (phase === "abort-replay") { const errno = continuation.abortErrno(); - (instance.exports.wpk_fork_abort_end as () => void)(); - continuation.finishAbortReplayAndRelease(); + runtime.coordinator.finishAbortReplay(); return -errno; } @@ -298,15 +319,14 @@ describe("instrumented ABORT_UNWINDING", () => { lowMemoryUntouched = scratchWordsAreUntouched(0); retiredStorageUntouched = scratchWordsAreUntouched(moduleBuffer); } - moduleBuffer = Number(continuation.beginUnwind()); - (instance.exports.wpk_fork_unwind_begin as (addr: number) => void)( - moduleBuffer, - ); + runtime.beginCapture(); + moduleBuffer = runtime.coordinator.rootFor(0); return 0; }, }, }; instance = new WebAssembly.Instance(module, imports); + runtime.register(instance); const run = instance.exports.run as () => number; const state = instance.exports.wpk_fork_state as () => number; @@ -326,15 +346,10 @@ describe("instrumented ABORT_UNWINDING", () => { fillScratchWords(moduleBuffer); failGrowth = false; nextAddress = 65_536; - const unwindResult = run(); - expect(unwindResult).toBe(0); + runtime.expectCaptureTransport(run); expect(state()).toBe(1); - (instance.exports.wpk_fork_unwind_end as () => void)(); - continuation.finishUnwind(); - continuation.beginReplay(); - (instance.exports.wpk_fork_rewind_begin as (addr: number) => void)( - moduleBuffer, - ); + runtime.coordinator.sealCapture(); + runtime.coordinator.beginParentReplay(); const successfulResult = run(); expect({ diff --git a/host/test/fork-activation-registry.test.ts b/host/test/fork-activation-registry.test.ts new file mode 100644 index 0000000000..8d5e35f14d --- /dev/null +++ b/host/test/fork-activation-registry.test.ts @@ -0,0 +1,433 @@ +import { execFileSync } from "node:child_process"; +import { mkdtempSync, readFileSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, expect, it, vi } from "vitest"; +import { + buildForkActivationStateImports, + ForkActivationRegistry, + type ForkActivationRegistration, +} from "../src/fork-activation-registry"; +import { + ForkModuleStateArena, + ForkTableDirtyTracker, +} from "../src/fork-module-state"; +import { + WPK_FORK_REFERENCE_IMPORT_GC_CAPTURE_LAYOUT, +} from "../src/generated/abi"; + +const PAGE_SIZE = 65_536; + +function makeArena(memory: WebAssembly.Memory, label: string): ForkModuleStateArena { + let next = PAGE_SIZE; + return new ForkModuleStateArena( + memory, + 4, + (size) => { + const address = next; + next += size; + if (next > memory.buffer.byteLength) { + memory.grow(Math.ceil((next - memory.buffer.byteLength) / PAGE_SIZE)); + } + return address; + }, + () => {}, + label, + ); +} + +function emptyCatalog(): WebAssembly.Table { + return new WebAssembly.Table({ + element: "anyfunc", + initial: 0, + maximum: 0, + }); +} + +function emptyStaticRootCatalog(): WebAssembly.Table { + return new WebAssembly.Table({ + element: "externref", + initial: 0, + maximum: 0, + }); +} + +function funcrefTableActivation(): { + readonly instance: WebAssembly.Instance; + readonly functionCatalog: WebAssembly.Table; + readonly mutableTable: WebAssembly.Table; +} { + const dir = mkdtempSync(join(tmpdir(), "kandelo-table-patch-")); + const wat = join(dir, "table-patch.wat"); + const wasm = join(dir, "table-patch.wasm"); + writeFileSync(wat, `(module + (table $catalog (export "__wpk_fork_function_catalog") 2 2 funcref) + (table $mutable (export "__wpk_fork_table_3") 2 4 funcref) + (func $first (result i32) i32.const 17) + (func $second (result i32) i32.const 29) + (elem (table $catalog) (i32.const 0) func $first $second) + )`); + execFileSync("wat2wasm", [wat, "-o", wasm]); + const instance = new WebAssembly.Instance( + new WebAssembly.Module(readFileSync(wasm)), + ); + return { + instance, + functionCatalog: + instance.exports.__wpk_fork_function_catalog as WebAssembly.Table, + mutableTable: instance.exports.__wpk_fork_table_3 as WebAssembly.Table, + }; +} + +function registration( + activationId: number, + calls: string[], + options: { + clear?: () => void; + abort?: () => void; + } = {}, +): ForkActivationRegistration { + return { + activationId, + instance: { exports: {} } as unknown as WebAssembly.Instance, + templateId: new Uint8Array(32).fill(activationId + 1), + functionCatalog: emptyCatalog(), + staticRootCatalog: emptyStaticRootCatalog(), + staticRootHarvest: () => {}, + tableDirty: new ForkTableDirtyTracker(), + moduleState: { + bootstrap: () => { calls.push(`bootstrap:${activationId}`); }, + save: (id) => { calls.push(`save:${id}`); }, + restore: (id) => { calls.push(`restore:${id}`); }, + finishRestore: (id) => { calls.push(`finish-restore:${id}`); }, + saveTables: (id) => { calls.push(`save-tables:${id}`); }, + restoreTables: (id) => { calls.push(`restore-tables:${id}`); }, + }, + typedReferenceProvider: { + clear: options.clear ?? (() => { calls.push(`clear:${activationId}`); }), + abort: options.abort ?? (() => { calls.push(`abort:${activationId}`); }), + }, + }; +} + +function registry(memory: WebAssembly.Memory, label: string): ForkActivationRegistry { + return new ForkActivationRegistry( + memory, + { + capture: () => { + throw new Error("fixture has no externrefs"); + }, + materialize: () => { + throw new Error("fixture has no externrefs"); + }, + }, + label, + ); +} + +describe("ForkActivationRegistry", () => { + it("binds GC layout capture in the generated slot/activation/layout order", () => { + const memory = new WebAssembly.Memory({ initial: 2 }); + const owner = registry(memory, "GC capture import"); + const capture = vi.spyOn(owner, "captureGcLayout").mockReturnValue(23); + const imports = buildForkActivationStateImports(7, owner); + const captureLayout = imports[ + WPK_FORK_REFERENCE_IMPORT_GC_CAPTURE_LAYOUT + ] as CallableFunction; + + expect(captureLayout(5, 7, 11)).toBe(23); + expect(capture).toHaveBeenCalledWith(7, 5, 11); + expect(() => captureLayout(7, 5, 11)).toThrow( + "activation 7 cannot select GC layout for activation 5", + ); + }); + + it("captures and restores every activation in deterministic id order", () => { + const memory = new WebAssembly.Memory({ initial: 8 }); + const calls: string[] = []; + const parent = registry(memory, "parent"); + parent.registerActivation(registration(7, calls)); + parent.registerActivation(registration(0, calls)); + parent.bootstrapActivation(0); + parent.bootstrapActivation(7); + + const arena = makeArena(memory, "parent arena"); + arena.begin(); + parent.beginCapture(arena); + expect(calls).toEqual([ + "bootstrap:0", + "bootstrap:7", + "save:0", + "save:7", + ]); + parent.sealCapture(); + parent.beginParentReplay(); + parent.restoreModuleState(); + parent.finishReplay(); + expect(calls.slice(-6)).toEqual([ + "restore:0", + "restore:7", + "finish-restore:0", + "finish-restore:7", + "clear:0", + "clear:7", + ]); + expect(parent.phaseName()).toBe("idle"); + }); + + it("attaches a fresh child only after the complete activation set exists", () => { + const parentMemory = new WebAssembly.Memory({ initial: 8 }); + const parent = registry(parentMemory, "parent"); + parent.registerActivation(registration(0, [])); + parent.registerActivation(registration(2, [])); + const parentArena = makeArena(parentMemory, "parent arena"); + const root = parentArena.begin(); + parent.beginCapture(parentArena); + parent.sealCapture(); + + const childMemory = new WebAssembly.Memory({ + initial: parentMemory.buffer.byteLength / PAGE_SIZE, + }); + new Uint8Array(childMemory.buffer).set(new Uint8Array(parentMemory.buffer)); + const childArena = new ForkModuleStateArena( + childMemory, + 4, + () => { throw new Error("attached child arena must not allocate"); }, + () => {}, + "child arena", + ); + childArena.attach(root); + + const childCalls: string[] = []; + const child = registry(childMemory, "child"); + child.registerActivation(registration(0, childCalls)); + expect(() => child.attachChild(childArena)).toThrow( + "copied module activations do not match", + ); + child.registerActivation(registration(2, childCalls)); + child.attachChild(childArena); + child.currentReferences().materializeAllTyped = () => { + childCalls.push("materialize-typed"); + }; + child.restoreModuleState(); + child.finishReplay(); + expect(childCalls).toEqual([ + "materialize-typed", + "restore:0", + "restore:2", + "finish-restore:0", + "finish-restore:2", + "clear:0", + "clear:2", + ]); + }); + + it("drops every provider root even when one cleanup reports an error", () => { + const memory = new WebAssembly.Memory({ initial: 8 }); + const calls: string[] = []; + const owner = registry(memory, "cleanup"); + owner.registerActivation(registration(0, calls, { + abort: () => { + calls.push("abort:0"); + throw new Error("first cleanup failed"); + }, + })); + owner.registerActivation(registration(1, calls)); + const arena = makeArena(memory, "cleanup arena"); + arena.begin(); + owner.beginCapture(arena); + + expect(() => owner.abort()).toThrow("first cleanup failed"); + expect(calls.slice(-2)).toEqual(["abort:0", "abort:1"]); + expect(owner.phaseName()).toBe("idle"); + }); + + it("keeps weak static identity across later forks and forgets it on unregister", () => { + const memory = new WebAssembly.Memory({ initial: 8 }); + const root = Object.freeze({ segment: "already dropped" }); + const harvest = new WebAssembly.Table({ + element: "externref", + initial: 1, + maximum: 1, + }); + harvest.set(0, root); + const owner = registry(memory, "static roots"); + owner.registerActivation({ + ...registration(0, []), + staticRootCatalog: harvest, + staticRootHarvest: () => {}, + }); + expect(harvest.get(0)).toBeNull(); + + for (const label of ["first fork", "later fork"]) { + const arena = makeArena(memory, label); + arena.begin(); + owner.beginCapture(arena); + expect(owner.currentReferences().encodeExternref(root)).toBe(1); + owner.abort(); + } + + owner.unregisterActivation(0); + const afterUnload = makeArena(memory, "after unload"); + afterUnload.begin(); + owner.beginCapture(afterUnload); + expect(() => owner.currentReferences().encodeExternref(root)).toThrow( + "fixture has no externrefs", + ); + owner.abort(); + }); + + it("keeps table owner ordinals activation-local", () => { + const memory = new WebAssembly.Memory({ initial: 2 }); + const owner = registry(memory, "tables"); + owner.registerActivation(registration(0, [])); + owner.registerActivation(registration(1, [])); + owner.tableDirty(0).markPages(3, 1n, 1n); + owner.tableDirty(1).markPages(3, 9n, 1n); + expect(owner.tableDirty(0).pageAt(3, 0)).toBe(1n); + expect(owner.tableDirty(1).pageAt(3, 0)).toBe(9n); + }); + + it("attributes host table mutations to every catalog alias", () => { + const memory = new WebAssembly.Memory({ initial: 2 }); + const table = new WebAssembly.Table({ + element: "anyfunc", + initial: 2_048, + maximum: 4_096, + }); + const owner = registry(memory, "host table mutation"); + const first = registration(0, []); + const second = registration(1, []); + owner.registerActivation({ + ...first, + instance: { + exports: { __wpk_fork_table_3: table }, + } as unknown as WebAssembly.Instance, + }); + owner.registerActivation({ + ...second, + instance: { + exports: { __wpk_fork_table_9: table }, + } as unknown as WebAssembly.Instance, + }); + expect(owner.tableDirty(0).ownsState(3)).toBe(true); + expect(owner.tableDirty(1).ownsState(9)).toBe(false); + + // The range crosses the ABI-defined 1,024-entry sparse-page boundary. + owner.markTableMutation(table, 1_023, 2); + for (const [activationId, tableOwner] of [[0, 3], [1, 9]] as const) { + expect(owner.tableDirty(activationId).pageCount(tableOwner)).toBe(2); + expect(owner.tableDirty(activationId).pageAt(tableOwner, 0)).toBe(0n); + expect(owner.tableDirty(activationId).pageAt(tableOwner, 1)).toBe(1n); + } + + // Catalog removal must neither retain nor keep writing the unloaded + // activation's coordinate. + owner.unregisterActivation(0); + expect(owner.tableDirty(1).ownsState(9)).toBe(true); + owner.markTableMutation(table, 2_048, 1); + expect(owner.tableDirty(1).pageAt(9, 2)).toBe(2n); + }); + + it("rejects host mutations of tables outside activation catalogs", () => { + const memory = new WebAssembly.Memory({ initial: 2 }); + const owner = registry(memory, "unknown host table"); + const table = new WebAssembly.Table({ + element: "anyfunc", + initial: 1, + maximum: 1, + }); + expect(() => owner.markTableMutation(table, 0, 1)).toThrow( + "outside the registered fork catalogs", + ); + }); + + it("replays funcref/null table patches through fresh activation catalogs", () => { + const parentMemory = new WebAssembly.Memory({ initial: 2 }); + const parent = registry(parentMemory, "parent table patch"); + const parentModule = funcrefTableActivation(); + parent.registerActivation({ + ...registration(0, []), + instance: parentModule.instance, + functionCatalog: parentModule.functionCatalog, + }); + // A second activation aliases the same physical process Table. The patch + // target deliberately uses that non-canonical coordinate. + parent.registerActivation({ + ...registration(1, []), + instance: { + exports: { __wpk_fork_table_9: parentModule.mutableTable }, + } as unknown as WebAssembly.Instance, + }); + + const parentFirst = parentModule.functionCatalog.get(0); + const parentSecond = parentModule.functionCatalog.get(1); + parentModule.mutableTable.set(0, parentFirst); + parentModule.mutableTable.set(1, parentFirst); + const first = parent.captureFuncrefTablePatch(1, 9, 0, 2); + expect(first).toEqual({ + activationId: 1, + ownerId: 9, + start: 0, + tableLength: 2, + runs: [{ + length: 2, + function: { activationId: 0, ordinal: 0 }, + }], + }); + parentModule.mutableTable.grow(2, parentSecond); + const growth = parent.captureFuncrefTablePatch(1, 9, 2, 2); + + const childMemory = new WebAssembly.Memory({ initial: 2 }); + const child = registry(childMemory, "child table patch"); + const childModule = funcrefTableActivation(); + child.registerActivation({ + ...registration(0, []), + instance: childModule.instance, + functionCatalog: childModule.functionCatalog, + }); + child.registerActivation({ + ...registration(1, []), + instance: { + exports: { __wpk_fork_table_9: childModule.mutableTable }, + } as unknown as WebAssembly.Instance, + }); + child.applyFuncrefTablePatch({ ...first!, generation: 1 }); + child.applyFuncrefTablePatch({ ...growth!, generation: 2 }); + + expect(childModule.mutableTable.length).toBe(4); + expect(childModule.mutableTable.get(0)).toBe( + childModule.functionCatalog.get(0), + ); + expect(childModule.mutableTable.get(1)).toBe( + childModule.functionCatalog.get(0), + ); + expect(childModule.mutableTable.get(2)).toBe( + childModule.functionCatalog.get(1), + ); + expect(childModule.mutableTable.get(3)).toBe( + childModule.functionCatalog.get(1), + ); + expect(childModule.mutableTable.get(0)).not.toBe(parentFirst); + expect(child.tableDirty(0).pageCount(3)).toBe(1); + expect(child.tableDirty(1).pageCount(9)).toBe(1); + }); + + it("routes non-funcref table values to the typed checkpoint", () => { + const memory = new WebAssembly.Memory({ initial: 2 }); + const owner = registry(memory, "typed table fallback"); + const table = new WebAssembly.Table({ + element: "externref", + initial: 1, + maximum: 1, + }); + table.set(0, Object.freeze({ processHandle: 47 })); + owner.registerActivation({ + ...registration(0, []), + instance: { + exports: { __wpk_fork_table_4: table }, + } as unknown as WebAssembly.Instance, + }); + expect(owner.captureFuncrefTablePatch(0, 4, 0, 1)).toBeNull(); + }); +}); diff --git a/host/test/fork-anyref-transit.test.ts b/host/test/fork-anyref-transit.test.ts new file mode 100644 index 0000000000..933b9596b4 --- /dev/null +++ b/host/test/fork-anyref-transit.test.ts @@ -0,0 +1,50 @@ +import { describe, expect, it } from "vitest"; +import { + FORK_ANYREF_TRANSIT_IMPORT, + ForkAnyrefTransitTable, + forkAnyrefTransitProviderBytes, +} from "../src/fork-anyref-transit"; + +describe("ForkAnyrefTransitTable", () => { + it("uses a closed audited Wasm provider with the exact table type", () => { + const bytes = forkAnyrefTransitProviderBytes(); + const module = new WebAssembly.Module(bytes as BufferSource); + + expect(WebAssembly.Module.imports(module)).toEqual([]); + expect(WebAssembly.Module.exports(module)).toEqual([ + { + name: FORK_ANYREF_TRANSIT_IMPORT, + kind: "table", + }, + { + name: `${FORK_ANYREF_TRANSIT_IMPORT}_clear`, + kind: "function", + }, + ]); + }); + + it("clears every grown slot and isolates workers", () => { + const first = new ForkAnyrefTransitTable(); + const second = new ForkAnyrefTransitTable(); + + expect(first.table).not.toBe(second.table); + expect(first.table.length).toBe(1); + first.table.grow(3); + expect(first.table.length).toBe(4); + + first.clear(); + expect( + Array.from( + { length: first.table.length }, + (_, index) => first.table.get(index), + ), + ).toEqual([null, null, null, null]); + expect(second.table.length).toBe(1); + }); + + it("does not expose mutable provider bytes", () => { + const first = forkAnyrefTransitProviderBytes(); + first[0] = 0xff; + expect(forkAnyrefTransitProviderBytes()[0]).toBe(0x00); + }); +}); diff --git a/host/test/fork-artifact-gc-types.test.ts b/host/test/fork-artifact-gc-types.test.ts new file mode 100644 index 0000000000..0923fed44f --- /dev/null +++ b/host/test/fork-artifact-gc-types.test.ts @@ -0,0 +1,88 @@ +import { describe, expect, it } from "vitest"; + +import { + detectPtrWidth, + describeWasmArtifactPolicyFailures, + readWasmImportNames, + wasmImportsKernelFork, +} from "../src/constants"; + +function uleb(value: number): number[] { + const bytes: number[] = []; + do { + let byte = value & 0x7f; + value >>>= 7; + if (value !== 0) byte |= 0x80; + bytes.push(byte); + } while (value !== 0); + return bytes; +} + +function name(value: string): number[] { + const bytes = new TextEncoder().encode(value); + return [...uleb(bytes.byteLength), ...bytes]; +} + +function section(id: number, payload: number[]): number[] { + return [id, ...uleb(payload.length), ...payload]; +} + +/** + * A structurally parseable module whose function type follows a recursive GC + * group. The fork artifact is intentionally incomplete: this fixture proves + * the policy reader reports the missing ABI contract instead of losing type + * indices or treating the struct/array definitions as malformed functions. + */ +function gcForkImportFixture(): ArrayBuffer { + const typeSection = [ + ...uleb(1), // one explicit recursive group + 0x4e, + ...uleb(3), + // type 0: (struct (field (mut (ref null 0)))) + 0x5f, ...uleb(1), 0x63, 0x00, 0x01, + // type 1: (array (mut i32)) + 0x5e, 0x7f, 0x01, + // type 2: (func (param (ref null 0)) (result i32)) + 0x60, ...uleb(1), 0x63, 0x00, ...uleb(1), 0x7f, + ]; + const imports = [ + ...uleb(4), + ...name("kernel"), ...name("kernel_fork"), 0x00, ...uleb(2), + ...name("env"), ...name("gc_table"), 0x01, + 0x63, 0x00, // concrete nullable table reference + 0x00, ...uleb(1), // limits + ...name("env"), ...name("gc_global"), 0x03, + 0x63, 0x00, // concrete nullable global reference + 0x01, // mutable + ...name("env"), ...name("memory"), 0x02, + 0x04, ...uleb(1), // memory64 limits + ]; + return new Uint8Array([ + 0x00, 0x61, 0x73, 0x6d, + 0x01, 0x00, 0x00, 0x00, + ...section(1, typeSection), + ...section(2, imports), + ]).buffer; +} + +describe("fork artifact parsing with recursive GC types", () => { + it("keeps function type indices aligned across struct and array types", () => { + const wasm = gcForkImportFixture(); + + expect(readWasmImportNames(wasm)).toEqual([ + "kernel.kernel_fork", + "env.gc_table", + "env.gc_global", + "env.memory", + ]); + expect(wasmImportsKernelFork(wasm)).toBe(true); + expect(detectPtrWidth(wasm)).toBe(8); + + const failures = describeWasmArtifactPolicyFailures(wasm); + expect(failures.join("\n")).not.toContain("cannot validate"); + expect(failures.join("\n")).not.toContain("non-function type"); + expect(failures).toContain( + "missing required kandelo.wpk_fork.capabilities capability", + ); + }); +}); diff --git a/host/test/fork-continuation.test.ts b/host/test/fork-continuation.test.ts index c0d54765ae..cfee45c449 100644 --- a/host/test/fork-continuation.test.ts +++ b/host/test/fork-continuation.test.ts @@ -210,7 +210,10 @@ describe("LinkedForkContinuation", () => { "child", ); child.attachForReplay(moduleBuffer); + const peekOuter = Number(child.peekFrame(24)); + expect(Number(child.peekFrame(24))).toBe(peekOuter); const replayOuter = Number(child.nextFrame(24)); + expect(replayOuter).toBe(peekOuter); const replayInner = Number(child.nextFrame(16)); expect(new Uint8Array(childMemory.buffer, replayOuter, 24)).toEqual( new Uint8Array(24).fill(0x22), diff --git a/host/test/fork-dlopen-replay-e2e.test.ts b/host/test/fork-dlopen-replay-e2e.test.ts index 7a5b0375d8..ad7960c8da 100644 --- a/host/test/fork-dlopen-replay-e2e.test.ts +++ b/host/test/fork-dlopen-replay-e2e.test.ts @@ -89,16 +89,21 @@ function buildSharedLib(source: string, name: string): string { const objPath = join(BUILD_DIR, `${name}.o`); const soPath = join(BUILD_DIR, `${name}.so`); - writeFileSync(srcPath, source); + writeFileSync(srcPath, `${source} + #include "abi_constants.h" + __attribute__((export_name("__abi_version"))) + unsigned __abi_version(void) { return WASM_POSIX_ABI_VERSION; } + `); execSync( - `${CLANG} --target=wasm32-unknown-unknown -fPIC -O2 -matomics -mbulk-memory -c ${srcPath} -o ${objPath}`, + `${CLANG} --target=wasm32-unknown-unknown -fPIC -O2 -matomics -mbulk-memory -I${GLUE_DIR} -c ${srcPath} -o ${objPath}`, { stdio: "pipe" }, ); execSync( `${WASM_LD} --experimental-pic --shared --shared-memory --export-all --allow-undefined -o ${soPath} ${objPath}`, { stdio: "pipe" }, ); + execSync(`${FORK_INSTRUMENT} ${soPath} -o ${soPath}`, { stdio: "pipe" }); return soPath; } @@ -109,7 +114,11 @@ function buildCppSharedLib(source: string, name: string): string { const srcPath = join(BUILD_DIR, `${name}.cpp`); const objPath = join(BUILD_DIR, `${name}.o`); const soPath = join(BUILD_DIR, `${name}.so`); - writeFileSync(srcPath, source); + writeFileSync(srcPath, `${source} + #include "abi_constants.h" + extern "C" __attribute__((export_name("__abi_version"))) + unsigned __abi_version(void) { return WASM_POSIX_ABI_VERSION; } + `); execFileSync(CLANGXX, [ "--target=wasm32-unknown-unknown", `--sysroot=${SYSROOT}`, @@ -119,6 +128,7 @@ function buildCppSharedLib(source: string, name: string): string { "-fwasm-exceptions", "-matomics", "-mbulk-memory", + `-I${GLUE_DIR}`, `-I${join(libcxxPrefix, "include", "c++", "v1")}`, "-c", srcPath, @@ -138,6 +148,12 @@ function buildCppSharedLib(source: string, name: string): string { join(libcxxPrefix, "lib", "libc++-pic.a"), join(libcxxPrefix, "lib", "libc++abi-pic.a"), ], { stdio: "pipe" }); + execFileSync("bash", [ + FORK_INSTRUMENT, + soPath, + "-o", + soPath, + ], { stdio: "pipe" }); return soPath; } @@ -270,60 +286,80 @@ describe.skipIf(!hasSysroot || !hasKernel)("fork after dlopen end-to-end", () => }); expect(result.stderr).not.toContain("table index is out of bounds"); - expect(result.exitCode).toBe(0); + expect(result.exitCode, JSON.stringify(result)).toBe(0); expect(result.stdout).toContain("ok"); }); - it("fails pthread dlopen and fork after process dlopen without creating a child", { timeout: 30_000 }, async () => { + it("replays pthread-hosted dlopen table state into a fresh fork child", { timeout: 30_000 }, async () => { const soPath = buildSharedLib( - `int pthread_boundary_fixture(void) { return 1; }`, - "libpthreadboundary", + ` + typedef int (*step_fn)(int); + static int increment(int value) { return value + 1; } + static step_fn relocated_step = increment; + int pthread_replay_value(int value) { return relocated_step(value); } + `, + "libpthreadreplay", ); const wasmPath = buildMainProgram(` #include - #include #include #include - #include #include + #include static const char *side_path; static int thread_result; + typedef int (*replay_fn)(int); static void *run_thread(void *unused) { (void)unused; - void *nested = dlopen(side_path, RTLD_NOW); - const char *error = dlerror(); - if (nested != NULL || error == NULL || strstr(error, "pthread workers") == NULL) { + void *side = dlopen(side_path, RTLD_NOW); + if (!side) { + fprintf(stderr, "pthread dlopen: %s\\n", dlerror()); thread_result = 1; return NULL; } - errno = 0; - pid_t child = fork(); - if (child != -1 || errno != ENOTSUP) { + replay_fn replay = (replay_fn)dlsym(side, "pthread_replay_value"); + if (!replay || replay(40) != 41) { thread_result = 2; return NULL; } + + pid_t child = fork(); + if (child == 0) { + _exit(replay(41) == 42 ? 0 : 3); + } + if (child < 0) { + thread_result = 4; + return NULL; + } + int status = 0; + if ( + waitpid(child, &status, 0) != child + || !WIFEXITED(status) + || WEXITSTATUS(status) != 0 + ) { + thread_result = 5; + return NULL; + } thread_result = 0; return NULL; } int main(int argc, char **argv) { side_path = argv[1]; - void *side = dlopen(side_path, RTLD_NOW); - if (!side) { fprintf(stderr, "main dlopen: %s\\n", dlerror()); return 2; } pthread_t thread; if (pthread_create(&thread, NULL, run_thread, NULL) != 0) return 3; if (pthread_join(thread, NULL) != 0) return 4; if (thread_result != 0) return 10 + thread_result; - puts("pthread dylink boundary ok"); + puts("pthread dlopen fork replay ok"); return 0; } - `, "test-pthread-dylink-boundary"); + `, "test-pthread-dlopen-fork-replay"); const result = await runCentralizedProgram({ programPath: wasmPath, - argv: ["pthread-dylink-boundary", soPath], + argv: ["pthread-dlopen-fork-replay", soPath], timeout: 30_000, io: io(), captureForkCount: true, @@ -331,8 +367,140 @@ describe.skipIf(!hasSysroot || !hasKernel)("fork after dlopen end-to-end", () => expect(result.stderr).toBe(""); expect(result.exitCode).toBe(0); - expect(result.stdout).toContain("pthread dylink boundary ok"); - expect(result.forkCount).toBe(0n); + expect(result.stdout).toContain("pthread dlopen fork replay ok"); + expect(result.forkCount).toBe(1n); + }); + + it("blocks a foreign pthread until the staged loader owner commits", { timeout: 30_000 }, async () => { + const slowPath = buildSharedLib( + ` + extern void loader_ctor_enter(void); + extern void loader_ctor_wait(void); + __attribute__((constructor)) + static void slow_constructor(void) { + loader_ctor_enter(); + loader_ctor_wait(); + } + int slow_value(void) { return 17; } + `, + "libpthread-loader-owner", + ); + const fastPath = buildSharedLib( + `int fast_value(void) { return 29; }`, + "libpthread-loader-waiter", + ); + const wasmPath = buildMainProgram(` + #include + #include + #include + #include + #include + #include + + static const char *slow_path; + static const char *fast_path; + static _Atomic int owner_ready; + static _Atomic int waiter_ready; + static _Atomic int start_owner; + static _Atomic int constructor_entered; + static _Atomic int release_constructor; + static _Atomic int waiter_entered; + static _Atomic int waiter_done; + static _Atomic int constructor_timeout; + static int owner_result; + static int waiter_result; + + static int wait_for(_Atomic int *value) { + for (int attempt = 0; attempt < 5000; attempt++) { + if (atomic_load_explicit(value, memory_order_acquire)) return 1; + usleep(1000); + } + return atomic_load_explicit(value, memory_order_acquire) != 0; + } + + void loader_ctor_enter(void) { + atomic_store_explicit(&constructor_entered, 1, memory_order_release); + } + + void loader_ctor_wait(void) { + if (!wait_for(&release_constructor)) { + atomic_store_explicit(&constructor_timeout, 1, memory_order_release); + } + } + + static void *run_owner(void *unused) { + (void)unused; + atomic_store_explicit(&owner_ready, 1, memory_order_release); + if (!wait_for(&start_owner)) { + owner_result = 2; + return NULL; + } + void *handle = dlopen(slow_path, RTLD_NOW | RTLD_GLOBAL); + if (!handle) fprintf(stderr, "owner dlopen: %s\\n", dlerror()); + owner_result = handle ? 0 : 1; + return NULL; + } + + static void *run_waiter(void *unused) { + (void)unused; + atomic_store_explicit(&waiter_ready, 1, memory_order_release); + if (!wait_for(&constructor_entered)) { + waiter_result = 2; + return NULL; + } + atomic_store_explicit(&waiter_entered, 1, memory_order_release); + void *handle = dlopen(fast_path, RTLD_NOW | RTLD_LOCAL); + if (!handle) fprintf(stderr, "waiter dlopen: %s\\n", dlerror()); + waiter_result = handle ? 0 : 1; + atomic_store_explicit(&waiter_done, 1, memory_order_release); + return NULL; + } + + int main(int argc, char **argv) { + slow_path = argv[1]; + fast_path = argv[2]; + pthread_t owner; + pthread_t waiter; + if (pthread_create(&owner, NULL, run_owner, NULL) != 0) return 2; + if (pthread_create(&waiter, NULL, run_waiter, NULL) != 0) return 3; + if (!wait_for(&owner_ready)) return 11; + if (!wait_for(&waiter_ready)) return 12; + atomic_store_explicit(&start_owner, 1, memory_order_release); + if (!wait_for(&constructor_entered)) return 8; + if (!wait_for(&waiter_entered)) return 9; + usleep(10000); + int waiter_completed_early = + atomic_load_explicit(&waiter_done, memory_order_acquire); + atomic_store_explicit( + &release_constructor, + 1, + memory_order_release + ); + if (pthread_join(owner, NULL) != 0) return 5; + if (pthread_join(waiter, NULL) != 0) return 6; + if (atomic_load_explicit(&constructor_timeout, memory_order_acquire)) { + return 10; + } + if (waiter_completed_early) return waiter_result == 0 ? 4 : 14; + if (owner_result != 0 || waiter_result != 0) return 7; + puts("pthread loader lease ok"); + return 0; + } + `, "test-pthread-loader-owner", [ + "loader_ctor_enter", + "loader_ctor_wait", + ]); + + const result = await runCentralizedProgram({ + programPath: wasmPath, + argv: ["pthread-loader-owner", slowPath, fastPath], + timeout: 30_000, + io: io(), + }); + + expect(result.stderr).toBe(""); + expect(result.exitCode, JSON.stringify(result)).toBe(0); + expect(result.stdout).toContain("pthread loader lease ok"); }); it.skipIf(!hasCppPrerequisites)( diff --git a/host/test/fork-early-reference-provider.test.ts b/host/test/fork-early-reference-provider.test.ts new file mode 100644 index 0000000000..77b8fbd9c2 --- /dev/null +++ b/host/test/fork-early-reference-provider.test.ts @@ -0,0 +1,1011 @@ +import { describe, expect, it } from "vitest"; +import { + WPK_FORK_IMPORTED_GLOBAL_BINDINGS_OWNER, + WPK_FORK_MODULE_STATE_GLOBAL_TYPE_ANYREF, + WPK_FORK_MODULE_STATE_GLOBAL_TYPE_EXNREF, + WPK_FORK_MODULE_STATE_GLOBAL_TYPE_EXTERNREF, + WPK_FORK_MODULE_STATE_GLOBAL_TYPE_FUNCREF, +} from "../src/generated/abi"; +import { + ForkEarlyChildReferenceProvider, + type ForkEarlyChildReferenceProviderOptions, + type ForkEarlyReferenceActivationDeclaration, + type ForkEarlyReferenceTransit, +} from "../src/fork-early-reference-provider"; +import { + FORK_GC_FIELD_ALLOCATION_DEPENDENCY, + FORK_GC_FIELD_MUTABLE, + FORK_GC_FIELD_NULLABLE, + FORK_GC_FIELD_REFERENCE, + FORK_GC_LAYOUT_DEFAULTABLE_SHELL, + ForkGcCodecDescriptor, + ForkGcConstructorKind, + ForkGcLayoutKind, + type ForkGcCodecProvider, +} from "../src/fork-gc-codec"; +import { ForkFunctionCatalog } from "../src/fork-function-catalog"; +import { + encodeForkImportedGlobalBindings, + ForkImportedGlobalBindingKind, + ForkModuleStateRecordKind, + type ForkImportedGlobalBinding, + type ForkModuleStateRecord, +} from "../src/fork-module-state"; +import { + type ForkReferenceRecipeGraph, +} from "../src/fork-reference-recipes"; +import { + decodeSegmentedForkReferenceTransaction, + encodeSegmentedForkReferenceRecords, + forkReferenceVectorFrom, + PagedForkReferenceVector, +} from "../src/fork-reference-segments"; +import { + FORK_HOST_EXCEPTION_ACTIVATION_ID, + FORK_REFERENCE_TRANSACTION_OWNER_ID, + ForkReferenceTransaction, + type ForkExternrefRecipeProvider, + type ForkTypedReferenceReplayOwner, +} from "../src/fork-reference-transaction"; + +function recordsFor( + graph: ForkReferenceRecipeGraph, + activationIds: readonly number[], + bindings: readonly ForkImportedGlobalBinding[] = [], + vectors: readonly (readonly number[])[] = [], +): ForkModuleStateRecord[] { + return [ + ...activationIds.map((activationId) => ({ + kind: ForkModuleStateRecordKind.Module, + activationId, + ownerId: 0, + payload: new Uint8Array(32).fill(activationId), + })), + ...encodeSegmentedForkReferenceRecords( + FORK_REFERENCE_TRANSACTION_OWNER_ID, + graph.nodes, + [ + PagedForkReferenceVector.empty, + ...vectors.map((vector) => + forkReferenceVectorFrom(vector, vector.length) + ), + ], + { segmentDataBytes: 19 }, + ), + { + kind: ForkModuleStateRecordKind.ImportedGlobalBindings, + activationId: 0, + ownerId: WPK_FORK_IMPORTED_GLOBAL_BINDINGS_OWNER, + payload: encodeForkImportedGlobalBindings(bindings), + }, + ]; +} + +class TestTransit implements ForkEarlyReferenceTransit { + readonly values = new Map(); + prepared: number[] = []; + aborts = 0; + + prepare(maxRecipeId: number): void { + this.prepared.push(maxRecipeId); + } + + read(recipeId: number): unknown { + return this.values.get(recipeId); + } + + publish(recipeId: number, value: unknown): void { + this.values.set(recipeId, value); + } + + abort(): void { + this.aborts++; + this.values.clear(); + } +} + +function scratchOwner(memory: WebAssembly.Memory): { + readonly allocate: (size: number) => number; + readonly deallocate: (addr: number, size: number) => void; + readonly deallocated: Array<{ addr: number; size: number }>; +} { + let next = 65_536; + const deallocated: Array<{ addr: number; size: number }> = []; + return { + allocate(size) { + const addr = next; + next += size; + if (next > memory.buffer.byteLength) { + throw new Error("test scratch memory exhausted"); + } + return addr; + }, + deallocate(addr, size) { + deallocated.push({ addr, size }); + }, + deallocated, + }; +} + +function externrefs(values: ReadonlyMap): { + readonly provider: ForkExternrefRecipeProvider; + readonly materializations: number[]; +} { + const materializations: number[] = []; + return { + provider: { + capture(): number { + throw new Error("capture is not available in a fresh child"); + }, + materialize(handle): unknown { + materializations.push(handle); + if (!values.has(handle)) throw new Error(`unknown externref handle ${handle}`); + return values.get(handle); + }, + }, + materializations, + }; +} + +function providerOptions( + records: readonly ForkModuleStateRecord[], + declarations: readonly ForkEarlyReferenceActivationDeclaration[], + provider: ForkExternrefRecipeProvider, + transit: TestTransit, +): ForkEarlyChildReferenceProviderOptions & { + readonly scratch: ReturnType; +} { + const memory = new WebAssembly.Memory({ initial: 8 }); + const scratch = scratchOwner(memory); + return { + records, + transaction: decodeSegmentedForkReferenceTransaction( + records, + FORK_REFERENCE_TRANSACTION_OWNER_ID, + ), + declarations, + externrefs: provider, + transit, + memory, + allocateScratch: scratch.allocate, + deallocateScratch: scratch.deallocate, + label: "test early references", + scratch, + }; +} + +function mutableReferenceStructDescriptor(): ForkGcCodecDescriptor { + return new ForkGcCodecDescriptor([{ + id: 1, + typeOrdinal: 0, + kind: ForkGcLayoutKind.Struct, + constructor: ForkGcConstructorKind.Struct, + flags: FORK_GC_LAYOUT_DEFAULTABLE_SHELL, + scalarLengthOrStride: 0, + fields: [{ + storage: 8, + flags: + FORK_GC_FIELD_REFERENCE + | FORK_GC_FIELD_MUTABLE + | FORK_GC_FIELD_NULLABLE, + scalarOffset: null, + referenceOrdinal: 0, + }], + superTypeOrdinal: null, + baseLayoutId: 1, + auxiliary: 0, + provenanceScalarLength: 0, + provenanceReferenceCount: 0, + }]); +} + +function immutableReferenceStructDescriptor(): ForkGcCodecDescriptor { + return new ForkGcCodecDescriptor([{ + id: 1, + typeOrdinal: 0, + kind: ForkGcLayoutKind.Struct, + constructor: ForkGcConstructorKind.Struct, + flags: 0, + scalarLengthOrStride: 0, + fields: [{ + storage: 8, + flags: + FORK_GC_FIELD_REFERENCE + | FORK_GC_FIELD_ALLOCATION_DEPENDENCY, + scalarOffset: null, + referenceOrdinal: 0, + }], + superTypeOrdinal: null, + baseLayoutId: 1, + auxiliary: 0, + provenanceScalarLength: 0, + provenanceReferenceCount: 0, + }]); +} + +function typedProvider( + activationId: number, + descriptor: ForkGcCodecDescriptor, + transit: TestTransit, + events: string[], + beforeAllocate?: (recipeId: number) => void, +): ForkGcCodecProvider { + return { + activationId, + descriptor, + probe: () => 0n, + encodeSlot: () => 0, + allocate(recipeId) { + beforeAllocate?.(recipeId); + events.push(`allocate:${recipeId}`); + transit.publish(recipeId, Object.freeze({ activationId, recipeId })); + }, + fill(recipeId) { + events.push(`fill:${recipeId}`); + }, + publishExternref(recipeId, value) { + transit.publish(recipeId, value); + }, + }; +} + +function nullGraph(): ForkReferenceRecipeGraph { + return { + roots: [0], + nodes: [{ id: 0, node: { kind: "null" } }], + }; +} + +describe("early child reference provider", () => { + it("materializes funcref, externref, and static-root recipes once", () => { + const callback = (() => 73) as CallableFunction; + const token = Object.freeze({ token: "child" }); + const staticRoot = Object.freeze({ root: "fresh activation" }); + const graph: ForkReferenceRecipeGraph = { + roots: [0, 1, 2, 3], + nodes: [ + { id: 0, node: { kind: "null" } }, + { + id: 1, + node: { + kind: "funcref", + moduleActivation: 1, + functionOrdinal: 4, + }, + }, + { id: 2, node: { kind: "externref", handle: 91 } }, + { + id: 3, + node: { + kind: "static-root", + moduleActivation: 2, + staticRootOrdinal: 7, + }, + }, + ], + }; + const refs = externrefs(new Map([[91, token]])); + const transit = new TestTransit(); + const options = providerOptions( + recordsFor(graph, [1, 2]), + [{ activationId: 1 }, { activationId: 2 }], + refs.provider, + transit, + ); + const provider = new ForkEarlyChildReferenceProvider(options); + const functionReads: number[] = []; + const staticReads: number[] = []; + provider.registerActivation({ + activationId: 1, + functions: { + decode(ordinal) { + functionReads.push(ordinal); + return callback; + }, + }, + }); + provider.registerActivation({ + activationId: 2, + staticRoots: { + decode(ordinal) { + staticReads.push(ordinal); + return staticRoot; + }, + }, + }); + + expect(provider.ownerActivation( + 1, + WPK_FORK_MODULE_STATE_GLOBAL_TYPE_FUNCREF, + )).toBe(1); + expect(provider.activationDependencies( + 2, + WPK_FORK_MODULE_STATE_GLOBAL_TYPE_EXTERNREF, + )).toEqual([]); + expect(provider.activationDependencies( + 3, + WPK_FORK_MODULE_STATE_GLOBAL_TYPE_ANYREF, + )).toEqual([2]); + + expect(provider.materialize( + 1, + WPK_FORK_MODULE_STATE_GLOBAL_TYPE_FUNCREF, + )).toBe(callback); + expect(provider.materialize( + 1, + WPK_FORK_MODULE_STATE_GLOBAL_TYPE_EXTERNREF, + )).toBe(callback); + expect(provider.materialize( + 2, + WPK_FORK_MODULE_STATE_GLOBAL_TYPE_EXTERNREF, + )).toBe(token); + expect(provider.materialize( + 2, + WPK_FORK_MODULE_STATE_GLOBAL_TYPE_EXTERNREF, + )).toBe(token); + expect(provider.materialize( + 3, + WPK_FORK_MODULE_STATE_GLOBAL_TYPE_ANYREF, + )).toBe(staticRoot); + expect(transit.read(3)).toBe(staticRoot); + expect(transit.prepared).toEqual([3]); + expect(functionReads).toEqual([4]); + expect(refs.materializations).toEqual([91]); + expect(staticReads).toEqual([7]); + }); + + it("publishes a static root before an immutable GC constructor consumes it", () => { + const descriptor = immutableReferenceStructDescriptor(); + const staticRoot = Object.freeze({ root: "fresh activation" }); + const graph: ForkReferenceRecipeGraph = { + roots: [0, 1, 2], + nodes: [ + { id: 0, node: { kind: "null" } }, + { + id: 1, + node: { + kind: "static-root", + moduleActivation: 1, + staticRootOrdinal: 0, + }, + }, + { + id: 2, + node: { + kind: "struct", + moduleActivation: 2, + typeOrdinal: 0, + layoutId: 1, + scalars: new Uint8Array(), + fields: [1], + }, + }, + ], + }; + const transit = new TestTransit(); + const options = providerOptions( + recordsFor(graph, [1, 2]), + [ + { activationId: 1 }, + { activationId: 2, gcDescriptor: descriptor }, + ], + externrefs(new Map()).provider, + transit, + ); + const events: string[] = []; + const provider = new ForkEarlyChildReferenceProvider(options); + provider.registerActivation({ + activationId: 1, + staticRoots: { + decode: () => staticRoot, + }, + }); + provider.registerActivation({ + activationId: 2, + typed: typedProvider( + 2, + descriptor, + transit, + events, + () => expect(transit.read(1)).toBe(staticRoot), + ), + }); + + expect(provider.activationDependencies( + 2, + WPK_FORK_MODULE_STATE_GLOBAL_TYPE_ANYREF, + )).toEqual([1, 2]); + expect(provider.materialize( + 2, + WPK_FORK_MODULE_STATE_GLOBAL_TYPE_ANYREF, + )).toBe(transit.read(2)); + expect(transit.read(1)).toBe(staticRoot); + expect(events).toEqual(["allocate:2", "fill:2"]); + }); + + it("publishes an owner token before an immutable GC constructor consumes it", () => { + const descriptor = immutableReferenceStructDescriptor(); + const token = Object.freeze({ token: "fresh child owner token" }); + const graph: ForkReferenceRecipeGraph = { + roots: [0, 1, 2], + nodes: [ + { id: 0, node: { kind: "null" } }, + { id: 1, node: { kind: "externref", handle: 71 } }, + { + id: 2, + node: { + kind: "struct", + moduleActivation: 2, + typeOrdinal: 0, + layoutId: 1, + scalars: new Uint8Array(), + fields: [1], + }, + }, + ], + }; + const transit = new TestTransit(); + const refs = externrefs(new Map([[71, token]])); + const options = providerOptions( + recordsFor(graph, [2]), + [{ activationId: 2, gcDescriptor: descriptor }], + refs.provider, + transit, + ); + const events: string[] = []; + const provider = new ForkEarlyChildReferenceProvider(options); + provider.registerActivation({ + activationId: 2, + typed: typedProvider( + 2, + descriptor, + transit, + events, + () => expect(transit.read(1)).toBe(token), + ), + }); + + expect(provider.materialize( + 2, + WPK_FORK_MODULE_STATE_GLOBAL_TYPE_ANYREF, + )).toBe(transit.read(2)); + expect(transit.read(1)).toBe(token); + expect(refs.materializations).toEqual([71]); + expect(events).toEqual(["allocate:2", "fill:2"]); + }); + + it("adopts cached undefined and typed identities without reconstructing them twice", () => { + const descriptor = mutableReferenceStructDescriptor(); + const graph: ForkReferenceRecipeGraph = { + roots: [0, 1, 2], + nodes: [ + { id: 0, node: { kind: "null" } }, + { id: 1, node: { kind: "externref", handle: 44 } }, + { + id: 2, + node: { + kind: "struct", + moduleActivation: 1, + typeOrdinal: 0, + layoutId: 1, + scalars: new Uint8Array(), + fields: [0], + }, + }, + ], + }; + const records = recordsFor(graph, [1]); + const refs = externrefs(new Map([[44, undefined]])); + const transit = new TestTransit(); + const options = providerOptions( + records, + [{ activationId: 1, gcDescriptor: descriptor }], + refs.provider, + transit, + ); + const events: string[] = []; + const typed = typedProvider(1, descriptor, transit, events); + const early = new ForkEarlyChildReferenceProvider(options); + early.registerActivation({ activationId: 1, typed }); + + expect(early.materialize( + 1, + WPK_FORK_MODULE_STATE_GLOBAL_TYPE_EXTERNREF, + )).toBeUndefined(); + const typedValue = early.materialize( + 2, + WPK_FORK_MODULE_STATE_GLOBAL_TYPE_ANYREF, + ); + expect(events).toEqual(["allocate:2", "fill:2"]); + + const typedOwner: ForkTypedReferenceReplayOwner = { + prepareTransit: () => {}, + publishTransit: () => {}, + publishExternref: (recipeId, value) => { + transit.publish(recipeId, value); + events.push(`publish-externref:${recipeId}`); + }, + provider: () => typed, + providers: () => [typed], + validateExceptionOwner: () => {}, + materializeException: () => {}, + }; + const transaction = new ForkReferenceTransaction( + new ForkFunctionCatalog(), + refs.provider, + options.memory, + options.allocateScratch, + options.deallocateScratch, + "adopted transaction", + undefined, + typedOwner, + ); + transaction.attachChild(options.transaction); + early.adoptInto(transaction); + + expect(transaction.decodeExternref(1)).toBeUndefined(); + transaction.materializeAllTyped(); + expect(events).toEqual([ + "allocate:2", + "fill:2", + "publish-externref:1", + ]); + expect(transit.read(2)).toBe(typedValue); + expect(refs.materializations).toEqual([44]); + transaction.finishReplay(); + expect(() => early.adoptInto(transaction)).toThrow("was adopted"); + }); + + it("allocates defaultable shells before filling cyclic and aliased GC edges", () => { + const descriptor = mutableReferenceStructDescriptor(); + const graph: ForkReferenceRecipeGraph = { + roots: [0, 1, 2], + nodes: [ + { id: 0, node: { kind: "null" } }, + { + id: 1, + node: { + kind: "struct", + moduleActivation: 1, + typeOrdinal: 0, + layoutId: 1, + scalars: new Uint8Array(), + fields: [2], + }, + }, + { + id: 2, + node: { + kind: "struct", + moduleActivation: 1, + typeOrdinal: 0, + layoutId: 1, + scalars: new Uint8Array(), + fields: [1], + }, + }, + ], + }; + const refs = externrefs(new Map()); + const transit = new TestTransit(); + const options = providerOptions( + recordsFor(graph, [1]), + [{ activationId: 1, gcDescriptor: descriptor }], + refs.provider, + transit, + ); + const events: string[] = []; + const provider = new ForkEarlyChildReferenceProvider(options); + provider.registerActivation({ + activationId: 1, + typed: typedProvider(1, descriptor, transit, events), + }); + + const first = provider.materialize( + 1, + WPK_FORK_MODULE_STATE_GLOBAL_TYPE_ANYREF, + ); + expect(provider.materialize( + 1, + WPK_FORK_MODULE_STATE_GLOBAL_TYPE_ANYREF, + )).toBe(first); + expect(events).toEqual([ + "allocate:1", + "allocate:2", + "fill:1", + "fill:2", + ]); + }); + + it("rejects an immutable constructor cycle and cleans partial replay roots", () => { + const descriptor = immutableReferenceStructDescriptor(); + const graph: ForkReferenceRecipeGraph = { + roots: [0, 1, 2], + nodes: [ + { id: 0, node: { kind: "null" } }, + { + id: 1, + node: { + kind: "struct", + moduleActivation: 1, + typeOrdinal: 0, + layoutId: 1, + scalars: new Uint8Array(), + fields: [2], + }, + }, + { + id: 2, + node: { + kind: "struct", + moduleActivation: 1, + typeOrdinal: 0, + layoutId: 1, + scalars: new Uint8Array(), + fields: [1], + }, + }, + ], + }; + const refs = externrefs(new Map()); + const transit = new TestTransit(); + const options = providerOptions( + recordsFor(graph, [1]), + [{ activationId: 1, gcDescriptor: descriptor }], + refs.provider, + transit, + ); + let activationAborts = 0; + const provider = new ForkEarlyChildReferenceProvider(options); + provider.registerActivation({ + activationId: 1, + typed: typedProvider(1, descriptor, transit, []), + abort: () => { activationAborts++; }, + }); + + expect(() => provider.materialize( + 1, + WPK_FORK_MODULE_STATE_GLOBAL_TYPE_ANYREF, + )).toThrow("unallocatable constructor cycle"); + expect(transit.aborts).toBe(1); + expect(activationAborts).toBe(1); + expect(() => provider.ownerActivation( + 1, + WPK_FORK_MODULE_STATE_GLOBAL_TYPE_ANYREF, + )).toThrow("was aborted"); + }); + + it("orders GC/exnref dependencies and serves generated replay callbacks", () => { + const descriptor = mutableReferenceStructDescriptor(); + const graph: ForkReferenceRecipeGraph = { + roots: [0, 1, 2, 3], + nodes: [ + { id: 0, node: { kind: "null" } }, + { + id: 1, + node: { + kind: "funcref", + moduleActivation: 1, + functionOrdinal: 0, + }, + }, + { + id: 2, + node: { + kind: "exnref", + moduleActivation: 2, + tagOrdinal: 0, + layoutId: 7, + scalars: Uint8Array.of(0xaa, 0xbb), + payloads: [1], + }, + }, + { + id: 3, + node: { + kind: "struct", + moduleActivation: 3, + typeOrdinal: 0, + layoutId: 1, + scalars: new Uint8Array(), + fields: [2], + }, + }, + ], + }; + const refs = externrefs(new Map()); + const transit = new TestTransit(); + const options = providerOptions( + recordsFor(graph, [1, 2, 3], [], [[1]]), + [ + { activationId: 1 }, + { + activationId: 2, + exceptionDescriptor: { + version: 1, + tags: [{ + tagOrdinal: 0, + layoutId: 7, + scalarByteLength: 2, + referenceCount: 1, + }], + }, + }, + { activationId: 3, gcDescriptor: descriptor }, + ], + refs.provider, + transit, + ); + const events: string[] = []; + const provider = new ForkEarlyChildReferenceProvider(options); + const callback = (() => 12) as CallableFunction; + provider.registerActivation({ + activationId: 1, + functions: { + decode(ordinal) { + events.push(`function:${ordinal}`); + return callback; + }, + }, + }); + provider.registerActivation({ + activationId: 2, + exceptions: { + throwSlot(): never { + throw new Error("unused exception slot"); + }, + throwRecipe(): never { + throw new Error("unused exception throw"); + }, + encodeIngress: () => 0, + materialize(recipeId) { + events.push(`exception:${recipeId}`); + expect(provider.routeException(recipeId, 2)).toBe(7); + expect(provider.exceptionCacheIndex(recipeId)).toBe(1); + const scratch = provider.reserveScratch(16); + new Uint8Array(options.memory.buffer, scratch, 16).fill(0xcc); + expect(provider.loadException( + recipeId, + 2, + 0, + 7, + scratch, + 2, + scratch + 4, + 1, + )).toBe(1); + expect([...new Uint8Array(options.memory.buffer, scratch, 2)]) + .toEqual([0xaa, 0xbb]); + expect(new DataView(options.memory.buffer).getUint32(scratch + 4, true)) + .toBe(1); + provider.releaseScratch(scratch, 16); + expect(new Uint8Array(options.memory.buffer, scratch, 16) + .every((byte) => byte === 0)).toBe(true); + }, + clear: () => {}, + abort: () => {}, + }, + }); + provider.registerActivation({ + activationId: 3, + typed: typedProvider( + 3, + descriptor, + transit, + events, + (recipeId) => { + expect(provider.routeGc(recipeId, 3)).toBe(1); + expect(provider.gcPayloadLength(recipeId, 3, 1)).toBe(0); + const scratch = provider.reserveScratch(1); + const vector = provider.loadGc( + recipeId, + 3, + 0, + 1, + 1, + scratch, + 0, + ); + expect(provider.getReferenceVector(vector, 0)).toBe(2); + provider.releaseScratch(scratch, 1); + }, + ), + }); + + expect(provider.ownerActivation( + 2, + WPK_FORK_MODULE_STATE_GLOBAL_TYPE_EXNREF, + )).toBe(2); + expect(provider.activationDependencies( + 3, + WPK_FORK_MODULE_STATE_GLOBAL_TYPE_ANYREF, + )).toEqual([1, 2, 3]); + expect(provider.getReferenceVector(1, 0)).toBe(1); + expect(() => provider.captureUnavailable("encode funcref")) + .toThrow("unavailable during pre-instantiation child replay"); + + provider.materialize( + 3, + WPK_FORK_MODULE_STATE_GLOBAL_TYPE_ANYREF, + ); + expect(events).toEqual([ + "allocate:3", + "function:0", + "exception:2", + "fill:3", + ]); + provider.abort(); + expect(options.scratch.deallocated).toEqual([ + { addr: 65_536, size: 65_536 }, + ]); + }); + + it("routes host-owned exception recipes through the canonical externref token", () => { + const hostException = Object.freeze({ host: "fresh child token" }); + const graph: ForkReferenceRecipeGraph = { + roots: [0, 1, 2], + nodes: [ + { id: 0, node: { kind: "null" } }, + { id: 1, node: { kind: "externref", handle: 73 } }, + { + id: 2, + node: { + kind: "exnref", + moduleActivation: FORK_HOST_EXCEPTION_ACTIVATION_ID, + tagOrdinal: 0, + layoutId: 0, + scalars: new Uint8Array(), + payloads: [1], + }, + }, + ], + }; + const refs = externrefs(new Map([[73, hostException]])); + const transit = new TestTransit(); + const options = providerOptions( + recordsFor(graph, [5]), + [{ + activationId: 5, + exceptionDescriptor: { version: 1, tags: [] }, + }], + refs.provider, + transit, + ); + const provider = new ForkEarlyChildReferenceProvider(options); + provider.registerActivation({ + activationId: 5, + exceptions: { + throwSlot(): never { + throw new Error("unused exception slot"); + }, + throwRecipe(): never { + throw new Error("unused exception throw"); + }, + encodeIngress: () => 0, + materialize: () => {}, + clear: () => {}, + abort: () => {}, + }, + }); + + expect(provider.activationDependencies( + 2, + WPK_FORK_MODULE_STATE_GLOBAL_TYPE_EXNREF, + )).toEqual([5]); + expect(provider.exceptionOwner(2)).toBe( + FORK_HOST_EXCEPTION_ACTIVATION_ID, + ); + expect(provider.materializeHostException(2)).toBe(hostException); + expect(provider.materializeHostException(2)).toBe(hostException); + expect(refs.materializations).toEqual([73]); + expect(() => provider.materializeHostException(1)) + .toThrow("is not an exception"); + }); + + it("rejects malformed recipe ownership and non-null raw exnref provenance", () => { + const missingOwner: ForkReferenceRecipeGraph = { + roots: [0, 1], + nodes: [ + { id: 0, node: { kind: "null" } }, + { + id: 1, + node: { + kind: "funcref", + moduleActivation: 9, + functionOrdinal: 0, + }, + }, + ], + }; + const refs = externrefs(new Map()); + const transit = new TestTransit(); + expect(() => new ForkEarlyChildReferenceProvider(providerOptions( + recordsFor(missingOwner, [1]), + [{ activationId: 1 }], + refs.provider, + transit, + ))).toThrow("names missing activation 9"); + + const exnGraph: ForkReferenceRecipeGraph = { + roots: [0, 1], + nodes: [ + { id: 0, node: { kind: "null" } }, + { + id: 1, + node: { + kind: "exnref", + moduleActivation: 1, + tagOrdinal: 0, + layoutId: 3, + scalars: new Uint8Array(), + payloads: [], + }, + }, + ], + }; + const rawExnBinding: ForkImportedGlobalBinding = { + consumerActivation: 1, + consumerOwner: 1, + sourceActivation: 0, + sourceOwner: 0, + reserved: 0, + recipeId: 1, + rawBits: 0n, + kind: ForkImportedGlobalBindingKind.RawReference, + mutable: false, + shared: false, + typeCode: WPK_FORK_MODULE_STATE_GLOBAL_TYPE_EXNREF, + }; + expect(() => new ForkEarlyChildReferenceProvider(providerOptions( + recordsFor(exnGraph, [1], [rawExnBinding]), + [{ + activationId: 1, + exceptionDescriptor: { + version: 1, + tags: [{ + tagOrdinal: 0, + layoutId: 3, + scalarByteLength: 0, + referenceCount: 0, + }], + }, + }], + refs.provider, + new TestTransit(), + ))).toThrow("non-null raw recipe"); + }); + + it("zeros live scratch and releases activation roots on abort", () => { + const refs = externrefs(new Map()); + const transit = new TestTransit(); + const options = providerOptions( + recordsFor(nullGraph(), [1]), + [{ activationId: 1 }], + refs.provider, + transit, + ); + let activationAborts = 0; + const provider = new ForkEarlyChildReferenceProvider(options); + provider.registerActivation({ + activationId: 1, + abort: () => { activationAborts++; }, + }); + const scratch = provider.reserveScratch(32); + new Uint8Array(options.memory.buffer, scratch, 32).fill(0x5a); + provider.abort(); + + expect(new Uint8Array(options.memory.buffer, scratch, 32) + .every((byte) => byte === 0)).toBe(true); + expect(options.scratch.deallocated).toEqual([ + { addr: 65_536, size: 65_536 }, + ]); + expect(activationAborts).toBe(1); + expect(transit.aborts).toBe(1); + provider.abort(); + expect(activationAborts).toBe(1); + expect(() => provider.materialize( + 0, + WPK_FORK_MODULE_STATE_GLOBAL_TYPE_EXTERNREF, + )).toThrow("was aborted"); + }); +}); diff --git a/host/test/fork-exception-provider.test.ts b/host/test/fork-exception-provider.test.ts new file mode 100644 index 0000000000..a07ea44319 --- /dev/null +++ b/host/test/fork-exception-provider.test.ts @@ -0,0 +1,336 @@ +import { describe, expect, it } from "vitest"; +import { + ForkActivationRegistry, + type ForkActivationExceptionProvider, + type ForkActivationRegistration, +} from "../src/fork-activation-registry"; +import { + ForkExceptionBroker, +} from "../src/fork-exception-provider"; +import { + ForkModuleStateArena, + ForkTableDirtyTracker, + writeForkModuleStateRoot, +} from "../src/fork-module-state"; +import { ForkExternrefProcessOwner } from "../src/fork-externref-process-owner"; +import { + ForkExternrefTokenCache, + ForkExternrefTokenRecipeProvider, +} from "../src/fork-reference-broker"; + +const PAGE_SIZE = 65_536; + +function emptyCatalog(): WebAssembly.Table { + return new WebAssembly.Table({ + element: "anyfunc", + initial: 0, + maximum: 0, + }); +} + +function registration( + activationId: number, + exceptionProvider: ForkActivationExceptionProvider, +): ForkActivationRegistration { + return { + activationId, + instance: { exports: {} } as unknown as WebAssembly.Instance, + templateId: new Uint8Array(32).fill(activationId + 1), + functionCatalog: emptyCatalog(), + staticRootCatalog: new WebAssembly.Table({ + element: "externref", + initial: 0, + maximum: 0, + }), + staticRootHarvest: () => {}, + moduleState: { + bootstrap: () => {}, + save: () => {}, + restore: () => {}, + finishRestore: () => {}, + saveTables: () => {}, + restoreTables: () => {}, + }, + exceptionProvider, + tableDirty: new ForkTableDirtyTracker(), + }; +} + +function activeRegistry( + providers: readonly [number, ForkActivationExceptionProvider][], +): { + registry: ForkActivationRegistry; + arena: ForkModuleStateArena; +} { + const memory = new WebAssembly.Memory({ initial: 16 }); + let next = PAGE_SIZE; + const allocate = (size: number): number => { + const address = next; + next += size; + return address; + }; + const registry = new ForkActivationRegistry( + memory, + { + capture: () => 41, + materialize: () => { + throw new Error("parent materialization should use captured identity"); + }, + }, + "exception broker test", + allocate, + () => {}, + ); + for (const [activationId, provider] of providers) { + registry.registerActivation(registration(activationId, provider)); + } + const arena = new ForkModuleStateArena( + memory, + 4, + allocate, + () => {}, + "exception broker arena", + ); + arena.begin(); + registry.beginCapture(arena); + return { registry, arena }; +} + +function provider(options: { + throwValue: unknown; + encodeIngress?: (token: number) => number; +}): ForkActivationExceptionProvider { + return { + throwSlot(): never { + throw options.throwValue; + }, + throwRecipe(): never { + throw options.throwValue; + }, + encodeIngress: options.encodeIngress ?? (() => 0), + clear: () => {}, + abort: () => {}, + }; +} + +describe("ForkExceptionBroker", () => { + it("probes providers in activation order and terminates nested unknown probes", () => { + const exception = Object.freeze({ exact: "exception" }); + const order: number[] = []; + let broker: ForkExceptionBroker; + const first = provider({ + throwValue: exception, + encodeIngress() { + order.push(1); + // This is the callback made by the candidate codec's CatchAllRef + // fallback. The broker recognizes the active identity and returns the + // explicit not-owned sentinel instead of recursing. + return broker.encodeFromSlot(1, 0); + }, + }); + const owner = provider({ + throwValue: exception, + encodeIngress() { + order.push(2); + return 17; + }, + }); + const source = provider({ throwValue: exception }); + const { registry } = activeRegistry([ + [2, owner], + [0, source], + [1, first], + ]); + broker = new ForkExceptionBroker(registry, "deterministic broker"); + + expect(broker.encodeFromSlot(0, 0)).toBe(17); + expect(order).toEqual([1, 2]); + registry.abort(); + }); + + it("owns raw JSTag-style values as recipes and rethrows parent identity", () => { + const exception = Object.freeze({ host: "error token" }); + const source = provider({ throwValue: exception }); + const { registry } = activeRegistry([[0, source]]); + const broker = new ForkExceptionBroker(registry, "host exception broker"); + const recipeId = broker.encodeFromSlot(0, 0); + expect(recipeId).toBe(1); + + registry.sealCapture(); + registry.beginParentReplay(); + let replayed: unknown; + try { + broker.throwRecipe(recipeId); + } catch (value) { + replayed = value; + } + expect(replayed).toBe(exception); + registry.finishReplay(); + }); + + it("keeps unclaimed object/primitive identity in the parent and tokenizes only the fresh child", () => { + const owner = new ForkExternrefProcessOwner(); + const parentGeneration = owner.startGeneration(101); + const rawException = new WebAssembly.Exception( + new WebAssembly.Tag({ parameters: ["i32"] }), + [73], + ); + const rawObject = Object.freeze({ + callback: () => 73, + }); + const rawValues: readonly unknown[] = [ + rawException, + rawObject, + -0, + ]; + const durableValues: readonly unknown[] = [ + Object.freeze({ opaqueWorkerException: true }), + Object.freeze({ opaqueWorkerObject: true }), + -0, + ]; + const handles = durableValues.map((value) => + owner.registerForWire( + parentGeneration.pid, + parentGeneration.id, + value, + ) + ); + const parentTokens = new ForkExternrefTokenCache(parentGeneration.id); + const parentTokensByValue = handles.map((handle) => + parentTokens.materialize(handle) + ); + + const memory = new WebAssembly.Memory({ initial: 16 }); + let next = PAGE_SIZE; + const allocate = (size: number): number => { + const address = next; + next += size; + return address; + }; + const parentRegistry = new ForkActivationRegistry( + memory, + new ForkExternrefTokenRecipeProvider(parentTokens), + "normalized JSTag parent", + allocate, + () => {}, + ); + const sourceProvider: ForkActivationExceptionProvider = { + throwSlot(slot): never { + if (!Number.isInteger(slot) || slot < 0 || slot >= rawValues.length) { + throw new Error(`invalid raw exception slot ${slot}`); + } + throw rawValues[slot]; + }, + throwRecipe(): never { + throw new Error("parent source provider does not decode host recipes"); + }, + encodeIngress: () => 0, + clear: () => {}, + abort: () => {}, + }; + parentRegistry.registerActivation( + registration(0, sourceProvider), + ); + const parentArena = new ForkModuleStateArena( + memory, + 4, + allocate, + () => {}, + "normalized JSTag parent arena", + ); + const root = parentArena.begin(); + parentRegistry.beginCapture(parentArena); + const parentBroker = new ForkExceptionBroker( + parentRegistry, + "normalized JSTag parent broker", + undefined, + (value) => { + const index = rawValues.findIndex((candidate) => + Object.is(candidate, value) + ); + if (index < 0) throw new Error("unknown raw host exception value"); + return parentTokensByValue[index]!; + }, + ); + const recipeIds = rawValues.map((_value, index) => + parentBroker.encodeFromSlot(0, index) + ); + parentRegistry.sealCapture(); + + const moduleBufferAddress = 0x1000; + writeForkModuleStateRoot(memory, moduleBufferAddress, 4, root); + const childGrant = owner.forkGenerationFromContinuation( + parentGeneration, + 102, + memory, + 4, + moduleBufferAddress, + ); + + const childMemory = new WebAssembly.Memory({ initial: 16 }); + new Uint8Array(childMemory.buffer).set(new Uint8Array(memory.buffer)); + + parentRegistry.beginParentReplay(); + for (let index = 0; index < recipeIds.length; index++) { + let parentReplay: unknown; + try { + parentBroker.throwRecipe(recipeIds[index]!); + } catch (value) { + parentReplay = value; + } + expect(Object.is(parentReplay, rawValues[index])).toBe(true); + } + parentRegistry.finishReplay(); + + const childArena = new ForkModuleStateArena( + childMemory, + 4, + () => { + throw new Error("fresh child arena must not allocate"); + }, + () => {}, + "normalized JSTag child arena", + ); + childArena.attach(root); + const childTokens = new ForkExternrefTokenCache(childGrant.generation.id); + const childRegistry = new ForkActivationRegistry( + childMemory, + new ForkExternrefTokenRecipeProvider(childTokens), + "normalized JSTag child", + ); + childRegistry.registerActivation(registration( + 0, + provider({ throwValue: null }), + )); + childRegistry.attachChild(childArena); + const childBroker = new ForkExceptionBroker( + childRegistry, + "normalized JSTag child broker", + ); + + for (let index = 0; index < recipeIds.length; index++) { + let childToken: unknown; + try { + childBroker.throwRecipe(recipeIds[index]!); + } catch (value) { + childToken = value; + } + expect(childToken).not.toBe(parentTokensByValue[index]); + expect(childTokens.encode(childToken)).toBe(handles[index]); + expect( + Object.is( + owner.authorizeForWire( + childGrant.generation.pid, + childGrant.generation.id, + handles[index]!, + ), + durableValues[index], + ), + ).toBe(true); + } + + childRegistry.finishReplay(); + owner.releaseGeneration(parentGeneration); + owner.releaseGeneration(childGrant.generation); + }); +}); diff --git a/host/test/fork-externref-host-parity.test.ts b/host/test/fork-externref-host-parity.test.ts new file mode 100644 index 0000000000..aaafcf8bc7 --- /dev/null +++ b/host/test/fork-externref-host-parity.test.ts @@ -0,0 +1,82 @@ +import { readFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { describe, expect, it } from "vitest"; + +const testDir = dirname(fileURLToPath(import.meta.url)); +const repoRoot = join(testDir, "..", ".."); + +function source(relativePath: string): string { + return readFileSync(join(repoRoot, relativePath), "utf8"); +} + +function functionSource( + text: string, + startName: string, + nextName: string, +): string { + const start = text.indexOf(startName); + const end = text.indexOf(nextName, start + startName.length); + expect(start, `missing ${startName}`).toBeGreaterThanOrEqual(0); + expect(end, `missing ${nextName} after ${startName}`).toBeGreaterThan(start); + return text.slice(start, end); +} + +describe.each([ + ["Node", "host/src/node-kernel-worker-entry.ts"], + ["browser", "host/src/browser-kernel-worker-entry.ts"], +])("%s externref process ownership", (_host, relativePath) => { + const entry = source(relativePath); + + it("replaces PID-stable authority only in the committed exec transition", () => { + const exec = functionSource( + entry, + "async function handleExec(", + "async function handlePosixSpawnResolve(", + ); + const commit = exec.indexOf("kernelWorker.prepareProcessForExec(pid)"); + const replace = exec.indexOf( + "externrefProcessOwner.replaceGeneration(", + commit, + ); + const replacementInit = exec.indexOf( + "externrefGenerationId: replacementExternrefGeneration.id", + replace, + ); + + expect(commit).toBeGreaterThanOrEqual(0); + expect(replace).toBeGreaterThan(commit); + expect(replacementInit).toBeGreaterThan(replace); + }); + + it("gives pthread Workers the main process image generation", () => { + const clone = functionSource( + entry, + "async function handleClone(", + "function handleThreadExit(", + ); + expect(clone).toContain( + "externrefGenerationId: processInfo.externrefGeneration.id", + ); + }); + + it("releases owner generations on exit, explicit terminate, and destroy", () => { + const release = + "externrefProcessOwner.releaseGeneration(info.externrefGeneration)"; + const terminateStart = relativePath.includes("browser") + ? "async function handleTerminateProcess(" + : "async function handleTerminate("; + const exit = functionSource( + entry, + "async function finishProcessExit(", + terminateStart, + ); + const destroyStart = "async function handleDestroy("; + const terminate = functionSource(entry, terminateStart, destroyStart); + const destroy = entry.slice(entry.indexOf(destroyStart)); + + expect(exit).toContain(release); + expect(terminate).toContain(release); + expect(destroy).toContain(release); + }); +}); diff --git a/host/test/fork-externref-import-mailbox.test.ts b/host/test/fork-externref-import-mailbox.test.ts new file mode 100644 index 0000000000..dbbe6af703 --- /dev/null +++ b/host/test/fork-externref-import-mailbox.test.ts @@ -0,0 +1,754 @@ +import { Worker } from "node:worker_threads"; +import { describe, expect, it } from "vitest"; +import { + createForkExternrefImportMailbox, + defineForkExternrefImport, + forkExternrefImportMailboxBytes, + ForkExternrefImportClosedError, + ForkExternrefImportFailureCode, + type ForkExternrefImportBinding, + type ForkExternrefImportDescriptor, + type ForkExternrefImportHandler, + type ForkExternrefImportValueType, + ForkExternrefImportOwnerCatalog, + ForkExternrefImportOwnerEndpoint, + ForkExternrefImportRemoteFailure, + type ForkExternrefImportWake, + ForkExternrefImportWorkerCaller, +} from "../src/fork-externref-import-mailbox"; +import { + ForkExternrefBroker, + type ForkExternrefGeneration, + ForkExternrefTokenCache, +} from "../src/fork-reference-broker"; + +class TestAuthority { + constructor( + readonly broker: ForkExternrefBroker, + readonly generation: ForkExternrefGeneration, + ) {} + + authorizeForWire( + pid: number, + generationId: number, + handle: number, + ): unknown { + this.assertBinding(pid, generationId); + return this.broker.authorize(this.generation, handle); + } + + registerForWire( + pid: number, + generationId: number, + value: unknown, + ): number { + this.assertBinding(pid, generationId); + return this.broker.register(this.generation, value); + } + + private assertBinding(pid: number, generationId: number): void { + if ( + pid !== this.generation.pid + || generationId !== this.generation.id + ) { + throw new Error( + `stale test authority pid=${pid} generation=${generationId}`, + ); + } + } +} + +interface Harness { + readonly broker: ForkExternrefBroker; + readonly generation: ForkExternrefGeneration; + readonly tokens: ForkExternrefTokenCache; + readonly binding: ForkExternrefImportBinding; + readonly mailbox: SharedArrayBuffer; + readonly catalog: ForkExternrefImportOwnerCatalog; + readonly endpoint: ForkExternrefImportOwnerEndpoint; + readonly caller: ForkExternrefImportWorkerCaller; + readonly wakes: ForkExternrefImportWake[]; +} + +function harness( + registrations: readonly [ + ForkExternrefImportDescriptor, + ForkExternrefImportHandler, + ][], + options: { + readonly authorizeSender?: ( + binding: ForkExternrefImportBinding, + ) => void; + readonly notify?: ( + wake: ForkExternrefImportWake, + endpoint: ForkExternrefImportOwnerEndpoint, + binding: ForkExternrefImportBinding, + ) => void; + readonly diagnostics?: Array<{ + error: unknown; + failure: ForkExternrefImportFailureCode; + }>; + readonly onDiagnostic?: ( + error: unknown, + failure: ForkExternrefImportFailureCode, + ) => void; + } = {}, +): Harness { + const broker = new ForkExternrefBroker(); + const generation = broker.createGeneration(101); + const tokens = new ForkExternrefTokenCache(generation.id); + const binding: ForkExternrefImportBinding = { + pid: generation.pid, + generationId: generation.id, + senderId: 17, + }; + const catalog = new ForkExternrefImportOwnerCatalog(); + for (const [descriptor, handler] of registrations) { + catalog.register(descriptor, handler); + } + const mailbox = createForkExternrefImportMailbox( + catalog.mailboxCapacity, + ); + const authority = new TestAuthority(broker, generation); + const endpoint = new ForkExternrefImportOwnerEndpoint( + mailbox, + binding, + catalog, + authority, + { + authorizeSender: options.authorizeSender ?? (() => {}), + onDiagnostic: (error, failure) => { + options.diagnostics?.push({ error, failure }); + options.onDiagnostic?.(error, failure); + }, + }, + ); + const wakes: ForkExternrefImportWake[] = []; + const caller = new ForkExternrefImportWorkerCaller( + mailbox, + binding, + tokens, + (wake) => { + wakes.push(wake); + if (options.notify) { + options.notify(wake, endpoint, binding); + } else if (!endpoint.dispatch(wake, binding)) { + throw new Error("test owner did not claim current wake"); + } + }, + ); + return { + broker, + generation, + tokens, + binding, + mailbox, + catalog, + endpoint, + caller, + wakes, + }; +} + +describe("fork externref host-import mailbox", () => { + it("allocates one catalog-sized mailbox per Worker", () => { + const empty = new ForkExternrefImportOwnerCatalog(); + const mailbox = createForkExternrefImportMailbox( + empty.mailboxCapacity, + ); + expect(mailbox).toBeInstanceOf(SharedArrayBuffer); + expect(mailbox.byteLength).toBe( + forkExternrefImportMailboxBytes({ params: 0, results: 0 }), + ); + expect(mailbox.byteLength).toBe(72); + + expect(() => + defineForkExternrefImport( + 1, + Array(257).fill("i32"), + [], + ) + ).not.toThrow(); + expect(() => + defineForkExternrefImport(1, ["v128" as never], []) + ).toThrow(/unsupported fork externref import value type v128/); + }); + + it("rejects forged capacity metadata without allocating from it", () => { + const mailbox = createForkExternrefImportMailbox({ + params: 0, + results: 0, + }); + // Header word 12 is the declared parameter capacity. A Worker receives an + // already allocated SAB; it validates this count against byteLength and + // never allocates storage based on the untrusted word. + new DataView(mailbox).setUint32(12 * 4, 0xffff_ffff, true); + const generation = new ForkExternrefBroker().createGeneration(102); + const tokens = new ForkExternrefTokenCache(generation.id); + + expect(() => + new ForkExternrefImportWorkerCaller( + mailbox, + { + pid: generation.pid, + generationId: generation.id, + senderId: 18, + }, + tokens, + () => {}, + ) + ).toThrow(/declared capacity requires exactly/); + }); + + it("round-trips signatures wider than 16 with exact tail validation", () => { + const params = Array.from( + { length: 40 }, + (_, index): ForkExternrefImportValueType => + index === 31 ? "i64" : "i32", + ); + const results = Array.from( + { length: 24 }, + (_, index): ForkExternrefImportValueType => + index === 22 ? "i64" : "i32", + ); + const descriptor = defineForkExternrefImport(41, params, results); + const resultValues = results.map((type, index) => + type === "i64" ? BigInt(index) : index === 0 ? 0 : -index + ); + const state = harness([[ + descriptor, + (_context, ...args) => { + expect(args).toHaveLength(params.length); + expect(args[31]).toBe(31n); + return resultValues; + }, + ]]); + expect(state.mailbox.byteLength).toBe( + forkExternrefImportMailboxBytes({ params: 40, results: 24 }), + ); + + const args = params.map((type, index) => + type === "i64" ? BigInt(index) : index + ); + expect(state.caller.call(descriptor, args)).toEqual(resultValues); + + const mismatchedParams = [...params]; + mismatchedParams[31] = "f64"; + const mismatched = defineForkExternrefImport( + descriptor.ordinal, + mismatchedParams, + results, + ); + const mismatchedArgs = mismatchedParams.map((type, index) => + type === "i64" ? BigInt(index) : index + ); + try { + state.caller.call(mismatched, mismatchedArgs); + throw new Error("expected wide signature mismatch"); + } catch (error) { + expect(error).toBeInstanceOf(ForkExternrefImportRemoteFailure); + expect((error as ForkExternrefImportRemoteFailure).failureCode).toBe( + ForkExternrefImportFailureCode.Protocol, + ); + } + }); + + it("round-trips scalar bit patterns and owner-authorized externref aliases", () => { + const descriptor = defineForkExternrefImport( + 1, + ["i32", "i64", "f32", "f64", "externref"], + ["i64", "f32", "f64", "externref"], + ); + const realValue = { owner: true }; + const state = harness([ + [ + descriptor, + (_context, i32, i64, f32, f64, externref) => { + expect(i32).toBe(-17); + expect(i64).toBe(-0x7fff_ffff_ffff_ffffn); + expect(f32).toBe(Math.fround(1 / 3)); + expect(Object.is(f64, -0)).toBe(true); + expect(externref).toBe(realValue); + return [i64, f32, f64, externref]; + }, + ], + ]); + const handle = state.broker.register(state.generation, realValue); + const token = state.tokens.materialize(handle); + + const result = state.caller.call( + descriptor, + [-17, -0x7fff_ffff_ffff_ffffn, 1 / 3, -0, token], + ); + expect(result).toEqual([ + -0x7fff_ffff_ffff_ffffn, + Math.fround(1 / 3), + -0, + token, + ]); + expect(Object.is((result as unknown[])[2], -0)).toBe(true); + }); + + it("normalizes a scalar-only host import exception into a forkable token", () => { + const descriptor = defineForkExternrefImport( + 2, + ["i32"], + ["i32"], + ); + const ownerError = new Error("owner-only failure"); + const state = harness([ + [ + descriptor, + () => { + throw ownerError; + }, + ], + ]); + + let parentToken: unknown; + try { + state.caller.call(descriptor, [41]); + throw new Error("expected owner exception"); + } catch (error) { + parentToken = error; + } + const handle = state.tokens.encode(parentToken); + expect(handle).not.toBeNull(); + expect(state.broker.authorize(state.generation, handle!)).toBe(ownerError); + + // A fork child creates a different canonical token for the same leased + // owner handle. CatchAllRef can retain that child-local identity without + // consulting the parent's Worker or copying the Error through postMessage. + const child = state.broker.createGeneration(102); + state.broker.acquireFork(state.generation, child, [handle!]); + const childTokens = new ForkExternrefTokenCache(child.id); + const childToken = childTokens.materialize(handle!); + expect(childToken).not.toBe(parentToken); + expect(childTokens.encode(childToken)).toBe(handle); + expect(state.broker.authorize(child, handle!)).toBe(ownerError); + }); + + it("gives thrown null a nonzero owner handle instead of the null sentinel", () => { + const descriptor = defineForkExternrefImport(15, [], []); + const state = harness([[descriptor, () => { + throw null; + }]]); + + let token: unknown; + try { + state.caller.call(descriptor, []); + throw new Error("expected owner exception"); + } catch (error) { + token = error; + } + expect(token).not.toBeNull(); + const handle = state.tokens.encode(token); + expect(handle).not.toBeNull(); + expect(state.broker.authorize(state.generation, handle!)).toBeNull(); + }); + + it("routes only numeric wake metadata and rejects stale duplicate wakes", () => { + const descriptor = defineForkExternrefImport(3, ["i32"], ["i32"]); + let previous: ForkExternrefImportWake | undefined; + const state = harness( + [[descriptor, (_context, value) => (value as number) + 1]], + { + notify: (wake, endpoint, binding) => { + if (previous) { + expect(endpoint.dispatch(previous, binding)).toBe(false); + } + expect(endpoint.dispatch(wake, binding)).toBe(true); + previous = wake; + }, + }, + ); + expect(state.caller.call(descriptor, [1])).toBe(2); + expect(state.caller.call(descriptor, [2])).toBe(3); + for (const wake of state.wakes) { + expect(Object.values(wake).every((value) => typeof value === "number")) + .toBe(true); + } + expect(state.wakes[1]!.sequenceLow).toBeGreaterThan( + state.wakes[0]!.sequenceLow, + ); + }); + + it("requires independently observed exact sender identity", () => { + const descriptor = defineForkExternrefImport(4, [], ["i32"]); + let calls = 0; + const state = harness( + [[descriptor, () => ++calls]], + { + notify: (wake, endpoint, binding) => { + expect(endpoint.dispatch(wake, { + ...binding, + senderId: binding.senderId + 1, + })).toBe(false); + expect(endpoint.dispatch(wake, binding)).toBe(true); + }, + }, + ); + + expect(state.caller.call(descriptor, [])).toBe(1); + expect(calls).toBe(1); + }); + + it("fails a replaced sender generation before invoking its handler", () => { + const descriptor = defineForkExternrefImport(5, [], ["i32"]); + const diagnostics: Array<{ + error: unknown; + failure: ForkExternrefImportFailureCode; + }> = []; + let current = false; + let invoked = false; + const state = harness( + [[descriptor, () => { + invoked = true; + return 1; + }]], + { + authorizeSender: () => { + if (!current) throw new Error("process image was replaced"); + }, + diagnostics, + }, + ); + + expect(() => state.caller.call(descriptor, [])).toThrow( + ForkExternrefImportRemoteFailure, + ); + try { + state.caller.call(descriptor, []); + } catch (error) { + expect((error as ForkExternrefImportRemoteFailure).failureCode).toBe( + ForkExternrefImportFailureCode.Unauthorized, + ); + } + expect(invoked).toBe(false); + expect(diagnostics.at(-1)?.failure).toBe( + ForkExternrefImportFailureCode.Unauthorized, + ); + + current = true; + expect(state.caller.call(descriptor, [])).toBe(1); + }); + + it("matches an ordinal's complete signature instead of trusting a hash", () => { + const ownerDescriptor = defineForkExternrefImport( + 6, + ["i32", "f64"], + ["i32"], + ); + const mismatchedWorkerDescriptor = defineForkExternrefImport( + 6, + ["f32", "f64"], + ["i32"], + ); + let invoked = false; + const state = harness([ + [ownerDescriptor, () => { + invoked = true; + return 1; + }], + ]); + + try { + state.caller.call(mismatchedWorkerDescriptor, [1, 2]); + throw new Error("expected signature rejection"); + } catch (error) { + expect(error).toBeInstanceOf(ForkExternrefImportRemoteFailure); + expect((error as ForkExternrefImportRemoteFailure).failureCode).toBe( + ForkExternrefImportFailureCode.Protocol, + ); + } + expect(invoked).toBe(false); + }); + + it("rejects wrong-generation raw tokens before notifying the owner", () => { + const descriptor = defineForkExternrefImport( + 7, + ["externref"], + ["externref"], + ); + const state = harness([ + [descriptor, (_context, value) => value], + ]); + const foreignTokens = new ForkExternrefTokenCache( + state.generation.id + 1, + ); + + expect(() => + state.caller.call(descriptor, [foreignTokens.materialize(9)]) + ).toThrow(/did not come from this process-image owner/); + expect(state.wakes).toHaveLength(0); + + const value = { valid: true }; + const handle = state.broker.register(state.generation, value); + const token = state.tokens.materialize(handle); + expect(state.caller.call(descriptor, [token])).toBe(token); + }); + + it("rejects mailbox reentrancy rather than overwriting the live request", () => { + const descriptor = defineForkExternrefImport(8, ["i32"], ["i32"]); + let nestedError: unknown; + let bound: (...args: Parameters< + ForkExternrefImportWorkerCaller["call"] + >) => unknown; + const state = harness( + [[descriptor, (_context, value) => value]], + { + notify: (wake, endpoint, binding) => { + try { + state.caller.call(descriptor, [99]); + } catch (error) { + nestedError = error; + } + expect(endpoint.dispatch(wake, binding)).toBe(true); + }, + }, + ); + bound = state.caller.call.bind(state.caller); + expect(bound(descriptor, [17])).toBe(17); + expect(String(nestedError)).toMatch(/reentrant/); + }); + + it("lets main and side-module wrappers share one Worker mailbox", () => { + const main = defineForkExternrefImport(9, ["i32"], ["i32"]); + const side = defineForkExternrefImport(10, ["i64"], ["i64"]); + const state = harness([ + [main, (_context, value) => (value as number) + 1], + [side, (_context, value) => (value as bigint) + 1n], + ]); + const mainImport = state.caller.bind(main); + const sideImport = state.caller.bind(side); + + expect(mainImport(4)).toBe(5); + expect(sideImport(9n)).toBe(10n); + expect(state.caller.mailbox).toBe(state.mailbox); + expect(state.wakes).toHaveLength(2); + }); + + it("wakes a pending caller when process teardown closes the mailbox", () => { + const descriptor = defineForkExternrefImport(11, [], []); + const state = harness( + [[descriptor, () => undefined]], + { + notify: (_wake, endpoint) => { + endpoint.close(ForkExternrefImportFailureCode.Teardown); + }, + }, + ); + + try { + state.caller.call(descriptor, []); + throw new Error("expected closed mailbox"); + } catch (error) { + expect(error).toBeInstanceOf(ForkExternrefImportClosedError); + expect((error as ForkExternrefImportClosedError).reasonCode).toBe( + ForkExternrefImportFailureCode.Teardown, + ); + } + expect(() => state.caller.call(descriptor, [])).toThrow( + ForkExternrefImportClosedError, + ); + }); + + it("does not resurrect a mailbox closed during owner dispatch", () => { + const descriptor = defineForkExternrefImport(12, [], ["i32"]); + let endpoint: ForkExternrefImportOwnerEndpoint; + const state = harness([ + [descriptor, () => { + endpoint.close(); + return 42; + }], + ]); + endpoint = state.endpoint; + + expect(() => state.caller.call(descriptor, [])).toThrow( + ForkExternrefImportClosedError, + ); + }); + + it("does not let losing dispatch failure overwrite the teardown reason", () => { + const descriptor = defineForkExternrefImport(14, [], ["i64"]); + let endpoint: ForkExternrefImportOwnerEndpoint; + const state = harness([ + [descriptor, () => { + endpoint.close( + ForkExternrefImportFailureCode.NotificationFailure, + ); + // This invalid i64 result makes dispatch publish HandlerContract after + // close. The caller must still observe the independently owned close + // reason, not that losing completion. + return 42; + }], + ]); + endpoint = state.endpoint; + + try { + state.caller.call(descriptor, []); + throw new Error("expected closed mailbox"); + } catch (error) { + expect(error).toBeInstanceOf(ForkExternrefImportClosedError); + expect((error as ForkExternrefImportClosedError).reasonCode).toBe( + ForkExternrefImportFailureCode.NotificationFailure, + ); + } + }); + + it("reports handler result-shape failures without publishing partial data", () => { + const descriptor = defineForkExternrefImport(13, [], ["i64", "i32"]); + const diagnostics: Array<{ + error: unknown; + failure: ForkExternrefImportFailureCode; + }> = []; + const state = harness( + [[descriptor, () => [1n]]], + { diagnostics }, + ); + + try { + state.caller.call(descriptor, []); + throw new Error("expected handler contract failure"); + } catch (error) { + expect((error as ForkExternrefImportRemoteFailure).failureCode).toBe( + ForkExternrefImportFailureCode.HandlerContract, + ); + } + expect(diagnostics.at(-1)?.failure).toBe( + ForkExternrefImportFailureCode.HandlerContract, + ); + }); + + it("completes a failure even when the diagnostic observer throws", () => { + const descriptor = defineForkExternrefImport(16, [], ["i64"]); + const state = harness( + [[descriptor, () => 42]], + { + onDiagnostic: () => { + throw new Error("broken diagnostic sink"); + }, + }, + ); + + try { + state.caller.call(descriptor, []); + throw new Error("expected handler contract failure"); + } catch (error) { + expect(error).toBeInstanceOf(ForkExternrefImportRemoteFailure); + expect((error as ForkExternrefImportRemoteFailure).failureCode).toBe( + ForkExternrefImportFailureCode.HandlerContract, + ); + } + }); + + it("blocks a real Worker while the owner returns only scalar wire data", async () => { + const aliasDescriptor = defineForkExternrefImport( + 30, + ["externref", "i64"], + ["externref", "i64"], + ); + const throwingDescriptor = defineForkExternrefImport( + 31, + ["i32"], + ["i32"], + ); + const broker = new ForkExternrefBroker(); + const generation = broker.createGeneration(201); + const binding: ForkExternrefImportBinding = { + pid: generation.pid, + generationId: generation.id, + senderId: 29, + }; + const realValue = { ownerOnly: true }; + const ownerError = new Error("owner-only exception"); + const inputHandle = broker.register(generation, realValue); + const catalog = new ForkExternrefImportOwnerCatalog(); + catalog.register( + aliasDescriptor, + (_context, value, scalar) => [ + value, + (scalar as bigint) - 1n, + ], + ); + catalog.register(throwingDescriptor, () => { + throw ownerError; + }); + const mailbox = createForkExternrefImportMailbox( + catalog.mailboxCapacity, + ); + const endpoint = new ForkExternrefImportOwnerEndpoint( + mailbox, + binding, + catalog, + new TestAuthority(broker, generation), + { authorizeSender: () => {} }, + ); + const worker = new Worker( + new URL( + "./fixtures/fork-externref-import-worker.ts", + import.meta.url, + ), + { + execArgv: ["--import", "tsx"], + workerData: { mailbox, binding, inputHandle }, + }, + ); + + try { + const complete = new Promise<{ + resultHandle: number; + resultScalar: bigint; + exceptionHandle: number; + }>((resolve, reject) => { + worker.on("message", (message: { + type: string; + wake?: ForkExternrefImportWake; + resultHandle?: number; + resultScalar?: bigint; + exceptionHandle?: number; + message?: string; + }) => { + if (message.type === "wake") { + if (!endpoint.dispatch(message.wake!, binding)) { + reject(new Error("owner rejected current Worker wake")); + } + } else if (message.type === "complete") { + resolve({ + resultHandle: message.resultHandle!, + resultScalar: message.resultScalar!, + exceptionHandle: message.exceptionHandle!, + }); + } else if (message.type === "failed") { + reject(new Error(message.message)); + } + }); + worker.once("error", reject); + worker.once("exit", (code) => { + if (code !== 0) { + reject(new Error(`externref import Worker exited ${code}`)); + } + }); + }); + const timeout = new Promise((_, reject) => { + setTimeout( + () => reject(new Error("externref import Worker watchdog expired")), + 5_000, + ); + }); + const result = await Promise.race([complete, timeout]); + + expect(result.resultHandle).toBe(inputHandle); + expect(result.resultScalar).toBe(-10n); + expect(broker.authorize(generation, result.resultHandle)).toBe( + realValue, + ); + expect(broker.authorize(generation, result.exceptionHandle)).toBe( + ownerError, + ); + } finally { + endpoint.close(); + await worker.terminate(); + } + }, 8_000); +}); diff --git a/host/test/fork-externref-process-owner.test.ts b/host/test/fork-externref-process-owner.test.ts new file mode 100644 index 0000000000..d593aa6b11 --- /dev/null +++ b/host/test/fork-externref-process-owner.test.ts @@ -0,0 +1,182 @@ +import { describe, expect, it, vi } from "vitest"; +import { ForkExternrefProcessOwner } from "../src/fork-externref-process-owner"; +import { + ForkModuleStateArena, + ForkModuleStateRecordKind, + writeForkModuleStateRoot, +} from "../src/fork-module-state"; +import { + type ForkReferenceRecipeGraph, +} from "../src/fork-reference-recipes"; +import { FORK_REFERENCE_TRANSACTION_OWNER_ID } from "../src/fork-reference-transaction"; +import { + encodeSegmentedForkReferenceRecords, + PagedForkReferenceVector, +} from "../src/fork-reference-segments"; + +function copiedContinuation( + graph: ForkReferenceRecipeGraph, +): { + memory: WebAssembly.Memory; + moduleBufferAddress: number; +} { + const memory = new WebAssembly.Memory({ initial: 8 }); + let next = 0x2_0000; + const arena = new ForkModuleStateArena( + memory, + 4, + (size) => { + const address = next; + next += Math.ceil(Number(size) / 0x1_0000) * 0x1_0000; + return address; + }, + () => {}, + "externref owner test arena", + ); + const root = arena.begin(); + arena.appendModule({ + activationId: 0, + templateId: new Uint8Array(32).fill(0x71), + }); + for (const record of encodeSegmentedForkReferenceRecords( + FORK_REFERENCE_TRANSACTION_OWNER_ID, + graph.nodes, + [PagedForkReferenceVector.empty], + { segmentDataBytes: 17 }, + )) { + arena.appendRecord(record); + } + arena.seal(); + + const moduleBufferAddress = 0x1_0000; + writeForkModuleStateRoot(memory, moduleBufferAddress, 4, root); + return { memory, moduleBufferAddress }; +} + +function graphForHandles(handles: readonly number[]): ForkReferenceRecipeGraph { + return { + roots: [0, ...handles.map((_, index) => index + 1)], + nodes: [ + { id: 0, node: { kind: "null" } }, + ...handles.map((handle, index) => ({ + id: index + 1, + node: { kind: "externref" as const, handle }, + })), + ], + }; +} + +describe("ForkExternrefProcessOwner", () => { + it("leases each aliased handle once before a fresh child starts", () => { + const owner = new ForkExternrefProcessOwner(); + const parent = owner.startGeneration(41); + const value = { opaque: true }; + const handle = owner.registerForWire( + 41, + owner.generationId(parent), + value, + ); + const copied = copiedContinuation( + graphForHandles([handle, handle, handle]), + ); + + const grant = owner.forkGenerationFromContinuation( + parent, + 42, + copied.memory, + 4, + copied.moduleBufferAddress, + ); + expect(grant.handleCount).toBe(1); + expect( + owner.authorizeForWire(42, grant.generation.id, handle), + ).toBe(value); + + owner.releaseGeneration(parent); + expect( + owner.authorizeForWire(42, grant.generation.id, handle), + ).toBe(value); + owner.releaseGeneration(grant.generation); + expect(() => + owner.authorizeForWire(42, grant.generation.id, handle) + ).toThrow("stale"); + }); + + it("retires PID-stable authority exactly when exec replaces an image", () => { + const owner = new ForkExternrefProcessOwner(); + const beforeExec = owner.startGeneration(51); + const beforeId = owner.generationId(beforeExec); + const handle = owner.registerForWire(51, beforeId, Symbol("old image")); + + const afterExec = owner.replaceGeneration(beforeExec); + expect(afterExec.pid).toBe(51); + expect(afterExec.id).not.toBe(beforeId); + expect(() => owner.authorizeForWire(51, beforeId, handle)).toThrow( + "stale", + ); + expect(() => + owner.authorizeForWire(51, afterExec.id, handle) + ).toThrow("retired"); + }); + + it("rolls back a provisional child generation when its graph is not owned", () => { + const owner = new ForkExternrefProcessOwner(); + const parent = owner.startGeneration(61); + const copied = copiedContinuation(graphForHandles([900])); + + expect(() => + owner.forkGenerationFromContinuation( + parent, + 62, + copied.memory, + 4, + copied.moduleBufferAddress, + ) + ).toThrow("unknown externref handle"); + + // A failed grant leaves no hidden child generation behind. + expect(owner.startGeneration(62).pid).toBe(62); + }); + + it("uses one process generation for main and pthread import adapters", () => { + const owner = new ForkExternrefProcessOwner(); + const generation = owner.startGeneration(71); + const idForMainWorker = owner.generationId(generation); + const idForPthreadWorker = owner.generationId(generation); + const handle = owner.registerForWire(71, idForMainWorker, "shared"); + + expect( + owner.authorizeForWire(71, idForPthreadWorker, handle), + ).toBe("shared"); + }); + + it("does not adopt or copy the complete module-state arena to grant a lease", () => { + const owner = new ForkExternrefProcessOwner(); + const parent = owner.startGeneration(81); + const handle = owner.registerForWire(81, parent.id, { opaque: true }); + const copied = copiedContinuation(graphForHandles([handle])); + const attach = vi.spyOn(ForkModuleStateArena.prototype, "attach") + .mockImplementation(() => { + throw new Error("full arena attachment is not allowed in the grant path"); + }); + const records = vi.spyOn(ForkModuleStateArena.prototype, "records") + .mockImplementation(() => { + throw new Error("full arena copying is not allowed in the grant path"); + }); + try { + const grant = owner.forkGenerationFromContinuation( + parent, + 82, + copied.memory, + 4, + copied.moduleBufferAddress, + ); + expect(grant.handleCount).toBe(1); + owner.releaseGeneration(grant.generation); + } finally { + attach.mockRestore(); + records.mockRestore(); + owner.releaseGeneration(parent); + } + }); +}); diff --git a/host/test/fork-from-dlopen-side-module-e2e.test.ts b/host/test/fork-from-dlopen-side-module-e2e.test.ts index aae67a2daa..e55dcb9cb6 100644 --- a/host/test/fork-from-dlopen-side-module-e2e.test.ts +++ b/host/test/fork-from-dlopen-side-module-e2e.test.ts @@ -14,6 +14,7 @@ import { NodePlatformIO } from "../src/platform/node"; import { FORK_CAP_DYLINK_MAIN, FORK_CAP_SIDE_ENTRY, + parseDylinkSection, readForkInstrumentCapabilities, } from "../src/dylink"; import { runCentralizedProgram } from "./centralized-test-helper"; @@ -57,17 +58,26 @@ function instrumentInPlace(wasmPath: string, entry?: string): void { renameSync(output, wasmPath); } -function buildSharedLibrary(source: string): string { - const sourcePath = join(buildDir, "libforkinside.c"); - const objectPath = join(buildDir, "libforkinside.o"); - const libraryPath = join(buildDir, "libforkinside.so"); - writeFileSync(sourcePath, source); +function buildSharedLibrary( + source: string, + name = "libforkinside", + dependencies: readonly string[] = [], +): string { + const sourcePath = join(buildDir, `${name}.c`); + const objectPath = join(buildDir, `${name}.o`); + const libraryPath = join(buildDir, `${name}.so`); + writeFileSync(sourcePath, `${source} + #include "abi_constants.h" + __attribute__((export_name("__abi_version"))) + unsigned __abi_version(void) { return WASM_POSIX_ABI_VERSION; } + `); execFileSync(llvmTool("clang"), [ "--target=wasm32-unknown-unknown", "-fPIC", "-O2", "-matomics", "-mbulk-memory", + `-I${glueDir}`, "-c", sourcePath, "-o", @@ -82,6 +92,7 @@ function buildSharedLibrary(source: string): string { "-o", libraryPath, objectPath, + ...(dependencies.length === 0 ? [] : ["--Bdynamic", ...dependencies]), ], { stdio: "pipe" }); instrumentInPlace(libraryPath, "env.fork"); return libraryPath; @@ -147,21 +158,31 @@ describe.skipIf(!hasPrerequisites)("fork from a dlopened side module", () => { `); const programPath = buildMainProgram(` #include + #include #include #include + #include typedef int (*side_fork_fn)(void); int main(int argc, char **argv) { void *lib = dlopen(argv[1], RTLD_NOW); - if (!lib) return 2; + if (!lib) { + fprintf(stderr, "dlopen failed: %s\\n", dlerror()); + return 2; + } side_fork_fn side_fork = (side_fork_fn)dlsym(lib, "side_fork"); if (!side_fork) return 3; for (int i = 0; i < 2; i++) { int pid = side_fork(); if (pid < 0) return 4; + if (pid == 0) { + if (dlclose(lib) != 0) exit(7); + exit(0); + } int status = 0; if (waitpid(pid, &status, 0) != pid) return 5; if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) return 6; } + if (dlclose(lib) != 0) return 8; puts("side fork ok"); return 0; } @@ -191,4 +212,215 @@ describe.skipIf(!hasPrerequisites)("fork from a dlopened side module", () => { expect(result.exitCode, `stderr:\n${result.stderr}`).toBe(0); expect(result.stdout).toContain("side fork ok"); }, 30_000); + + it("replays a fork issued while dlopen runs a side-module constructor", async () => { + const libraryPath = buildSharedLibrary(` + extern int fork(void); + extern void exit(int); + static int constructor_child = -1; + __attribute__((constructor)) + static void fork_during_constructor(void) { + volatile int preserved = 73; + int pid = fork(); + if (preserved != 73) exit(92); + if (pid == 0) exit(0); + constructor_child = pid; + } + int constructor_child_pid(void) { + return constructor_child; + } + `); + const programPath = buildMainProgram(` + #include + #include + #include + typedef int (*constructor_child_pid_fn)(void); + int main(int argc, char **argv) { + void *lib = dlopen(argv[1], RTLD_NOW); + if (!lib) { + fprintf(stderr, "constructor dlopen failed: %s\\n", dlerror()); + return 2; + } + constructor_child_pid_fn child_pid = + (constructor_child_pid_fn)dlsym(lib, "constructor_child_pid"); + if (!child_pid) return 3; + int pid = child_pid(); + if (pid <= 0) return 4; + int status = 0; + if (waitpid(pid, &status, 0) != pid) return 5; + if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) return 6; + puts("constructor fork ok"); + return 0; + } + `); + + const result = await runCentralizedProgram({ + programPath, + argv: ["fork-from-constructor-main", libraryPath], + timeout: 30_000, + io: new NodePlatformIO(), + }); + expect(result.exitCode, `stderr:\n${result.stderr}`).toBe(0); + expect(result.stdout).toContain("constructor fork ok"); + }, 30_000); + + it("lowers the original two-argument loader before a constructor can fork", async () => { + const libraryPath = buildSharedLibrary(` + extern int fork(void); + extern void exit(int); + static int constructor_child = -1; + __attribute__((constructor)) + static void fork_during_legacy_load(void) { + volatile int preserved = 89; + int pid = fork(); + if (preserved != 89) exit(93); + if (pid == 0) exit(0); + constructor_child = pid; + } + int legacy_constructor_child_pid(void) { + return constructor_child; + } + `, "liblegacy-constructor-fork"); + const programPath = buildMainProgram(` + #include + #include + #include + #include + #include + #include + #include + #include + + __attribute__((import_module("env"), import_name("__wasm_dlopen"))) + extern int legacy_host_dlopen(const void *, int); + + static int legacy_open(const char *path) { + struct stat st; + if (stat(path, &st) != 0 || st.st_size <= 0) return 0; + int fd = open(path, O_RDONLY); + if (fd < 0) return 0; + void *bytes = malloc((size_t)st.st_size); + if (!bytes) { + close(fd); + return 0; + } + ssize_t total = 0; + while (total < st.st_size) { + ssize_t count = read( + fd, (char *)bytes + total, (size_t)(st.st_size - total)); + if (count <= 0) break; + total += count; + } + close(fd); + int handle = total == st.st_size + ? legacy_host_dlopen(bytes, (int)st.st_size) + : 0; + free(bytes); + return handle; + } + + typedef int (*child_pid_fn)(void); + int main(int argc, char **argv) { + int handle = legacy_open(argv[1]); + if (handle <= 0) { + fprintf(stderr, "legacy loader failed: %s\\n", dlerror()); + return 2; + } + child_pid_fn child_pid = (child_pid_fn)dlsym( + (void *)(long)handle, "legacy_constructor_child_pid"); + if (!child_pid) return 3; + int pid = child_pid(); + if (pid <= 0) return 4; + int status = 0; + if (waitpid(pid, &status, 0) != pid) return 5; + if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) return 6; + puts("lowered legacy constructor fork ok"); + return 0; + } + `); + + const programModule = new WebAssembly.Module( + new Uint8Array(readFileSync(programPath)) as unknown as BufferSource, + ); + const imports = WebAssembly.Module.imports(programModule); + expect(imports.some( + (entry) => entry.module === "env" && entry.name === "__wasm_dlopen", + )).toBe(false); + expect(imports.some( + (entry) => entry.module === "env" + && entry.name === "__wasm_dlopen_prepare", + )).toBe(true); + expect(WebAssembly.Module.exports(programModule).map((entry) => entry.name)) + .toContain("__wasm_posix_signal_checkpoint"); + + const result = await runCentralizedProgram({ + programPath, + argv: ["legacy-constructor-main", libraryPath], + timeout: 30_000, + io: new NodePlatformIO(), + }); + expect(result.exitCode, `stderr:\n${result.stderr}`).toBe(0); + expect(result.stdout).toContain("lowered legacy constructor fork ok"); + }, 30_000); + + it("resolves and replays a real DT_NEEDED closure from the process VFS", async () => { + const providerPath = buildSharedLibrary(` + int dependency_value(void) { + return 41; + } + `, "libneeded-provider"); + const consumerPath = buildSharedLibrary(` + extern int dependency_value(void); + int needed_value(void) { + return dependency_value() + 1; + } + `, "libneeded-consumer", [providerPath]); + const consumerBytes = new Uint8Array(readFileSync(consumerPath)); + const metadata = parseDylinkSection(consumerBytes); + expect(metadata?.neededDynlibs.some( + (dependency) => dependency.endsWith("libneeded-provider.so"), + )).toBe(true); + + const programPath = buildMainProgram(` + #include + #include + #include + #include + #include + typedef int (*needed_value_fn)(void); + int main(int argc, char **argv) { + void *lib = dlopen(argv[1], RTLD_NOW | RTLD_LOCAL); + if (!lib) { + fprintf(stderr, "needed dlopen failed: %s\\n", dlerror()); + return 2; + } + needed_value_fn needed_value = + (needed_value_fn)dlsym(lib, "needed_value"); + if (!needed_value || needed_value() != 42) return 3; + int pid = fork(); + if (pid < 0) return 4; + if (pid == 0) { + if (needed_value() != 42) exit(5); + if (dlclose(lib) != 0) exit(6); + exit(0); + } + int status = 0; + if (waitpid(pid, &status, 0) != pid) return 7; + if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) return 8; + if (needed_value() != 42) return 9; + if (dlclose(lib) != 0) return 10; + puts("needed fork ok"); + return 0; + } + `); + + const result = await runCentralizedProgram({ + programPath, + argv: ["fork-needed-main", consumerPath], + timeout: 30_000, + io: new NodePlatformIO(), + }); + expect(result.exitCode, `stderr:\n${result.stderr}`).toBe(0); + expect(result.stdout).toContain("needed fork ok"); + }, 30_000); }); diff --git a/host/test/fork-from-thread.test.ts b/host/test/fork-from-thread.test.ts index 63ee648f48..1a26c7f7f6 100644 --- a/host/test/fork-from-thread.test.ts +++ b/host/test/fork-from-thread.test.ts @@ -33,6 +33,10 @@ describe("fork-from-non-main-thread", () => { programPath: forkFromThreadBinary!, argv: ["fork-from-thread"], timeout: 15_000, + // This fixture exercises worker/continuation ownership only and never + // touches the VFS. Keep the pthread fork proof independent of the + // separately versioned rootfs package rebuild. + useDefaultRootfs: false, }); expect(result.exitCode, `stderr=${result.stderr}\nstdout=${result.stdout}`).toBe(0); @@ -65,6 +69,7 @@ describe("fork-from-non-main-thread", () => { programPath: concurrentForkBinary!, argv: ["fork-from-concurrent-threads"], timeout: 60_000, + useDefaultRootfs: false, }); expect(result.exitCode, `stderr=${result.stderr}\nstdout=${result.stdout}`).toBe(0); diff --git a/host/test/fork-function-catalog.test.ts b/host/test/fork-function-catalog.test.ts new file mode 100644 index 0000000000..99313ca73f --- /dev/null +++ b/host/test/fork-function-catalog.test.ts @@ -0,0 +1,149 @@ +import { execFileSync } from "node:child_process"; +import { mkdtempSync, readFileSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; +import { ForkFunctionCatalog } from "../src/fork-function-catalog"; + +function catalogModule(): WebAssembly.Module { + const dir = mkdtempSync(join(tmpdir(), "kandelo-funcref-catalog-")); + const wat = join(dir, "catalog.wat"); + const wasm = join(dir, "catalog.wasm"); + writeFileSync(wat, `(module + (table $catalog (export "__wpk_fork_function_catalog") 2 2 funcref) + (func $first (result i32) i32.const 17) + (func $second (result i32) i32.const 29) + (elem (table $catalog) (i32.const 0) func $first $second) + )`); + execFileSync("wat2wasm", [wat, "-o", wasm]); + return new WebAssembly.Module(readFileSync(wasm)); +} + +describe("ForkFunctionCatalog", () => { + it("reconstructs the same logical function from a fresh module instance", () => { + const module = catalogModule(); + const parentInstance = new WebAssembly.Instance(module); + const childInstance = new WebAssembly.Instance(module); + const parentTable = parentInstance.exports.__wpk_fork_function_catalog as WebAssembly.Table; + const childTable = childInstance.exports.__wpk_fork_function_catalog as WebAssembly.Table; + + const parent = new ForkFunctionCatalog(); + parent.register(0, parentTable); + const recipe = parent.encode(parentTable.get(1)); + expect(recipe).toEqual({ moduleActivation: 0, ordinal: 1 }); + + const child = new ForkFunctionCatalog(); + child.register(0, childTable); + const reconstructed = child.decode(recipe); + expect(reconstructed).not.toBe(parentTable.get(1)); + expect(reconstructed).toBe(childTable.get(1)); + expect((reconstructed as () => number)()).toBe(29); + }); + + it("keeps side-module activation identities distinct", () => { + const module = catalogModule(); + const mainTable = new WebAssembly.Instance(module).exports + .__wpk_fork_function_catalog as WebAssembly.Table; + const sideTable = new WebAssembly.Instance(module).exports + .__wpk_fork_function_catalog as WebAssembly.Table; + const catalog = new ForkFunctionCatalog(); + catalog.register(0, mainTable); + catalog.register(7, sideTable); + + expect(catalog.encode(mainTable.get(0))).toEqual({ + moduleActivation: 0, + ordinal: 0, + }); + expect(catalog.encode(sideTable.get(0))).toEqual({ + moduleActivation: 7, + ordinal: 0, + }); + }); + + it("reconstructs a side-module funcref written into the shared process table", () => { + const module = catalogModule(); + const parentMain = new WebAssembly.Instance(module); + const parentSide = new WebAssembly.Instance(module); + const parentSideCatalog = parentSide.exports + .__wpk_fork_function_catalog as WebAssembly.Table; + const processTable = new WebAssembly.Table({ + element: "anyfunc", + initial: 1, + maximum: 1, + }); + processTable.set(0, parentSideCatalog.get(1)); + + const parent = new ForkFunctionCatalog(); + parent.register( + 0, + parentMain.exports.__wpk_fork_function_catalog as WebAssembly.Table, + ); + parent.register(9, parentSideCatalog); + const recipe = parent.encode(processTable.get(0)); + expect(recipe).toEqual({ moduleActivation: 9, ordinal: 1 }); + + const childMain = new WebAssembly.Instance(module); + const childSide = new WebAssembly.Instance(module); + const childSideCatalog = childSide.exports + .__wpk_fork_function_catalog as WebAssembly.Table; + const child = new ForkFunctionCatalog(); + child.register( + 0, + childMain.exports.__wpk_fork_function_catalog as WebAssembly.Table, + ); + child.register(9, childSideCatalog); + const reconstructed = child.decode(recipe); + expect(reconstructed).toBe(childSideCatalog.get(1)); + expect(reconstructed).not.toBe(processTable.get(0)); + expect((reconstructed as () => number)()).toBe(29); + }); + + it("rejects an unregistered foreign-instance function instead of encoding the wrong module", () => { + const module = catalogModule(); + const first = new WebAssembly.Instance(module); + const second = new WebAssembly.Instance(module); + const catalog = new ForkFunctionCatalog(); + catalog.register( + 0, + first.exports.__wpk_fork_function_catalog as WebAssembly.Table, + ); + const foreign = ( + second.exports.__wpk_fork_function_catalog as WebAssembly.Table + ).get(0); + expect(() => catalog.encode(foreign)).toThrow("absent from"); + }); + + it("rebinds shared function aliases when a module is unloaded", () => { + const module = catalogModule(); + const source = new WebAssembly.Instance(module).exports + .__wpk_fork_function_catalog as WebAssembly.Table; + const value = source.get(0); + const first = new WebAssembly.Table({ + element: "anyfunc", + initial: 1, + maximum: 1, + }); + const second = new WebAssembly.Table({ + element: "anyfunc", + initial: 1, + maximum: 1, + }); + first.set(0, value); + second.set(0, value); + + const catalog = new ForkFunctionCatalog(); + catalog.register(7, second); + catalog.register(2, first); + expect(catalog.encode(value)).toEqual({ + moduleActivation: 2, + ordinal: 0, + }); + catalog.unregister(2); + expect(catalog.encode(value)).toEqual({ + moduleActivation: 7, + ordinal: 0, + }); + catalog.unregister(7); + expect(() => catalog.encode(value)).toThrow("absent from"); + }); +}); diff --git a/host/test/fork-gc-codec.test.ts b/host/test/fork-gc-codec.test.ts new file mode 100644 index 0000000000..ec1cd64ea5 --- /dev/null +++ b/host/test/fork-gc-codec.test.ts @@ -0,0 +1,255 @@ +import { describe, expect, it } from "vitest"; + +import { ForkAnyrefTransitTable } from "../src/fork-anyref-transit"; +import { + FORK_GC_FIELD_MUTABLE, + FORK_GC_FIELD_REFERENCE, + FORK_GC_LAYOUT_REQUIRES_PROVENANCE, + ForkGcCodecDescriptor, + ForkGcConstructorKind, + ForkGcLayoutKind, + ForkGcProvenanceRegistry, + decodeForkGcCodecDescriptor, +} from "../src/fork-gc-codec"; +import { + WPK_FORK_GC_CODEC_FIELD_RECORD_SIZE, + WPK_FORK_GC_CODEC_HEADER_SIZE, + WPK_FORK_GC_CODEC_LAYOUT_RECORD_SIZE, + WPK_FORK_GC_CODEC_MAGIC, + WPK_FORK_GC_CODEC_VERSION, +} from "../src/generated/abi"; + +const GC_OBJECT_MODULE = Uint8Array.of( + 0, 97, 115, 109, 1, 0, 0, 0, 1, 9, 2, 95, 1, 127, 1, 96, 1, 127, 0, + 2, 35, 1, 3, 101, 110, 118, 25, 95, 95, 119, 112, 107, 95, 102, 111, + 114, 107, 95, 114, 101, 102, 95, 103, 99, 95, 116, 114, 97, 110, 115, + 105, 116, 1, 110, 0, 1, 3, 2, 1, 1, 7, 11, 1, 7, 112, 117, 98, 108, + 105, 115, 104, 0, 0, 10, 13, 1, 11, 0, 65, 0, 32, 0, 251, 0, 0, 38, + 0, 11, +); + +function descriptorBytes(): Uint8Array { + const bytes = new Uint8Array( + WPK_FORK_GC_CODEC_HEADER_SIZE + + WPK_FORK_GC_CODEC_LAYOUT_RECORD_SIZE + + WPK_FORK_GC_CODEC_FIELD_RECORD_SIZE, + ); + bytes.set(WPK_FORK_GC_CODEC_MAGIC); + const view = new DataView(bytes.buffer); + view.setUint16(4, WPK_FORK_GC_CODEC_VERSION, true); + view.setUint16(6, WPK_FORK_GC_CODEC_HEADER_SIZE, true); + view.setUint32(8, 1, true); + view.setUint32(12, 1, true); + const layout = WPK_FORK_GC_CODEC_HEADER_SIZE; + view.setUint32(layout, 1, true); + view.setUint32(layout + 4, 0, true); + view.setUint8(layout + 8, ForkGcLayoutKind.Struct); + view.setUint8(layout + 9, ForkGcConstructorKind.Struct); + view.setUint16(layout + 10, FORK_GC_LAYOUT_REQUIRES_PROVENANCE, true); + view.setUint32(layout + 12, 0, true); + view.setUint32(layout + 16, 0, true); + view.setUint32(layout + 20, 1, true); + view.setUint32(layout + 24, 0xffff_ffff, true); + view.setUint32(layout + 28, 1, true); + view.setUint32(layout + 32, 0, true); + view.setUint32(layout + 36, 0, true); + view.setUint32(layout + 40, 1, true); + const field = layout + WPK_FORK_GC_CODEC_LAYOUT_RECORD_SIZE; + view.setUint8(field, 8); + view.setUint8( + field + 1, + FORK_GC_FIELD_MUTABLE | FORK_GC_FIELD_REFERENCE, + ); + view.setUint32(field + 4, 0xffff_ffff, true); + view.setUint32(field + 8, 0, true); + return bytes; +} + +function objectFixture(): { + transit: ForkAnyrefTransitTable; + publish(value: number): void; +} { + const transit = new ForkAnyrefTransitTable(); + const instance = new WebAssembly.Instance( + new WebAssembly.Module(GC_OBJECT_MODULE), + { env: { __wpk_fork_ref_gc_transit: transit.table } }, + ); + return { + transit, + publish: instance.exports.publish as (value: number) => void, + }; +} + +describe("fork GC codec metadata", () => { + it("decodes the canonical structural/provenance layout", () => { + const descriptor = decodeForkGcCodecDescriptor(descriptorBytes()); + expect(descriptor.require(1)).toMatchObject({ + id: 1, + kind: ForkGcLayoutKind.Struct, + provenanceReferenceCount: 1, + }); + }); + + it("rejects malformed magic, field order, and base coordinates", () => { + const magic = descriptorBytes(); + magic[0] ^= 1; + expect(() => decodeForkGcCodecDescriptor(magic)).toThrow(/magic/); + + const fields = descriptorBytes(); + new DataView(fields.buffer).setUint32( + WPK_FORK_GC_CODEC_HEADER_SIZE + 16, + 1, + true, + ); + expect(() => decodeForkGcCodecDescriptor(fields)).toThrow(); + + expect(() => new ForkGcCodecDescriptor([{ + ...decodeForkGcCodecDescriptor(descriptorBytes()).require(1), + baseLayoutId: 2, + }])).toThrow(/invalid base/); + }); +}); + +describe("ForkGcProvenanceRegistry", () => { + it("records exact activation/base evidence without retaining a pending root", () => { + const descriptor = decodeForkGcCodecDescriptor(descriptorBytes()); + const provenance = new ForkGcProvenanceRegistry(); + const { transit, publish } = objectFixture(); + publish(11); + const object = transit.get(0); + const token = provenance.begin( + transit.table, + descriptor, + 7, + 0, + 7, + 1, + 1, + 0n, + 0n, + 1, + ); + publish(12); + const seed = transit.get(0); + provenance.appendReference(transit.table, token, 0, 0); + transit.clearSlot(0); + provenance.end(token); + + expect(provenance.lookup(object, 7, descriptor, 1)).toMatchObject({ + activationId: 7, + baseLayoutId: 1, + layoutId: 1, + references: [seed], + }); + }); + + it("retains a nullable zero-length constructor seed as recipe-zero evidence", () => { + const descriptor = decodeForkGcCodecDescriptor(descriptorBytes()); + const provenance = new ForkGcProvenanceRegistry(); + const { transit, publish } = objectFixture(); + publish(13); + const object = transit.get(0); + const token = provenance.begin( + transit.table, + descriptor, + 7, + 0, + 7, + 1, + 1, + 0n, + 0n, + 1, + ); + transit.clearSlot(0); + provenance.appendReference(transit.table, token, 0, 0); + provenance.end(token); + + expect(provenance.lookup(object, 7, descriptor, 1)).toMatchObject({ + references: [null], + }); + }); + + it("fails closed for wrong activation/base and reentrant/interleaved hooks", () => { + const descriptor = decodeForkGcCodecDescriptor(descriptorBytes()); + const provenance = new ForkGcProvenanceRegistry(); + const { transit, publish } = objectFixture(); + publish(1); + expect(() => provenance.begin( + transit.table, + descriptor, + 3, + 0, + 4, + 1, + 1, + 0n, + 0n, + 1, + )).toThrow(/cannot register/); + expect(transit.get(0)).toBeNull(); + + publish(2); + expect(() => provenance.begin( + transit.table, + descriptor, + 3, + 0, + 3, + 1, + 2, + 0n, + 0n, + 1, + )).toThrow(/unknown GC layout/); + expect(transit.get(0)).toBeNull(); + + publish(3); + const token = provenance.begin( + transit.table, + descriptor, + 3, + 0, + 3, + 1, + 1, + 0n, + 0n, + 1, + ); + publish(4); + expect(() => provenance.begin( + transit.table, + descriptor, + 3, + 0, + 3, + 1, + 1, + 0n, + 0n, + 1, + )).toThrow(/still pending/); + expect(() => provenance.end(token)).toThrow(/not active/); + + publish(5); + const next = provenance.begin( + transit.table, + descriptor, + 3, + 0, + 3, + 1, + 1, + 0n, + 0n, + 1, + ); + publish(6); + expect(() => + provenance.appendReference(transit.table, next, 1, 0) + ).toThrow(/canonical order/); + expect(transit.get(0)).toBeNull(); + expect(() => provenance.end(next)).toThrow(/not active/); + }); +}); diff --git a/host/test/fork-host-import-runtime.test.ts b/host/test/fork-host-import-runtime.test.ts new file mode 100644 index 0000000000..c4b9347fa2 --- /dev/null +++ b/host/test/fork-host-import-runtime.test.ts @@ -0,0 +1,700 @@ +import { execFileSync } from "node:child_process"; +import { + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; +import { + readWasmFunctionImports, +} from "../src/constants"; +import { + defineForkExternrefImport, + forkExternrefImportMailboxBytes, + type ForkExternrefImportWake, +} from "../src/fork-externref-import-mailbox"; +import { + ForkHostImportOwnerRuntime, + ForkHostImportWorkerRuntime, +} from "../src/fork-host-import-runtime"; +import { ForkExternrefProcessOwner } from "../src/fork-externref-process-owner"; +import { + ForkExternrefTokenCache, +} from "../src/fork-reference-broker"; +import { + isForkWorkerExceptionCapability, +} from "../src/fork-worker-exception-capability"; + +function uleb(value: number): number[] { + const bytes: number[] = []; + do { + let byte = value & 0x7f; + value >>>= 7; + if (value !== 0) byte |= 0x80; + bytes.push(byte); + } while (value !== 0); + return bytes; +} + +function name(value: string): number[] { + const bytes = [...new TextEncoder().encode(value)]; + return [...uleb(bytes.length), ...bytes]; +} + +function section(id: number, payload: number[]): number[] { + return [id, ...uleb(payload.length), ...payload]; +} + +function importedFunctionsModule(): ArrayBuffer { + const typeSection = section(1, [ + 2, + 0x60, 2, 0x7f, 0x63, 0x6f, 1, 0x64, 0x6f, + 0x60, 1, 0x7f, 1, 0x7f, + ]); + const importSection = section(2, [ + 2, + ...name("host"), ...name("opaque"), 0, 0, + ...name("env"), ...name("local"), 0, 1, + ]); + return new Uint8Array([ + 0x00, 0x61, 0x73, 0x6d, + 0x01, 0x00, 0x00, 0x00, + ...typeSection, + ...importSection, + ]).buffer; +} + +function taggedImportModule(): ArrayBuffer { + const typeSection = section(1, [ + 1, + 0x60, 0, 0, + ]); + const importSection = section(2, [ + 1, + ...name("env"), ...name("throw_tagged"), 0, 0, + ]); + return new Uint8Array([ + 0x00, 0x61, 0x73, 0x6d, + 0x01, 0x00, 0x00, 0x00, + ...typeSection, + ...importSection, + ]).buffer; +} + +function typedBoundaryImportsModule(): ArrayBuffer { + const typeSection = section(1, [ + 1, + 0x60, + 12, + 0x7b, // v128 + 0x70, // funcref + 0x6f, // externref + 0x6e, // anyref + 0x6d, // eqref + 0x6c, // i31ref + 0x6b, // structref + 0x6a, // arrayref + 0x69, // exnref + 0x63, 0x00, // (ref null 0) + 0x64, 0x00, // (ref 0) + 0x63, 0x65, 0x00, // (ref null shared 0) + 2, + 0x68, // contref + 0x74, // noexnref + ]); + const importSection = section(2, [ + 1, + ...name("typed"), ...name("all"), 0, 0, + ]); + return new Uint8Array([ + 0x00, 0x61, 0x73, 0x6d, + 0x01, 0x00, 0x00, 0x00, + ...typeSection, + ...importSection, + ]).buffer; +} + +function compileWat( + directory: string, + stem: string, + source: string, + flags: readonly string[] = [], +): ArrayBuffer { + const watPath = join(directory, `${stem}.wat`); + const wasmPath = join(directory, `${stem}.wasm`); + writeFileSync(watPath, source); + execFileSync("wat2wasm", [ + ...flags, + watPath, + "-o", + wasmPath, + ]); + const file = readFileSync(wasmPath); + return file.buffer.slice( + file.byteOffset, + file.byteOffset + file.byteLength, + ) as ArrayBuffer; +} + +function thrownBy(fn: () => unknown): unknown { + let didThrow = false; + let thrown: unknown; + try { + fn(); + } catch (value) { + didThrow = true; + thrown = value; + } + expect(didThrow).toBe(true); + return thrown; +} + +describe("production fork host-import routing", () => { + it("retains complete scalar, vector, abstract, and concrete import types", () => { + const [imported] = readWasmFunctionImports( + typedBoundaryImportsModule(), + ); + expect(imported?.signature.params).toEqual([ + 0x7b, 0x70, 0x6f, 0x6e, 0x6d, 0x6c, 0x6b, 0x6a, 0x69, + 0x63, 0x64, 0x63, + ]); + expect(imported?.signature.paramTypes).toEqual([ + { code: 0x7b, shared: false }, + { code: 0x70, shared: false }, + { code: 0x6f, shared: false }, + { code: 0x6e, shared: false }, + { code: 0x6d, shared: false }, + { code: 0x6c, shared: false }, + { code: 0x6b, shared: false }, + { code: 0x6a, shared: false }, + { code: 0x69, shared: false }, + { code: 0x63, heapType: 0, shared: false }, + { code: 0x64, heapType: 0, shared: false }, + { code: 0x63, heapType: 0, shared: true }, + ]); + expect(imported?.signature.resultTypes).toEqual([ + { code: 0x68, shared: false }, + { code: 0x74, shared: false }, + ]); + }); + + it("keeps v128 and exnref imports on a direct Wasm-to-Wasm boundary", () => { + const directory = mkdtempSync(join(tmpdir(), "kandelo-typed-import-")); + try { + const vectorProviderBytes = compileWat( + directory, + "vector-provider", + `(module + (func (export "id") (param v128) (result v128) + local.get 0))`, + ); + const vectorConsumerBytes = compileWat( + directory, + "vector-consumer", + `(module + (import "m" "id" (func $id (param v128) (result v128))) + (func (export "run") (result i32) + v128.const i32x4 1 2 3 4 + call $id + i32x4.extract_lane 2))`, + ); + const exceptionProviderBytes = compileWat( + directory, + "exception-provider", + `(module + (func (export "id") (param exnref) (result exnref) + local.get 0))`, + ["--enable-exceptions"], + ); + const exceptionConsumerBytes = compileWat( + directory, + "exception-consumer", + `(module + (import "m" "id" (func $id (param exnref) (result exnref))) + (tag $tag (param i32)) + (func (export "run") (result i32) + (block $done (result i32) + (try_table (result i32) (catch $tag $done) + (block $captured (result i32 exnref) + (try_table (result i32 exnref) + (catch_ref $tag $captured) + i32.const 77 + throw $tag)) + call $id + throw_ref))))`, + ["--enable-exceptions"], + ); + + const processOwner = new ForkExternrefProcessOwner(); + const generation = processOwner.startGeneration(408); + const ownerRuntime = new ForkHostImportOwnerRuntime(processOwner); + const ownerWorker = ownerRuntime.createWorker({ + pid: generation.pid, + generationId: generation.id, + authorizeSender: () => {}, + }); + const workerRuntime = new ForkHostImportWorkerRuntime( + ownerWorker.init, + generation.pid, + generation.id, + new ForkExternrefTokenCache(generation.id), + (wake) => expect(ownerWorker.dispatch(wake)).toBe(true), + ); + + const vectorProvider = new WebAssembly.Instance( + new WebAssembly.Module(vectorProviderBytes), + ); + const vectorId = + vectorProvider.exports.id as CallableFunction; + const routedVector = workerRuntime.routeImportObject( + vectorConsumerBytes, + { m: { id: vectorId } }, + ); + expect(routedVector.m!.id).toBe(vectorId); + const vectorConsumer = new WebAssembly.Instance( + new WebAssembly.Module(vectorConsumerBytes), + routedVector, + ); + expect((vectorConsumer.exports.run as CallableFunction)()).toBe(3); + + const exceptionProvider = new WebAssembly.Instance( + new WebAssembly.Module(exceptionProviderBytes), + ); + const exceptionId = + exceptionProvider.exports.id as CallableFunction; + const routedException = workerRuntime.routeImportObject( + exceptionConsumerBytes, + { m: { id: exceptionId } }, + ); + expect(routedException.m!.id).toBe(exceptionId); + const exceptionConsumer = new WebAssembly.Instance( + new WebAssembly.Module(exceptionConsumerBytes), + routedException, + ); + expect((exceptionConsumer.exports.run as CallableFunction)()).toBe(77); + ownerWorker.close(); + } finally { + rmSync(directory, { recursive: true, force: true }); + } + }); + + it("parses exact artifact signatures and routes only registered opaque imports", () => { + const bytes = importedFunctionsModule(); + expect(readWasmFunctionImports(bytes)).toEqual([ + { + module: "host", + name: "opaque", + importOrdinal: 0, + functionIndex: 0, + signature: { + params: [0x7f, 0x63], + results: [0x64], + paramTypes: [ + { code: 0x7f, shared: false }, + { code: 0x63, heapType: -17, shared: false }, + ], + resultTypes: [{ code: 0x64, heapType: -17, shared: false }], + }, + }, + { + module: "env", + name: "local", + importOrdinal: 1, + functionIndex: 1, + signature: { + params: [0x7f], + results: [0x7f], + paramTypes: [{ code: 0x7f, shared: false }], + resultTypes: [{ code: 0x7f, shared: false }], + }, + }, + ]); + + const processOwner = new ForkExternrefProcessOwner(); + const generation = processOwner.startGeneration(401); + const ownerRuntime = new ForkHostImportOwnerRuntime(processOwner); + const opaque = defineForkExternrefImport( + 100, + ["i32", "externref"], + ["externref"], + ); + const seen: unknown[] = []; + ownerRuntime.register( + "host", + "opaque", + opaque, + (_context, scalar, value) => { + expect(scalar).toBe(9); + seen.push(value); + return value; + }, + ); + + let current = true; + const ownerWorker = ownerRuntime.createWorker({ + pid: generation.pid, + generationId: generation.id, + authorizeSender: () => { + if (!current) throw new Error("replaced Worker"); + }, + }); + const tokens = new ForkExternrefTokenCache(generation.id); + const wakes: ForkExternrefImportWake[] = []; + const workerRuntime = new ForkHostImportWorkerRuntime( + structuredClone(ownerWorker.init), + generation.pid, + generation.id, + tokens, + (wake) => { + wakes.push(wake); + expect(ownerWorker.dispatch(wake)).toBe(true); + }, + ); + const local = (value: number): number => value + 1; + const routed = workerRuntime.routeImportObject(bytes, { + host: { opaque: () => "unsafe local fallback" }, + env: { local }, + }); + const realValue = { ownerOnly: true }; + const handle = processOwner.registerForWire( + generation.pid, + generation.id, + realValue, + ); + const token = tokens.materialize(handle); + + expect( + (routed.host!.opaque as CallableFunction)(9, token), + ).toBe(token); + expect(seen).toEqual([realValue]); + const wakeCount = wakes.length; + expect((routed.env!.local as CallableFunction)(4)).toBe(5); + // The scalar memory-local fast path does not call the owner. + expect(wakes).toHaveLength(wakeCount); + + current = false; + expect(() => + (routed.host!.opaque as CallableFunction)(9, token) + ).toThrow(/Unauthorized/); + ownerWorker.close(); + }); + + it("preserves a primitive rethrow and unwraps its capture-time child token", () => { + const bytes = importedFunctionsModule(); + const processOwner = new ForkExternrefProcessOwner(); + const generation = processOwner.startGeneration(402); + const ownerRuntime = new ForkHostImportOwnerRuntime(processOwner); + const opaque = defineForkExternrefImport( + 101, + ["i32", "externref"], + ["externref"], + ); + const seen: unknown[] = []; + ownerRuntime.register( + "host", + "opaque", + opaque, + (_context, _scalar, value) => { + seen.push(value); + return value; + }, + ); + const ownerWorker = ownerRuntime.createWorker({ + pid: generation.pid, + generationId: generation.id, + authorizeSender: () => {}, + }); + const tokens = new ForkExternrefTokenCache(generation.id); + const workerRuntime = new ForkHostImportWorkerRuntime( + ownerWorker.init, + generation.pid, + generation.id, + tokens, + (wake) => { + expect(ownerWorker.dispatch(wake)).toBe(true); + }, + ); + const routed = workerRuntime.routeImportObject(bytes, { + host: { opaque: () => undefined }, + env: { + local: () => { + throw null; + }, + }, + }); + const importThrown = thrownBy( + routed.env!.local as CallableFunction, + ); + expect(importThrown).toBeNull(); + const normalizedNull = + workerRuntime.localExceptions.normalizeUnclaimedForkException( + importThrown, + ); + expect(tokens.encode(normalizedNull)).not.toBeNull(); + + expect( + (routed.host!.opaque as CallableFunction)(1, normalizedNull), + ).toBeNull(); + expect(seen).toEqual([null]); + ownerWorker.close(); + }); + + it("preserves an Error rethrow and captures a stable child capability", () => { + const bytes = importedFunctionsModule(); + const processOwner = new ForkExternrefProcessOwner(); + const generation = processOwner.startGeneration(403); + const ownerRuntime = new ForkHostImportOwnerRuntime(processOwner); + const opaque = defineForkExternrefImport( + 102, + ["i32", "externref"], + ["externref"], + ); + let observed: unknown; + ownerRuntime.register( + "host", + "opaque", + opaque, + (_context, _scalar, value) => { + observed = value; + return value; + }, + ); + const ownerWorker = ownerRuntime.createWorker({ + pid: generation.pid, + generationId: generation.id, + authorizeSender: () => {}, + }); + const tokens = new ForkExternrefTokenCache(generation.id); + const workerRuntime = new ForkHostImportWorkerRuntime( + ownerWorker.init, + generation.pid, + generation.id, + tokens, + (wake) => { + expect(ownerWorker.dispatch(wake)).toBe(true); + }, + ); + const error = new RangeError("Worker-local range failure"); + const routed = workerRuntime.routeImportObject(bytes, { + host: { opaque: () => undefined }, + env: { + local: () => { + throw error; + }, + }, + }); + const importThrown = thrownBy(routed.env!.local as CallableFunction); + expect(importThrown).toBe(error); + const normalized = + workerRuntime.localExceptions.normalizeUnclaimedForkException( + importThrown, + ); + const echoed = (routed.host!.opaque as CallableFunction)(1, normalized); + + expect(echoed).toBe(normalized); + expect(isForkWorkerExceptionCapability(observed)).toBe(true); + expect(observed).toMatchObject({ + kind: "error", + name: "RangeError", + message: error.message, + }); + ownerWorker.close(); + }); + + it("preserves exact imported-tag exception semantics before fork capture", () => { + const processOwner = new ForkExternrefProcessOwner(); + const generation = processOwner.startGeneration(405); + const ownerRuntime = new ForkHostImportOwnerRuntime(processOwner); + const ownerWorker = ownerRuntime.createWorker({ + pid: generation.pid, + generationId: generation.id, + authorizeSender: () => {}, + }); + const workerRuntime = new ForkHostImportWorkerRuntime( + ownerWorker.init, + generation.pid, + generation.id, + new ForkExternrefTokenCache(generation.id), + (wake) => { + expect(ownerWorker.dispatch(wake)).toBe(true); + }, + ); + const exception = new WebAssembly.Exception( + new WebAssembly.Tag({ parameters: [] }), + [], + ); + const routed = workerRuntime.routeImportObject( + taggedImportModule(), + { + env: { + throw_tagged: () => { + throw exception; + }, + }, + }, + ); + + expect( + thrownBy(routed.env!.throw_tagged as CallableFunction), + ).toBe(exception); + ownerWorker.close(); + }); + + it("keeps imported-tag Catch and CatchRef matching on the real Wasm boundary", () => { + const directory = mkdtempSync(join(tmpdir(), "kandelo-import-tag-")); + try { + const watPath = join(directory, "import-tag.wat"); + const wasmPath = join(directory, "import-tag.wasm"); + writeFileSync(watPath, `(module + (import "env" "tag" (tag $tag (param i32))) + (import "env" "throw_tagged" (func $throw_tagged)) + (import "env" "throw_any" (func $throw_any)) + (func (export "catch_plain") (result i32) + (block $caught (result i32) + (try_table (result i32) (catch $tag $caught) + call $throw_tagged + i32.const -1))) + (func (export "catch_ref") (result i32) + (block $caught (result i32 exnref) + (try_table (result i32 exnref) (catch_ref $tag $caught) + call $throw_tagged + i32.const -1 + ref.null exn)) + drop) + (func (export "catch_all_rethrow") + (block $caught (result exnref) + (try_table (result exnref) (catch_all_ref $caught) + call $throw_any + unreachable)) + (throw_ref)))`); + execFileSync("wat2wasm", [ + "--enable-exceptions", + watPath, + "-o", + wasmPath, + ]); + const file = readFileSync(wasmPath); + const bytes = file.buffer.slice( + file.byteOffset, + file.byteOffset + file.byteLength, + ) as ArrayBuffer; + const processOwner = new ForkExternrefProcessOwner(); + const generation = processOwner.startGeneration(406); + const ownerRuntime = new ForkHostImportOwnerRuntime(processOwner); + const ownerWorker = ownerRuntime.createWorker({ + pid: generation.pid, + generationId: generation.id, + authorizeSender: () => {}, + }); + const workerRuntime = new ForkHostImportWorkerRuntime( + ownerWorker.init, + generation.pid, + generation.id, + new ForkExternrefTokenCache(generation.id), + (wake) => { + expect(ownerWorker.dispatch(wake)).toBe(true); + }, + ); + const tag = new WebAssembly.Tag({ parameters: ["i32"] }); + let arbitraryThrown: unknown; + const routed = workerRuntime.routeImportObject(bytes, { + env: { + tag, + throw_tagged: () => { + throw new WebAssembly.Exception(tag, [37]); + }, + throw_any: () => { + throw arbitraryThrown; + }, + }, + }); + const instance = new WebAssembly.Instance( + new WebAssembly.Module(bytes), + routed, + ); + + expect((instance.exports.catch_plain as CallableFunction)()).toBe(37); + expect((instance.exports.catch_ref as CallableFunction)()).toBe(37); + const workerObject = Object.freeze({ callback: () => 1 }); + for (const value of [workerObject, -0, "exact primitive"]) { + arbitraryThrown = value; + expect( + Object.is( + thrownBy( + instance.exports.catch_all_rethrow as CallableFunction, + ), + value, + ), + ).toBe(true); + } + ownerWorker.close(); + } finally { + rmSync(directory, { recursive: true, force: true }); + } + }); + + it("allocates one distinct catalog-sized mailbox per pthread Worker", () => { + const processOwner = new ForkExternrefProcessOwner(); + const generation = processOwner.startGeneration(404); + const ownerRuntime = new ForkHostImportOwnerRuntime(processOwner); + const wideDescriptor = defineForkExternrefImport( + 91, + Array(33).fill("i32"), + Array(19).fill("i32"), + ); + ownerRuntime.register( + "host", + "wide", + wideDescriptor, + (_context, ...args) => args.slice(0, 19), + ); + const main = ownerRuntime.createWorker({ + pid: generation.pid, + generationId: generation.id, + authorizeSender: () => {}, + }); + const pthread = ownerRuntime.createWorker({ + pid: generation.pid, + generationId: generation.id, + authorizeSender: () => {}, + }); + + expect(main.init.mailbox).not.toBe(pthread.init.mailbox); + const expectedBytes = forkExternrefImportMailboxBytes({ + params: 33, + results: 19, + }); + expect(main.init.mailbox.byteLength).toBe(expectedBytes); + expect(pthread.init.mailbox.byteLength).toBe(expectedBytes); + expect(main.init.senderId).not.toBe(pthread.init.senderId); + const mainRuntime = new ForkHostImportWorkerRuntime( + main.init, + generation.pid, + generation.id, + new ForkExternrefTokenCache(generation.id), + (wake) => expect(main.dispatch(wake)).toBe(true), + ); + const pthreadRuntime = new ForkHostImportWorkerRuntime( + pthread.init, + generation.pid, + generation.id, + new ForkExternrefTokenCache(generation.id), + (wake) => expect(pthread.dispatch(wake)).toBe(true), + ); + const args = Array.from({ length: 33 }, (_, index) => index); + const expectedResults = args.slice(0, 19); + expect(mainRuntime.caller.call(wideDescriptor, args)).toEqual( + expectedResults, + ); + expect(pthreadRuntime.caller.call(wideDescriptor, args)).toEqual( + expectedResults, + ); + mainRuntime.clear(); + pthreadRuntime.clear(); + main.close(); + pthread.close(); + }); +}); diff --git a/host/test/fork-imported-globals.test.ts b/host/test/fork-imported-globals.test.ts new file mode 100644 index 0000000000..616d0a83e8 --- /dev/null +++ b/host/test/fork-imported-globals.test.ts @@ -0,0 +1,795 @@ +import { execFileSync } from "node:child_process"; +import { + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; +import { + ForkImportedGlobalCapture, + ForkImportedGlobalPlanner, + type ForkImportedReferenceProvider, + type ForkWasmImports, +} from "../src/fork-imported-globals"; +import { + decodeForkImportedTableBindings, + ForkImportedGlobalBindingKind, + ForkImportedTableBindingKind, + ForkModuleStateArena, + ForkModuleStateRecordKind, + ForkTableDirtyTracker, +} from "../src/fork-module-state"; +import { + WPK_FORK_IMPORTED_GLOBALS_HEADER_SIZE, + WPK_FORK_IMPORTED_GLOBALS_MAGIC, + WPK_FORK_IMPORTED_GLOBALS_RECORD_HEADER_SIZE, + WPK_FORK_IMPORTED_GLOBALS_SECTION, + WPK_FORK_IMPORTED_GLOBALS_VERSION, + WPK_FORK_IMPORTED_TABLES_HEADER_SIZE, + WPK_FORK_IMPORTED_TABLES_MAGIC, + WPK_FORK_IMPORTED_TABLES_RECORD_HEADER_SIZE, + WPK_FORK_IMPORTED_TABLES_SECTION, + WPK_FORK_IMPORTED_TABLES_VERSION, + WPK_FORK_MODULE_STATE_GLOBAL_HEADER_SIZE, + WPK_FORK_MODULE_STATE_GLOBAL_TYPE_EXNREF, + WPK_FORK_MODULE_STATE_GLOBAL_TYPE_EXTERNREF, + WPK_FORK_MODULE_STATE_GLOBAL_TYPE_FUNCREF, + WPK_FORK_MODULE_STATE_GLOBAL_TYPE_I32, +} from "../src/generated/abi"; + +const PAGE_SIZE = 65_536; + +function uleb128(value: number): number[] { + const result: number[] = []; + do { + let byte = value & 0x7f; + value >>>= 7; + if (value !== 0) byte |= 0x80; + result.push(byte); + } while (value !== 0); + return result; +} + +function importedGlobalsSection( + records: ReadonlyArray<{ + module: string; + name: string; + importOrdinal?: number; + ownerId: number; + typeCode: number; + }>, +): Uint8Array { + const encoder = new TextEncoder(); + const encoded = records.map((record, importOrdinal) => ({ + ...record, + importOrdinal: record.importOrdinal ?? importOrdinal, + moduleBytes: encoder.encode(record.module), + nameBytes: encoder.encode(record.name), + })); + const size = WPK_FORK_IMPORTED_GLOBALS_HEADER_SIZE + + encoded.reduce( + (sum, record) => + sum + WPK_FORK_IMPORTED_GLOBALS_RECORD_HEADER_SIZE + + record.moduleBytes.byteLength + record.nameBytes.byteLength, + 0, + ); + const bytes = new Uint8Array(size); + const view = new DataView(bytes.buffer); + bytes.set(WPK_FORK_IMPORTED_GLOBALS_MAGIC); + view.setUint16(4, WPK_FORK_IMPORTED_GLOBALS_VERSION, true); + view.setUint16(6, WPK_FORK_IMPORTED_GLOBALS_HEADER_SIZE, true); + view.setUint32(8, encoded.length, true); + let offset = WPK_FORK_IMPORTED_GLOBALS_HEADER_SIZE; + for (const record of encoded) { + const recordSize = WPK_FORK_IMPORTED_GLOBALS_RECORD_HEADER_SIZE + + record.moduleBytes.byteLength + record.nameBytes.byteLength; + view.setUint32(offset, recordSize, true); + view.setUint32(offset + 4, record.ownerId, true); + view.setUint8(offset + 8, record.typeCode); + view.setUint32(offset + 12, record.moduleBytes.byteLength, true); + view.setUint32(offset + 16, record.nameBytes.byteLength, true); + view.setUint32(offset + 20, record.importOrdinal, true); + bytes.set( + record.moduleBytes, + offset + WPK_FORK_IMPORTED_GLOBALS_RECORD_HEADER_SIZE, + ); + bytes.set( + record.nameBytes, + offset + WPK_FORK_IMPORTED_GLOBALS_RECORD_HEADER_SIZE + + record.moduleBytes.byteLength, + ); + offset += recordSize; + } + return bytes; +} + +function importedTablesSection( + records: ReadonlyArray<{ + module: string; + name: string; + importOrdinal?: number; + ownerId: number; + typeCode: number; + table64?: boolean; + }>, +): Uint8Array { + const encoder = new TextEncoder(); + const encoded = records.map((record, importOrdinal) => ({ + ...record, + importOrdinal: record.importOrdinal ?? importOrdinal, + moduleBytes: encoder.encode(record.module), + nameBytes: encoder.encode(record.name), + })); + const size = WPK_FORK_IMPORTED_TABLES_HEADER_SIZE + + encoded.reduce( + (sum, record) => + sum + WPK_FORK_IMPORTED_TABLES_RECORD_HEADER_SIZE + + record.moduleBytes.byteLength + record.nameBytes.byteLength, + 0, + ); + const bytes = new Uint8Array(size); + const view = new DataView(bytes.buffer); + bytes.set(WPK_FORK_IMPORTED_TABLES_MAGIC); + view.setUint16(4, WPK_FORK_IMPORTED_TABLES_VERSION, true); + view.setUint16(6, WPK_FORK_IMPORTED_TABLES_HEADER_SIZE, true); + view.setUint32(8, encoded.length, true); + let offset = WPK_FORK_IMPORTED_TABLES_HEADER_SIZE; + for (const record of encoded) { + const recordSize = WPK_FORK_IMPORTED_TABLES_RECORD_HEADER_SIZE + + record.moduleBytes.byteLength + record.nameBytes.byteLength; + view.setUint32(offset, recordSize, true); + view.setUint32(offset + 4, record.ownerId, true); + view.setUint8(offset + 8, record.typeCode); + view.setUint8(offset + 9, record.table64 ? 1 : 0); + view.setUint32(offset + 12, record.moduleBytes.byteLength, true); + view.setUint32(offset + 16, record.nameBytes.byteLength, true); + view.setUint32(offset + 20, record.importOrdinal, true); + bytes.set( + record.moduleBytes, + offset + WPK_FORK_IMPORTED_TABLES_RECORD_HEADER_SIZE, + ); + bytes.set( + record.nameBytes, + offset + WPK_FORK_IMPORTED_TABLES_RECORD_HEADER_SIZE + + record.moduleBytes.byteLength, + ); + offset += recordSize; + } + return bytes; +} + +function appendCustomSection( + wasm: Uint8Array, + name: string, + data: Uint8Array, +): Uint8Array { + const nameBytes = new TextEncoder().encode(name); + const payload = new Uint8Array( + uleb128(nameBytes.byteLength).length + nameBytes.byteLength + data.byteLength, + ); + const nameLength = uleb128(nameBytes.byteLength); + payload.set(nameLength); + payload.set(nameBytes, nameLength.length); + payload.set(data, nameLength.length + nameBytes.byteLength); + const sectionSize = uleb128(payload.byteLength); + const result = new Uint8Array(wasm.byteLength + 1 + sectionSize.length + payload.byteLength); + result.set(wasm); + let offset = wasm.byteLength; + result[offset++] = 0; + result.set(sectionSize, offset); + offset += sectionSize.length; + result.set(payload, offset); + return result; +} + +function compileModule( + wat: string, + descriptor: Uint8Array, + tableDescriptor = importedTablesSection([]), +): WebAssembly.Module { + const directory = mkdtempSync(join(tmpdir(), "kandelo-imported-globals-")); + try { + const watPath = join(directory, "fixture.wat"); + const wasmPath = join(directory, "fixture.wasm"); + writeFileSync(watPath, wat); + execFileSync("wat2wasm", [ + "--enable-exceptions", + watPath, + "-o", + wasmPath, + ]); + const withGlobals = appendCustomSection( + readFileSync(wasmPath), + WPK_FORK_IMPORTED_GLOBALS_SECTION, + descriptor, + ); + const bytes = appendCustomSection( + withGlobals, + WPK_FORK_IMPORTED_TABLES_SECTION, + tableDescriptor, + ); + // Node Buffers are typed as ArrayBufferLike, while the WebAssembly + // constructor correctly requires an owned, non-shared BufferSource. + const owned = new Uint8Array(bytes.byteLength); + owned.set(bytes); + return new WebAssembly.Module(owned); + } finally { + rmSync(directory, { recursive: true, force: true }); + } +} + +function allocator(memory: WebAssembly.Memory) { + let next = PAGE_SIZE; + return { + allocate(size: number): number { + const address = next; + next += Math.ceil(size / 8) * 8; + if (next > memory.buffer.byteLength) { + memory.grow(Math.ceil((next - memory.buffer.byteLength) / PAGE_SIZE)); + } + return address; + }, + deallocate(): void {}, + }; +} + +function referenceGlobal(typeCode: number, recipeId: number): Uint8Array { + const payload = new Uint8Array(WPK_FORK_MODULE_STATE_GLOBAL_HEADER_SIZE + 4); + const view = new DataView(payload.buffer); + view.setUint8(0, typeCode); + view.setUint8(1, 4); + view.setUint32(WPK_FORK_MODULE_STATE_GLOBAL_HEADER_SIZE, recipeId, true); + return payload; +} + +describe("fork imported-global provider planning", () => { + it("rebinds fresh funcref/externref/exnref providers before const initialization", () => { + const providerModule = compileModule( + `(module + (func (export "callback") (result i32) i32.const 73) + (global (export "__wpk_fork_global_1") exnref (ref.null exn)))`, + importedGlobalsSection([]), + ); + const descriptors = [ + { + module: "provider", + name: "callback", + ownerId: 1, + typeCode: WPK_FORK_MODULE_STATE_GLOBAL_TYPE_FUNCREF, + }, + { + module: "provider", + name: "token", + ownerId: 2, + typeCode: WPK_FORK_MODULE_STATE_GLOBAL_TYPE_EXTERNREF, + }, + { + module: "provider", + name: "exception", + ownerId: 3, + typeCode: WPK_FORK_MODULE_STATE_GLOBAL_TYPE_EXNREF, + }, + ] as const; + const consumerModule = compileModule( + `(module + (import "provider" "callback" (global $callback funcref)) + (import "provider" "token" (global $token externref)) + (import "provider" "exception" (global $exception exnref)) + (global $callback_alias funcref (global.get $callback)) + (global $token_alias externref (global.get $token)) + (global $exception_alias exnref (global.get $exception)) + (export "__wpk_fork_global_1" (global $callback)) + (export "__wpk_fork_global_2" (global $token)) + (export "__wpk_fork_global_3" (global $exception)) + (export "callback_global" (global $callback)) + (export "callback_alias" (global $callback_alias)) + (export "token_global" (global $token)) + (export "token_alias" (global $token_alias)) + (export "exception_global" (global $exception)) + (export "exception_alias" (global $exception_alias)))`, + importedGlobalsSection(descriptors), + ); + + const parentToken = Object.freeze({ generation: "parent" }); + const capture = new ForkImportedGlobalCapture("parent imported globals"); + const preparedProvider = capture.prepareActivation(1, providerModule, {}); + // Use the wrapped imports for the real instantiation boundary even though + // this provider has no imported globals. + const capturedParentProvider = new WebAssembly.Instance( + providerModule, + preparedProvider.imports as WebAssembly.Imports, + ); + preparedProvider.complete(capturedParentProvider); + const parentProvider = capturedParentProvider; + const preparedConsumer = capture.prepareActivation(2, consumerModule, { + provider: { + callback: parentProvider.exports.callback, + token: parentToken, + exception: parentProvider.exports.__wpk_fork_global_1, + }, + }); + const parentConsumer = new WebAssembly.Instance( + consumerModule, + preparedConsumer.imports as WebAssembly.Imports, + ); + preparedConsumer.complete(parentConsumer); + + const memory = new WebAssembly.Memory({ initial: 4 }); + const allocations = allocator(memory); + const arena = new ForkModuleStateArena( + memory, + 4, + allocations.allocate, + allocations.deallocate, + "imported-global capture", + ); + arena.begin(); + arena.appendModule({ activationId: 1, templateId: new Uint8Array(32).fill(1) }); + arena.appendModule({ activationId: 2, templateId: new Uint8Array(32).fill(2) }); + for (const [ownerId, typeCode, recipeId] of [ + [1, WPK_FORK_MODULE_STATE_GLOBAL_TYPE_FUNCREF, 1], + [2, WPK_FORK_MODULE_STATE_GLOBAL_TYPE_EXTERNREF, 2], + [3, WPK_FORK_MODULE_STATE_GLOBAL_TYPE_EXNREF, 0], + ] as const) { + arena.appendRecord({ + kind: ForkModuleStateRecordKind.MutableGlobal, + activationId: 2, + ownerId, + payload: referenceGlobal(typeCode, recipeId), + }); + } + const capturedBindings = capture.appendTo(arena); + expect(capturedBindings.map((binding) => binding.kind)).toEqual([ + ForkImportedGlobalBindingKind.RawReference, + ForkImportedGlobalBindingKind.RawReference, + ForkImportedGlobalBindingKind.ActivationGlobal, + ]); + expect(capturedBindings[2]).toMatchObject({ + sourceActivation: 1, + sourceOwner: 1, + }); + arena.seal(); + + const childInstances = new Map(); + const childToken = Object.freeze({ generation: "child" }); + const references: ForkImportedReferenceProvider = { + ownerActivation(recipeId) { + return recipeId === 1 ? 1 : null; + }, + materialize(recipeId) { + if (recipeId === 1) return childInstances.get(1)!.exports.callback; + if (recipeId === 2) return childToken; + throw new Error(`unknown test recipe ${recipeId}`); + }, + }; + const planner = new ForkImportedGlobalPlanner( + arena.records(), + new Map([ + [1, providerModule], + [2, consumerModule], + ]), + references, + "child imported globals", + ); + expect(planner.instantiationOrder()).toEqual([1, 2]); + for (const activationId of planner.instantiationOrder()) { + const module = activationId === 1 ? providerModule : consumerModule; + const imports = planner.importsForActivation(activationId, {}); + const instance = new WebAssembly.Instance( + module, + imports as WebAssembly.Imports, + ); + childInstances.set(activationId, instance); + planner.registerInstance(activationId, instance); + } + + const childProvider = childInstances.get(1)!; + const childConsumer = childInstances.get(2)!; + expect(childProvider.exports.callback).not.toBe(parentProvider.exports.callback); + expect(childConsumer.exports.callback_global).toBeInstanceOf(WebAssembly.Global); + expect((childConsumer.exports.callback_global as WebAssembly.Global).value) + .toBe(childProvider.exports.callback); + expect((childConsumer.exports.callback_alias as WebAssembly.Global).value) + .toBe(childProvider.exports.callback); + expect((childConsumer.exports.token_global as WebAssembly.Global).value) + .toBe(childToken); + expect((childConsumer.exports.token_alias as WebAssembly.Global).value) + .toBe(childToken); + expect(childToken).not.toBe(parentToken); + expect(childConsumer.exports.exception_global) + .toBe(childProvider.exports.__wpk_fork_global_1); + // The alias is a distinct immutable Global cell initialized from the exact + // provider exnref. JavaScript cannot read exnref values, which is precisely + // why the provider Global wrapper is the pre-instantiation transport. + expect(childConsumer.exports.exception_alias) + .toBeInstanceOf(WebAssembly.Global); + expect(() => (childConsumer.exports.exception_alias as WebAssembly.Global).value) + .toThrow(); + }); + + it("rebinds an imported mutable table to its activation owner before restore", () => { + const providerModule = compileModule( + `(module + (type $callback (func (result i32))) + (func $initial (type $callback) (result i32) i32.const 11) + (func $mutated (type $callback) (result i32) i32.const 22) + (table $dispatch 2 funcref) + (elem (i32.const 0) $initial) + (export "__wpk_fork_table_1" (table $dispatch)) + (export "mutated" (func $mutated)))`, + importedGlobalsSection([]), + ); + const consumerModule = compileModule( + `(module + (type $callback (func (result i32))) + (import "provider" "dispatch" (table $dispatch 2 funcref)) + (export "__wpk_fork_table_1" (table $dispatch)) + (func (export "call") (result i32) + i32.const 0 + call_indirect (type $callback)))`, + importedGlobalsSection([]), + importedTablesSection([{ + module: "provider", + name: "dispatch", + importOrdinal: 0, + ownerId: 1, + typeCode: WPK_FORK_MODULE_STATE_GLOBAL_TYPE_FUNCREF, + }]), + ); + + const capture = new ForkImportedGlobalCapture("parent imported tables"); + const preparedProvider = capture.prepareActivation(1, providerModule, {}); + const parentProvider = new WebAssembly.Instance( + providerModule, + preparedProvider.imports as WebAssembly.Imports, + ); + preparedProvider.complete(parentProvider); + const preparedConsumer = capture.prepareActivation(2, consumerModule, { + provider: { dispatch: parentProvider.exports.__wpk_fork_table_1 }, + }); + const parentConsumer = new WebAssembly.Instance( + consumerModule, + preparedConsumer.imports as WebAssembly.Imports, + ); + preparedConsumer.complete(parentConsumer); + + const memory = new WebAssembly.Memory({ initial: 4 }); + const allocations = allocator(memory); + const arena = new ForkModuleStateArena( + memory, + 4, + allocations.allocate, + allocations.deallocate, + "imported-table capture", + ); + arena.begin(); + arena.appendModule({ activationId: 1, templateId: new Uint8Array(32).fill(1) }); + arena.appendModule({ activationId: 2, templateId: new Uint8Array(32).fill(2) }); + capture.appendTo(arena); + const tableRecord = arena.recordsForCapture().find( + (record) => record.kind === ForkModuleStateRecordKind.ImportedTableBindings, + )!; + expect(decodeForkImportedTableBindings(tableRecord.payload)).toEqual([{ + consumerActivation: 2, + consumerOwner: 1, + sourceActivation: 1, + sourceOwner: 1, + reserved: 0, + kind: ForkImportedTableBindingKind.ActivationTable, + }]); + arena.seal(); + + const planner = new ForkImportedGlobalPlanner( + arena.records(), + new Map([[1, providerModule], [2, consumerModule]]), + { + ownerActivation: () => null, + materialize: () => null, + }, + "child imported tables", + ); + expect(planner.instantiationOrder()).toEqual([1, 2]); + const children = new Map(); + for (const activationId of planner.instantiationOrder()) { + const module = activationId === 1 ? providerModule : consumerModule; + const instance = new WebAssembly.Instance( + module, + planner.importsForActivation(activationId, {}) as WebAssembly.Imports, + ); + children.set(activationId, instance); + planner.registerInstance(activationId, instance); + } + const childProvider = children.get(1)!; + const childConsumer = children.get(2)!; + expect(childConsumer.exports.__wpk_fork_table_1) + .toBe(childProvider.exports.__wpk_fork_table_1); + expect((childConsumer.exports.call as () => number)()).toBe(11); + + // KFMS restores table overrides through the consumer's imported alias. + // Exercising the same identity here proves the mutation is immediately + // visible to continuation code after the restore phase. + (childProvider.exports.__wpk_fork_table_1 as WebAssembly.Table).set( + 0, + childProvider.exports.mutated, + ); + expect((childConsumer.exports.call as () => number)()).toBe(22); + }); + + it("re-resolves one fresh base-import table for every captured alias", () => { + const module = compileModule( + `(module + (import "host" "dispatch" (table $dispatch 1 4 funcref)) + (export "__wpk_fork_table_1" (table $dispatch)))`, + importedGlobalsSection([]), + importedTablesSection([{ + module: "host", + name: "dispatch", + importOrdinal: 0, + ownerId: 1, + typeCode: WPK_FORK_MODULE_STATE_GLOBAL_TYPE_FUNCREF, + }]), + ); + const parentTable = new WebAssembly.Table({ + element: "anyfunc", + initial: 1, + maximum: 4, + }); + const capture = new ForkImportedGlobalCapture("base table parent"); + for (const activationId of [1, 2]) { + const prepared = capture.prepareActivation(activationId, module, { + host: { dispatch: parentTable }, + }); + const instance = new WebAssembly.Instance( + module, + prepared.imports as WebAssembly.Imports, + ); + prepared.complete(instance); + } + const parentTrackers = new Map([ + [1, new ForkTableDirtyTracker()], + [2, new ForkTableDirtyTracker()], + ]); + parentTrackers.get(1)!.markPages(1, 2n, 1n); + parentTrackers.get(2)!.markPages(1, 7n, 1n); + capture.bindTableDirtyTrackers(parentTrackers); + expect(parentTrackers.get(1)!.ownsState(1)).toBe(true); + expect(parentTrackers.get(2)!.ownsState(1)).toBe(false); + parentTrackers.get(1)!.markPages(1, 11n, 1n); + expect( + [0, 1, 2].map((ordinal) => + parentTrackers.get(2)!.pageAt(1, ordinal) + ), + ).toEqual([2n, 7n, 11n]); + + const memory = new WebAssembly.Memory({ initial: 4 }); + const allocations = allocator(memory); + const arena = new ForkModuleStateArena( + memory, + 4, + allocations.allocate, + allocations.deallocate, + "host table aliases", + ); + arena.begin(); + arena.appendModule({ activationId: 1, templateId: new Uint8Array(32).fill(1) }); + arena.appendModule({ activationId: 2, templateId: new Uint8Array(32).fill(2) }); + capture.appendTo(arena); + const tableBindingRecord = arena.recordsForCapture().find( + (record) => record.kind === ForkModuleStateRecordKind.ImportedTableBindings, + )!; + expect(decodeForkImportedTableBindings(tableBindingRecord.payload)).toEqual([ + { + consumerActivation: 1, + consumerOwner: 1, + sourceActivation: 0, + sourceOwner: 0, + reserved: 0, + kind: ForkImportedTableBindingKind.BaseImport, + }, + { + consumerActivation: 2, + consumerOwner: 1, + sourceActivation: 0, + sourceOwner: 0, + reserved: 0, + kind: ForkImportedTableBindingKind.BaseImport, + }, + ]); + arena.seal(); + + const planner = new ForkImportedGlobalPlanner( + arena.records(), + new Map([[1, module], [2, module]]), + { ownerActivation: () => null, materialize: () => null }, + "base table child", + ); + const childTable = new WebAssembly.Table({ + element: "anyfunc", + initial: 1, + maximum: 4, + }); + const instances = new Map(); + for (const activationId of planner.instantiationOrder()) { + const instance = new WebAssembly.Instance( + module, + planner.importsForActivation(activationId, { + host: { dispatch: childTable }, + }) as WebAssembly.Imports, + ); + instances.set(activationId, instance); + planner.registerInstance(activationId, instance); + } + const childTrackers = new Map([ + [1, new ForkTableDirtyTracker()], + [2, new ForkTableDirtyTracker()], + ]); + childTrackers.get(1)!.markPages(1, 3n, 1n); + childTrackers.get(2)!.markPages(1, 9n, 1n); + planner.bindTableDirtyTrackers(childTrackers); + expect(childTrackers.get(1)!.ownsState(1)).toBe(true); + expect(childTrackers.get(2)!.ownsState(1)).toBe(false); + childTrackers.get(2)!.markPages(1, 12n, 1n); + expect( + [0, 1, 2].map((ordinal) => + childTrackers.get(1)!.pageAt(1, ordinal) + ), + ).toEqual([3n, 9n, 12n]); + expect(instances.get(1)!.exports.__wpk_fork_table_1) + .toBe(instances.get(2)!.exports.__wpk_fork_table_1); + expect(instances.get(1)!.exports.__wpk_fork_table_1).toBe(childTable); + expect(instances.get(1)!.exports.__wpk_fork_table_1).not.toBe(parentTable); + planner.clear(); + }); + + it("rejects a provider cycle deterministically before instantiation", () => { + const cycleModule = compileModule( + `(module + (import "peer" "value" (global $value i32)) + (export "__wpk_fork_global_1" (global $value)))`, + importedGlobalsSection([{ + module: "peer", + name: "value", + ownerId: 1, + typeCode: WPK_FORK_MODULE_STATE_GLOBAL_TYPE_I32, + }]), + ); + const memory = new WebAssembly.Memory({ initial: 3 }); + const allocations = allocator(memory); + const arena = new ForkModuleStateArena( + memory, + 4, + allocations.allocate, + allocations.deallocate, + "cycle", + ); + arena.begin(); + arena.appendModule({ activationId: 1, templateId: new Uint8Array(32).fill(1) }); + arena.appendModule({ activationId: 2, templateId: new Uint8Array(32).fill(2) }); + arena.appendImportedGlobalBindings([ + { + consumerActivation: 1, + consumerOwner: 1, + sourceActivation: 2, + sourceOwner: 1, + reserved: 0, + recipeId: 0, + rawBits: 0n, + kind: ForkImportedGlobalBindingKind.ActivationGlobal, + mutable: false, + shared: false, + typeCode: WPK_FORK_MODULE_STATE_GLOBAL_TYPE_I32, + }, + { + consumerActivation: 2, + consumerOwner: 1, + sourceActivation: 1, + sourceOwner: 1, + reserved: 0, + recipeId: 0, + rawBits: 0n, + kind: ForkImportedGlobalBindingKind.ActivationGlobal, + mutable: false, + shared: false, + typeCode: WPK_FORK_MODULE_STATE_GLOBAL_TYPE_I32, + }, + ]); + arena.appendImportedTableBindings([]); + arena.seal(); + const planner = new ForkImportedGlobalPlanner( + arena.records(), + new Map([[1, cycleModule], [2, cycleModule]]), + { + ownerActivation: () => null, + materialize: () => null, + }, + "cycle", + ); + expect(() => planner.instantiationOrder()) + .toThrow("provider cycle among activations 1, 2"); + }); + + it("orders the complete typed-reference provider closure before its consumer", () => { + const emptyModule = compileModule( + `(module)`, + importedGlobalsSection([]), + ); + const consumerModule = compileModule( + `(module + (import "env" "token" (global $token externref)) + (export "token" (global $token)))`, + importedGlobalsSection([{ + module: "env", + name: "token", + ownerId: 1, + typeCode: WPK_FORK_MODULE_STATE_GLOBAL_TYPE_EXTERNREF, + }]), + ); + const memory = new WebAssembly.Memory({ initial: 3 }); + const allocations = allocator(memory); + const arena = new ForkModuleStateArena( + memory, + 4, + allocations.allocate, + allocations.deallocate, + "typed reference dependency closure", + ); + arena.begin(); + for (const activationId of [1, 2, 3]) { + arena.appendModule({ + activationId, + templateId: new Uint8Array(32).fill(activationId), + }); + } + arena.appendImportedGlobalBindings([{ + consumerActivation: 3, + consumerOwner: 1, + sourceActivation: 0, + sourceOwner: 0, + reserved: 0, + recipeId: 1, + rawBits: 0n, + kind: ForkImportedGlobalBindingKind.RawReference, + mutable: false, + shared: false, + typeCode: WPK_FORK_MODULE_STATE_GLOBAL_TYPE_EXTERNREF, + }]); + arena.appendImportedTableBindings([]); + arena.seal(); + + const registered = new Set(); + const token = Object.freeze({ child: true }); + const planner = new ForkImportedGlobalPlanner( + arena.records(), + new Map([ + [1, emptyModule], + [2, emptyModule], + [3, consumerModule], + ]), + { + ownerActivation: () => 1, + activationDependencies: () => [2, 1], + materialize: () => { + expect([...registered].sort()).toEqual([1, 2]); + return token; + }, + }, + "typed reference dependency closure", + ); + expect(planner.instantiationOrder()).toEqual([1, 2, 3]); + for (const activationId of planner.instantiationOrder()) { + const module = activationId === 3 ? consumerModule : emptyModule; + const instance = new WebAssembly.Instance( + module, + planner.importsForActivation( + activationId, + activationId === 3 ? { env: {} } : {}, + ) as WebAssembly.Imports, + ); + planner.registerInstance(activationId, instance); + registered.add(activationId); + } + }); +}); diff --git a/host/test/fork-instrument-coverage.test.ts b/host/test/fork-instrument-coverage.test.ts index a1e0a24006..8f4a69a10c 100644 --- a/host/test/fork-instrument-coverage.test.ts +++ b/host/test/fork-instrument-coverage.test.ts @@ -2,35 +2,31 @@ * fork_instrument_coverage — comprehensive regression matrix for * `wasm-fork-instrument`. * - * Source of truth: docs/plans/2026-05-13-fork-instrument-megaPR-eliminate-guard-dispatch-and-modern-EH-plan.md + * The test IDs originated in: + * docs/plans/2026-05-13-fork-instrument-megaPR-eliminate-guard-dispatch-and-modern-EH-plan.md * - * Six categories, 41 test IDs: + * Six categories, 51 test IDs: * D-* (10) dispatch coverage — switch-dispatch and the runtime * trampoline that replaces guard-dispatch. * C-* (11) catch-handler resume — B1/A2/A3/A4 patterns. (C-01..C-10 * from the matrix plus C-11 post-catch fork.) * S-* (8) side-effects-during-rewind — atomic ops, table.*, * non-nullable funcref, throw-from-outside. - * K-* (4) callback-registration fork roots — sigaction, signal, - * pthread_cleanup_push, qsort comparator. + * K-* (7) callback-registration and asynchronous fork roots. * P-* (11) process / threading patterns — main thread, blocked * cond, held mutex, popen, posix_spawn, deep and failed * continuation allocation. - * F-* (4) accepted-limit failure modes — ucontext, wasm-GC refs. + * F-* (4) explicit ucontext boundaries and Wasm-GC ownership. * - * Pre-refactor expected behaviour is encoded with vitest modifiers: - * - it() — should pass today AND after the architectural - * pivot. Regression gate against the refactor - * accidentally breaking working features. - * - it.fails() — expected to fail today; should pass after the - * named commit lands. When CI flags it as - * unexpectedly passing, flip to it(). - * - it.todo() — fixture not yet written (e.g. needs WAT). Marked - * for tracking; no assertion runs. + * Modifiers describe the current ownership of each proof: + * - it() — this file executes the process-runtime gate. + * - it.fails() — an explicit platform boundary is expected to fail + * truthfully; an unexpected pass requires review. + * - it.skip() — another named suite owns the executable proof because + * the shape has no C/C++ source fixture. * - * The whole file must stay green until the architectural pivot ships - * (commits 2-N of the mega-PR). Each pivot commit should flip the - * relevant tests from it.fails() to it(). + * Supported compiler/reference shapes must not be hidden behind a skip whose + * label still claims that ABI 43 rejects them. */ import { describe, it, expect } from "vitest"; import { runCentralizedProgram } from "./centralized-test-helper"; @@ -190,79 +186,59 @@ describe("fork_instrument_coverage / D-* dispatch", () => { // --------------------------------------------------------------------------- describe("fork_instrument_coverage / C-* catch-handler resume", () => { - it("C-01 try { fork() } catch (int) — no throw, fork in try body", async () => { + // LLVM 21 emits exnref locals and untagged cleanup catches for these C++ + // functions. ABI 43 gives both forms deterministic exception recipes, so + // keep the compiler output in the real process-runtime gate. + it("C-01 fork in compiler EH try body", async () => { await runFixture("programs/c_01_fork_in_try_no_throw.wasm", { contains: ["IN_TRY", "PRE_FORK", "CHILD: ok", "PASS: C-01"], }); }); - // C-02: B1 plain catch, single arm — fork inside catch handler. - // The B1-stages-1+2 machinery (Phase 6 rewind-throw stub + capture - // block + exnref stash) handles this correctly under modern wasm-EH - // lowering. Was `it.fails` pre-2026-05-14 because the SDK emitted - // legacy `try`/`catch`; the B1 machinery is structured for modern - // `try_table`/`catch_ref`/`throw_ref` only. Commit 9's SDK flip - // (with the empirical 2026-05-14 follow-up adding - // `-wasm-use-legacy-eh=false` explicitly) made this case actually - // exercise the existing modern-EH path. - it("C-02 fork inside single-arm plain catch (B1)", async () => { + it("C-02 fork in compiler EH catch", async () => { await runFixture("programs/c_02_fork_in_catch.wasm", { contains: ["THROWING", "CAUGHT: 7", "PRE_FORK", "CHILD: ok", "PASS: C-02"], }); }); - // C-03: multi-arm plain-catch try_tables. The B1 stage 2 machinery's - // per-arm capture-block emission handles multi-arm under modern EH. - it("C-03 fork in multi-arm plain catch", async () => { + it("C-03 fork in a distinct multi-arm catch target", async () => { await runFixture("programs/c_03_fork_in_multi_arm_catch.wasm", { contains: ["THROWING", "CAUGHT_STR: x", "PRE_FORK", "CHILD: ok", "PASS: C-03"], }); }); - // C-04: throw originates outside the instrumented region. Switch- - // dispatch's body-skip-on-REWIND construction means the throw - // doesn't re-fire on REWIND — no gating needed. - it("C-04 fork in catch where throw originates outside instrumented region (B2)", async () => { + it("C-04 fork after an external throw reaches a catch", async () => { await runFixture("programs/c_04_fork_in_catch_external_throw.wasm", { contains: ["CALLING_HELPER", "IN_HELPER", "CAUGHT: 99", "PRE_FORK", "CHILD: ok", "PASS: C-04"], }); }); - // C-05..C-07: modern wasm-EH variants. Post-commit-9 + 2026-05-14 - // follow-up, ALL C++ programs lower via modern EH, so these are - // effectively duplicates of C-02 / C-03 / multi-typed-catch under - // the unified lowering — but kept distinct in case future toolchain - // versions reintroduce divergence. - it("C-05 modern EH single-clause typed catch + fork", async () => { + it("C-05 fork in a single modern-EH catch", async () => { await runFixture("programs/c_05_fork_modern_eh_single.wasm", { contains: ["THROWING", "CAUGHT: 1", "PRE_FORK", "CHILD: ok", "PASS: C-05"], }); }); - it("C-06 modern EH multi-target *_ref try_table + fork", async () => { + it("C-06 fork in a reference-form multi-arm catch", async () => { await runFixture("programs/c_06_fork_modern_eh_multi_ref.wasm", { contains: ["THROWING", "CAUGHT_DOUBLE: 3.14", "PRE_FORK", "CHILD: ok", "PASS: C-06"], }); }); - it("C-07 modern EH multi-arm plain catches + fork", async () => { + it("C-07 fork in a plain-form multi-arm catch", async () => { await runFixture("programs/c_07_fork_modern_eh_multi_plain.wasm", { contains: ["THROWING", "CAUGHT_LONG: 1234567", "PRE_FORK", "CHILD: ok", "PASS: C-07"], }); }); - // C-08, C-09 — A4 funcref/externref catch operands. No C-source - // surface; covered by `crates/fork-instrument/tests/coverage_wat.rs` - // which verifies fork-instrument doesn't panic on these patterns. - // Full A4 implementation (per-arm aux-table spilling for ref-typed - // catch operands) is future work — today the affected function is - // carved out of the fork-path set via b2_carveout. - it.skip("C-08 plain catch arm with funcref operand [tested via crates/fork-instrument/tests/coverage_wat.rs]", () => {}); - it.skip("C-09 plain catch arm with externref operand [tested via crates/fork-instrument/tests/coverage_wat.rs]", () => {}); + // C-08, C-09 — funcref/externref catch operands. There is no C-source + // surface, so `crates/fork-instrument/tests/coverage_wat.rs` verifies the + // ABI 43 boundary directly: reference payloads become complete exception + // recipes and never enter module-instance scratch state. + it.skip("C-08 funcref catch operand [coverage_wat.rs + catch-ref-fresh-worker.test.ts]", () => {}); + it.skip("C-09 externref catch operand [coverage_wat.rs + catch-ref-fresh-worker.test.ts]", () => {}); - // C-10: fork in BOTH try body and catch handler. Combines D-06 with - // C-02. Passes under modern EH. - it("C-10 fork in both try body and catch handler", async () => { + it("C-10 forks in both a try body and its catch", async () => { await runFixture("programs/c_10_fork_in_try_and_catch.wasm", { contains: [ "IN_TRY", "PRE_FORK_TRY", "CHILD_TRY: ok", @@ -272,10 +248,7 @@ describe("fork_instrument_coverage / C-* catch-handler resume", () => { }); }); - // C-11: post-catch fork (catch frame fully popped). Repro of the - // SpiderMonkey spike test (b). Closed by commit 9 + follow-up - // alongside C-02 — same root cause (modern-EH-only B1 machinery). - it("C-11 fork after fully-popped catch frame (spike test b)", async () => { + it("C-11 forks after a compiler catch has completed", async () => { await runFixture("programs/c_11_post_catch_fork.wasm", { contains: ["CAUGHT: 42", "PRE_FORK", "CHILD: ok", "PASS: C-11"], }); @@ -321,10 +294,9 @@ describe("fork_instrument_coverage / S-* side effects during rewind", () => { it.skip("S-06 table.grow before fork [tested via crates/fork-instrument/tests/coverage_wat.rs]", () => {}); it.skip("S-07 non-nullable funcref direct-call result before fork [tested via crates/fork-instrument/tests/coverage_wat.rs]", () => {}); - // S-08: throw from outside instrumented region, caught inside, - // fork in catch. Sibling of C-04. Closed by commit 9 + 2026-05-14 - // follow-up (explicit modern EH). - it("S-08 throw from outside instrumented region, fork in catch (B2)", async () => { + // LLVM retains an exnref local across this external-throw path. Its + // activation-owned recipe must survive the child instance boundary. + it("S-08 external throw with live compiler exnref state", async () => { await runFixture("programs/s_08_external_throw_fork_in_catch.wasm", { contains: ["ENTER_OUTER", "ENTER_INNER", "THROWING", "CAUGHT: 73", "PRE_FORK", "CHILD: ok", "PASS: S-08"], }); @@ -379,10 +351,9 @@ describe("fork_instrument_coverage / K-* callback fork roots", () => { }); }); - // K-06: fork() from a C++ destructor. Unusual but legal RAII - // pattern. The dtor is called as part of stack unwinding when - // the object goes out of scope; fork() inside it must work. - it("K-06 fork from C++ destructor (RAII)", async () => { + // K-06 lowers destructor cleanup to an untagged CatchAll. ABI 43 captures + // the complete exception recipe rather than relying on the parent instance. + it("K-06 fork from destructor through compiler CatchAll cleanup", async () => { await runFixture("programs/k_06_fork_from_dtor.wasm", { contains: ["IN_SCOPE", "IN_DTOR", "PRE_FORK", "CHILD: ok", "PARENT: child=", "PASS: K-06"], }); @@ -480,7 +451,7 @@ describe("fork_instrument_coverage / P-* process & threading", () => { // P-10: 4,096 live recursive activations require more frame payload than // ABI 41's retired 60 KiB contiguous reserve. This is the end-to-end guard - // that the ABI 42 host grows a linked continuation and replays it safely. + // that the current host grows a linked continuation and replays it safely. it("P-10 continuation grows beyond the retired fixed reserve", async () => { await runFixture("programs/p_10_deep_linked_continuation.wasm", { contains: ["PRE_DEEP_FORK", "DEEP_CHILD: ok", "DEEP_PARENT: child=", "PASS: P-10"], @@ -514,10 +485,10 @@ describe("fork_instrument_coverage / P-* process & threading", () => { }); // --------------------------------------------------------------------------- -// F-* accepted-limit failure modes +// F-* explicit boundaries and Wasm-GC ownership // --------------------------------------------------------------------------- -describe("fork_instrument_coverage / F-* accepted limits", () => { +describe("fork_instrument_coverage / F-* boundaries and Wasm-GC", () => { // F-01: getcontext(). Empirically: musl's wasm sysroot exposes // the symbol via an `env.getcontext` import that the kernel // doesn't implement — the program traps at first call with @@ -540,11 +511,9 @@ describe("fork_instrument_coverage / F-* accepted limits", () => { }); }); - // F-03, F-04 — wasm-GC anyref / struct.new. No C-source surface - // (LLVM-emitted C doesn't produce these); covered by cargo-level - // tests in `crates/fork-instrument/tests/coverage_wat.rs` which - // verify fork-instrument rejects the accepted-limit shapes with a - // clear diagnostic rather than silently accepting them. - it.skip("F-03 wasm-GC anyref accepted limit [tested via crates/fork-instrument/tests/coverage_wat.rs]", () => {}); - it.skip("F-04 wasm-GC struct.new accepted limit [tested via crates/fork-instrument/tests/coverage_wat.rs]", () => {}); + // F-03, F-04 — wasm-GC anyref / struct.new have no C-source surface. + // `coverage_wat.rs` verifies that both are accepted, encoded into + // activation-owned recipes, and emitted as independently valid Wasm. + it.skip("F-03 wasm-GC anyref [coverage_wat.rs + gc-reference-state-fresh-worker.test.ts]", () => {}); + it.skip("F-04 wasm-GC struct.new [coverage_wat.rs + gc-reference-state-fresh-worker.test.ts]", () => {}); }); diff --git a/host/test/fork-instrument-runtime-harness.ts b/host/test/fork-instrument-runtime-harness.ts new file mode 100644 index 0000000000..8e8304fa02 --- /dev/null +++ b/host/test/fork-instrument-runtime-harness.ts @@ -0,0 +1,159 @@ +import { + buildForkActivationStateImports, + ForkActivationRegistry, + forkActivationRegistrationFromInstance, +} from "../src/fork-activation-registry"; +import type { LinkedForkContinuation } from "../src/fork-continuation"; +import { + buildForkExceptionImports, + ForkExceptionBroker, + forkExceptionProviderFromInstance, + type ForkExceptionProvider, +} from "../src/fork-exception-provider"; +import { + computeForkModuleTemplateIdSync, + ForkModuleStateArena, +} from "../src/fork-module-state"; +import { ForkProcessContinuationCoordinator } from "../src/fork-process-continuation"; +import { forkResumeTargetsFromInstance } from "../src/fork-resume-catalog"; +import { + createForkUnwindTag, + FORK_UNWIND_TAG_IMPORT_NAME, + isForkUnwindException, +} from "../src/fork-unwind-transport"; + +export interface SingleActivationForkRuntimeOptions { + readonly module: WebAssembly.Module; + readonly moduleBytes: ArrayBufferView; + readonly memory: WebAssembly.Memory; + readonly continuation: LinkedForkContinuation; + readonly newArena: () => ForkModuleStateArena; + readonly label: string; +} + +/** + * Production-shaped ABI 43 owner for direct instrumenter tests. + * + * WHY: these tests instantiate generated Wasm without a process Worker. They + * still need the real activation registry, resume-event journal, module-state + * arena, and typed codecs; inert zero stubs would let ABI drift pass while + * bypassing the ownership protocol the test is supposed to exercise. + */ +export class SingleActivationForkRuntime { + readonly registry: ForkActivationRegistry; + readonly coordinator: ForkProcessContinuationCoordinator; + readonly envImports: Record; + + private readonly unwindTag = createForkUnwindTag(); + private instance: WebAssembly.Instance | null = null; + private exceptionProvider: ForkExceptionProvider | null = null; + private processLaunchRoot = 0; + + constructor( + private readonly options: SingleActivationForkRuntimeOptions, + ) { + const { memory, continuation, label } = options; + this.registry = new ForkActivationRegistry( + memory, + { + capture: () => { + throw new Error(`${label}: fixture unexpectedly captured externref`); + }, + materialize: () => { + throw new Error(`${label}: fixture unexpectedly replayed externref`); + }, + }, + `${label}: activation registry`, + ); + this.coordinator = new ForkProcessContinuationCoordinator( + memory, + this.registry, + `${label}: process continuation`, + ); + this.coordinator.prepareActivation({ + activationId: 0, + continuation, + publishProcessLaunchRoot: (address) => { + this.processLaunchRoot = address; + }, + readProcessLaunchRoot: () => this.processLaunchRoot, + }); + const exceptionBroker = new ForkExceptionBroker( + this.registry, + `${label}: exception broker`, + ); + this.envImports = { + [FORK_UNWIND_TAG_IMPORT_NAME]: + this.unwindTag as unknown as WebAssembly.ImportValue, + ...this.coordinator.continuationImports(0, (errno) => { + this.coordinator.beginCaptureAbort(errno); + }), + ...buildForkActivationStateImports(0, this.registry), + ...buildForkExceptionImports({ + activationId: 0, + ptrWidth: continuation.format.ptrWidth, + registry: this.registry, + broker: exceptionBroker, + provider: () => { + if (!this.exceptionProvider) { + throw new Error(`${label}: exception provider is not registered`); + } + return this.exceptionProvider; + }, + }), + }; + } + + register( + instance: WebAssembly.Instance, + options: { readonly bootstrap?: boolean } = {}, + ): void { + if (this.instance) { + throw new Error(`${this.options.label}: activation is already registered`); + } + this.instance = instance; + this.exceptionProvider = forkExceptionProviderFromInstance(0, instance); + this.coordinator.registerActivation( + forkActivationRegistrationFromInstance({ + activationId: 0, + module: this.options.module, + instance, + templateId: computeForkModuleTemplateIdSync(this.options.moduleBytes), + exceptionProvider: this.exceptionProvider, + }), + forkResumeTargetsFromInstance(this.options.module, instance), + ); + if (options.bootstrap ?? true) this.registry.bootstrapActivation(0); + } + + beginCapture(): void { + const arena = this.options.newArena(); + arena.begin(); + this.coordinator.beginCapture(arena); + } + + setCopiedProcessLaunchRoot(address: number): void { + if (!Number.isSafeInteger(address) || address <= 0) { + throw new RangeError( + `${this.options.label}: copied process launch root is invalid`, + ); + } + this.processLaunchRoot = address; + } + + isForkUnwind(value: unknown): boolean { + return isForkUnwindException(value, this.unwindTag); + } + + expectCaptureTransport(invoke: () => unknown): void { + try { + invoke(); + } catch (error) { + if (this.isForkUnwind(error)) return; + throw error; + } + throw new Error( + `${this.options.label}: capture returned instead of transporting unwind`, + ); + } +} diff --git a/host/test/fork-module-state.test.ts b/host/test/fork-module-state.test.ts new file mode 100644 index 0000000000..3309f01132 --- /dev/null +++ b/host/test/fork-module-state.test.ts @@ -0,0 +1,1345 @@ +import { describe, expect, it } from "vitest"; +import { + activationContinuationsForChild, + decodeForkActivationContinuations, + decodeForkModuleStateDescriptor, + decodeForkImportedGlobalBindings, + decodeForkImportedTableBindings, + encodeForkActivationContinuations, + encodeForkImportedGlobalBindings, + encodeForkImportedTableBindings, + encodeForkModuleStateDescriptor, + computeForkModuleTemplateId, + computeForkModuleTemplateIdSync, + ForkImportedGlobalBindingKind, + ForkImportedTableBindingKind, + ForkModuleStateArena, + ForkModuleStateRecordKind, + ForkTableDirtyTracker, + FORK_MODULE_STATE_DESCRIPTOR_SIZE, + FORK_MODULE_STATE_DESCRIPTOR_VERSION, + FORK_MODULE_STATE_REQUIRED_FLAGS, + FORK_MODULE_STATE_ROOT_POINTER_WORD_OFFSET, + FORK_MODULE_STATE_SECTION, + readForkModuleStateDescriptor, + readForkImportedGlobals, + readForkImportedTables, + readForkModuleStateRoot, + writeForkModuleStateRoot, + replayEventsForChild, + type ForkSparseTableSnapshot, +} from "../src/fork-module-state"; +import { + type ForkReplayEvent, + ForkReplayEventJournal, +} from "../src/fork-replay-events"; +import { + WPK_FORK_MODULE_STATE_GLOBAL_HEADER_SIZE, + WPK_FORK_ACTIVATION_CONTINUATIONS_HEADER_SIZE, + WPK_FORK_MODULE_STATE_GLOBAL_TYPE_I32, + WPK_FORK_MODULE_STATE_GLOBAL_TYPE_F64, + WPK_FORK_MODULE_STATE_GLOBAL_TYPE_EXNREF, + WPK_FORK_MODULE_STATE_GLOBAL_TYPE_EXTERNREF, + WPK_FORK_MODULE_STATE_GLOBAL_TYPE_FUNCREF, + WPK_FORK_MODULE_STATE_GLOBAL_TYPE_I64, + WPK_FORK_IMPORTED_GLOBALS_FLAG_MUTABLE, + WPK_FORK_IMPORTED_GLOBALS_FLAG_SHARED, + WPK_FORK_IMPORTED_GLOBALS_HEADER_SIZE, + WPK_FORK_IMPORTED_GLOBALS_MAGIC, + WPK_FORK_IMPORTED_GLOBALS_RECORD_HEADER_SIZE, + WPK_FORK_IMPORTED_GLOBALS_SECTION, + WPK_FORK_IMPORTED_GLOBALS_VERSION, + WPK_FORK_IMPORTED_TABLES_HEADER_SIZE, + WPK_FORK_IMPORTED_TABLES_MAGIC, + WPK_FORK_IMPORTED_TABLES_RECORD_HEADER_SIZE, + WPK_FORK_IMPORTED_TABLES_SECTION, + WPK_FORK_IMPORTED_TABLES_VERSION, +} from "../src/generated/abi"; + +const PAGE_SIZE = 65_536; + +function hex(bytes: Uint8Array): string { + return Array.from(bytes, (byte) => byte.toString(16).padStart(2, "0")).join(""); +} + +function replayEventCapture( + events: readonly ForkReplayEvent[], +): ForkReplayEventJournal { + const journal = new ForkReplayEventJournal(); + journal.beginCapture(); + for (const event of events) { + journal.recordCommit(event.activationId, event.functionOrdinal); + } + journal.sealCapture(); + return journal; +} + +function replayEventRecords(events: readonly ForkReplayEvent[]) { + const journal = replayEventCapture(events); + return [ + ...[...journal.capturedSegmentPayloads()].map((payload) => ({ + kind: ForkModuleStateRecordKind.ReplayEventSegment, + activationId: 0, + ownerId: 1, + payload, + })), + { + kind: ForkModuleStateRecordKind.ReplayEvents, + activationId: 0, + ownerId: 1, + payload: journal.capturedManifestPayload(), + }, + ]; +} + +describe("fork module template identity", () => { + it("matches the published SHA-256 empty and abc vectors synchronously", () => { + expect(hex(computeForkModuleTemplateIdSync(new Uint8Array()))) + .toBe("e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"); + expect(hex(computeForkModuleTemplateIdSync(new TextEncoder().encode("abc")))) + .toBe("ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"); + }); + + it("matches WebCrypto for sliced inputs spanning several blocks", async () => { + const storage = new Uint8Array(271); + for (let index = 0; index < storage.length; index++) { + storage[index] = (index * 73 + 19) & 0xff; + } + const source = storage.subarray(7, 264); + expect(computeForkModuleTemplateIdSync(source)) + .toEqual(await computeForkModuleTemplateId(source)); + }); +}); + +describe("fork table dirty journal", () => { + it("merges mutation ranges and enumerates pages deterministically", () => { + const tracker = new ForkTableDirtyTracker(); + tracker.markPages(7, 20n, 3n); + tracker.markPages(7, 4n, 2n); + tracker.markPages(7, 6n, 4n); + tracker.markPages(7, 9n, 12n); + tracker.markPages(7, 5n, 0n); + + expect(tracker.pageCount(7)).toBe(19); + expect( + Array.from({ length: tracker.pageCount(7) }, (_, ordinal) => + tracker.pageAt(7, ordinal) + ), + ).toEqual(Array.from({ length: 19 }, (_, index) => BigInt(index + 4))); + expect(() => tracker.pageAt(7, 19)).toThrow("has no page ordinal"); + }); + + it("round-trips unsigned i64 page bits through signed Wasm BigInt", () => { + const tracker = new ForkTableDirtyTracker(); + tracker.markPages(1, -1n, 1n); + expect(tracker.pageCount(1)).toBe(1); + expect(tracker.pageAt(1, 0)).toBe(-1n); + }); + + it("unions per-activation journals for one imported Table identity", () => { + const provider = new ForkTableDirtyTracker(); + const consumer = new ForkTableDirtyTracker(); + provider.markPages(4, 1n, 2n); + consumer.markPages(9, 7n, 1n); + consumer.aliasOwner(9, provider, 4); + expect(provider.ownsState(4)).toBe(true); + expect(consumer.ownsState(9)).toBe(false); + provider.markPages(4, 12n, 2n); + consumer.markPages(9, 20n, 1n); + + const expected = [1n, 2n, 7n, 12n, 13n, 20n]; + for (const [tracker, owner] of [[provider, 4], [consumer, 9]] as const) { + expect(tracker.pageCount(owner)).toBe(expected.length); + expect(expected.map((_, index) => tracker.pageAt(owner, index))) + .toEqual(expected); + } + + // A provider can be dlclosed while its physical Table remains reachable + // through a consumer import. Ownership moves without severing the merged + // mutation journal or losing pages written through the old provider. + provider.setStateOwner(4, false); + consumer.setStateOwner(9, true); + expect(provider.ownsState(4)).toBe(false); + expect(consumer.ownsState(9)).toBe(true); + expect(expected.map((_, index) => consumer.pageAt(9, index))) + .toEqual(expected); + }); +}); + +function allocator(memory: WebAssembly.Memory) { + let next = PAGE_SIZE; + const allocations: Array<{ addr: number; size: number }> = []; + const releases: Array<{ addr: number; size: number }> = []; + return { + allocations, + releases, + allocate(size: number): number { + const addr = next; + next += size; + if (next > memory.buffer.byteLength) { + memory.grow(Math.ceil((next - memory.buffer.byteLength) / PAGE_SIZE)); + } + allocations.push({ addr, size }); + return addr; + }, + deallocate(addr: number, size: number): void { + releases.push({ addr, size }); + }, + }; +} + +function cloneMemory(memory: WebAssembly.Memory): WebAssembly.Memory { + const clone = new WebAssembly.Memory({ + initial: memory.buffer.byteLength / PAGE_SIZE, + }); + new Uint8Array(clone.buffer).set(new Uint8Array(memory.buffer)); + return clone; +} + +function uleb128(value: number): number[] { + const bytes: number[] = []; + do { + let byte = value & 0x7f; + value >>>= 7; + if (value !== 0) byte |= 0x80; + bytes.push(byte); + } while (value !== 0); + return bytes; +} + +function moduleWithDescriptors(...descriptors: Uint8Array[]): WebAssembly.Module { + return moduleWithCustomSections(FORK_MODULE_STATE_SECTION, ...descriptors); +} + +function moduleWithCustomSections( + sectionName: string, + ...descriptors: Uint8Array[] +): WebAssembly.Module { + const name = [...new TextEncoder().encode(sectionName)]; + const sections = descriptors.flatMap((descriptor) => { + const payload = [...uleb128(name.length), ...name, ...descriptor]; + return [0, ...uleb128(payload.length), ...payload]; + }); + return new WebAssembly.Module(new Uint8Array([ + 0x00, 0x61, 0x73, 0x6d, 0x01, 0x00, 0x00, 0x00, + ...sections, + ])); +} + +function importedGlobalsSection( + records: ReadonlyArray<{ + module: string; + name: string; + ownerId: number; + importOrdinal?: number; + typeCode: number; + mutable?: boolean; + shared?: boolean; + }>, +): Uint8Array { + const encoder = new TextEncoder(); + const encoded = records.map((record, importOrdinal) => ({ + ...record, + importOrdinal: record.importOrdinal ?? importOrdinal, + moduleBytes: encoder.encode(record.module), + nameBytes: encoder.encode(record.name), + })); + const size = WPK_FORK_IMPORTED_GLOBALS_HEADER_SIZE + + encoded.reduce( + (sum, record) => + sum + + WPK_FORK_IMPORTED_GLOBALS_RECORD_HEADER_SIZE + + record.moduleBytes.byteLength + + record.nameBytes.byteLength, + 0, + ); + const bytes = new Uint8Array(size); + const view = new DataView(bytes.buffer); + bytes.set(WPK_FORK_IMPORTED_GLOBALS_MAGIC, 0); + view.setUint16(4, WPK_FORK_IMPORTED_GLOBALS_VERSION, true); + view.setUint16(6, WPK_FORK_IMPORTED_GLOBALS_HEADER_SIZE, true); + view.setUint32(8, encoded.length, true); + let offset = WPK_FORK_IMPORTED_GLOBALS_HEADER_SIZE; + for (const record of encoded) { + const recordSize = WPK_FORK_IMPORTED_GLOBALS_RECORD_HEADER_SIZE + + record.moduleBytes.byteLength + + record.nameBytes.byteLength; + view.setUint32(offset, recordSize, true); + view.setUint32(offset + 4, record.ownerId, true); + view.setUint8(offset + 8, record.typeCode); + view.setUint8( + offset + 9, + (record.mutable ? WPK_FORK_IMPORTED_GLOBALS_FLAG_MUTABLE : 0) + | (record.shared ? WPK_FORK_IMPORTED_GLOBALS_FLAG_SHARED : 0), + ); + view.setUint32(offset + 12, record.moduleBytes.byteLength, true); + view.setUint32(offset + 16, record.nameBytes.byteLength, true); + view.setUint32(offset + 20, record.importOrdinal, true); + bytes.set( + record.moduleBytes, + offset + WPK_FORK_IMPORTED_GLOBALS_RECORD_HEADER_SIZE, + ); + bytes.set( + record.nameBytes, + offset + + WPK_FORK_IMPORTED_GLOBALS_RECORD_HEADER_SIZE + + record.moduleBytes.byteLength, + ); + offset += recordSize; + } + return bytes; +} + +function importedTablesSection( + records: ReadonlyArray<{ + module: string; + name: string; + ownerId: number; + importOrdinal?: number; + typeCode: number; + table64?: boolean; + }>, +): Uint8Array { + const encoder = new TextEncoder(); + const encoded = records.map((record, importOrdinal) => ({ + ...record, + importOrdinal: record.importOrdinal ?? importOrdinal, + moduleBytes: encoder.encode(record.module), + nameBytes: encoder.encode(record.name), + })); + const size = WPK_FORK_IMPORTED_TABLES_HEADER_SIZE + + encoded.reduce( + (sum, record) => + sum + + WPK_FORK_IMPORTED_TABLES_RECORD_HEADER_SIZE + + record.moduleBytes.byteLength + + record.nameBytes.byteLength, + 0, + ); + const bytes = new Uint8Array(size); + const view = new DataView(bytes.buffer); + bytes.set(WPK_FORK_IMPORTED_TABLES_MAGIC, 0); + view.setUint16(4, WPK_FORK_IMPORTED_TABLES_VERSION, true); + view.setUint16(6, WPK_FORK_IMPORTED_TABLES_HEADER_SIZE, true); + view.setUint32(8, encoded.length, true); + let offset = WPK_FORK_IMPORTED_TABLES_HEADER_SIZE; + for (const record of encoded) { + const recordSize = WPK_FORK_IMPORTED_TABLES_RECORD_HEADER_SIZE + + record.moduleBytes.byteLength + + record.nameBytes.byteLength; + view.setUint32(offset, recordSize, true); + view.setUint32(offset + 4, record.ownerId, true); + view.setUint8(offset + 8, record.typeCode); + view.setUint8(offset + 9, record.table64 ? 1 : 0); + view.setUint32(offset + 12, record.moduleBytes.byteLength, true); + view.setUint32(offset + 16, record.nameBytes.byteLength, true); + view.setUint32(offset + 20, record.importOrdinal, true); + bytes.set( + record.moduleBytes, + offset + WPK_FORK_IMPORTED_TABLES_RECORD_HEADER_SIZE, + ); + bytes.set( + record.nameBytes, + offset + + WPK_FORK_IMPORTED_TABLES_RECORD_HEADER_SIZE + + record.moduleBytes.byteLength, + ); + offset += recordSize; + } + return bytes; +} + +function templateId(byte: number): Uint8Array { + return new Uint8Array(32).fill(byte); +} + +function mutableI32(value: number): Uint8Array { + const payload = new Uint8Array(WPK_FORK_MODULE_STATE_GLOBAL_HEADER_SIZE + 4); + const view = new DataView(payload.buffer); + view.setUint8(0, WPK_FORK_MODULE_STATE_GLOBAL_TYPE_I32); + view.setUint8(1, 4); + view.setInt32(WPK_FORK_MODULE_STATE_GLOBAL_HEADER_SIZE, value, true); + return payload; +} + +function sparseTable( + overrides: Partial = {}, +): ForkSparseTableSnapshot { + return { + activationId: 0, + ownerId: 4, + indexWidth: 4, + pageShift: 4, + length: 40, + baselineLength: 16, + baselineFingerprint: new Uint8Array(32).fill(0x5a), + pages: [ + { + pageIndex: 0, + runs: [ + { start: 2, recipeIds: [10, 11] }, + { start: 8, recipeIds: new Uint32Array([12, 13, 14]) }, + ], + }, + { + pageIndex: 2, + runs: [{ start: 0, recipeIds: [20, 21, 22, 23] }], + }, + ], + ...overrides, + }; +} + +describe("fork module-state descriptor", () => { + it.each([4, 8] as const)( + "round-trips the exact wasm%s descriptor and custom section", + (ptrWidth) => { + const bytes = encodeForkModuleStateDescriptor(ptrWidth); + expect(bytes).toHaveLength(FORK_MODULE_STATE_DESCRIPTOR_SIZE); + expect(decodeForkModuleStateDescriptor(bytes)).toEqual({ + version: FORK_MODULE_STATE_DESCRIPTOR_VERSION, + ptrWidth, + alignment: 8, + flags: FORK_MODULE_STATE_REQUIRED_FLAGS, + arenaVersion: 1, + recordVersion: 1, + rootPointerWordOffset: FORK_MODULE_STATE_ROOT_POINTER_WORD_OFFSET, + }); + expect(readForkModuleStateDescriptor(moduleWithDescriptors(bytes))).toEqual( + decodeForkModuleStateDescriptor(bytes), + ); + }, + ); + + it("rejects duplicate, unknown, and noncanonical descriptors", () => { + const exact = encodeForkModuleStateDescriptor(4); + expect(() => readForkModuleStateDescriptor( + moduleWithDescriptors(exact, exact), + )).toThrow("expected one kandelo.wpk_fork.module_state section, found 2"); + + const unknownFlags = exact.slice(); + new DataView(unknownFlags.buffer).setUint16( + 10, + FORK_MODULE_STATE_REQUIRED_FLAGS | 0x8000, + true, + ); + expect(() => decodeForkModuleStateDescriptor(unknownFlags)) + .toThrow("unknown module-state descriptor flags"); + + const wrongRootWord = exact.slice(); + new DataView(wrongRootWord.buffer).setUint32(16, 2, true); + expect(() => decodeForkModuleStateDescriptor(wrongRootWord)) + .toThrow("unsupported module-state root-pointer word offset 2"); + + const reserved = exact.slice(); + new DataView(reserved.buffer).setUint32(20, 1, true); + expect(() => decodeForkModuleStateDescriptor(reserved)) + .toThrow("reserved field is nonzero"); + }); +}); + +describe("fork imported-global ownership", () => { + it("parses immutable and mutable pre-instantiation recipes exactly", () => { + const descriptor = importedGlobalsSection([ + { + module: "callbacks", + name: "handler", + importOrdinal: 0, + ownerId: 2, + typeCode: WPK_FORK_MODULE_STATE_GLOBAL_TYPE_FUNCREF, + }, + { + module: "env", + name: "counter", + importOrdinal: 1, + ownerId: 7, + typeCode: WPK_FORK_MODULE_STATE_GLOBAL_TYPE_I64, + mutable: true, + shared: true, + }, + ]); + expect(readForkImportedGlobals( + moduleWithCustomSections(WPK_FORK_IMPORTED_GLOBALS_SECTION, descriptor), + )).toEqual([ + { + module: "callbacks", + name: "handler", + importOrdinal: 0, + ownerId: 2, + typeCode: WPK_FORK_MODULE_STATE_GLOBAL_TYPE_FUNCREF, + mutable: false, + shared: false, + }, + { + module: "env", + name: "counter", + importOrdinal: 1, + ownerId: 7, + typeCode: WPK_FORK_MODULE_STATE_GLOBAL_TYPE_I64, + mutable: true, + shared: true, + }, + ]); + }); + + it("preserves repeated bindings but rejects ambiguous owners and trailing bytes", () => { + const duplicate = importedGlobalsSection([ + { + module: "env", + name: "value", + ownerId: 1, + typeCode: WPK_FORK_MODULE_STATE_GLOBAL_TYPE_I32, + }, + { + module: "env", + name: "value", + ownerId: 2, + typeCode: WPK_FORK_MODULE_STATE_GLOBAL_TYPE_I32, + }, + ]); + expect(readForkImportedGlobals( + moduleWithCustomSections(WPK_FORK_IMPORTED_GLOBALS_SECTION, duplicate), + )).toHaveLength(2); + + const duplicateOwner = importedGlobalsSection([ + { + module: "env", + name: "first", + ownerId: 1, + typeCode: WPK_FORK_MODULE_STATE_GLOBAL_TYPE_I32, + }, + { + module: "env", + name: "second", + ownerId: 1, + typeCode: WPK_FORK_MODULE_STATE_GLOBAL_TYPE_I32, + }, + ]); + expect(() => readForkImportedGlobals( + moduleWithCustomSections(WPK_FORK_IMPORTED_GLOBALS_SECTION, duplicateOwner), + )).toThrow("duplicates owner 1"); + + const trailing = new Uint8Array(duplicate.byteLength + 1); + trailing.set(importedGlobalsSection([])); + expect(() => readForkImportedGlobals( + moduleWithCustomSections(WPK_FORK_IMPORTED_GLOBALS_SECTION, trailing), + )).toThrow("trailing bytes"); + }); +}); + +describe("fork imported-table ownership", () => { + it("parses exact import ordinals, reference classes, and table64 flags", () => { + const descriptor = importedTablesSection([ + { + module: "env", + name: "dispatch", + importOrdinal: 2, + ownerId: 3, + typeCode: WPK_FORK_MODULE_STATE_GLOBAL_TYPE_FUNCREF, + }, + { + module: "shared", + name: "objects", + importOrdinal: 7, + ownerId: 8, + typeCode: WPK_FORK_MODULE_STATE_GLOBAL_TYPE_EXTERNREF, + table64: true, + }, + ]); + expect(readForkImportedTables( + moduleWithCustomSections(WPK_FORK_IMPORTED_TABLES_SECTION, descriptor), + )).toEqual([ + { + module: "env", + name: "dispatch", + importOrdinal: 2, + ownerId: 3, + typeCode: WPK_FORK_MODULE_STATE_GLOBAL_TYPE_FUNCREF, + table64: false, + }, + { + module: "shared", + name: "objects", + importOrdinal: 7, + ownerId: 8, + typeCode: WPK_FORK_MODULE_STATE_GLOBAL_TYPE_EXTERNREF, + table64: true, + }, + ]); + }); + + it("rejects duplicate owners, unordered ordinals, and non-reference elements", () => { + expect(() => readForkImportedTables(moduleWithCustomSections( + WPK_FORK_IMPORTED_TABLES_SECTION, + importedTablesSection([ + { + module: "env", + name: "a", + importOrdinal: 1, + ownerId: 1, + typeCode: WPK_FORK_MODULE_STATE_GLOBAL_TYPE_FUNCREF, + }, + { + module: "env", + name: "b", + importOrdinal: 0, + ownerId: 2, + typeCode: WPK_FORK_MODULE_STATE_GLOBAL_TYPE_FUNCREF, + }, + ]), + ))).toThrow("duplicated or unordered import ordinal"); + expect(() => readForkImportedTables(moduleWithCustomSections( + WPK_FORK_IMPORTED_TABLES_SECTION, + importedTablesSection([ + { + module: "env", + name: "a", + ownerId: 1, + typeCode: WPK_FORK_MODULE_STATE_GLOBAL_TYPE_FUNCREF, + }, + { + module: "env", + name: "b", + ownerId: 1, + typeCode: WPK_FORK_MODULE_STATE_GLOBAL_TYPE_EXTERNREF, + }, + ]), + ))).toThrow("duplicates owner 1"); + expect(() => readForkImportedTables(moduleWithCustomSections( + WPK_FORK_IMPORTED_TABLES_SECTION, + importedTablesSection([{ + module: "env", + name: "bad", + ownerId: 1, + typeCode: WPK_FORK_MODULE_STATE_GLOBAL_TYPE_I32, + }]), + ))).toThrow("unknown element type"); + }); +}); + +describe("fork tail replay manifest", () => { + it("round-trips commit order and exposes reverse replay order", () => { + const events = [ + { activationId: 0, functionOrdinal: 9 }, + { activationId: 4, functionOrdinal: 3 }, + { activationId: 4, functionOrdinal: 3 }, + ]; + const memory = new WebAssembly.Memory({ initial: 3 }); + const arenaAllocator = allocator(memory); + const arena = new ForkModuleStateArena( + memory, + 4, + arenaAllocator.allocate, + arenaAllocator.deallocate, + "tail-events", + ); + arena.begin(); + arena.appendModule({ activationId: 2, templateId: templateId(2) }); + arena.appendModule({ activationId: 4, templateId: templateId(4) }); + arena.appendReplayEvents(replayEventCapture(events)); + arena.seal(); + const replay = new ForkReplayEventJournal(); + replay.attachChild(replayEventsForChild(arena.recordViews())); + for (const event of [...events].reverse()) { + expect(replay.peek()).toEqual(event); + replay.consume(event.activationId, event.functionOrdinal); + } + replay.finishReplay(); + arena.release(); + }); + + it("requires at most one process-owned manifest", () => { + const memory = new WebAssembly.Memory({ initial: 3 }); + const arenaAllocator = allocator(memory); + const arena = new ForkModuleStateArena( + memory, + 4, + arenaAllocator.allocate, + arenaAllocator.deallocate, + "duplicate-tail-events", + ); + arena.begin(); + arena.appendModule({ activationId: 1, templateId: templateId(1) }); + const empty = replayEventCapture([]); + arena.appendReplayEvents(empty); + arena.appendReplayEvents(empty); + expect(() => arena.seal()).toThrow("duplicate process replay-event record"); + arena.release(); + }); + + it("requires every ordered segment to precede the final manifest", () => { + const records = replayEventRecords([ + { activationId: 0, functionOrdinal: 1 }, + ]); + expect(() => replayEventsForChild([...records].reverse())) + .toThrow("segment follows its final manifest"); + expect(() => replayEventsForChild(records.slice(0, -1))) + .toThrow("no process replay-event manifest"); + }); +}); + +describe("fork imported-global binding manifest", () => { + const bindings = [ + { + consumerActivation: 1, + consumerOwner: 1, + sourceActivation: 0, + sourceOwner: 0, + reserved: 0, + recipeId: 0, + rawBits: 0x7ff8_0000_0000_0042n, + kind: ForkImportedGlobalBindingKind.RawNumber, + mutable: false, + shared: false, + typeCode: WPK_FORK_MODULE_STATE_GLOBAL_TYPE_F64, + }, + { + consumerActivation: 1, + consumerOwner: 2, + sourceActivation: 0, + sourceOwner: 0, + reserved: 0, + recipeId: 0, + rawBits: 0xffff_ffff_ffff_fffen, + kind: ForkImportedGlobalBindingKind.RawBigInt, + mutable: false, + shared: false, + typeCode: WPK_FORK_MODULE_STATE_GLOBAL_TYPE_I64, + }, + { + consumerActivation: 2, + consumerOwner: 1, + sourceActivation: 0, + sourceOwner: 0, + reserved: 0, + recipeId: 17, + rawBits: 0n, + kind: ForkImportedGlobalBindingKind.RawReference, + mutable: false, + shared: false, + typeCode: WPK_FORK_MODULE_STATE_GLOBAL_TYPE_FUNCREF, + }, + { + consumerActivation: 2, + consumerOwner: 2, + sourceActivation: 7, + sourceOwner: 4, + reserved: 0, + recipeId: 0, + rawBits: 0n, + kind: ForkImportedGlobalBindingKind.ActivationGlobal, + mutable: true, + shared: false, + typeCode: WPK_FORK_MODULE_STATE_GLOBAL_TYPE_EXNREF, + }, + { + consumerActivation: 3, + consumerOwner: 1, + sourceActivation: 0, + sourceOwner: 0, + reserved: 0, + recipeId: 0, + rawBits: 0n, + kind: ForkImportedGlobalBindingKind.BaseImport, + mutable: true, + shared: true, + typeCode: WPK_FORK_MODULE_STATE_GLOBAL_TYPE_EXTERNREF, + }, + ] as const; + + it("round-trips every deterministic provider kind with exact scalar bits", () => { + expect(decodeForkImportedGlobalBindings( + encodeForkImportedGlobalBindings(bindings), + )).toEqual(bindings); + }); + + it("rejects duplicate declarations, inconsistent owners, and reserved bytes", () => { + expect(() => encodeForkImportedGlobalBindings([ + bindings[0], + bindings[0], + ])).toThrow("unique and strictly ordered"); + expect(() => encodeForkImportedGlobalBindings([{ + ...bindings[3], + sourceOwner: 0, + }])).toThrow("activation-global binding fields are inconsistent"); + + const wire = encodeForkImportedGlobalBindings(bindings); + wire[24 + 35] = 1; + expect(() => decodeForkImportedGlobalBindings(wire)) + .toThrow("reserved fields are nonzero"); + }); +}); + +describe("fork imported-table binding manifest", () => { + const bindings = [ + { + consumerActivation: 1, + consumerOwner: 2, + sourceActivation: 4, + sourceOwner: 3, + reserved: 0, + kind: ForkImportedTableBindingKind.ActivationTable, + }, + { + consumerActivation: 7, + consumerOwner: 1, + sourceActivation: 0, + sourceOwner: 0, + reserved: 0, + kind: ForkImportedTableBindingKind.BaseImport, + }, + ] as const; + + it("round-trips activation and base-import table identities", () => { + expect(decodeForkImportedTableBindings( + encodeForkImportedTableBindings(bindings), + )).toEqual(bindings); + }); + + it("rejects duplicate consumers, inconsistent owners, and reserved bytes", () => { + expect(() => encodeForkImportedTableBindings([ + bindings[0], + bindings[0], + ])).toThrow("unique and strictly ordered"); + expect(() => encodeForkImportedTableBindings([{ + ...bindings[0], + sourceOwner: 0, + }])).toThrow("activation-table binding fields are inconsistent"); + expect(() => encodeForkImportedTableBindings([{ + ...bindings[1], + sourceActivation: 1, + }])).toThrow("base-import binding fields are inconsistent"); + + const wire = encodeForkImportedTableBindings(bindings); + wire[24 + 21] = 1; + expect(() => decodeForkImportedTableBindings(wire)) + .toThrow("reserved fields are nonzero"); + }); +}); + +describe("fork activation-continuation manifest", () => { + const continuations = [ + { activationId: 0, root: 0x1_0000n }, + { activationId: 7, root: 0x1_0000_0040n }, + ] as const; + const events = [ + { activationId: 0, functionOrdinal: 3 }, + { activationId: 7, functionOrdinal: 11 }, + { activationId: 7, functionOrdinal: 9 }, + ] as const; + + it("round-trips sorted nonzero u64 roots and checks the exact replay set", () => { + const payload = encodeForkActivationContinuations(continuations); + expect(decodeForkActivationContinuations(payload)).toEqual(continuations); + const records = [ + ...replayEventRecords(events), + { + kind: ForkModuleStateRecordKind.ActivationContinuations, + activationId: 0, + ownerId: 3, + payload, + }, + ]; + expect(activationContinuationsForChild(records, 8)).toEqual(continuations); + expect(() => activationContinuationsForChild(records, 4)) + .toThrow("does not fit wasm32"); + }); + + it("accepts side-only stacks and rejects empty, zero, unordered, or drifted manifests", () => { + expect(decodeForkActivationContinuations( + encodeForkActivationContinuations([ + { activationId: 7, root: 1n }, + ]), + )).toEqual([{ activationId: 7, root: 1n }]); + expect(() => encodeForkActivationContinuations([])) + .toThrow("must not be empty"); + expect(() => encodeForkActivationContinuations([ + { activationId: 0, root: 0n }, + ])).toThrow("root is zero"); + expect(() => encodeForkActivationContinuations([ + continuations[1], + continuations[0], + ])).toThrow("strictly ordered"); + + const zeroRoot = encodeForkActivationContinuations(continuations); + zeroRoot.fill( + 0, + WPK_FORK_ACTIVATION_CONTINUATIONS_HEADER_SIZE + 8, + WPK_FORK_ACTIVATION_CONTINUATIONS_HEADER_SIZE + 16, + ); + expect(() => decodeForkActivationContinuations(zeroRoot)) + .toThrow("continuation root is zero"); + + const records = [ + ...replayEventRecords(events.slice(0, 1)), + { + kind: ForkModuleStateRecordKind.ActivationContinuations, + activationId: 0, + ownerId: 3, + payload: encodeForkActivationContinuations(continuations), + }, + ]; + expect(() => activationContinuationsForChild(records, 8)) + .toThrow("does not exactly match replay events"); + }); + + it("seals only manifests whose active roots have module descriptors", () => { + const memory = new WebAssembly.Memory({ initial: 2 }); + const owner = allocator(memory); + const arena = new ForkModuleStateArena( + memory, + 8, + owner.allocate, + owner.deallocate, + "activation-continuations", + ); + arena.begin(); + arena.appendModule({ activationId: 0, templateId: templateId(1) }); + arena.appendModule({ activationId: 7, templateId: templateId(7) }); + arena.appendReplayEvents(replayEventCapture(events)); + arena.appendActivationContinuations(continuations); + expect(() => arena.seal()).not.toThrow(); + expect(activationContinuationsForChild(arena.records(), 8)) + .toEqual(continuations); + arena.release(); + }); +}); + +describe("module-state root-prefix ownership", () => { + it.each([4, 8] as const)( + "stores a wasm%s arena root in the reserved +P word and supports clearing it", + (ptrWidth) => { + const memory = new WebAssembly.Memory({ initial: 2 }); + const moduleBuffer = 256; + writeForkModuleStateRoot(memory, moduleBuffer, ptrWidth, PAGE_SIZE); + expect(readForkModuleStateRoot(memory, moduleBuffer, ptrWidth)).toBe(PAGE_SIZE); + + const view = new DataView(memory.buffer); + const slot = moduleBuffer + ptrWidth; + expect( + ptrWidth === 8 + ? view.getBigUint64(slot, true) + : BigInt(view.getUint32(slot, true)), + ).toBe(BigInt(PAGE_SIZE)); + + writeForkModuleStateRoot(memory, moduleBuffer, ptrWidth, 0); + expect(readForkModuleStateRoot(memory, moduleBuffer, ptrWidth)).toBe(0); + }, + ); + + it("rejects roots that cannot own page-aligned arena mappings", () => { + const memory = new WebAssembly.Memory({ initial: 1 }); + expect(() => writeForkModuleStateRoot(memory, 128, 4, 1234)) + .toThrow("arena root must be page-aligned"); + }); +}); + +describe("ForkModuleStateArena", () => { + it.each([4, 8] as const)( + "streams, validates, clones, and releases typed wasm%s state", + (ptrWidth) => { + const parentMemory = new WebAssembly.Memory({ initial: 4 }); + const parentAllocator = allocator(parentMemory); + const parent = new ForkModuleStateArena( + parentMemory, + ptrWidth, + parentAllocator.allocate, + parentAllocator.deallocate, + `parent-wasm${ptrWidth * 8}`, + ); + const root = parent.begin(); + parent.appendModule({ + activationId: 0, + templateId: templateId(0xa0), + }); + parent.appendRecord({ + kind: ForkModuleStateRecordKind.ReferenceRecipe, + activationId: 0, + ownerId: 1, + payload: new Uint8Array(70_000).fill(0x91), + }); + parent.appendRecord({ + kind: ForkModuleStateRecordKind.MutableGlobal, + activationId: 0, + ownerId: 2, + payload: mutableI32(0x0908_0706), + }); + parent.appendElementSegmentState({ + activationId: 0, + ownerId: 3, + segmentCount: 10, + dropped: new Uint8Array([0b0101_0101, 0b0000_0010]), + }); + parent.appendDataSegmentState({ + activationId: 0, + ownerId: 5, + segmentCount: 3, + dropped: new Uint8Array([0b0000_0101]), + }); + parent.appendSparseTable(sparseTable({ + indexWidth: ptrWidth, + length: ptrWidth === 8 ? 40n : 40, + })); + parent.seal(); + expect(parentAllocator.allocations.length).toBeGreaterThan(1); + + const moduleBuffer = 512; + writeForkModuleStateRoot(parentMemory, moduleBuffer, ptrWidth, root); + const childMemory = cloneMemory(parentMemory); + + const parentRecords = parent.records(); + expect(parentRecords.map(({ kind, ownerId }) => [kind, ownerId])).toEqual([ + [ForkModuleStateRecordKind.Module, 0], + [ForkModuleStateRecordKind.ReferenceRecipe, 1], + [ForkModuleStateRecordKind.MutableGlobal, 2], + [ForkModuleStateRecordKind.ElementSegments, 3], + [ForkModuleStateRecordKind.DataSegments, 5], + [ForkModuleStateRecordKind.Table, 4], + [ForkModuleStateRecordKind.TablePage, 4], + [ForkModuleStateRecordKind.TablePage, 4], + ]); + expect(parentRecords[1]!.payload[0]).toBe(0x91); + expect(parentRecords[1]!.payload.at(-1)).toBe(0x91); + const parentGlobalPayload = parent.findRecord( + ForkModuleStateRecordKind.MutableGlobal, + 0, + 2, + 0, + ); + expect(typeof parentGlobalPayload).toBe(ptrWidth === 8 ? "bigint" : "number"); + expect( + new Uint8Array( + parentMemory.buffer, + Number(parentGlobalPayload), + WPK_FORK_MODULE_STATE_GLOBAL_HEADER_SIZE + 4, + ), + ).toEqual(mutableI32(0x0908_0706)); + + const childReleases: Array<{ addr: number; size: number }> = []; + const child = new ForkModuleStateArena( + childMemory, + ptrWidth, + () => { throw new Error("attached arena must not allocate"); }, + (addr, size) => childReleases.push({ addr, size }), + `child-wasm${ptrWidth * 8}`, + ); + child.attach(readForkModuleStateRoot(childMemory, moduleBuffer, ptrWidth)); + expect(child.records()).toEqual(parentRecords); + const childGlobalPayload = child.findRecord( + ForkModuleStateRecordKind.MutableGlobal, + 0, + 2, + 0, + ); + expect( + new Uint8Array( + childMemory.buffer, + Number(childGlobalPayload), + WPK_FORK_MODULE_STATE_GLOBAL_HEADER_SIZE + 4, + ), + ).toEqual(mutableI32(0x0908_0706)); + expect(() => child.findRecord( + ForkModuleStateRecordKind.MutableGlobal, + 0, + 2, + 1, + )).toThrow("missing module-state record"); + expect(child.sparseTables()).toEqual([ + { + activationId: 0, + ownerId: 4, + indexWidth: ptrWidth, + pageShift: 4, + length: 40n, + baselineLength: 16n, + baselineFingerprint: new Uint8Array(32).fill(0x5a), + pages: [ + { + pageIndex: 0n, + runs: [ + { start: 2, recipeIds: new Uint32Array([10, 11]) }, + { start: 8, recipeIds: new Uint32Array([12, 13, 14]) }, + ], + }, + { + pageIndex: 2n, + runs: [ + { start: 0, recipeIds: new Uint32Array([20, 21, 22, 23]) }, + ], + }, + ], + }, + ]); + + child.release(); + expect(child.hasActiveArena()).toBe(false); + expect(childReleases).toEqual([...parentAllocator.allocations].reverse()); + + parent.release(); + expect(parentAllocator.releases).toEqual( + [...parentAllocator.allocations].reverse(), + ); + }, + ); + + it("rejects ownership ambiguity and undeclared activation state before sealing", () => { + const memory = new WebAssembly.Memory({ initial: 3 }); + const arenaAllocator = allocator(memory); + const arena = new ForkModuleStateArena( + memory, + 4, + arenaAllocator.allocate, + arenaAllocator.deallocate, + "ambiguous", + ); + arena.begin(); + arena.appendModule({ activationId: 0, templateId: templateId(1) }); + arena.appendRecord({ + kind: ForkModuleStateRecordKind.MutableGlobal, + activationId: 0, + ownerId: 8, + payload: mutableI32(1), + }); + arena.appendRecord({ + kind: ForkModuleStateRecordKind.MutableGlobal, + activationId: 0, + ownerId: 8, + payload: mutableI32(2), + }); + expect(() => arena.seal()).toThrow("duplicate owner 8"); + expect(arena.hasActiveArena()).toBe(true); + arena.release(); + + const second = new ForkModuleStateArena( + memory, + 4, + arenaAllocator.allocate, + arenaAllocator.deallocate, + "undeclared", + ); + second.begin(); + second.appendRecord({ + kind: ForkModuleStateRecordKind.ReferenceRecipe, + activationId: 99, + ownerId: 1, + payload: new Uint8Array(), + }); + expect(() => second.seal()).toThrow("undeclared module activation 99"); + second.release(); + }); + + it.each([4, 8] as const)( + "publishes wasm%s guest-written records only after their exact reservation commits", + (ptrWidth) => { + const memory = new WebAssembly.Memory({ initial: 3 }); + const arenaAllocator = allocator(memory); + const arena = new ForkModuleStateArena( + memory, + ptrWidth, + arenaAllocator.allocate, + arenaAllocator.deallocate, + `transactional-wasm${ptrWidth * 8}`, + ); + arena.begin(); + arena.appendModule({ activationId: 0, templateId: templateId(6) }); + const payload = arena.reserveRecord( + ForkModuleStateRecordKind.MutableGlobal, + 0, + 7, + ptrWidth === 8 ? 12n : 12, + ); + expect(typeof payload).toBe(ptrWidth === 8 ? "bigint" : "number"); + new Uint8Array(memory.buffer, Number(payload), 12).set(mutableI32(0x0907_0503)); + expect(() => arena.seal()).toThrow("pending module-state record"); + expect(() => arena.commitRecord(Number(payload) + 8)) + .toThrow("does not match reservation"); + arena.commitRecord(payload); + arena.seal(); + expect(arena.records().at(-1)).toEqual({ + kind: ForkModuleStateRecordKind.MutableGlobal, + activationId: 0, + ownerId: 7, + payload: mutableI32(0x0907_0503), + }); + arena.release(); + }, + ); + + it("rejects sparse pages that are unordered, overlapping, or outside final length", () => { + const memory = new WebAssembly.Memory({ initial: 3 }); + const arenaAllocator = allocator(memory); + const arena = new ForkModuleStateArena( + memory, + 4, + arenaAllocator.allocate, + arenaAllocator.deallocate, + "bad-sparse-table", + ); + arena.begin(); + arena.appendModule({ activationId: 0, templateId: templateId(2) }); + + expect(() => arena.appendSparseTable(sparseTable({ + pages: [{ + pageIndex: 0, + runs: [ + { start: 4, recipeIds: [1, 2] }, + { start: 5, recipeIds: [3] }, + ], + }], + }))).toThrow("unordered or out of bounds"); + arena.release(); + }); + + it("rejects tampered sparse page counts during fresh-instance attachment", () => { + const memory = new WebAssembly.Memory({ initial: 4 }); + const arenaAllocator = allocator(memory); + const arena = new ForkModuleStateArena( + memory, + 4, + arenaAllocator.allocate, + arenaAllocator.deallocate, + "table-count-parent", + ); + const root = arena.begin(); + arena.appendModule({ activationId: 0, templateId: templateId(3) }); + arena.appendSparseTable(sparseTable({ pages: [sparseTable().pages[0]!] })); + arena.seal(); + + // wasm32 chunk header is 40 bytes. The module record occupies 64 bytes; + // the following table record's payload begins after its 24-byte TLV header. + const tablePayload = root + 40 + 64 + 24; + new DataView(memory.buffer).setUint32(tablePayload + 4, 2, true); + const releases: Array<{ addr: number; size: number }> = []; + const child = new ForkModuleStateArena( + cloneMemory(memory), + 4, + () => { throw new Error("attachment must not allocate"); }, + (addr, size) => releases.push({ addr, size }), + "table-count-child", + ); + expect(() => child.attach(root)).toThrow("declares 2 sparse pages, found 1"); + expect(child.hasActiveArena()).toBe(false); + expect(releases).toEqual([]); + arena.release(); + }); + + it("does not adopt or release unsealed or malformed guest arenas", () => { + const memory = new WebAssembly.Memory({ initial: 3 }); + const arenaAllocator = allocator(memory); + const parent = new ForkModuleStateArena( + memory, + 4, + arenaAllocator.allocate, + arenaAllocator.deallocate, + "unsealed-parent", + ); + const root = parent.begin(); + parent.appendModule({ activationId: 0, templateId: templateId(4) }); + const childReleases: Array<{ addr: number; size: number }> = []; + const unsealedChild = new ForkModuleStateArena( + cloneMemory(memory), + 4, + () => { throw new Error("attachment must not allocate"); }, + (addr, size) => childReleases.push({ addr, size }), + "unsealed-child", + ); + expect(() => unsealedChild.attach(root)).toThrow("invalid or unsealed"); + expect(childReleases).toEqual([]); + + parent.seal(); + // Record kind is at root + wasm32 chunk header + 6. + new DataView(memory.buffer).setUint16(root + 40 + 6, 0xffff, true); + const malformedChild = new ForkModuleStateArena( + cloneMemory(memory), + 4, + () => { throw new Error("attachment must not allocate"); }, + (addr, size) => childReleases.push({ addr, size }), + "malformed-child", + ); + expect(() => malformedChild.attach(root)).toThrow("invalid record header"); + expect(malformedChild.hasActiveArena()).toBe(false); + expect(childReleases).toEqual([]); + parent.release(); + }); + + it("rejects a copied multi-chunk cycle without adopting forged ownership", () => { + const memory = new WebAssembly.Memory({ initial: 4 }); + const arenaAllocator = allocator(memory); + const parent = new ForkModuleStateArena( + memory, + 4, + arenaAllocator.allocate, + arenaAllocator.deallocate, + "cycle-parent", + ); + const root = parent.begin(); + parent.appendModule({ activationId: 0, templateId: templateId(7) }); + parent.appendRecord({ + kind: ForkModuleStateRecordKind.ReferenceRecipe, + activationId: 0, + ownerId: 1, + payload: new Uint8Array(70_000), + }); + parent.seal(); + expect(arenaAllocator.allocations).toHaveLength(2); + const tail = arenaAllocator.allocations[1]!.addr; + // wasm32 chunk next pointer is at +8 + 2P. + new DataView(memory.buffer).setUint32(tail + 16, root, true); + + const releases: Array<{ addr: number; size: number }> = []; + const child = new ForkModuleStateArena( + cloneMemory(memory), + 4, + () => { throw new Error("attachment must not allocate"); }, + (addr, size) => releases.push({ addr, size }), + "cycle-child", + ); + expect(() => child.attach(root)).toThrow("module-state chunk cycle"); + expect(child.hasActiveArena()).toBe(false); + expect(releases).toEqual([]); + parent.release(); + }); + + it("drops ownership before reporting cleanup failure", () => { + const memory = new WebAssembly.Memory({ initial: 3 }); + const arenaAllocator = allocator(memory); + const arena = new ForkModuleStateArena( + memory, + 4, + arenaAllocator.allocate, + () => { throw new Error("synthetic munmap failure"); }, + "release-failure", + ); + arena.begin(); + arena.appendModule({ activationId: 0, templateId: templateId(5) }); + arena.seal(); + expect(() => arena.release()).toThrow("synthetic munmap failure"); + expect(arena.hasActiveArena()).toBe(false); + expect(arena.isSealed()).toBe(false); + }); + + it("releases an uncommitted guest reservation during abort cleanup", () => { + const memory = new WebAssembly.Memory({ initial: 3 }); + const arenaAllocator = allocator(memory); + const arena = new ForkModuleStateArena( + memory, + 4, + arenaAllocator.allocate, + arenaAllocator.deallocate, + "pending-abort", + ); + arena.begin(); + arena.appendModule({ activationId: 0, templateId: templateId(8) }); + arena.reserveRecord( + ForkModuleStateRecordKind.ReferenceRecipe, + 0, + 1, + 128, + ); + arena.release(); + expect(arena.hasActiveArena()).toBe(false); + expect(arenaAllocator.releases).toEqual( + [...arenaAllocator.allocations].reverse(), + ); + }); +}); diff --git a/host/test/fork-process-continuation.test.ts b/host/test/fork-process-continuation.test.ts new file mode 100644 index 0000000000..8c29f31fad --- /dev/null +++ b/host/test/fork-process-continuation.test.ts @@ -0,0 +1,592 @@ +import { describe, expect, it } from "vitest"; +import { + ForkActivationRegistry, + type ForkActivationRegistration, +} from "../src/fork-activation-registry"; +import { + type LinkedFrameFormatDescriptor, + LinkedForkContinuation, +} from "../src/fork-continuation"; +import { + ForkModuleStateArena, + ForkTableDirtyTracker, +} from "../src/fork-module-state"; +import { ForkProcessContinuationCoordinator } from "../src/fork-process-continuation"; +import { FORK_REPLAY_EVENT_SEGMENT_CAPACITY } from "../src/fork-replay-events"; + +const PAGE_SIZE = 65_536; + +interface AllocationOwner { + allocate(size: number): number; + deallocate(addr: number, size: number): void; +} + +function allocationOwner(memory: WebAssembly.Memory): AllocationOwner { + let next = PAGE_SIZE; + return { + allocate(size) { + const address = next; + next += size; + if (next > memory.buffer.byteLength) { + memory.grow(Math.ceil((next - memory.buffer.byteLength) / PAGE_SIZE)); + } + return address; + }, + deallocate() {}, + }; +} + +function linkedFormat(): LinkedFrameFormatDescriptor { + return { + version: 1, + ptrWidth: 4, + alignment: 16, + flags: 1, + chunkHeaderSize: 32, + nodeHeaderSize: 32, + fixedPrefixSize: 64, + }; +} + +function externrefs() { + return { + capture(): number { + throw new Error("fixture has no externrefs"); + }, + materialize(): unknown { + throw new Error("fixture has no externrefs"); + }, + }; +} + +function emptyFunctionCatalog(): WebAssembly.Table { + return new WebAssembly.Table({ + element: "anyfunc", + initial: 0, + maximum: 0, + }); +} + +function wasmThunk(): CallableFunction { + const module = new WebAssembly.Module(new Uint8Array([ + 0x00, 0x61, 0x73, 0x6d, 0x01, 0x00, 0x00, 0x00, + 0x01, 0x05, 0x01, 0x60, 0x00, 0x01, 0x7f, + 0x03, 0x02, 0x01, 0x00, + 0x07, 0x05, 0x01, 0x01, 0x66, 0x00, 0x00, + 0x0a, 0x06, 0x01, 0x04, 0x00, 0x41, 0x07, 0x0b, + ])); + return new WebAssembly.Instance(module).exports.f as CallableFunction; +} + +function fakeActivation( + activationId: number, + calls: string[], +): { + registration: ForkActivationRegistration; + state: () => number; +} { + let state = 0; + const exports = { + wpk_fork_state: () => state, + wpk_fork_unwind_begin: () => { + expect(state).toBe(0); + state = 1; + calls.push(`unwind-begin:${activationId}`); + }, + wpk_fork_unwind_end: () => { + expect(state).toBe(1); + state = 0; + calls.push(`unwind-end:${activationId}`); + }, + wpk_fork_rewind_begin: () => { + expect(state).toBe(0); + state = 2; + calls.push(`rewind-begin:${activationId}`); + }, + wpk_fork_rewind_end: () => { + expect(state).toBe(2); + state = 0; + calls.push(`rewind-end:${activationId}`); + }, + wpk_fork_abort_begin: () => { + expect([0, 1]).toContain(state); + state = 3; + calls.push(`abort-begin:${activationId}`); + }, + wpk_fork_abort_end: () => { + expect(state).toBe(3); + state = 0; + calls.push(`abort-end:${activationId}`); + }, + }; + return { + registration: { + activationId, + instance: { exports } as unknown as WebAssembly.Instance, + templateId: new Uint8Array(32).fill(activationId + 1), + functionCatalog: emptyFunctionCatalog(), + staticRootCatalog: new WebAssembly.Table({ + element: "externref", + initial: 0, + maximum: 0, + }), + staticRootHarvest: () => {}, + moduleState: { + bootstrap: () => {}, + save: (id) => { calls.push(`save:${id}`); }, + restore: (id) => { calls.push(`restore:${id}`); }, + finishRestore: (id) => { calls.push(`finish-restore:${id}`); }, + saveTables: (id) => { calls.push(`save-tables:${id}`); }, + restoreTables: (id) => { calls.push(`restore-tables:${id}`); }, + }, + tableDirty: new ForkTableDirtyTracker(), + }, + state: () => state, + }; +} + +function makeCoordinator( + memory: WebAssembly.Memory, + owner: AllocationOwner, + calls: string[], + roots: Map, + label: string, +): { + coordinator: ForkProcessContinuationCoordinator; + arena: ForkModuleStateArena; +} { + const registry = new ForkActivationRegistry(memory, externrefs(), `${label}: registry`); + const coordinator = new ForkProcessContinuationCoordinator( + memory, + registry, + label, + ); + for (const activationId of [0, 4, 9]) { + const continuation = new LinkedForkContinuation( + memory, + linkedFormat(), + owner.allocate, + owner.deallocate, + `${label}: activation ${activationId}`, + ); + coordinator.prepareActivation({ + activationId, + continuation, + ...(activationId === 0 + ? { + publishProcessLaunchRoot: (root: number) => { + roots.set(0, root); + }, + readProcessLaunchRoot: () => roots.get(0) ?? 0, + } + : {}), + }); + const activation = fakeActivation(activationId, calls); + coordinator.registerActivation(activation.registration, [{ + functionOrdinal: activationId === 0 ? 11 : activationId === 4 ? 8 : 3, + thunk: wasmThunk(), + }]); + } + return { + coordinator, + arena: new ForkModuleStateArena( + memory, + 4, + owner.allocate, + owner.deallocate, + `${label}: arena`, + ), + }; +} + +function writeOrdinal( + memory: WebAssembly.Memory, + payload: number | bigint, + ordinal: number, +): void { + new DataView(memory.buffer).setUint32(Number(payload), ordinal, true); +} + +describe("ForkProcessContinuationCoordinator", () => { + it("reconstructs cross-activation frame order in a fresh child", () => { + const parentMemory = new WebAssembly.Memory({ initial: 16 }); + const parentOwner = allocationOwner(parentMemory); + const parentCalls: string[] = []; + const parentRoots = new Map(); + const parent = makeCoordinator( + parentMemory, + parentOwner, + parentCalls, + parentRoots, + "parent", + ); + const arenaRoot = parent.arena.begin(); + parent.coordinator.beginCapture(parent.arena); + + const sideImports = parent.coordinator.continuationImports(4); + const sidePayload = ( + sideImports.__wpk_fork_frame_reserve as (size: number) => number + )(16); + writeOrdinal(parentMemory, sidePayload, 8); + ( + sideImports.__wpk_fork_frame_commit as (payload: number) => void + )(sidePayload); + + const mainImports = parent.coordinator.continuationImports(0); + const mainPayload = ( + mainImports.__wpk_fork_frame_reserve as (size: number) => number + )(16); + writeOrdinal(parentMemory, mainPayload, 11); + ( + mainImports.__wpk_fork_frame_commit as (payload: number) => void + )(mainPayload); + parent.coordinator.sealCapture(); + + // Activation 9 exists in the process but was not on this thread's stack. + // Its runtime prefix is discarded before memory is copied. + expect(parent.coordinator.rootFor(9)).toBe(0); + expect(parent.coordinator.rootFor(0)).toBeGreaterThan(0); + expect(parent.coordinator.rootFor(4)).toBeGreaterThan(0); + // Only one process launch root crosses through the channel anchor. Side + // roots are reconstructed from the copied KFMS manifest, not JS/archive + // auxiliary state. + expect([...parentRoots.keys()]).toEqual([0]); + + const copiedBytes = new Uint8Array(parentMemory.buffer).slice(); + const copiedRoots = new Map(parentRoots); + const childMemory = new WebAssembly.Memory({ + initial: copiedBytes.byteLength / PAGE_SIZE, + }); + new Uint8Array(childMemory.buffer).set(copiedBytes); + const childOwner = allocationOwner(childMemory); + const childCalls: string[] = []; + const child = makeCoordinator( + childMemory, + childOwner, + childCalls, + copiedRoots, + "child", + ); + child.arena.attach(arenaRoot); + child.coordinator.attachChild(child.arena); + + const childMainImports = child.coordinator.continuationImports(0); + const childSideImports = child.coordinator.continuationImports(4); + const mainSlot = ( + childMainImports.__wpk_fork_resume_peek as (diagnostic: number) => number + )(0); + expect(mainSlot).toBeGreaterThan(0); + expect( + Number(( + childMainImports.__wpk_fork_frame_peek as (size: number) => number + )(16)), + ).toBe(Number(mainPayload)); + ( + childMainImports.__wpk_fork_frame_next as (size: number) => number + )(16); + + const sideSlot = ( + childSideImports.__wpk_fork_resume_peek as (diagnostic: number) => number + )(0); + expect(sideSlot).toBeGreaterThan(0); + ( + childSideImports.__wpk_fork_frame_peek as (size: number) => number + )(16); + ( + childSideImports.__wpk_fork_frame_next as (size: number) => number + )(16); + expect(( + childSideImports.__wpk_fork_resume_peek as (diagnostic: number) => number + )(0)).toBe(0); + child.coordinator.finishReplay(); + + expect(childCalls.filter((call) => call.startsWith("restore:"))).toEqual([ + "restore:0", + "restore:4", + "restore:9", + ]); + expect(copiedRoots.get(0)).toBe(0); + expect(child.coordinator.phaseName()).toBe("idle"); + }); + + it("does not consume a frame when the selected activation is wrong", () => { + const memory = new WebAssembly.Memory({ initial: 16 }); + const owner = allocationOwner(memory); + const roots = new Map(); + const fixture = makeCoordinator(memory, owner, [], roots, "mismatch"); + fixture.arena.begin(); + fixture.coordinator.beginCapture(fixture.arena); + const sideImports = fixture.coordinator.continuationImports(4); + const payload = ( + sideImports.__wpk_fork_frame_reserve as (size: number) => number + )(16); + writeOrdinal(memory, payload, 8); + ( + sideImports.__wpk_fork_frame_commit as (payload: number) => void + )(payload); + fixture.coordinator.sealCapture(); + fixture.coordinator.beginParentReplay(); + + const mainImports = fixture.coordinator.continuationImports(0); + ( + mainImports.__wpk_fork_resume_peek as (diagnostic: number) => number + )(0); + expect(() => ( + mainImports.__wpk_fork_frame_next as (size: number) => number + )(16)).toThrow("cannot consume frame for activation 4"); + + ( + sideImports.__wpk_fork_frame_next as (size: number) => number + )(16); + fixture.coordinator.finishReplay(); + }); + + it("launches a fresh child from a side-only continuation manifest", () => { + const parentMemory = new WebAssembly.Memory({ initial: 16 }); + const parentOwner = allocationOwner(parentMemory); + const parentRoots = new Map(); + const parent = makeCoordinator( + parentMemory, + parentOwner, + [], + parentRoots, + "side-only parent", + ); + const arenaRoot = parent.arena.begin(); + parent.coordinator.beginCapture(parent.arena); + const sideImports = parent.coordinator.continuationImports(4); + const payload = ( + sideImports.__wpk_fork_frame_reserve as (size: number) => number + )(16); + writeOrdinal(parentMemory, payload, 8); + ( + sideImports.__wpk_fork_frame_commit as (payload: number) => void + )(payload); + parent.coordinator.sealCapture(); + + const sideRoot = parent.coordinator.rootFor(4); + expect(parent.coordinator.rootFor(0)).toBe(0); + expect(sideRoot).toBeGreaterThan(0); + expect(parentRoots.get(0)).toBe(sideRoot); + expect([...parentRoots.keys()]).toEqual([0]); + + const copiedBytes = new Uint8Array(parentMemory.buffer).slice(); + const copiedRoots = new Map(parentRoots); + const childMemory = new WebAssembly.Memory({ + initial: copiedBytes.byteLength / PAGE_SIZE, + }); + new Uint8Array(childMemory.buffer).set(copiedBytes); + const child = makeCoordinator( + childMemory, + allocationOwner(childMemory), + [], + copiedRoots, + "side-only child", + ); + child.arena.attach(arenaRoot); + child.coordinator.attachChild(child.arena); + + expect(child.coordinator.rootFor(0)).toBe(0); + expect(child.coordinator.rootFor(4)).toBe(sideRoot); + ( + child.coordinator.continuationImports(4) + .__wpk_fork_frame_next as (size: number) => number + )(16); + child.coordinator.finishReplay(); + expect(copiedRoots.get(0)).toBe(0); + }); + + it("replays a partial unwind after continuation allocation failure", () => { + const memory = new WebAssembly.Memory({ initial: 16 }); + const owner = allocationOwner(memory); + const calls: string[] = []; + const roots = new Map(); + const fixture = makeCoordinator(memory, owner, calls, roots, "partial abort"); + fixture.arena.begin(); + fixture.coordinator.beginCapture(fixture.arena); + + const sideImports = fixture.coordinator.continuationImports(4); + const payload = ( + sideImports.__wpk_fork_frame_reserve as (size: number) => number + )(16); + writeOrdinal(memory, payload, 8); + ( + sideImports.__wpk_fork_frame_commit as (payload: number) => void + )(payload); + + fixture.coordinator.beginCaptureAbort(12); + expect(fixture.coordinator.phaseName()).toBe("abort-replay"); + expect(calls.filter((call) => call.startsWith("abort-begin:"))).toEqual([ + "abort-begin:0", + "abort-begin:4", + "abort-begin:9", + ]); + expect( + (sideImports.__wpk_fork_resume_peek as (diagnostic: number) => number)(0), + ).toBeGreaterThan(0); + ( + sideImports.__wpk_fork_frame_next as (size: number) => number + )(16); + fixture.coordinator.finishAbortReplay(); + + expect(calls.filter((call) => call.startsWith("abort-end:"))).toEqual([ + "abort-end:0", + "abort-end:4", + "abort-end:9", + ]); + expect(roots.get(0)).toBe(0); + expect([...roots.keys()]).toEqual([0]); + expect(fixture.coordinator.phaseName()).toBe("idle"); + }); + + it("replays an arbitrarily nested main-to-side-to-side stack", () => { + const parentMemory = new WebAssembly.Memory({ initial: 16 }); + const parentOwner = allocationOwner(parentMemory); + const parentRoots = new Map(); + const parent = makeCoordinator( + parentMemory, + parentOwner, + [], + parentRoots, + "nested parent", + ); + const arenaRoot = parent.arena.begin(); + parent.coordinator.beginCapture(parent.arena); + + const commit = (activationId: number, functionOrdinal: number): void => { + const imports = parent.coordinator.continuationImports(activationId); + const payload = ( + imports.__wpk_fork_frame_reserve as (size: number) => number + )(16); + writeOrdinal(parentMemory, payload, functionOrdinal); + ( + imports.__wpk_fork_frame_commit as (payload: number) => void + )(payload); + }; + // Unwind walks from the fork leaf outward. + commit(9, 3); + commit(4, 8); + commit(0, 11); + parent.coordinator.sealCapture(); + + const copiedBytes = new Uint8Array(parentMemory.buffer).slice(); + const copiedRoots = new Map(parentRoots); + const childMemory = new WebAssembly.Memory({ + initial: copiedBytes.byteLength / PAGE_SIZE, + }); + new Uint8Array(childMemory.buffer).set(copiedBytes); + const child = makeCoordinator( + childMemory, + allocationOwner(childMemory), + [], + copiedRoots, + "nested child", + ); + child.arena.attach(arenaRoot); + child.coordinator.attachChild(child.arena); + + for (const activationId of [0, 4, 9]) { + const imports = child.coordinator.continuationImports(activationId); + expect( + (imports.__wpk_fork_resume_peek as (diagnostic: number) => number)(0), + ).toBeGreaterThan(0); + ( + imports.__wpk_fork_frame_next as (size: number) => number + )(16); + } + expect(( + child.coordinator.continuationImports(9) + .__wpk_fork_resume_peek as (diagnostic: number) => number + )(0)).toBe(0); + child.coordinator.finishReplay(); + + expect(copiedRoots.get(0)).toBe(0); + expect([...copiedRoots.keys()]).toEqual([0]); + }); + + it("replays a continuation spanning multiple event pages in a fresh child", () => { + const eventCount = FORK_REPLAY_EVENT_SEGMENT_CAPACITY + 2; + const parentMemory = new WebAssembly.Memory({ initial: 16 }); + const parentOwner = allocationOwner(parentMemory); + const parentRoots = new Map(); + const parent = makeCoordinator( + parentMemory, + parentOwner, + [], + parentRoots, + "paged parent", + ); + const arenaRoot = parent.arena.begin(); + parent.coordinator.beginCapture(parent.arena); + const sideImports = parent.coordinator.continuationImports(4); + for (let index = 0; index < eventCount; index++) { + const payload = ( + sideImports.__wpk_fork_frame_reserve as (size: number) => number + )(16); + writeOrdinal(parentMemory, payload, 8); + ( + sideImports.__wpk_fork_frame_commit as (payload: number) => void + )(payload); + } + parent.coordinator.sealCapture(); + + const copiedBytes = new Uint8Array(parentMemory.buffer).slice(); + const childMemory = new WebAssembly.Memory({ + initial: copiedBytes.byteLength / PAGE_SIZE, + }); + new Uint8Array(childMemory.buffer).set(copiedBytes); + const child = makeCoordinator( + childMemory, + allocationOwner(childMemory), + [], + new Map(parentRoots), + "paged child", + ); + child.arena.attach(arenaRoot); + child.coordinator.attachChild(child.arena); + const childSideImports = child.coordinator.continuationImports(4); + for (let index = 0; index < eventCount; index++) { + expect(( + childSideImports.__wpk_fork_resume_peek as (diagnostic: number) => number + )(0)).toBeGreaterThan(0); + ( + childSideImports.__wpk_fork_frame_next as (size: number) => number + )(16); + } + expect(( + childSideImports.__wpk_fork_resume_peek as (diagnostic: number) => number + )(0)).toBe(0); + child.coordinator.finishReplay(); + }); + + it("aborts a partial capture spanning multiple event pages", () => { + const eventCount = FORK_REPLAY_EVENT_SEGMENT_CAPACITY + 1; + const memory = new WebAssembly.Memory({ initial: 16 }); + const owner = allocationOwner(memory); + const roots = new Map(); + const fixture = makeCoordinator(memory, owner, [], roots, "paged abort"); + fixture.arena.begin(); + fixture.coordinator.beginCapture(fixture.arena); + const sideImports = fixture.coordinator.continuationImports(4); + for (let index = 0; index < eventCount; index++) { + const payload = ( + sideImports.__wpk_fork_frame_reserve as (size: number) => number + )(16); + writeOrdinal(memory, payload, 8); + ( + sideImports.__wpk_fork_frame_commit as (payload: number) => void + )(payload); + } + fixture.coordinator.beginCaptureAbort(12); + for (let index = 0; index < eventCount; index++) { + ( + sideImports.__wpk_fork_resume_peek as (diagnostic: number) => number + )(0); + ( + sideImports.__wpk_fork_frame_next as (size: number) => number + )(16); + } + fixture.coordinator.finishAbortReplay(); + expect(fixture.coordinator.phaseName()).toBe("idle"); + }); +}); diff --git a/host/test/fork-reference-broker.test.ts b/host/test/fork-reference-broker.test.ts new file mode 100644 index 0000000000..6881f87f61 --- /dev/null +++ b/host/test/fork-reference-broker.test.ts @@ -0,0 +1,333 @@ +import { describe, expect, it } from "vitest"; +import { + ForkExternrefBroker, + ForkExternrefTokenRecipeProvider, + ForkExternrefTokenCache, +} from "../src/fork-reference-broker"; + +describe("ForkExternrefBroker", () => { + it("owns aliases once per execution generation and leases them once on fork", () => { + const broker = new ForkExternrefBroker(); + const parent = broker.createGeneration(11); + const child = broker.createGeneration(12); + const value = { opaque: true }; + const first = broker.register(parent, value); + const alias = broker.register(parent, value); + expect(alias).toBe(first); + expect(broker.holderCount(first, parent)).toBe(1); + + const lease = broker.acquireFork(parent, child, [first, first, first]); + expect(lease.handleCount).toBe(1); + expect(broker.authorize(child, first)).toBe(value); + expect(broker.holderCount(first, child)).toBe(1); + + expect(broker.releaseGeneration(parent)).toBe(true); + expect(() => broker.authorize(parent, first)).toThrow("stale"); + expect(broker.authorize(child, first)).toBe(value); + lease.release(); + expect(() => lease.release()).toThrow("already released"); + expect(() => broker.authorize(child, first)).toThrow("retired"); + }); + + it("publishes no partial child ownership when a fork recipe is invalid", () => { + const broker = new ForkExternrefBroker(); + const parent = broker.createGeneration(21); + const child = broker.createGeneration(22); + const handle = broker.register(parent, Symbol("opaque")); + expect(() => + broker.acquireFork(parent, child, [handle, handle + 1]) + ).toThrow("unknown externref handle"); + expect(broker.holderCount(handle, child)).toBe(0); + }); + + it("rolls back every child handle when fork publication fails mid-mutation", () => { + const broker = new ForkExternrefBroker(); + const parent = broker.createGeneration(23); + const child = broker.createGeneration(24); + const first = broker.register(parent, "first"); + const second = broker.register(parent, "second"); + const state = ( + broker as unknown as { + generations: WeakMap< + object, + { forkHandleCounts: Map } + >; + } + ).generations.get(child)!; + const originalSet = state.forkHandleCounts.set.bind( + state.forkHandleCounts, + ); + let writes = 0; + state.forkHandleCounts.set = (handle, count) => { + writes++; + if (writes === 2) throw new Error("injected fork publication failure"); + return originalSet(handle, count); + }; + + expect(() => broker.acquireFork(parent, child, [first, second])).toThrow( + "injected fork publication failure", + ); + expect(broker.holderCount(first, child)).toBe(0); + expect(broker.holderCount(second, child)).toBe(0); + expect(broker.authorize(parent, first)).toBe("first"); + expect(broker.authorize(parent, second)).toBe("second"); + }); + + it("tracks primitive externrefs without conflating distinct values", () => { + const broker = new ForkExternrefBroker(); + const generation = broker.createGeneration(31); + const one = broker.register(generation, 1); + const oneAlias = broker.register(generation, 1); + const text = broker.register(generation, "1"); + expect(oneAlias).toBe(one); + expect(text).not.toBe(one); + + const positiveZero = broker.register(generation, 0); + const negativeZero = broker.register(generation, -0); + expect(negativeZero).not.toBe(positiveZero); + expect(Object.is(broker.authorize(generation, positiveZero), 0)).toBe(true); + expect(Object.is(broker.authorize(generation, negativeZero), -0)).toBe(true); + + const firstNanBytes = new ArrayBuffer(8); + const firstNanView = new DataView(firstNanBytes); + firstNanView.setBigUint64(0, 0x7ff8_0000_0000_0001n, true); + const secondNanBytes = new ArrayBuffer(8); + const secondNanView = new DataView(secondNanBytes); + secondNanView.setBigUint64(0, 0x7ff8_0000_0000_0002n, true); + const firstNan = firstNanView.getFloat64(0, true); + const secondNan = secondNanView.getFloat64(0, true); + const firstNanHandle = broker.register(generation, firstNan); + const firstNanAlias = broker.register(generation, firstNan); + const secondNanHandle = broker.register(generation, secondNan); + expect(firstNanAlias).toBe(firstNanHandle); + expect(secondNanHandle).not.toBe(firstNanHandle); + const resultBits = (handle: number): bigint => { + const bytes = new ArrayBuffer(8); + const view = new DataView(bytes); + view.setFloat64( + 0, + broker.authorize(generation, handle) as number, + true, + ); + return view.getBigUint64(0, true); + }; + expect(resultBits(firstNanHandle)).toBe(0x7ff8_0000_0000_0001n); + expect(resultBits(secondNanHandle)).toBe(0x7ff8_0000_0000_0002n); + }); + + it("accepts every ordinary JavaScript externref shape", () => { + const broker = new ForkExternrefBroker(); + const generation = broker.createGeneration(32); + const values = [ + undefined, + null, + true, + 7, + 8n, + "opaque", + Symbol("opaque"), + () => 1, + { opaque: true }, + ]; + const handles = values.map((value) => broker.register(generation, value)); + expect(new Set(handles).size).toBe(values.length); + values.forEach((value, index) => { + expect(broker.authorize(generation, handles[index]!)).toBe(value); + }); + }); + + it("tombstones a replaced generation even when its PID is reused", () => { + const broker = new ForkExternrefBroker(); + const oldGeneration = broker.createGeneration(41); + const value = { image: "old" }; + const oldHandle = broker.register(oldGeneration, value); + + const replacement = broker.createGeneration(41); + expect(replacement.id).toBeGreaterThan(oldGeneration.id); + expect(() => broker.authorize(oldGeneration, oldHandle)).toThrow("stale"); + expect(broker.holderCount(oldHandle, oldGeneration)).toBe(0); + expect(() => broker.authorize(replacement, oldHandle)).toThrow("retired"); + + const replacementHandle = broker.register(replacement, value); + expect(replacementHandle).toBeGreaterThan(oldHandle); + expect(broker.authorize(replacement, replacementHandle)).toBe(value); + }); + + it("keeps independent fork leases without multiplying graph aliases", () => { + const broker = new ForkExternrefBroker(); + const parent = broker.createGeneration(51); + const child = broker.createGeneration(52); + const handle = broker.register(parent, { opaque: true }); + const first = broker.acquireFork(parent, child, [handle, handle]); + const second = broker.acquireFork(parent, child, [handle]); + + first.release(); + expect(broker.holderCount(handle, child)).toBe(1); + second.release(); + expect(broker.holderCount(handle, child)).toBe(0); + expect(broker.holderCount(handle, parent)).toBe(1); + }); + + it("verifies a complete fork lease before releasing any handle", () => { + const broker = new ForkExternrefBroker(); + const parent = broker.createGeneration(53); + const child = broker.createGeneration(54); + const first = broker.register(parent, "first"); + const second = broker.register(parent, "second"); + const lease = broker.acquireFork(parent, child, [first, second]); + const state = ( + broker as unknown as { + generations: WeakMap< + object, + { forkHandleCounts: Map } + >; + } + ).generations.get(child)!; + state.forkHandleCounts.delete(second); + + expect(() => lease.release()).toThrow("no longer owns fork lease"); + expect(broker.holderCount(first, child)).toBe(1); + expect(broker.holderCount(second, child)).toBe(1); + + state.forkHandleCounts.set(second, 1); + lease.release(); + expect(broker.holderCount(first, child)).toBe(0); + expect(broker.holderCount(second, child)).toBe(0); + }); + + it("does not let a fork lease release a direct generation lease", () => { + const broker = new ForkExternrefBroker(); + const parent = broker.createGeneration(61); + const child = broker.createGeneration(62); + const value = { opaque: true }; + const handle = broker.register(parent, value); + broker.acquire(child, handle); + const forkLease = broker.acquireFork(parent, child, [handle]); + + forkLease.release(); + expect(broker.authorize(child, handle)).toBe(value); + broker.release(child, handle); + expect(() => broker.authorize(child, handle)).toThrow("not authorized"); + }); + + it("permanently tombstones explicitly closed handles in every generation", () => { + const broker = new ForkExternrefBroker(); + const parent = broker.createGeneration(71); + const child = broker.createGeneration(72); + const value = { resource: "closed" }; + const handle = broker.register(parent, value); + const lease = broker.acquireFork(parent, child, [handle]); + + broker.tombstone(child, handle); + expect(() => broker.authorize(parent, handle)).toThrow("retired"); + expect(() => broker.authorize(child, handle)).toThrow("retired"); + lease.release(); + + const replacement = broker.register(parent, value); + expect(replacement).toBeGreaterThan(handle); + expect(broker.authorize(parent, replacement)).toBe(value); + }); + + it("never reuses wire handles and fails before overflowing u32", () => { + const broker = new ForkExternrefBroker({ maxHandle: 2 }); + const generation = broker.createGeneration(81); + const first = broker.register(generation, "first"); + const second = broker.register(generation, "second"); + expect([first, second]).toEqual([1, 2]); + broker.release(generation, first); + expect(() => broker.register(generation, "third")).toThrow( + "handle space exhausted", + ); + expect(() => broker.authorize(generation, first)).toThrow("retired"); + }); + + it("checks generation exhaustion before replacing a live generation", () => { + const broker = new ForkExternrefBroker({ maxGeneration: 1 }); + const generation = broker.createGeneration(91); + const handle = broker.register(generation, "still-owned"); + expect(() => broker.createGeneration(91)).toThrow( + "generation space exhausted", + ); + expect(broker.authorize(generation, handle)).toBe("still-owned"); + }); + + it("rejects a generation token issued by another broker", () => { + const firstBroker = new ForkExternrefBroker(); + const secondBroker = new ForkExternrefBroker(); + const foreign = firstBroker.createGeneration(101); + expect(() => secondBroker.register(foreign, "opaque")).toThrow( + "another broker", + ); + }); +}); + +describe("ForkExternrefTokenCache", () => { + it("reconstructs one worker-local identity per stable handle", () => { + const parent = new ForkExternrefTokenCache(11); + const child = new ForkExternrefTokenCache(12); + + const parentValue = parent.materialize(7); + const childValue = child.materialize(7); + expect(parent.materialize(7)).toBe(parentValue); + expect(child.materialize(7)).toBe(childValue); + expect(childValue).not.toBe(parentValue); + expect(parent.encode(parentValue)).toBe(7); + expect(child.encode(childValue)).toBe(7); + expect(parent.encode(childValue)).toBeNull(); + expect(child.encode(parentValue)).toBeNull(); + expect(child.encode({})).toBeNull(); + }); + + it("rejects handles that cannot round-trip through the u32 recipe contract", () => { + const cache = new ForkExternrefTokenCache(13); + expect(() => cache.materialize(0x1_0000_0000)).toThrow( + "invalid externref handle", + ); + }); + + it("rejects worker generation ids outside the u32 wire contract", () => { + expect(() => new ForkExternrefTokenCache(0)).toThrow( + "externref worker generation", + ); + expect(() => new ForkExternrefTokenCache(0x1_0000_0000)).toThrow( + "externref worker generation", + ); + }); +}); + +describe("ForkExternrefTokenRecipeProvider", () => { + it("round-trips owner handles through canonical worker tokens", () => { + const cache = new ForkExternrefTokenCache(14); + const provider = new ForkExternrefTokenRecipeProvider(cache); + const token = provider.materialize(41); + expect(provider.capture(token)).toBe(41); + expect(provider.materialize(41)).toBe(token); + }); + + it("adopts a raw Worker externref only when fork capture needs a child recipe", () => { + const cache = new ForkExternrefTokenCache(15); + const raw = Object.freeze({ workerLocal: true }); + const token = cache.materialize(43); + const normalized: unknown[] = []; + const provider = new ForkExternrefTokenRecipeProvider( + cache, + (value) => { + normalized.push(value); + return token; + }, + ); + + expect(provider.capture(raw)).toBe(43); + expect(normalized).toEqual([raw]); + expect(provider.materialize(43)).toBe(token); + }); + + it("detects a host import that bypassed process ownership", () => { + const provider = new ForkExternrefTokenRecipeProvider( + new ForkExternrefTokenCache(16), + ); + expect(() => provider.capture({ raw: true })).toThrow( + /without passing through the process reference owner/, + ); + }); +}); diff --git a/host/test/fork-reference-recipes.test.ts b/host/test/fork-reference-recipes.test.ts new file mode 100644 index 0000000000..064d5faf1c --- /dev/null +++ b/host/test/fork-reference-recipes.test.ts @@ -0,0 +1,719 @@ +import { execFileSync } from "node:child_process"; +import { mkdtempSync, readFileSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; +import { ForkFunctionCatalog } from "../src/fork-function-catalog"; +import { ForkExternrefBroker } from "../src/fork-reference-broker"; +import { ForkStaticRootCatalog } from "../src/fork-static-root-catalog"; +import { + DEFAULT_FORK_REFERENCE_RECIPE_LIMITS, + FORK_REFERENCE_RECIPE_VERSION, + ForkReferenceRecipeCoordinator, + ForkReferenceTypeCatalog, + decodeForkReferenceRecipes, + encodeForkReferenceRecipes, + type ForkReferenceRecipeGraph, + type ForkReferenceReplayArena, + type ForkReferenceReplayTarget, +} from "../src/fork-reference-recipes"; + +interface MaterializedNode { + kind: "externref" | "exnref" | "i31" | "struct" | "array"; + coordinate?: string; + value?: number; + edges: unknown[]; +} + +class RecordingArena implements ForkReferenceReplayArena { + readonly staged = new Set(); + readonly externrefs = new Map(); + committedRoots: readonly unknown[] | undefined; + aborted = false; + failAt: "connect" | "commit" | undefined; + + materializeExternref(handle: number): unknown { + let value = this.externrefs.get(handle); + if (!value) { + value = { kind: "externref", value: handle, edges: [] }; + this.externrefs.set(handle, value); + this.staged.add(value); + } + return value; + } + + materializeI31(value: number): unknown { + return this.add({ kind: "i31", value, edges: [] }); + } + + allocateException( + moduleActivation: number, + tagOrdinal: number, + payloadCount: number, + ): unknown { + return this.add({ + kind: "exnref", + coordinate: `${moduleActivation}:${tagOrdinal}`, + edges: new Array(payloadCount), + }); + } + + allocateStruct( + moduleActivation: number, + typeOrdinal: number, + fieldCount: number, + ): unknown { + return this.add({ + kind: "struct", + coordinate: `${moduleActivation}:${typeOrdinal}`, + edges: new Array(fieldCount), + }); + } + + allocateArray( + moduleActivation: number, + typeOrdinal: number, + length: number, + ): unknown { + return this.add({ + kind: "array", + coordinate: `${moduleActivation}:${typeOrdinal}`, + edges: new Array(length), + }); + } + + setExceptionPayload(exception: unknown, index: number, value: unknown): void { + this.connect(exception, index, value); + } + + setStructField(struct: unknown, index: number, value: unknown): void { + this.connect(struct, index, value); + } + + setArrayElement(array: unknown, index: number, value: unknown): void { + this.connect(array, index, value); + } + + commit(roots: readonly unknown[]): void { + if (this.failAt === "commit") throw new Error("injected commit failure"); + this.committedRoots = [...roots]; + this.staged.clear(); + } + + abort(): void { + this.aborted = true; + this.committedRoots = undefined; + this.staged.clear(); + this.externrefs.clear(); + } + + private add(node: MaterializedNode): MaterializedNode { + this.staged.add(node); + return node; + } + + private connect(container: unknown, index: number, value: unknown): void { + if (this.failAt === "connect") throw new Error("injected connect failure"); + (container as MaterializedNode).edges[index] = value; + } +} + +class RecordingTarget implements ForkReferenceReplayTarget { + beginCount = 0; + readonly arenas: RecordingArena[] = []; + + constructor( + readonly functions: ForkFunctionCatalog, + readonly types: ForkReferenceTypeCatalog, + private readonly failAt?: "connect" | "commit", + readonly staticRoots?: ForkStaticRootCatalog, + ) {} + + beginReferenceReplay(_nodeCount: number): RecordingArena { + this.beginCount++; + const arena = new RecordingArena(); + arena.failAt = this.failAt; + this.arenas.push(arena); + return arena; + } +} + +function emptyFunctions(): ForkFunctionCatalog { + return new ForkFunctionCatalog(); +} + +function referenceTypes(): ForkReferenceTypeCatalog { + const types = new ForkReferenceTypeCatalog(); + types.register(7, { + tags: [{ ordinal: 5, payloadCount: 2 }], + structs: [{ ordinal: 2, fieldCount: 3 }], + arrays: [{ ordinal: 3 }], + }); + return types; +} + +function graphWithEveryKind(handle: number): ForkReferenceRecipeGraph { + return { + roots: [100, 100, 60, 80, 90, 30], + nodes: [ + { + id: 100, + node: { + kind: "struct", + moduleActivation: 7, + typeOrdinal: 2, + layoutId: 12, + scalars: Uint8Array.of(0x78, 0x56, 0x34, 0x12), + fields: [20, 30, 60], + }, + }, + { + id: 20, + node: { + kind: "array", + moduleActivation: 7, + typeOrdinal: 3, + layoutId: 13, + scalars: Uint8Array.of(0xaa, 0xbb), + elements: [100, 60], + }, + }, + { id: 30, node: { kind: "externref", handle } }, + { + id: 60, + node: { + kind: "exnref", + moduleActivation: 7, + tagOrdinal: 5, + layoutId: 15, + scalars: Uint8Array.of(0, 1, 2, 3, 4, 5, 6, 7), + payloads: [100, 70], + }, + }, + { id: 70, node: { kind: "i31", value: -17 } }, + { + id: 80, + node: { + kind: "funcref", + moduleActivation: 7, + functionOrdinal: 0, + }, + }, + { id: 90, node: { kind: "null" } }, + ], + }; +} + +function catalogModule(): WebAssembly.Module { + const dir = mkdtempSync(join(tmpdir(), "kandelo-reference-recipes-")); + const wat = join(dir, "catalog.wat"); + const wasm = join(dir, "catalog.wasm"); + writeFileSync(wat, `(module + (table $catalog (export "__wpk_fork_function_catalog") 1 1 funcref) + (func $value (result i32) i32.const 43) + (elem (table $catalog) (i32.const 0) func $value) + )`); + execFileSync("wat2wasm", [wat, "-o", wasm]); + return new WebAssembly.Module(readFileSync(wasm)); +} + +function freshFunctionCatalogs(): { + source: ForkFunctionCatalog; + target: ForkFunctionCatalog; + sourceFunction: CallableFunction; + targetFunction: CallableFunction; +} { + const module = catalogModule(); + const sourceTable = new WebAssembly.Instance(module).exports + .__wpk_fork_function_catalog as WebAssembly.Table; + const targetTable = new WebAssembly.Instance(module).exports + .__wpk_fork_function_catalog as WebAssembly.Table; + const source = new ForkFunctionCatalog(); + const target = new ForkFunctionCatalog(); + source.register(7, sourceTable); + target.register(7, targetTable); + return { + source, + target, + sourceFunction: sourceTable.get(0) as CallableFunction, + targetFunction: targetTable.get(0) as CallableFunction, + }; +} + +describe("fork reference recipe wire codec", () => { + it("uses wire-format bounds rather than arbitrary production quotas", () => { + expect(DEFAULT_FORK_REFERENCE_RECIPE_LIMITS).toEqual({ + maxWireBytes: 0xffff_ffff, + maxNodes: 0xffff_ffff, + maxRoots: 0xffff_ffff, + maxEdges: 0xffff_ffff, + }); + }); + + it("canonicalizes input IDs and preserves cycles, aliases, and every kind", () => { + const graph = graphWithEveryKind(9); + const reversed: ForkReferenceRecipeGraph = { + roots: graph.roots, + nodes: [...graph.nodes].reverse(), + }; + + const first = encodeForkReferenceRecipes(graph); + const second = encodeForkReferenceRecipes(reversed); + expect(second).toEqual(first); + expect(new DataView(first.buffer).getUint16(4, true)).toBe( + FORK_REFERENCE_RECIPE_VERSION, + ); + + const decoded = decodeForkReferenceRecipes( + new Uint8Array(first.buffer, first.byteOffset, first.byteLength), + ); + expect(decoded.nodes.map(({ node }) => node.kind)).toEqual([ + "array", + "externref", + "exnref", + "i31", + "funcref", + "null", + "struct", + ]); + const array = decoded.nodes[0]!.node; + const exception = decoded.nodes[2]!.node; + const struct = decoded.nodes[6]!.node; + expect(array.kind === "array" && array.elements[0]).toBe(6); + expect(array.kind === "array" && array.scalars).toEqual( + Uint8Array.of(0xaa, 0xbb), + ); + expect(exception.kind === "exnref" && exception.layoutId).toBe(15); + expect(exception.kind === "exnref" && exception.scalars).toEqual( + Uint8Array.of(0, 1, 2, 3, 4, 5, 6, 7), + ); + expect(struct.kind === "struct" && struct.fields).toEqual([0, 1, 2]); + expect(struct.kind === "struct" && struct.layoutId).toBe(12); + expect(decoded.roots[0]).toBe(decoded.roots[1]); + expect(encodeForkReferenceRecipes(decoded)).toEqual(first); + }); + + it("rejects malformed versions, kinds, reserved fields, edges, and reachability", () => { + const scalar = encodeForkReferenceRecipes({ + roots: [0], + nodes: [{ id: 0, node: { kind: "null" } }], + }); + const mutate = (offset: number, value: number, width: 1 | 2 | 4): Uint8Array => { + const bytes = scalar.slice(); + const view = new DataView(bytes.buffer); + if (width === 1) view.setUint8(offset, value); + else if (width === 2) view.setUint16(offset, value, true); + else view.setUint32(offset, value, true); + return bytes; + }; + expect(() => decodeForkReferenceRecipes(mutate(4, 99, 2))).toThrow( + "unsupported reference recipe version", + ); + expect(() => decodeForkReferenceRecipes(mutate(40, 99, 1))).toThrow( + "unknown kind", + ); + expect(() => decodeForkReferenceRecipes(mutate(41, 1, 1))).toThrow( + "nonzero flags", + ); + expect(() => decodeForkReferenceRecipes(mutate(56, 1, 4))).toThrow( + "edge range exceeds", + ); + expect(() => decodeForkReferenceRecipes(mutate(32, 1, 4))).toThrow( + "reserved header", + ); + + const aggregate = encodeForkReferenceRecipes({ + roots: [0], + nodes: [ + { + id: 0, + node: { + kind: "array", + moduleActivation: 1, + typeOrdinal: 0, + elements: [1], + }, + }, + { id: 1, node: { kind: "null" } }, + ], + }); + const badEdge = aggregate.slice(); + const edgeOffset = 40 + 2 * 32 + 4; + new DataView(badEdge.buffer).setUint32(edgeOffset, 2, true); + expect(() => decodeForkReferenceRecipes(badEdge)).toThrow( + "targets missing node", + ); + + const reachable = encodeForkReferenceRecipes({ + roots: [0, 1], + nodes: [ + { id: 0, node: { kind: "null" } }, + { id: 1, node: { kind: "i31", value: 1 } }, + ], + }); + const unreachable = reachable.slice(); + const rootOffset = 40 + 2 * 32; + new DataView(unreachable.buffer).setUint32(rootOffset + 4, 0, true); + expect(() => decodeForkReferenceRecipes(unreachable)).toThrow( + "unreachable from every root", + ); + }); + + it("enforces bounded, exact layouts and i31/handle domains", () => { + expect(() => + encodeForkReferenceRecipes( + { + roots: [0], + nodes: [{ id: 0, node: { kind: "i31", value: 0x4000_0000 } }], + }, + ) + ).toThrow("invalid i31"); + expect(() => + encodeForkReferenceRecipes({ + roots: [0], + nodes: [{ id: 0, node: { kind: "externref", handle: 0 } }], + }) + ).toThrow("positive unsigned 32-bit integer"); + expect(() => + encodeForkReferenceRecipes({ + roots: [0], + nodes: [{ + id: 0, + node: { kind: "externref", handle: 0x1_0000_0000 }, + }], + }) + ).toThrow("positive unsigned 32-bit integer"); + const externref = encodeForkReferenceRecipes({ + roots: [0], + nodes: [{ id: 0, node: { kind: "externref", handle: 1 } }], + }); + const nonU32Externref = externref.slice(); + new DataView(nonU32Externref.buffer).setUint32(48, 1, true); + expect(() => decodeForkReferenceRecipes(nonU32Externref)).toThrow( + "positive unsigned 32-bit integer", + ); + + const i31 = encodeForkReferenceRecipes({ + roots: [0, 1], + nodes: [ + { id: 0, node: { kind: "i31", value: -0x4000_0000 } }, + { id: 1, node: { kind: "i31", value: 0x3fff_ffff } }, + ], + }); + expect( + decodeForkReferenceRecipes(i31).nodes.map(({ node }) => + node.kind === "i31" ? node.value : undefined + ), + ).toEqual([-0x4000_0000, 0x3fff_ffff]); + const noncanonicalI31 = i31.slice(); + new DataView(noncanonicalI31.buffer).setUint32(44, 0x4000_0000, true); + expect(() => decodeForkReferenceRecipes(noncanonicalI31)).toThrow( + "invalid i31", + ); + + const bytes = encodeForkReferenceRecipes({ + roots: [], + nodes: [], + }); + const trailing = new Uint8Array(bytes.length + 1); + trailing.set(bytes); + new DataView(trailing.buffer).setUint32(8, trailing.length, true); + expect(() => decodeForkReferenceRecipes(trailing)).toThrow( + "layout needs", + ); + expect(() => + decodeForkReferenceRecipes(bytes, { + maxWireBytes: 40, + maxNodes: 0, + maxRoots: 0, + maxEdges: 0, + }) + ).not.toThrow(); + expect(() => + encodeForkReferenceRecipes( + { roots: [0], nodes: [{ id: 0, node: { kind: "null" } }] }, + { + maxWireBytes: 40, + maxNodes: 1, + maxRoots: 1, + maxEdges: 0, + }, + ) + ).toThrow("needs"); + }); +}); + +describe("ForkReferenceRecipeCoordinator", () => { + it("resolves static-root recipes against the fresh child activation", () => { + const sourceValue = Object.freeze({ instance: "source" }); + const targetValue = Object.freeze({ instance: "target" }); + const sourceTable = new WebAssembly.Table({ + element: "externref", + initial: 1, + maximum: 1, + }); + const targetTable = new WebAssembly.Table({ + element: "externref", + initial: 1, + maximum: 1, + }); + sourceTable.set(0, sourceValue); + targetTable.set(0, targetValue); + const sourceRoots = new ForkStaticRootCatalog(); + const targetRoots = new ForkStaticRootCatalog(); + sourceRoots.register(6, sourceTable); + targetRoots.register(6, targetTable); + const target = new RecordingTarget( + emptyFunctions(), + new ForkReferenceTypeCatalog(), + undefined, + targetRoots, + ); + const broker = new ForkExternrefBroker(); + const parentGeneration = broker.createGeneration(80); + const childGeneration = broker.createGeneration(81); + const coordinator = new ForkReferenceRecipeCoordinator( + emptyFunctions(), + new ForkReferenceTypeCatalog(), + broker, + DEFAULT_FORK_REFERENCE_RECIPE_LIMITS, + sourceRoots, + ); + const ownership = coordinator.replay({ + parentGeneration, + childGeneration, + wire: encodeForkReferenceRecipes({ + roots: [0], + nodes: [{ + id: 0, + node: { + kind: "static-root", + moduleActivation: 6, + staticRootOrdinal: 0, + }, + }], + }), + target, + }); + expect(target.arenas[0]!.committedRoots).toEqual([targetValue]); + expect(target.arenas[0]!.committedRoots![0]).not.toBe(sourceValue); + ownership.release(); + }); + + it("reconstructs fresh-instance identities and cyclic typed graphs transactionally", () => { + const functions = freshFunctionCatalogs(); + expect(functions.targetFunction).not.toBe(functions.sourceFunction); + const sourceTypes = referenceTypes(); + const targetTypes = referenceTypes(); + const broker = new ForkExternrefBroker(); + const parentGeneration = broker.createGeneration(41); + const childGeneration = broker.createGeneration(42); + const opaque = { owned: "outside workers" }; + const handle = broker.register(parentGeneration, opaque); + expect(broker.register(parentGeneration, opaque)).toBe(handle); + + const target = new RecordingTarget(functions.target, targetTypes); + const coordinator = new ForkReferenceRecipeCoordinator( + functions.source, + sourceTypes, + broker, + ); + const ownership = coordinator.replay({ + parentGeneration, + childGeneration, + wire: encodeForkReferenceRecipes(graphWithEveryKind(handle)), + target, + }); + + const roots = target.arenas[0]!.committedRoots!; + const struct = roots[0] as MaterializedNode; + const exception = roots[2] as MaterializedNode; + expect(roots[1]).toBe(struct); + expect((struct.edges[0] as MaterializedNode).edges[0]).toBe(struct); + expect(struct.edges[2]).toBe(exception); + expect(exception.edges[0]).toBe(struct); + expect((exception.edges[1] as MaterializedNode).kind).toBe("i31"); + expect(roots[3]).toBe(functions.targetFunction); + expect(roots[4]).toBeNull(); + expect(roots[5]).toBe(struct.edges[1]); + expect(broker.holderCount(handle, childGeneration)).toBe(1); + expect(target.arenas[0]!.staged.size).toBe(0); + + ownership.release(); + expect(broker.holderCount(handle, childGeneration)).toBe(0); + expect(() => ownership.release()).toThrow("already released"); + }); + + it.each(["connect", "commit"] as const)( + "aborts the arena and releases every acquired handle after a %s failure", + (failAt) => { + const broker = new ForkExternrefBroker(); + const parentGeneration = broker.createGeneration(51); + const childGeneration = broker.createGeneration(52); + const handle = broker.register(parentGeneration, { opaque: true }); + const sourceTypes = new ForkReferenceTypeCatalog(); + const targetTypes = new ForkReferenceTypeCatalog(); + sourceTypes.register(0, { arrays: [{ ordinal: 0 }] }); + targetTypes.register(0, { arrays: [{ ordinal: 0 }] }); + const target = new RecordingTarget( + emptyFunctions(), + targetTypes, + failAt, + ); + const coordinator = new ForkReferenceRecipeCoordinator( + emptyFunctions(), + sourceTypes, + broker, + ); + const wire = encodeForkReferenceRecipes({ + roots: [0], + nodes: [ + { + id: 0, + node: { + kind: "array", + moduleActivation: 0, + typeOrdinal: 0, + elements: [1], + }, + }, + { id: 1, node: { kind: "externref", handle } }, + ], + }); + + expect(() => + coordinator.replay({ + parentGeneration, + childGeneration, + wire, + target, + }) + ).toThrow(`injected ${failAt} failure`); + expect(target.arenas[0]!.aborted).toBe(true); + expect(target.arenas[0]!.staged.size).toBe(0); + expect(target.arenas[0]!.externrefs.size).toBe(0); + expect(broker.holderCount(handle, childGeneration)).toBe(0); + expect(broker.resolve(parentGeneration, handle)).toEqual({ opaque: true }); + }, + ); + + it("validates source and target coordinate ownership before acquisition", () => { + const broker = new ForkExternrefBroker(); + const parentGeneration = broker.createGeneration(61); + const childGeneration = broker.createGeneration(62); + const handle = broker.register(parentGeneration, "opaque"); + const sourceTypes = new ForkReferenceTypeCatalog(); + const targetTypes = new ForkReferenceTypeCatalog(); + sourceTypes.register(1, { structs: [{ ordinal: 2, fieldCount: 1 }] }); + targetTypes.register(1, { structs: [{ ordinal: 9, fieldCount: 1 }] }); + const target = new RecordingTarget(emptyFunctions(), targetTypes); + const coordinator = new ForkReferenceRecipeCoordinator( + emptyFunctions(), + sourceTypes, + broker, + ); + const wire = encodeForkReferenceRecipes({ + roots: [0], + nodes: [ + { + id: 0, + node: { + kind: "struct", + moduleActivation: 1, + typeOrdinal: 2, + fields: [1], + }, + }, + { id: 1, node: { kind: "externref", handle } }, + ], + }); + + expect(() => + coordinator.replay({ + parentGeneration, + childGeneration, + wire, + target, + }) + ).toThrow("target catalog rejected"); + expect(target.beginCount).toBe(0); + expect(broker.holderCount(handle, childGeneration)).toBe(0); + }); + + it("acquires one host lease across every aliased occurrence in the graph", () => { + const broker = new ForkExternrefBroker(); + const parentGeneration = broker.createGeneration(66); + const childGeneration = broker.createGeneration(67); + const handle = broker.register(parentGeneration, { opaque: true }); + const target = new RecordingTarget( + emptyFunctions(), + new ForkReferenceTypeCatalog(), + ); + const coordinator = new ForkReferenceRecipeCoordinator( + emptyFunctions(), + new ForkReferenceTypeCatalog(), + broker, + ); + + const ownership = coordinator.replay({ + parentGeneration, + childGeneration, + wire: encodeForkReferenceRecipes({ + roots: [0, 0], + nodes: [{ id: 0, node: { kind: "externref", handle } }], + }), + target, + }); + expect(target.arenas[0]!.committedRoots![0]).toBe( + target.arenas[0]!.committedRoots![1], + ); + expect(broker.holderCount(handle, childGeneration)).toBe(1); + ownership.release(); + expect(broker.holderCount(handle, childGeneration)).toBe(0); + }); + + it("releases all replay leases when the child generation is retired", () => { + const broker = new ForkExternrefBroker(); + const parentGeneration = broker.createGeneration(71); + const childGeneration = broker.createGeneration(73); + const first = broker.register(parentGeneration, "first"); + const second = broker.register(parentGeneration, "second"); + + const coordinator = new ForkReferenceRecipeCoordinator( + emptyFunctions(), + new ForkReferenceTypeCatalog(), + broker, + ); + const replayOwnership = coordinator.replay({ + parentGeneration, + childGeneration, + wire: encodeForkReferenceRecipes({ + roots: [0, 1], + nodes: [ + { id: 0, node: { kind: "externref", handle: first } }, + { id: 1, node: { kind: "externref", handle: second } }, + ], + }), + target: new RecordingTarget( + emptyFunctions(), + new ForkReferenceTypeCatalog(), + ), + }); + expect(broker.releaseGeneration(childGeneration)).toBe(true); + expect(broker.holderCount(first, childGeneration)).toBe(0); + expect(broker.holderCount(second, childGeneration)).toBe(0); + expect(() => replayOwnership.release()).toThrow("already released"); + }); +}); + +describe("ForkReferenceTypeCatalog", () => { + it("binds tag and aggregate arity to one module activation", () => { + const types = referenceTypes(); + expect(() => types.validateTag(7, 5, 2)).not.toThrow(); + expect(() => types.validateTag(7, 5, 1)).toThrow("expects 2"); + expect(() => types.validateStruct(7, 2, 2)).toThrow("expects 3"); + expect(() => types.validateArray(7, 3)).not.toThrow(); + expect(() => types.validateArray(8, 3)).toThrow("not registered"); + expect(() => types.register(7, {})).toThrow("already registered"); + }); +}); diff --git a/host/test/fork-reference-segments.test.ts b/host/test/fork-reference-segments.test.ts new file mode 100644 index 0000000000..4b841d0c0c --- /dev/null +++ b/host/test/fork-reference-segments.test.ts @@ -0,0 +1,359 @@ +import { describe, expect, it } from "vitest"; +import { + ForkModuleStateRecordKind, + type ForkModuleStateRecord, +} from "../src/fork-module-state"; +import type { + ForkReferenceRecipeEntry, +} from "../src/fork-reference-recipes"; +import { + advanceForkReferenceLogicalOffset, + decodeSegmentedForkReferenceTransaction, + encodeSegmentedForkReferenceRecords, + findForkReferenceVectorOrdinal, + forkReferenceVectorFrom, + ForkReferenceDirectoryOverlay, + PagedForkReferenceVector, + scanSegmentedForkReferenceExternrefHandles, +} from "../src/fork-reference-segments"; +import { + FORK_REFERENCE_TRANSACTION_OWNER_ID, +} from "../src/fork-reference-transaction"; + +function graph(): ForkReferenceRecipeEntry[] { + return [ + { id: 0, node: { kind: "null" } }, + { id: 1, node: { kind: "externref", handle: 17 } }, + { + id: 2, + node: { + kind: "struct", + moduleActivation: 4, + typeOrdinal: 9, + layoutId: 12, + scalars: Uint8Array.of(1, 2, 3, 4, 5, 6, 7, 8, 9), + fields: [1, 2], + }, + }, + { + id: 3, + node: { + kind: "exnref", + moduleActivation: 4, + tagOrdinal: 3, + layoutId: 8, + scalars: Uint8Array.of(0xaa, 0xbb, 0xcc), + payloads: [2, 1], + }, + }, + { + id: 4, + node: { + kind: "funcref", + moduleActivation: 4, + functionOrdinal: 21, + }, + }, + { id: 5, node: { kind: "i31", value: -37 } }, + { + id: 6, + node: { + kind: "static-root", + moduleActivation: 4, + staticRootOrdinal: 5, + }, + }, + ]; +} + +function vectors(): [ + PagedForkReferenceVector, + PagedForkReferenceVector, + PagedForkReferenceVector, +] { + return [ + PagedForkReferenceVector.empty, + forkReferenceVectorFrom([1, 2, 0, 3], 4), + forkReferenceVectorFrom([6, 4, 5], 3), + ]; +} + +function records(segmentDataBytes = 7): ForkModuleStateRecord[] { + return encodeSegmentedForkReferenceRecords( + FORK_REFERENCE_TRANSACTION_OWNER_ID, + graph(), + vectors(), + { segmentDataBytes }, + ); +} + +function cloneRecords( + source: readonly ForkModuleStateRecord[], +): ForkModuleStateRecord[] { + return source.map((record) => ({ + ...record, + payload: record.payload.slice(), + })); +} + +function segmentRecords( + source: readonly ForkModuleStateRecord[], +): ForkModuleStateRecord[] { + return source.filter( + ({ kind }) => kind === ForkModuleStateRecordKind.ReferenceRecipeSegment, + ); +} + +function manifestRecord( + source: readonly ForkModuleStateRecord[], +): ForkModuleStateRecord { + return source.find( + ({ kind }) => kind === ForkModuleStateRecordKind.ReferenceRecipe, + )!; +} + +function sectionSegment( + source: readonly ForkModuleStateRecord[], + section: number, +): ForkModuleStateRecord { + return segmentRecords(source).find((record) => + new DataView( + record.payload.buffer, + record.payload.byteOffset, + record.payload.byteLength, + ).getUint16(8, true) === section + )!; +} + +describe("segmented KFRV v2", () => { + it("decodes fields split across many records without concatenating them", () => { + const encoded = records(); + expect(segmentRecords(encoded).length).toBeGreaterThan(40); + + const decoded = decodeSegmentedForkReferenceTransaction( + encoded, + FORK_REFERENCE_TRANSACTION_OWNER_ID, + ); + expect(decoded.graph.roots).toEqual([]); + expect(decoded.graph.nodes.length).toBe(7); + expect(decoded.graph.nodes.get(1)?.node).toEqual({ + kind: "externref", + handle: 17, + }); + expect(decoded.graph.nodes.get(2)?.node).toEqual({ + kind: "struct", + moduleActivation: 4, + typeOrdinal: 9, + layoutId: 12, + scalars: Uint8Array.of(1, 2, 3, 4, 5, 6, 7, 8, 9), + fields: [1, 2], + }); + expect(decoded.graph.nodes.get(3)?.node).toEqual({ + kind: "exnref", + moduleActivation: 4, + tagOrdinal: 3, + layoutId: 8, + scalars: Uint8Array.of(0xaa, 0xbb, 0xcc), + payloads: [2, 1], + }); + expect([...decoded.vectors.get(1)!]).toEqual([1, 2, 0, 3]); + expect([...decoded.vectors.get(2)!]).toEqual([6, 4, 5]); + expect(scanSegmentedForkReferenceExternrefHandles( + encoded, + FORK_REFERENCE_TRANSACTION_OWNER_ID, + )).toEqual(new Set([17])); + }); + + it("preserves deep cycles and paged vectors across hundreds of segments", () => { + const nodeCount = 2_000; + const nodes: ForkReferenceRecipeEntry[] = [ + { id: 0, node: { kind: "null" } }, + ]; + for (let id = 1; id < nodeCount; id++) { + nodes.push({ + id, + node: { + kind: "struct", + moduleActivation: 1, + typeOrdinal: 0, + fields: [id + 1 === nodeCount ? 1 : id + 1, 1], + }, + }); + } + const vectorValues = Array.from( + { length: 10_000 }, + (_, index) => index % nodeCount, + ); + const encoded = encodeSegmentedForkReferenceRecords( + FORK_REFERENCE_TRANSACTION_OWNER_ID, + nodes, + [ + PagedForkReferenceVector.empty, + forkReferenceVectorFrom(vectorValues, vectorValues.length), + ], + { segmentDataBytes: 127 }, + ); + expect(segmentRecords(encoded).length).toBeGreaterThan(800); + + const decoded = decodeSegmentedForkReferenceTransaction( + encoded, + FORK_REFERENCE_TRANSACTION_OWNER_ID, + ); + expect(decoded.graph.nodes.length).toBe(nodeCount); + expect(decoded.graph.nodes.get(nodeCount - 1)?.node).toMatchObject({ + kind: "struct", + fields: [1, 1], + }); + const vector = decoded.vectors.get(1)!; + expect(vector.length).toBe(10_000); + expect(vector.get(0)).toBe(0); + expect(vector.get(4_096)).toBe(96); + expect(vector.get(9_999)).toBe(1_999); + }); + + it("shares decoded vectors through an append-only replay overlay", () => { + const decoded = decodeSegmentedForkReferenceTransaction( + records(), + FORK_REFERENCE_TRANSACTION_OWNER_ID, + ); + const overlay = new ForkReferenceDirectoryOverlay(decoded.vectors); + expect(overlay.get(1)).toBe(decoded.vectors.get(1)); + expect(findForkReferenceVectorOrdinal( + [decoded.vectorIntern], + overlay, + forkReferenceVectorFrom([1, 2, 0, 3]), + )).toBe(1); + + const appended = forkReferenceVectorFrom([5, 4], 2); + const ordinal = overlay.length; + overlay.push(appended); + expect(overlay.get(ordinal)).toBe(appended); + expect(decoded.vectors.length).toBe(3); + }); + + it.each([ + { + name: "a missing segment", + mutate(source: ForkModuleStateRecord[]) { + source.splice(source.indexOf(segmentRecords(source)[2]!), 1); + }, + message: "ordinal", + }, + { + name: "a duplicate or reordered ordinal", + mutate(source: ForkModuleStateRecord[]) { + const segments = segmentRecords(source); + const firstIndex = source.indexOf(segments[0]!); + const secondIndex = source.indexOf(segments[1]!); + [source[firstIndex], source[secondIndex]] = [ + source[secondIndex]!, + source[firstIndex]!, + ]; + }, + message: "ordinal", + }, + { + name: "a gap", + mutate(source: ForkModuleStateRecord[]) { + const segment = segmentRecords(source)[1]!; + const view = new DataView(segment.payload.buffer); + view.setBigUint64(24, view.getBigUint64(24, true) + 1n, true); + }, + message: "gap, overlap, or duplicate", + }, + { + name: "an overlap", + mutate(source: ForkModuleStateRecord[]) { + const segment = segmentRecords(source)[1]!; + const view = new DataView(segment.payload.buffer); + view.setBigUint64(24, view.getBigUint64(24, true) - 1n, true); + }, + message: "gap, overlap, or duplicate", + }, + { + name: "trailing segment bytes", + mutate(source: ForkModuleStateRecord[]) { + const segment = segmentRecords(source)[0]!; + const payload = new Uint8Array(segment.payload.byteLength + 1); + payload.set(segment.payload); + segment.payload = payload; + }, + message: "invalid data length", + }, + { + name: "a segment after the manifest", + mutate(source: ForkModuleStateRecord[]) { + const manifest = manifestRecord(source); + const segment = source.splice( + source.indexOf(manifest) - 1, + 1, + )[0]!; + source.push(segment); + }, + message: "follows its final manifest", + }, + ])("rejects $name before materialization", ({ mutate, message }) => { + const malformed = cloneRecords(records()); + mutate(malformed); + expect(() => decodeSegmentedForkReferenceTransaction( + malformed, + FORK_REFERENCE_TRANSACTION_OWNER_ID, + )).toThrow(message); + }); + + it("rejects invalid semantic edges and noncanonical vector indexes", () => { + const badEdge = cloneRecords(records(64)); + const edge = sectionSegment(badEdge, 2); + new DataView( + edge.payload.buffer, + edge.payload.byteOffset, + edge.payload.byteLength, + ).setUint32(40, 0xffff_ffff, true); + expect(() => decodeSegmentedForkReferenceTransaction( + badEdge, + FORK_REFERENCE_TRANSACTION_OWNER_ID, + )).toThrow("missing recipe"); + + const badIndex = cloneRecords(records(64)); + const index = sectionSegment(badIndex, 4); + new DataView( + index.payload.buffer, + index.payload.byteOffset, + index.payload.byteLength, + ).setBigUint64(40 + 16, 99n, true); + expect(() => decodeSegmentedForkReferenceTransaction( + badIndex, + FORK_REFERENCE_TRANSACTION_OWNER_ID, + )).toThrow("expected"); + }); + + it("rejects wrong ownership and a duplicate canonical vector", () => { + const wrongOwner = cloneRecords(records()); + wrongOwner[0]!.ownerId++; + expect(() => decodeSegmentedForkReferenceTransaction( + wrongOwner, + FORK_REFERENCE_TRANSACTION_OWNER_ID, + )).toThrow("invalid process ownership"); + + expect(() => encodeSegmentedForkReferenceRecords( + FORK_REFERENCE_TRANSACTION_OWNER_ID, + graph(), + [ + PagedForkReferenceVector.empty, + forkReferenceVectorFrom([1, 2], 2), + forkReferenceVectorFrom([1, 2], 2), + ], + )).toThrow("duplicates canonical vector"); + }); + + it("uses u64 logical offsets without a 4-GiB allocation", () => { + expect(advanceForkReferenceLogicalOffset( + 0xffff_fff0n, + 0x40, + )).toBe(0x1_0000_0030n); + expect(() => advanceForkReferenceLogicalOffset( + 0xffff_ffff_ffff_fff0n, + 0x40, + )).toThrow("exceeds u64"); + }); +}); diff --git a/host/test/fork-reference-transaction.test.ts b/host/test/fork-reference-transaction.test.ts new file mode 100644 index 0000000000..3267ab020b --- /dev/null +++ b/host/test/fork-reference-transaction.test.ts @@ -0,0 +1,936 @@ +import { describe, expect, it } from "vitest"; +import { ForkFunctionCatalog } from "../src/fork-function-catalog"; +import { + ForkReferenceTransaction, + type ForkExternrefRecipeProvider, +} from "../src/fork-reference-transaction"; +import { + ForkModuleStateArena, + ForkModuleStateRecordKind, + type ForkModuleStateRecord, +} from "../src/fork-module-state"; +import { ForkStaticRootCatalog } from "../src/fork-static-root-catalog"; +import { + FORK_GC_FIELD_ALLOCATION_DEPENDENCY, + FORK_GC_FIELD_MUTABLE, + FORK_GC_FIELD_NULLABLE, + FORK_GC_FIELD_REFERENCE, + FORK_GC_LAYOUT_DEFAULTABLE_SHELL, + FORK_GC_LAYOUT_REQUIRES_PROVENANCE, + ForkGcCodecDescriptor, + ForkGcConstructorKind, + ForkGcLayoutKind, + type ForkGcCodecProvider, + type ForkGcLayoutDescriptor, +} from "../src/fork-gc-codec"; + +function makeFunctionCatalog( + moduleActivation: number, + functions: readonly CallableFunction[], +): ForkFunctionCatalog { + const table = new WebAssembly.Table({ + element: "anyfunc", + initial: functions.length, + maximum: functions.length, + }); + functions.forEach((fn, index) => table.set(index, fn)); + const catalog = new ForkFunctionCatalog(); + catalog.register(moduleActivation, table); + return catalog; +} + +function makeExternrefs(): { + provider: ForkExternrefRecipeProvider; + values: Map; +} { + let next = 1; + const values = new Map(); + const handles = new WeakMap(); + const provider: ForkExternrefRecipeProvider = { + capture(value) { + if ((typeof value === "object" && value !== null) || typeof value === "function") { + const known = handles.get(value as object); + if (known) return known; + const handle = next++; + handles.set(value as object, handle); + values.set(handle, value); + return handle; + } + const handle = next++; + values.set(handle, value); + return handle; + }, + materialize(handle) { + if (!values.has(handle)) throw new Error(`missing handle ${handle}`); + return values.get(handle); + }, + }; + return { provider, values }; +} + +function withArena( + run: (arena: ForkModuleStateArena) => void, +): ForkModuleStateRecord[] { + const memory = new WebAssembly.Memory({ initial: 16 }); + let next = 0x1_0000; + const arena = new ForkModuleStateArena( + memory, + 4, + (size) => { + const addr = next; + next += Number(size); + return addr; + }, + () => {}, + "reference transaction test", + ); + arena.begin(); + arena.appendModule({ + activationId: 0, + templateId: new Uint8Array(32), + }); + run(arena); + arena.seal(); + return arena.records(); +} + +describe("ForkReferenceTransaction", () => { + it("returns original identities in the parent and fresh catalog identities in the child", () => { + const parentFunction = new WebAssembly.Instance( + new WebAssembly.Module( + Uint8Array.from([ + 0, 97, 115, 109, 1, 0, 0, 0, + 1, 4, 1, 96, 0, 0, + 3, 2, 1, 0, + 7, 5, 1, 1, 102, 0, 0, + 10, 4, 1, 2, 0, 11, + ]), + ), + ).exports.f as CallableFunction; + const childFunction = new WebAssembly.Instance( + new WebAssembly.Module( + Uint8Array.from([ + 0, 97, 115, 109, 1, 0, 0, 0, + 1, 4, 1, 96, 0, 0, + 3, 2, 1, 0, + 7, 5, 1, 1, 102, 0, 0, + 10, 4, 1, 2, 0, 11, + ]), + ), + ).exports.f as CallableFunction; + const extern = Object.freeze({ owner: "process" }); + const parentExternrefs = makeExternrefs(); + const parent = new ForkReferenceTransaction( + makeFunctionCatalog(0, [parentFunction]), + parentExternrefs.provider, + ); + parent.beginCapture(); + const functionId = parent.encodeFuncref(parentFunction); + const functionAsExternId = parent.encodeExternref(parentFunction); + const externId = parent.encodeExternref(extern); + expect(functionAsExternId).toBe(functionId); + + const records = withArena((arena) => parent.sealInto(arena)); + parent.beginParentReplay(); + expect(parent.decodeFuncref(functionId)).toBe(parentFunction); + expect(parent.decodeExternref(functionAsExternId)).toBe(parentFunction); + expect(parent.decodeExternref(externId)).toBe(extern); + parent.finishReplay(); + + const childTokens = new Map(); + const child = new ForkReferenceTransaction( + makeFunctionCatalog(0, [childFunction]), + { + capture() { + throw new Error("child must not capture parent externrefs"); + }, + materialize(handle) { + let token = childTokens.get(handle); + if (!token) { + token = Object.freeze({ handle }); + childTokens.set(handle, token); + } + return token; + }, + }, + ); + child.attachChild(records); + expect(child.decodeFuncref(functionId)).toBe(childFunction); + expect(child.decodeFuncref(functionId)).not.toBe(parentFunction); + expect(child.decodeExternref(functionAsExternId)).toBe(childFunction); + expect(child.decodeExternref(externId)).toBe(child.decodeExternref(externId)); + expect(child.decodeExternref(externId)).not.toBe(extern); + child.finishReplay(); + }); + + it("deduplicates aliases across typed slots and reserves zero for null", () => { + const externrefs = makeExternrefs(); + const transaction = new ForkReferenceTransaction( + makeFunctionCatalog(0, []), + externrefs.provider, + ); + const shared = { value: 1 }; + transaction.beginCapture(); + expect(transaction.encodeExternref(null)).toBe(0); + expect(transaction.encodeExternref(shared)).toBe(1); + expect(transaction.encodeExternref(shared)).toBe(1); + const records = withArena((arena) => transaction.sealInto(arena)); + transaction.beginParentReplay(); + expect(transaction.decodeExternref(0)).toBeNull(); + expect(transaction.decodeExternref(1)).toBe(shared); + transaction.finishReplay(); + expect(records.filter( + ({ kind }) => kind === ForkModuleStateRecordKind.ReferenceRecipe, + )).toHaveLength(1); + expect(records.some( + ({ kind }) => kind === ForkModuleStateRecordKind.ReferenceRecipeSegment, + )).toBe(true); + }); + + it("round-trips compact call-specific recipe vectors with O(1) lookup", () => { + const parent = new ForkReferenceTransaction( + makeFunctionCatalog(0, []), + makeExternrefs().provider, + ); + parent.beginCapture(); + const first = parent.encodeExternref({ value: 1 }); + const second = parent.encodeExternref({ value: 2 }); + const builder = parent.beginReferenceVector(2); + expect(builder).toBe(1); + parent.appendReferenceVector(builder, first); + parent.appendReferenceVector(builder, second); + const vector = parent.finishReferenceVector(builder); + expect(vector).toBe(1); + const duplicateBuilder = parent.beginReferenceVector(2); + // Completed builder handles are reused, while the frame-visible result is + // the canonical content ordinal. + expect(duplicateBuilder).toBe(builder); + parent.appendReferenceVector(duplicateBuilder, first); + parent.appendReferenceVector(duplicateBuilder, second); + expect(parent.finishReferenceVector(duplicateBuilder)).toBe(vector); + const records = withArena((arena) => parent.sealInto(arena)); + parent.beginParentReplay(); + expect(parent.getReferenceVector(vector, 0)).toBe(first); + expect(parent.getReferenceVector(vector, 1)).toBe(second); + + const child = new ForkReferenceTransaction( + makeFunctionCatalog(0, []), + makeExternrefs().provider, + ); + child.attachChild(records); + expect((child as unknown as { + decodedReferenceVectors: Array; + }).decodedReferenceVectors).toHaveLength(2); + expect(child.getReferenceVector(vector, 0)).toBe(first); + expect(child.getReferenceVector(vector, 1)).toBe(second); + expect(() => child.getReferenceVector(vector, 2)).toThrow(/out of bounds/); + child.finishReplay(); + parent.finishReplay(); + }); + + it("does not seal a partially appended reference vector", () => { + const transaction = new ForkReferenceTransaction( + makeFunctionCatalog(0, []), + makeExternrefs().provider, + ); + transaction.beginCapture(); + const recipe = transaction.encodeExternref({ value: 1 }); + const builder = transaction.beginReferenceVector(2); + transaction.appendReferenceVector(builder, recipe); + expect(() => transaction.finishReferenceVector(builder)).toThrow( + /expected 2/, + ); + expect(() => withArena((arena) => transaction.sealInto(arena))).toThrow( + /unfinished reference vector/, + ); + transaction.abort(); + }); + + it("encodes a module-static root before opaque capture and resolves the child root", () => { + const parentRoot = Object.freeze({ instance: "parent" }); + const parentTable = new WebAssembly.Table({ + element: "externref", + initial: 2, + maximum: 2, + }); + parentTable.set(0, parentRoot); + parentTable.set(1, parentRoot); + const parentRoots = new ForkStaticRootCatalog(); + parentRoots.register(5, parentTable); + const parentExternrefs = makeExternrefs(); + const parent = new ForkReferenceTransaction( + makeFunctionCatalog(0, []), + parentExternrefs.provider, + undefined, + undefined, + undefined, + "static-root parent", + parentRoots, + ); + parent.beginCapture(); + const recipeId = parent.encodeExternref(parentRoot); + const builder = parent.beginReferenceVector(1); + parent.appendReferenceVector(builder, recipeId); + const vector = parent.finishReferenceVector(builder); + expect(parentExternrefs.values.size).toBe(0); + const records = withArena((arena) => parent.sealInto(arena)); + + const childRoot = Object.freeze({ instance: "child" }); + const childTable = new WebAssembly.Table({ + element: "externref", + initial: 2, + maximum: 2, + }); + childTable.set(0, childRoot); + childTable.set(1, childRoot); + const childRoots = new ForkStaticRootCatalog(); + childRoots.register(5, childTable); + const child = new ForkReferenceTransaction( + makeFunctionCatalog(0, []), + makeExternrefs().provider, + undefined, + undefined, + undefined, + "static-root child", + childRoots, + ); + child.attachChild(records); + const restoredRecipeId = child.getReferenceVector(vector, 0); + expect(restoredRecipeId).toBe(recipeId); + expect(child.decodeExternref(restoredRecipeId)).toBe(childRoot); + expect(child.decodeExternref(restoredRecipeId)).not.toBe(parentRoot); + child.finishReplay(); + parent.abort(); + }); + + it("upgrades an earlier external view to one structural exception identity", () => { + const memory = new WebAssembly.Memory({ initial: 2 }); + const tag = new WebAssembly.Tag({ parameters: ["i32"] }); + const thrown = new WebAssembly.Exception(tag, [29]); + const transaction = new ForkReferenceTransaction( + makeFunctionCatalog(0, []), + makeExternrefs().provider, + memory, + ); + const provider = { + throwSlot(_slot: number): never { + throw thrown; + }, + clearSlots(): void {}, + }; + transaction.beginCapture(); + const externalView = transaction.encodeExternref(thrown); + expect(transaction.lookupExceptionSlot(0, provider)).toBe(0); + const exceptionView = transaction.claimExceptionSlot(0, provider); + expect(exceptionView).toBe(externalView); + expect(transaction.encodeExternref(thrown)).toBe(externalView); + transaction.defineException( + exceptionView, + 8, + 3, + 4, + 0, + 0, + 0, + 0, + ); + expect(transaction.exceptionOwner(externalView)).toBe(8); + withArena((arena) => transaction.sealInto(arena)); + transaction.abort(); + }); + + it("drops strong temporary roots after abort", () => { + const externrefs = makeExternrefs(); + const transaction = new ForkReferenceTransaction( + makeFunctionCatalog(0, []), + externrefs.provider, + ); + transaction.beginCapture(); + const reused = { value: 1 }; + transaction.encodeExternref(reused); + transaction.abort(); + transaction.beginCapture(); + expect(transaction.encodeExternref(reused)).toBe(1); + transaction.abort(); + }); + + it("owns reentrant shared-memory scratch with LIFO release and zeroing", () => { + const memory = new WebAssembly.Memory({ initial: 8 }); + let next = 0x1_0000; + const released: Array<[number, number]> = []; + const transaction = new ForkReferenceTransaction( + makeFunctionCatalog(0, []), + makeExternrefs().provider, + memory, + (size) => { + const addr = next; + next += size; + return addr; + }, + (addr, size) => released.push([addr, size]), + "scratch test", + ); + transaction.beginCapture(); + const outer = transaction.reserveScratch(24); + const inner = transaction.reserveScratch(32); + expect(inner).toBe(outer + 32); + new Uint8Array(memory.buffer, outer, 24).fill(0xaa); + new Uint8Array(memory.buffer, inner, 32).fill(0xbb); + + expect(() => transaction.releaseScratch(outer, 24)).toThrow( + /most recent reservation/, + ); + transaction.releaseScratch(inner, 32); + expect(new Uint8Array(memory.buffer, inner, 32)).toEqual(new Uint8Array(32)); + transaction.releaseScratch(outer, 24); + expect(new Uint8Array(memory.buffer, outer, 32)).toEqual(new Uint8Array(32)); + + // The common page remains transaction-owned for reuse, then is cleared + // and returned exactly once on abort. + expect(transaction.reserveScratch(16)).toBe(outer); + new Uint8Array(memory.buffer, outer, 16).fill(0xcc); + transaction.abort(); + expect(new Uint8Array(memory.buffer, outer, 16)).toEqual(new Uint8Array(16)); + expect(released).toEqual([[0x1_0000, 65_536]]); + }); + + it("interns Wasm-only exception identity and transfers exact scalar/reference payloads", () => { + const memory = new WebAssembly.Memory({ initial: 2 }); + const thrown = new WebAssembly.Exception( + new WebAssembly.Tag({ parameters: ["i32"] }), + [17], + ); + let cleared = 0; + const externrefs = makeExternrefs(); + const parent = new ForkReferenceTransaction( + makeFunctionCatalog(0, []), + externrefs.provider, + memory, + ); + parent.setExceptionSlotProvider({ + throwSlot(slot): never { + if (slot !== 3 && slot !== 4) throw new Error(`invalid slot ${slot}`); + throw thrown; + }, + clearSlots() { + cleared++; + }, + }); + parent.beginCapture(); + expect(parent.lookupExceptionSlot(3)).toBe(0); + const exceptionId = parent.claimExceptionSlot(3); + expect(exceptionId).toBe(1); + expect(parent.lookupExceptionSlot(4)).toBe(exceptionId); + expect(parent.claimExceptionSlot(4)).toBe(exceptionId); + + const sourceScalars = Uint8Array.of( + 0x78, 0x56, 0x34, 0x12, + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15, + ); + new Uint8Array(memory.buffer, 0x100, sourceScalars.length).set(sourceScalars); + new DataView(memory.buffer).setUint32(0x200, 0, true); + parent.defineException( + exceptionId, + 7, + 5, + 9, + 0x100, + sourceScalars.length, + 0x200, + 1, + ); + const records = withArena((arena) => parent.sealInto(arena)); + parent.beginParentReplay(); + expect(parent.routeException(exceptionId, 7)).toBe(9); + expect(parent.routeException(exceptionId, 8)).toBe(-1); + expect( + parent.loadException( + exceptionId, + 7, + 5, + 9, + 0x300, + sourceScalars.length, + 0x400, + 1, + ), + ).toBe(1); + expect(new Uint8Array(memory.buffer, 0x300, sourceScalars.length)).toEqual( + sourceScalars, + ); + expect(new DataView(memory.buffer).getUint32(0x400, true)).toBe(0); + parent.finishReplay(); + expect(cleared).toBe(1); + + const childMemory = new WebAssembly.Memory({ initial: 2 }); + const child = new ForkReferenceTransaction( + makeFunctionCatalog(0, []), + makeExternrefs().provider, + childMemory, + ); + child.attachChild(records); + expect(child.routeException(exceptionId, 7)).toBe(9); + expect( + child.loadException( + exceptionId, + 7, + 5, + 9, + 0x100, + sourceScalars.length, + 0x200, + 1, + ), + ).toBe(1); + expect( + new Uint8Array(childMemory.buffer, 0x100, sourceScalars.length), + ).toEqual(sourceScalars); + expect(() => + child.loadException( + exceptionId, + 7, + 5, + 10, + 0x100, + sourceScalars.length, + 0x200, + 1, + ) + ).toThrow(/coordinate does not match/); + child.finishReplay(); + }); +}); + +function gcStructDescriptor(options: { + defaultable?: boolean; + mutable?: boolean; + nullable?: boolean; + dependency?: boolean; +}): ForkGcCodecDescriptor { + const fieldFlags = + FORK_GC_FIELD_REFERENCE + | (options.mutable ? FORK_GC_FIELD_MUTABLE : 0) + | (options.nullable ? FORK_GC_FIELD_NULLABLE : 0) + | (options.dependency ? FORK_GC_FIELD_ALLOCATION_DEPENDENCY : 0); + const layout: ForkGcLayoutDescriptor = { + id: 1, + typeOrdinal: 0, + kind: ForkGcLayoutKind.Struct, + constructor: ForkGcConstructorKind.Struct, + flags: options.defaultable ? FORK_GC_LAYOUT_DEFAULTABLE_SHELL : 0, + scalarLengthOrStride: 0, + fields: [{ + storage: 8, + flags: fieldFlags, + scalarOffset: null, + referenceOrdinal: 0, + }], + superTypeOrdinal: null, + baseLayoutId: 1, + auxiliary: 0, + provenanceScalarLength: 0, + provenanceReferenceCount: 0, + }; + return new ForkGcCodecDescriptor([layout]); +} + +function captureGcStructGraph(options: { + descriptor: ForkGcCodecDescriptor; + edges: readonly (readonly number[])[]; +}): ForkModuleStateRecord[] { + const memory = new WebAssembly.Memory({ initial: 2 }); + const transaction = new ForkReferenceTransaction( + makeFunctionCatalog(0, []), + makeExternrefs().provider, + memory, + ); + const table = new WebAssembly.Table({ + element: "externref", + initial: 1, + }); + transaction.beginCapture(); + const recipeIds = options.edges.map((_, index) => { + table.set(0, { index }); + return transaction.claimGcSlot(table, 0); + }); + options.edges.forEach((edgeIndexes, index) => { + const builder = transaction.beginReferenceVector(edgeIndexes.length); + edgeIndexes.forEach((edgeIndex) => { + const edgeRecipe = edgeIndex === -1 ? 0 : recipeIds[edgeIndex]; + if (edgeRecipe === undefined) { + throw new Error(`test GC edge ${edgeIndex} is out of bounds`); + } + transaction.appendReferenceVector(builder, edgeRecipe); + }); + const vector = transaction.finishReferenceVector(builder); + transaction.defineGc( + recipeIds[index]!, + 0, + 0, + 1, + ForkGcLayoutKind.Struct, + 0, + 0, + vector, + options.descriptor, + null, + ); + }); + return withArena((arena) => transaction.sealInto(arena)); +} + +describe("ForkReferenceTransaction typed replay barrier", () => { + it("reuses canonical GC edge vectors without rebuilding the vector directory", () => { + const descriptor = gcStructDescriptor({ + defaultable: true, + mutable: true, + nullable: true, + }); + const records = captureGcStructGraph({ + descriptor, + edges: [[0], [1], [2]], + }); + const child = new ForkReferenceTransaction( + makeFunctionCatalog(0, []), + makeExternrefs().provider, + new WebAssembly.Memory({ initial: 2 }), + ); + child.attachChild(records); + + const internal = child as unknown as { + decodedReferenceVectors: Array; + }; + const directory = internal.decodedReferenceVectors; + const firstOrdinal = child.loadGc( + 1, + 0, + 0, + 1, + ForkGcLayoutKind.Struct, + 0, + 0, + ); + const secondOrdinal = child.loadGc( + 2, + 0, + 0, + 1, + ForkGcLayoutKind.Struct, + 0, + 0, + ); + const thirdOrdinal = child.loadGc( + 3, + 0, + 0, + 1, + ForkGcLayoutKind.Struct, + 0, + 0, + ); + + expect(internal.decodedReferenceVectors).toBe(directory); + expect([firstOrdinal, secondOrdinal, thirdOrdinal]).toEqual([1, 2, 3]); + expect(child.loadGc( + 1, + 0, + 0, + 1, + ForkGcLayoutKind.Struct, + 0, + 0, + )).toBe(firstOrdinal); + expect(internal.decodedReferenceVectors).toHaveLength(4); + child.abort(); + }); + + it("materializes a deep immutable dependency chain without host stack recursion", () => { + const nodeCount = 6_000; + const descriptor = gcStructDescriptor({ + nullable: true, + dependency: true, + }); + const records = captureGcStructGraph({ + descriptor, + edges: Array.from( + { length: nodeCount }, + (_, index) => [index + 1 === nodeCount ? -1 : index + 1], + ), + }); + let allocations = 0; + let fills = 0; + let firstAllocated = 0; + let lastAllocated = 0; + const provider: ForkGcCodecProvider = { + activationId: 0, + descriptor, + probe: () => 0n, + encodeSlot: () => 0, + allocate(recipeId) { + if (allocations === 0) firstAllocated = recipeId; + lastAllocated = recipeId; + allocations++; + }, + fill: () => { fills++; }, + publishExternref: () => {}, + }; + const child = new ForkReferenceTransaction( + makeFunctionCatalog(0, []), + makeExternrefs().provider, + new WebAssembly.Memory({ initial: 2 }), + undefined, + undefined, + "deep typed child", + undefined, + { + prepareTransit: () => {}, + publishTransit: () => {}, + publishExternref: () => {}, + provider: () => provider, + providers: () => [provider], + validateExceptionOwner: () => {}, + materializeException: () => {}, + }, + ); + child.attachChild(records); + child.materializeAllTyped(); + + expect(allocations).toBe(nodeCount); + expect(fills).toBe(nodeCount); + expect(firstAllocated).toBe(nodeCount); + expect(lastAllocated).toBe(1); + child.finishReplay(); + }); + + it("publishes fresh static roots before dynamic GC constructors consume them", () => { + const descriptor = gcStructDescriptor({ dependency: true }); + const parentRoot = Object.freeze({ instance: "parent-static-root" }); + const parentRoots = new ForkStaticRootCatalog(); + const parentCatalogTable = new WebAssembly.Table({ + element: "externref", + initial: 1, + maximum: 1, + }); + parentCatalogTable.set(0, parentRoot); + parentRoots.register(5, parentCatalogTable); + const captureTable = new WebAssembly.Table({ + element: "externref", + initial: 1, + }); + const memory = new WebAssembly.Memory({ initial: 2 }); + const parent = new ForkReferenceTransaction( + makeFunctionCatalog(0, []), + makeExternrefs().provider, + memory, + undefined, + undefined, + "static-root graph parent", + parentRoots, + ); + parent.beginCapture(); + captureTable.set(0, parentRoot); + const staticRecipe = parent.lookupGcSlot(captureTable, 0); + captureTable.set(0, { dynamic: true }); + const dynamicRecipe = parent.claimGcSlot(captureTable, 0); + const builder = parent.beginReferenceVector(1); + parent.appendReferenceVector(builder, staticRecipe); + const vector = parent.finishReferenceVector(builder); + parent.defineGc( + dynamicRecipe, + 0, + 0, + 1, + ForkGcLayoutKind.Struct, + 0, + 0, + vector, + descriptor, + null, + ); + const records = withArena((arena) => parent.sealInto(arena)); + + const childRoot = Object.freeze({ instance: "child-static-root" }); + const childRoots = new ForkStaticRootCatalog(); + const childCatalogTable = new WebAssembly.Table({ + element: "externref", + initial: 1, + maximum: 1, + }); + childCatalogTable.set(0, childRoot); + childRoots.register(5, childCatalogTable); + const transit = new Map(); + const calls: string[] = []; + const provider: ForkGcCodecProvider = { + activationId: 0, + descriptor, + probe: () => 0n, + encodeSlot: () => 0, + allocate(recipeId) { + expect(transit.get(staticRecipe)).toBe(childRoot); + expect(transit.get(staticRecipe)).not.toBe(parentRoot); + calls.push(`allocate:${recipeId}`); + }, + fill: (recipeId) => { calls.push(`fill:${recipeId}`); }, + publishExternref: () => {}, + }; + const child = new ForkReferenceTransaction( + makeFunctionCatalog(0, []), + makeExternrefs().provider, + new WebAssembly.Memory({ initial: 2 }), + undefined, + undefined, + "static-root graph child", + childRoots, + { + prepareTransit: (max) => { calls.push(`prepare:${max}`); }, + publishTransit(recipeId, value) { + transit.set(recipeId, value); + calls.push(`publish:${recipeId}`); + }, + publishExternref: () => {}, + provider: () => provider, + providers: () => [provider], + validateExceptionOwner: () => {}, + materializeException: () => {}, + }, + ); + child.attachChild(records); + child.materializeAllTyped(); + expect(calls).toEqual([ + `prepare:${dynamicRecipe}`, + `publish:${staticRecipe}`, + `allocate:${dynamicRecipe}`, + `fill:${dynamicRecipe}`, + ]); + child.finishReplay(); + parent.abort(); + }); + + it("allocates all defaultable shells before filling cyclic mutable edges", () => { + const descriptor = gcStructDescriptor({ + defaultable: true, + mutable: true, + nullable: true, + }); + const records = captureGcStructGraph({ + descriptor, + edges: [[0]], + }); + const calls: string[] = []; + const provider: ForkGcCodecProvider = { + activationId: 0, + descriptor, + probe: () => 0n, + encodeSlot: () => 0, + allocate: (recipeId) => { calls.push(`allocate:${recipeId}`); }, + fill: (recipeId) => { calls.push(`fill:${recipeId}`); }, + publishExternref: () => {}, + }; + const memory = new WebAssembly.Memory({ initial: 2 }); + const child = new ForkReferenceTransaction( + makeFunctionCatalog(0, []), + makeExternrefs().provider, + memory, + undefined, + undefined, + "typed child", + undefined, + { + prepareTransit: (max) => { calls.push(`prepare:${max}`); }, + publishTransit: () => {}, + publishExternref: () => {}, + provider: () => provider, + providers: () => [provider], + validateExceptionOwner: () => {}, + materializeException: () => {}, + }, + ); + child.attachChild(records); + child.materializeAllTyped(); + expect(calls).toEqual(["prepare:1", "allocate:1", "fill:1"]); + child.finishReplay(); + }); + + it("rejects an immutable constructor cycle before allocating any object", () => { + const descriptor = gcStructDescriptor({ + dependency: true, + }); + const records = captureGcStructGraph({ + descriptor, + edges: [[1], [0]], + }); + const calls: string[] = []; + const provider: ForkGcCodecProvider = { + activationId: 0, + descriptor, + probe: () => 0n, + encodeSlot: () => 0, + allocate: (recipeId) => { calls.push(`allocate:${recipeId}`); }, + fill: (recipeId) => { calls.push(`fill:${recipeId}`); }, + publishExternref: () => {}, + }; + const child = new ForkReferenceTransaction( + makeFunctionCatalog(0, []), + makeExternrefs().provider, + new WebAssembly.Memory({ initial: 2 }), + undefined, + undefined, + "typed cycle", + undefined, + { + prepareTransit: () => {}, + publishTransit: () => {}, + publishExternref: () => {}, + provider: () => provider, + providers: () => [provider], + validateExceptionOwner: () => {}, + materializeException: () => {}, + }, + ); + child.attachChild(records); + expect(() => child.materializeAllTyped()).toThrow( + /unallocatable constructor cycle/, + ); + expect(calls).toEqual([]); + child.abort(); + }); + + it("requires constructor provenance when the selected layout declares it", () => { + const base = gcStructDescriptor({}).require(1); + const descriptor = new ForkGcCodecDescriptor([{ + ...base, + flags: FORK_GC_LAYOUT_REQUIRES_PROVENANCE, + provenanceReferenceCount: 1, + }]); + const memory = new WebAssembly.Memory({ initial: 2 }); + const transaction = new ForkReferenceTransaction( + makeFunctionCatalog(0, []), + makeExternrefs().provider, + memory, + ); + const table = new WebAssembly.Table({ + element: "externref", + initial: 1, + }); + table.set(0, {}); + transaction.beginCapture(); + const recipe = transaction.claimGcSlot(table, 0); + const builder = transaction.beginReferenceVector(1); + transaction.appendReferenceVector(builder, 0); + const vector = transaction.finishReferenceVector(builder); + expect(() => transaction.defineGc( + recipe, + 0, + 0, + 1, + ForkGcLayoutKind.Struct, + 0, + 0, + vector, + descriptor, + null, + )).toThrow(/missing constructor provenance/); + transaction.abort(); + }); +}); diff --git a/host/test/fork-replay-events.test.ts b/host/test/fork-replay-events.test.ts new file mode 100644 index 0000000000..18ada8c0c5 --- /dev/null +++ b/host/test/fork-replay-events.test.ts @@ -0,0 +1,289 @@ +import { describe, expect, it } from "vitest"; +import { + FORK_REPLAY_EVENT_SEGMENT_CAPACITY, + type ForkReplayEventWire, + encodeForkReplayEventManifest, + encodeForkReplayEventSegment, + ForkReplayEventJournal, + ForkResumeTable, + validateForkReplayEventWire, +} from "../src/fork-replay-events"; + +function wasmFunction(): CallableFunction { + const module = new WebAssembly.Module(new Uint8Array([ + 0x00, 0x61, 0x73, 0x6d, 0x01, 0x00, 0x00, 0x00, + 0x01, 0x05, 0x01, 0x60, 0x00, 0x01, 0x7f, + 0x03, 0x02, 0x01, 0x00, + 0x07, 0x05, 0x01, 0x01, 0x66, 0x00, 0x00, + 0x0a, 0x06, 0x01, 0x04, 0x00, 0x41, 0x07, 0x0b, + ])); + return new WebAssembly.Instance(module).exports.f as CallableFunction; +} + +function sealedWire( + journal: ForkReplayEventJournal, +): ForkReplayEventWire & { segments: Uint8Array[] } { + journal.sealCapture(); + return { + manifest: journal.capturedManifestPayload(), + segments: [...journal.capturedSegmentPayloads()], + }; +} + +describe("ForkReplayEventJournal", () => { + it("replays the exact reverse of cross-module frame commit order", () => { + const parent = new ForkReplayEventJournal(); + parent.beginCapture(); + parent.recordCommit(3, 8); + parent.recordCommit(3, 4); + parent.recordCommit(0, 11); + const wire = sealedWire(parent); + parent.beginParentReplay(); + + expect(parent.peek()).toEqual({ activationId: 0, functionOrdinal: 11 }); + parent.consume(0, 11); + expect(parent.peek()).toEqual({ activationId: 3, functionOrdinal: 4 }); + parent.consume(3, 4); + expect(parent.peek()).toEqual({ activationId: 3, functionOrdinal: 8 }); + parent.consume(3, 8); + expect(parent.peek()).toBeNull(); + parent.finishReplay(); + + const child = new ForkReplayEventJournal(); + child.attachChild(wire); + expect(child.peek()).toEqual({ activationId: 0, functionOrdinal: 11 }); + }); + + it("streams multi-page capture and child replay without concatenation", () => { + const eventCount = FORK_REPLAY_EVENT_SEGMENT_CAPACITY + 3; + const parent = new ForkReplayEventJournal(); + parent.beginCapture(); + for (let index = 0; index < eventCount; index++) { + parent.recordCommit(index % 3, index); + } + const wire = sealedWire(parent); + const summary = validateForkReplayEventWire(wire); + expect(summary.eventCount).toBe(BigInt(eventCount)); + expect(summary.segmentCount).toBe(2n); + expect(summary.activationIds).toEqual(new Set([0, 1, 2])); + expect(wire.segments).toHaveLength(2); + + parent.beginParentReplay(); + const child = new ForkReplayEventJournal(); + child.attachChild(wire); + for (let index = eventCount - 1; index >= 0; index--) { + const expected = { activationId: index % 3, functionOrdinal: index }; + expect(parent.peek()).toEqual(expected); + parent.consume(expected.activationId, expected.functionOrdinal); + expect(child.peek()).toEqual(expected); + child.consume(expected.activationId, expected.functionOrdinal); + } + expect(parent.peek()).toBeNull(); + expect(child.peek()).toBeNull(); + parent.finishReplay(); + child.finishReplay(); + }); + + it("requires peek and consume to name the same frame atomically", () => { + const journal = new ForkReplayEventJournal(); + journal.beginCapture(); + journal.recordCommit(1, 2); + sealedWire(journal); + journal.beginParentReplay(); + expect(() => journal.consume(1, 2)).toThrow("without selecting"); + journal.peek(); + expect(() => journal.consume(1, 3)).toThrow("expected 1:2"); + }); + + it("drops every page on abort and can begin another capture", () => { + const journal = new ForkReplayEventJournal(); + journal.beginCapture(); + for ( + let index = 0; + index < FORK_REPLAY_EVENT_SEGMENT_CAPACITY + 1; + index++ + ) { + journal.recordCommit(7, index); + } + journal.abort(); + expect(journal.phaseName()).toBe("idle"); + journal.beginCapture(); + journal.recordCommit(1, 2); + expect(validateForkReplayEventWire(sealedWire(journal)).eventCount).toBe(1n); + }); +}); + +describe("fork replay event segmented wire", () => { + function twoPageWire(): ForkReplayEventWire & { segments: Uint8Array[] } { + const journal = new ForkReplayEventJournal(); + journal.beginCapture(); + for ( + let index = 0; + index < FORK_REPLAY_EVENT_SEGMENT_CAPACITY + 2; + index++ + ) { + journal.recordCommit(index % 2, index); + } + return sealedWire(journal); + } + + it("rejects out-of-order segment sequence numbers", () => { + const wire = twoPageWire(); + const segments = wire.segments.map((segment) => segment.slice()); + new DataView(segments[1]!.buffer).setBigUint64(8, 0n, true); + expect(() => + validateForkReplayEventWire({ manifest: wire.manifest, segments }) + ).toThrow("out of order"); + }); + + it("rejects reordered, duplicated, missing, gapped, and trailing segments", () => { + const wire = twoPageWire(); + expect(() => + validateForkReplayEventWire({ + manifest: wire.manifest, + segments: [wire.segments[1]!, wire.segments[0]!], + }) + ).toThrow("out of order"); + expect(() => + validateForkReplayEventWire({ + manifest: wire.manifest, + segments: [wire.segments[0]!, wire.segments[0]!], + }) + ).toThrow("out of order"); + expect(() => + validateForkReplayEventWire({ + manifest: wire.manifest, + segments: wire.segments.slice(0, -1), + }) + ).toThrow("expected 2"); + + const gapped = wire.segments.map((segment) => segment.slice()); + new DataView(gapped[1]!.buffer).setBigUint64(8, 2n, true); + expect(() => + validateForkReplayEventWire({ manifest: wire.manifest, segments: gapped }) + ).toThrow("out of order"); + + const trailingWords = new Uint32Array([8, 13]); + const trailing = encodeForkReplayEventSegment(trailingWords, 1, 2n); + expect(() => + validateForkReplayEventWire({ + manifest: wire.manifest, + segments: [...wire.segments, trailing], + }) + ).toThrow("after its declared segment count 2"); + }); + + it("requires every non-final page to be full", () => { + const wire = twoPageWire(); + const segments = wire.segments.map((segment) => segment.slice()); + new DataView(segments[0]!.buffer).setUint32( + 16, + FORK_REPLAY_EVENT_SEGMENT_CAPACITY - 1, + true, + ); + expect(() => + validateForkReplayEventWire({ manifest: wire.manifest, segments }) + ).toThrow(`expected ${FORK_REPLAY_EVENT_SEGMENT_CAPACITY}`); + }); + + it("requires the final page count and bounds to match the manifest", () => { + const wire = twoPageWire(); + const segments = wire.segments.map((segment) => segment.slice()); + new DataView(segments[1]!.buffer).setUint32(16, 1, true); + expect(() => + validateForkReplayEventWire({ manifest: wire.manifest, segments }) + ).toThrow("expected 2"); + + const truncated = wire.segments.map((segment, index) => + index === 1 ? segment.subarray(0, segment.byteLength - 1) : segment + ); + expect(() => + validateForkReplayEventWire({ + manifest: wire.manifest, + segments: truncated, + }) + ).toThrow("inconsistent bounds"); + }); + + it("rejects manifest trailing bytes and nonzero reserved fields", () => { + const wire = twoPageWire(); + const trailing = new Uint8Array(wire.manifest.byteLength + 1); + trailing.set(wire.manifest); + expect(() => + validateForkReplayEventWire({ manifest: trailing, segments: wire.segments }) + ).toThrow("inconsistent bounds"); + const reserved = wire.manifest.slice(); + new DataView(reserved.buffer).setUint32(20, 1, true); + expect(() => + validateForkReplayEventWire({ manifest: reserved, segments: wire.segments }) + ).toThrow("reserved"); + }); + + it("represents event totals beyond the old contiguous u32 boundary", () => { + const eventCount = 0x1_0000_0001n; + const capacity = BigInt(FORK_REPLAY_EVENT_SEGMENT_CAPACITY); + const segmentCount = (eventCount + capacity - 1n) / capacity; + const manifest = encodeForkReplayEventManifest(eventCount, segmentCount); + const view = new DataView(manifest.buffer); + expect(view.getBigUint64(24, true)).toBe(segmentCount); + expect(view.getBigUint64(32, true)) + .toBe(eventCount); + }); + + it("rejects unavailable u64 segment totals without lossy number conversion", () => { + const segmentCount = 0x1_0000_0000n; + const eventCount = + (segmentCount - 1n) * BigInt(FORK_REPLAY_EVENT_SEGMENT_CAPACITY) + 1n; + const manifest = encodeForkReplayEventManifest(eventCount, segmentCount); + expect(() => + validateForkReplayEventWire({ manifest, segments: [] }) + ).toThrow(`expected ${segmentCount}`); + }); + + it("rejects inexact numeric u64 inputs instead of rounding them", () => { + expect(() => + encodeForkReplayEventManifest( + BigInt(FORK_REPLAY_EVENT_SEGMENT_CAPACITY), + Number.MAX_SAFE_INTEGER + 1, + ) + ).toThrow("exact nonnegative integer"); + expect(() => + encodeForkReplayEventSegment( + new Uint32Array([1, 2]), + 1, + Number.MAX_SAFE_INTEGER + 1, + ) + ).toThrow("exact nonnegative integer"); + }); +}); + +describe("ForkResumeTable", () => { + it("reconstructs slots from activation coordinates", () => { + const first = wasmFunction(); + const second = wasmFunction(); + const table = new ForkResumeTable(); + table.registerActivation(4, [ + { functionOrdinal: 9, thunk: second }, + { functionOrdinal: 3, thunk: first }, + ]); + const firstSlot = table.slotFor({ activationId: 4, functionOrdinal: 3 }); + expect(firstSlot).toBeGreaterThan(0); + expect(table.table.get(firstSlot)).toBe(first); + expect(table.slotFor(null)).toBe(0); + expect(table.slotFor({ activationId: 4, functionOrdinal: 9 })).toBeGreaterThan(0); + }); + + it("clears unloaded activation roots and reuses private slots", () => { + const table = new ForkResumeTable(); + table.registerActivation(1, [ + { functionOrdinal: 1, thunk: wasmFunction() }, + ]); + const slot = table.slotFor({ activationId: 1, functionOrdinal: 1 }); + table.unregisterActivation(1); + expect(table.table.get(slot)).toBeNull(); + table.registerActivation(2, [ + { functionOrdinal: 7, thunk: wasmFunction() }, + ]); + expect(table.slotFor({ activationId: 2, functionOrdinal: 7 })).toBe(slot); + }); +}); diff --git a/host/test/fork-replay-gate.test.ts b/host/test/fork-replay-gate.test.ts new file mode 100644 index 0000000000..9277030bdc --- /dev/null +++ b/host/test/fork-replay-gate.test.ts @@ -0,0 +1,175 @@ +import { describe, expect, it } from "vitest"; +import { + cancelForkReplayGate, + commitForkReplayGate, + createForkReplayGate, + ForkReplayGateCoordinator, + observeForkReplayWorker, +} from "../src/fork-replay-gate"; +import { MockWorkerAdapter } from "../src/worker-adapter"; + +describe("fork replay two-phase gate", () => { + it("commits a pending child exactly once", () => { + const gate = createForkReplayGate(); + expect(Atomics.load(new Int32Array(gate), 0)).toBe(0); + commitForkReplayGate(gate); + expect(Atomics.load(new Int32Array(gate), 0)).toBe(1); + expect(() => commitForkReplayGate(gate)).toThrow(/no longer pending/); + }); + + it("cancels only while reconstruction is pending", () => { + const pending = createForkReplayGate(); + cancelForkReplayGate(pending); + expect(Atomics.load(new Int32Array(pending), 0)).toBe(-1); + + const committed = createForkReplayGate(); + commitForkReplayGate(committed); + cancelForkReplayGate(committed); + expect(Atomics.load(new Int32Array(committed), 0)).toBe(1); + }); + + it("rejects malformed gate storage before publishing state", () => { + expect(() => commitForkReplayGate(new SharedArrayBuffer(8))) + .toThrow(/one shared i32/); + }); +}); + +describe("fork replay readiness coordinator", () => { + it("keeps the child blocked after readiness until the host commits", async () => { + const coordinator = new ForkReplayGateCoordinator("pid=41"); + const waiting = coordinator.waitUntilReady(); + + coordinator.ready(); + await waiting; + + expect(coordinator.currentPhase).toBe("ready"); + expect(Atomics.load(new Int32Array(coordinator.gate), 0)).toBe(0); + coordinator.commit(); + expect(coordinator.currentPhase).toBe("committed"); + expect(Atomics.load(new Int32Array(coordinator.gate), 0)).toBe(1); + }); + + it("does not allow commit before the Worker proves replay readiness", () => { + const coordinator = new ForkReplayGateCoordinator("pid=41"); + expect(() => coordinator.commit()).toThrow( + /cannot commit fork replay while pending/, + ); + expect(Atomics.load(new Int32Array(coordinator.gate), 0)).toBe(0); + }); + + it("cancels a deferred launch and rejects its pending readiness", async () => { + const coordinator = new ForkReplayGateCoordinator("pid=41"); + const waiting = coordinator.waitUntilReady(); + + coordinator.cancel(new Error("deferred Worker launch was cancelled")); + + await expect(waiting).rejects.toThrow(/deferred Worker launch was cancelled/); + expect(coordinator.currentPhase).toBe("cancelled"); + expect(Atomics.load(new Int32Array(coordinator.gate), 0)).toBe(-1); + }); + + it("preserves cancellation for a waiter attached after launch rollback", async () => { + const coordinator = new ForkReplayGateCoordinator("pid=41"); + coordinator.cancel(new Error("Worker constructor failed")); + + await expect(coordinator.waitUntilReady()).rejects.toThrow( + /Worker constructor failed/, + ); + expect(Atomics.load(new Int32Array(coordinator.gate), 0)).toBe(-1); + }); + + it("cancellation between ready and commit wins the transaction", async () => { + const coordinator = new ForkReplayGateCoordinator("pid=41"); + coordinator.ready(); + await coordinator.waitUntilReady(); + + coordinator.cancel(new Error("generation was replaced")); + + expect(() => coordinator.commit()).toThrow(/generation was replaced/); + expect(Atomics.load(new Int32Array(coordinator.gate), 0)).toBe(-1); + }); +}); + +describe("fork replay Worker lifecycle observer", () => { + function observed(isCurrentGeneration = () => true) { + const adapter = new MockWorkerAdapter(); + const worker = adapter.createWorker({ pid: 41 }); + const coordinator = new ForkReplayGateCoordinator("fork child pid=41"); + observeForkReplayWorker( + coordinator, + worker, + 41, + isCurrentGeneration, + ); + return { coordinator, worker: adapter.lastWorker! }; + } + + it("accepts readiness only from the exact current child generation", async () => { + const current = observed(); + current.worker.simulateMessage({ type: "fork_replay_ready", pid: 41 }); + await current.coordinator.waitUntilReady(); + expect(current.coordinator.currentPhase).toBe("ready"); + + const stale = observed(() => false); + const staleWaiting = stale.coordinator.waitUntilReady(); + stale.worker.simulateMessage({ type: "fork_replay_ready", pid: 41 }); + await expect(staleWaiting).rejects.toThrow(/stale Worker generation/); + expect(Atomics.load(new Int32Array(stale.coordinator.gate), 0)).toBe(-1); + + const wrongPid = observed(); + const wrongPidWaiting = wrongPid.coordinator.waitUntilReady(); + wrongPid.worker.simulateMessage({ type: "fork_replay_ready", pid: 99 }); + await expect(wrongPidWaiting).rejects.toThrow(/expected pid=41/); + }); + + it.each([ + { + label: "worker-main error message", + fire: (worker: ReturnType["worker"]) => + worker.simulateMessage({ + type: "error", + pid: 41, + message: "instantiation failed", + }), + diagnostic: /instantiation failed/, + }, + { + label: "worker-main exit message", + fire: (worker: ReturnType["worker"]) => + worker.simulateMessage({ type: "exit", pid: 41, status: 7 }), + diagnostic: /status=7/, + }, + { + label: "Worker error event", + fire: (worker: ReturnType["worker"]) => + worker.simulateError(new Error("worker crashed")), + diagnostic: /worker crashed/, + }, + { + label: "Worker exit event", + fire: (worker: ReturnType["worker"]) => + worker.simulateExit(9), + diagnostic: /code=9/, + }, + ])("cancels on $label before readiness", async ({ fire, diagnostic }) => { + const { coordinator, worker } = observed(); + const waiting = coordinator.waitUntilReady(); + fire(worker); + await expect(waiting).rejects.toThrow(diagnostic); + expect(coordinator.currentPhase).toBe("cancelled"); + expect(Atomics.load(new Int32Array(coordinator.gate), 0)).toBe(-1); + }); + + it("ignores later terminal events after a committed replay", async () => { + const { coordinator, worker } = observed(); + worker.simulateMessage({ type: "fork_replay_ready", pid: 41 }); + await coordinator.waitUntilReady(); + coordinator.commit(); + + worker.simulateMessage({ type: "exit", pid: 41, status: 0 }); + worker.simulateExit(0); + + expect(coordinator.currentPhase).toBe("committed"); + expect(Atomics.load(new Int32Array(coordinator.gate), 0)).toBe(1); + }); +}); diff --git a/host/test/fork-replay-host-parity.test.ts b/host/test/fork-replay-host-parity.test.ts new file mode 100644 index 0000000000..e2bf9f58a1 --- /dev/null +++ b/host/test/fork-replay-host-parity.test.ts @@ -0,0 +1,83 @@ +import { readFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { describe, expect, it } from "vitest"; + +const testDir = dirname(fileURLToPath(import.meta.url)); +const repoRoot = join(testDir, "..", ".."); + +function forkHandlerSource(relativePath: string): string { + const path = join(repoRoot, relativePath); + const source = readFileSync(path, "utf8"); + const start = source.indexOf("async function handleFork("); + const end = source.indexOf("\nasync function handleExec(", start); + expect(start, `${relativePath} must define handleFork`).toBeGreaterThanOrEqual(0); + expect(end, `${relativePath} must define handleExec after handleFork`) + .toBeGreaterThan(start); + return source.slice(start, end); +} + +describe.each([ + ["Node", "host/src/node-kernel-worker-entry.ts"], + ["browser", "host/src/browser-kernel-worker-entry.ts"], +])("%s fork replay launch transaction", (_host, relativePath) => { + it("waits for the exact child generation before committing and resolving", () => { + const handler = forkHandlerSource(relativePath); + const wait = handler.indexOf("await forkReplay.waitUntilReady()"); + const generationCheck = handler.indexOf( + "processes.get(childPid)?.worker !== launchedWorker", + wait, + ); + const commit = handler.indexOf("forkReplay.commit()", generationCheck); + const resolve = handler.lastIndexOf("return [childChannelOffset]"); + + expect(handler).toContain("forkReplayGate: forkReplay.gate"); + expect(handler).toContain("observeForkReplayWorker("); + expect(wait).toBeGreaterThanOrEqual(0); + expect(generationCheck).toBeGreaterThan(wait); + expect(commit).toBeGreaterThan(generationCheck); + expect(resolve).toBeGreaterThan(commit); + }); + + it("cancels both a deferred launch and the rollback path", () => { + const handler = forkHandlerSource(relativePath); + const launchGate = handler.indexOf("startProcessWorkerWhenRunnable("); + const launchCancellation = handler.indexOf("forkReplay.cancel(", launchGate); + const rollback = handler.indexOf("} catch (error)"); + const rollbackCancellation = handler.indexOf("forkReplay.cancel(error)", rollback); + + expect(launchGate).toBeGreaterThanOrEqual(0); + expect(launchCancellation).toBeGreaterThan(launchGate); + expect(launchCancellation).toBeLessThan(rollback); + expect(rollbackCancellation).toBeGreaterThan(rollback); + expect(handler.indexOf("void childWorker.terminate()", rollbackCancellation)) + .toBeGreaterThan(rollbackCancellation); + }); + + it("grants the exact copied externref graph before launch and retires rollback", () => { + const handler = forkHandlerSource(relativePath); + const grant = handler.indexOf( + "externrefProcessOwner.forkGenerationFromContinuation(", + ); + const childInit = handler.indexOf( + "const childInitData: CentralizedWorkerInitMessage", + grant, + ); + const start = handler.indexOf("startProcessWorkerWhenRunnable(", childInit); + const rollback = handler.indexOf("} catch (error)", start); + const release = handler.indexOf( + "externrefProcessOwner.releaseGeneration(externrefGrant.generation)", + rollback, + ); + + expect(grant).toBeGreaterThanOrEqual(0); + expect(childInit).toBeGreaterThan(grant); + expect(handler.slice(childInit, start)).toContain( + "externrefGenerationId: externrefGrant.generation.id", + ); + expect(start).toBeGreaterThan(childInit); + expect(release).toBeGreaterThan(rollback); + expect(handler.indexOf("void childWorker.terminate()", release)) + .toBeGreaterThan(release); + }); +}); diff --git a/host/test/fork-resume-catalog.test.ts b/host/test/fork-resume-catalog.test.ts new file mode 100644 index 0000000000..037def0b9e --- /dev/null +++ b/host/test/fork-resume-catalog.test.ts @@ -0,0 +1,194 @@ +import { execFileSync } from "node:child_process"; +import { mkdtempSync, readFileSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; +import { + FORK_RESUME_CATALOG_EXPORT, + FORK_RESUME_CATALOG_HEADER_SIZE, + FORK_RESUME_CATALOG_SECTION, + FORK_RESUME_CATALOG_VERSION, + forkResumeTargetsFromInstance, + readForkResumeCatalog, +} from "../src/fork-resume-catalog"; + +function uleb128(value: number): number[] { + const bytes: number[] = []; + do { + let byte = value & 0x7f; + value >>>= 7; + if (value !== 0) byte |= 0x80; + bytes.push(byte); + } while (value !== 0); + return bytes; +} + +function appendCustomSection( + wasm: Uint8Array, + name: string, + payload: Uint8Array, +): Uint8Array { + const nameBytes = new TextEncoder().encode(name); + const contents = new Uint8Array( + uleb128(nameBytes.byteLength).length + + nameBytes.byteLength + + payload.byteLength, + ); + const encodedNameLength = uleb128(nameBytes.byteLength); + contents.set(encodedNameLength, 0); + contents.set(nameBytes, encodedNameLength.length); + contents.set(payload, encodedNameLength.length + nameBytes.byteLength); + const encodedSectionLength = uleb128(contents.byteLength); + const result = new Uint8Array( + wasm.byteLength + 1 + encodedSectionLength.length + contents.byteLength, + ); + result.set(wasm, 0); + result[wasm.byteLength] = 0; + result.set(encodedSectionLength, wasm.byteLength + 1); + result.set(contents, wasm.byteLength + 1 + encodedSectionLength.length); + return result; +} + +function descriptor( + records: readonly { + functionOrdinal: number; + localCatalogSlot: number; + }[], +): Uint8Array { + const bytes = new Uint8Array( + FORK_RESUME_CATALOG_HEADER_SIZE + records.length * 8, + ); + const view = new DataView(bytes.buffer); + bytes.set(new TextEncoder().encode("KFRC"), 0); + view.setUint16(4, FORK_RESUME_CATALOG_VERSION, true); + view.setUint16(6, FORK_RESUME_CATALOG_HEADER_SIZE, true); + view.setUint32(8, records.length, true); + records.forEach((record, index) => { + const offset = FORK_RESUME_CATALOG_HEADER_SIZE + index * 8; + view.setUint32(offset, record.functionOrdinal, true); + view.setUint32(offset + 4, record.localCatalogSlot, true); + }); + return bytes; +} + +function baseCatalogBytes(tableSize = 2): Uint8Array { + const directory = mkdtempSync(join(tmpdir(), "kandelo-resume-catalog-")); + const wat = join(directory, "catalog.wat"); + const wasm = join(directory, "catalog.wasm"); + const elements = tableSize === 0 ? "" : `(elem (i32.const 0) ${[ + "$first", + "$second", + ].slice(0, tableSize).join(" ")})`; + writeFileSync(wat, `(module + (table $catalog (export "${FORK_RESUME_CATALOG_EXPORT}") ${tableSize} ${tableSize} funcref) + (func $first (result i32) i32.const 17) + (func $second (result i32) i32.const 29) + ${elements} + )`); + execFileSync("wat2wasm", [wat, "-o", wasm]); + return readFileSync(wasm); +} + +function moduleWithDescriptor( + records: readonly { + functionOrdinal: number; + localCatalogSlot: number; + }[], + tableSize = 2, +): WebAssembly.Module { + return new WebAssembly.Module( + appendCustomSection( + baseCatalogBytes(tableSize), + FORK_RESUME_CATALOG_SECTION, + descriptor(records), + ), + ); +} + +describe("fork resume catalog", () => { + it("pairs deterministic ordinals with fresh-instance thunk objects", () => { + const module = moduleWithDescriptor([ + { functionOrdinal: 3, localCatalogSlot: 0 }, + { functionOrdinal: 9, localCatalogSlot: 1 }, + ]); + const first = new WebAssembly.Instance(module); + const second = new WebAssembly.Instance(module); + const firstTargets = forkResumeTargetsFromInstance(module, first); + const secondTargets = forkResumeTargetsFromInstance(module, second); + + expect(firstTargets.map(({ functionOrdinal, localCatalogSlot }) => ({ + functionOrdinal, + localCatalogSlot, + }))).toEqual([ + { functionOrdinal: 3, localCatalogSlot: 0 }, + { functionOrdinal: 9, localCatalogSlot: 1 }, + ]); + expect(firstTargets[0]!.thunk).not.toBe(secondTargets[0]!.thunk); + expect((firstTargets[0]!.thunk as () => number)()).toBe(17); + expect((secondTargets[1]!.thunk as () => number)()).toBe(29); + }); + + it("rejects malformed or ambiguous KFRC metadata", () => { + const base = baseCatalogBytes(); + expect(() => readForkResumeCatalog(new WebAssembly.Module(base))) + .toThrow(`expected one ${FORK_RESUME_CATALOG_SECTION}`); + + const valid = descriptor([ + { functionOrdinal: 3, localCatalogSlot: 0 }, + { functionOrdinal: 9, localCatalogSlot: 1 }, + ]); + const duplicate = appendCustomSection( + appendCustomSection(base, FORK_RESUME_CATALOG_SECTION, valid), + FORK_RESUME_CATALOG_SECTION, + valid, + ); + expect(() => readForkResumeCatalog(new WebAssembly.Module(duplicate))) + .toThrow("found 2"); + + const badMagic = valid.slice(); + badMagic[0] = 0; + expect(() => readForkResumeCatalog(new WebAssembly.Module( + appendCustomSection(base, FORK_RESUME_CATALOG_SECTION, badMagic), + ))).toThrow("invalid magic"); + + const truncated = valid.slice(0, valid.byteLength - 1); + expect(() => readForkResumeCatalog(new WebAssembly.Module( + appendCustomSection(base, FORK_RESUME_CATALOG_SECTION, truncated), + ))).toThrow("invalid size"); + + expect(() => readForkResumeCatalog(moduleWithDescriptor([ + { functionOrdinal: 9, localCatalogSlot: 0 }, + { functionOrdinal: 3, localCatalogSlot: 1 }, + ]))).toThrow("not strictly ordered"); + + expect(() => readForkResumeCatalog(moduleWithDescriptor([ + { functionOrdinal: 3, localCatalogSlot: 0 }, + { functionOrdinal: 9, localCatalogSlot: 0 }, + ]))).toThrow("repeats local slot"); + }); + + it("rejects metadata that cannot resolve against the instance table", () => { + const wrongLength = moduleWithDescriptor([ + { functionOrdinal: 3, localCatalogSlot: 0 }, + ]); + expect(() => forkResumeTargetsFromInstance( + wrongLength, + new WebAssembly.Instance(wrongLength), + )).toThrow("length 2, expected 1"); + + const outOfBounds = moduleWithDescriptor([ + { functionOrdinal: 3, localCatalogSlot: 0 }, + { functionOrdinal: 9, localCatalogSlot: 2 }, + ]); + expect(() => forkResumeTargetsFromInstance( + outOfBounds, + new WebAssembly.Instance(outOfBounds), + )).toThrow("out of bounds"); + + const nullModule = moduleWithDescriptor([], 0); + expect(forkResumeTargetsFromInstance( + nullModule, + new WebAssembly.Instance(nullModule), + )).toEqual([]); + }); +}); diff --git a/host/test/fork-save-buffer-overrun.test.ts b/host/test/fork-save-buffer-overrun.test.ts index 7b51742bb8..6ac6360629 100644 --- a/host/test/fork-save-buffer-overrun.test.ts +++ b/host/test/fork-save-buffer-overrun.test.ts @@ -19,16 +19,10 @@ * without implying that current linked continuations have the old ceiling. */ import { describe, it, expect } from "vitest"; -import type { SideModuleForkState } from "../src/dylink"; -import { - finalizeSideModuleForkUnwind, - forkSaveBufferOverrun, -} from "../src/worker-main"; +import { forkSaveBufferOverrun } from "../src/worker-main"; import { FORK_SAVE_BUFFER_SIZE } from "../src/process-memory"; -import type { LinkedForkContinuation } from "../src/fork-continuation"; const FORK_BUF_ADDR = 65536; // arbitrary page-aligned buffer base for the test -const SIDE_FORK_BUF_ADDR = 32768; // separate from the process-main test buffer function writeCurrentPos( memory: WebAssembly.Memory, @@ -41,31 +35,6 @@ function writeCurrentPos( else view.setUint32(addr, value, true); } -function createSideForkState( - name: string, - forkBufAddr: number, - finishUnwind: () => void = () => {}, -): { state: SideModuleForkState; runtimeState: () => number } { - let value = 1; // UNWINDING - const instance = { - exports: { - wpk_fork_state: () => value, - wpk_fork_unwind_end: () => { - value = 0; // NORMAL - }, - }, - } as unknown as WebAssembly.Instance; - return { - state: { - name, - instance, - forkBufAddr, - continuation: { finishUnwind } as unknown as LinkedForkContinuation, - }, - runtimeState: () => value, - }; -} - describe("forkSaveBufferOverrun", () => { it("reports no overrun when the save fits within the buffer", () => { const memory = new WebAssembly.Memory({ initial: 3 }); @@ -131,32 +100,4 @@ describe("forkSaveBufferOverrun", () => { forkSaveBufferOverrun(memory, FORK_BUF_ADDR, 8, FORK_SAVE_BUFFER_SIZE), ).toBe(1); }); - - it("finalizes the side-module linked continuation", () => { - const memory = new WebAssembly.Memory({ initial: 3 }); - let finalized = false; - const side = createSideForkState( - "libintl.so", - SIDE_FORK_BUF_ADDR, - () => { finalized = true; }, - ); - - expect(() => finalizeSideModuleForkUnwind(memory, side.state, 4)) - .not.toThrow(); - expect(finalized).toBe(true); - expect(side.runtimeState()).toBe(0); - }); - - it("propagates linked continuation validation before fork dispatch", () => { - const memory = new WebAssembly.Memory({ initial: 3 }); - const side = createSideForkState( - "libintl.so", - SIDE_FORK_BUF_ADDR, - () => { throw new Error("uncommitted linked frame"); }, - ); - - expect(() => finalizeSideModuleForkUnwind(memory, side.state, 4)) - .toThrow("uncommitted linked frame"); - expect(side.runtimeState()).toBe(0); - }); }); diff --git a/host/test/fork-static-root-catalog.test.ts b/host/test/fork-static-root-catalog.test.ts new file mode 100644 index 0000000000..47df1b77b1 --- /dev/null +++ b/host/test/fork-static-root-catalog.test.ts @@ -0,0 +1,69 @@ +import { describe, expect, it } from "vitest"; +import { + ForkStaticRootCatalog, +} from "../src/fork-static-root-catalog"; + +function externrefTable(values: readonly unknown[]): WebAssembly.Table { + const table = new WebAssembly.Table({ + element: "externref", + initial: values.length, + maximum: values.length, + }); + values.forEach((value, index) => table.set(index, value)); + return table; +} + +describe("ForkStaticRootCatalog", () => { + it("canonicalizes aliases and resolves the fresh child's root", () => { + const parentRoot = Object.freeze({ activation: "parent" }); + const parent = new ForkStaticRootCatalog(); + const parentHarvest = externrefTable([parentRoot, parentRoot]); + parent.register(4, parentHarvest); + expect(parentHarvest.get(0)).toBeNull(); + expect(parentHarvest.get(1)).toBeNull(); + + expect(parent.encode(parentRoot)).toEqual({ + moduleActivation: 4, + ordinal: 0, + }); + + const childRoot = Object.freeze({ activation: "child" }); + const child = new ForkStaticRootCatalog(); + child.register(4, externrefTable([childRoot, childRoot])); + const decoded = child.decode(parent.encode(parentRoot)!); + + expect(decoded).toBe(childRoot); + expect(decoded).not.toBe(parentRoot); + }); + + it("uses the first activation coordinate for an imported shared root", () => { + const imported = Object.freeze({ imported: true }); + const catalogs = new ForkStaticRootCatalog(); + catalogs.register(2, externrefTable([imported])); + catalogs.register(7, externrefTable([imported])); + + expect(catalogs.encode(imported)).toEqual({ + moduleActivation: 2, + ordinal: 0, + }); + catalogs.unregister(2); + expect(catalogs.encode(imported)).toEqual({ + moduleActivation: 7, + ordinal: 0, + }); + }); + + it("rejects duplicate activation registration and invalid recipes", () => { + const catalogs = new ForkStaticRootCatalog(); + catalogs.register(3, externrefTable([null])); + expect(() => catalogs.register(3, externrefTable([]))).toThrow( + /already registered/, + ); + expect(() => + catalogs.decode({ moduleActivation: 3, ordinal: 1 }) + ).toThrow(/out of bounds/); + expect(() => + catalogs.decode({ moduleActivation: 9, ordinal: 0 }) + ).toThrow(/not registered/); + }); +}); diff --git a/host/test/fork-unwind-transport.test.ts b/host/test/fork-unwind-transport.test.ts new file mode 100644 index 0000000000..be72d84007 --- /dev/null +++ b/host/test/fork-unwind-transport.test.ts @@ -0,0 +1,62 @@ +import { execFileSync } from "node:child_process"; +import { mkdtempSync, readFileSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; +import { + createForkUnwindTag, + FORK_UNWIND_TAG_IMPORT_MODULE, + FORK_UNWIND_TAG_IMPORT_NAME, + isForkUnwindException, +} from "../src/fork-unwind-transport"; + +function throwingModule(): WebAssembly.Module { + const dir = mkdtempSync(join(tmpdir(), "kandelo-unwind-tag-")); + const wat = join(dir, "transport.wat"); + const wasm = join(dir, "transport.wasm"); + writeFileSync(wat, `(module + (tag $unwind (import "${FORK_UNWIND_TAG_IMPORT_MODULE}" "${FORK_UNWIND_TAG_IMPORT_NAME}")) + (func (export "throw_unwind") throw $unwind) + )`); + execFileSync("wat2wasm", ["--enable-exceptions", wat, "-o", wasm]); + return new WebAssembly.Module(readFileSync(wasm)); +} + +describe.skipIf( + typeof WebAssembly.Tag !== "function" + || typeof WebAssembly.Exception !== "function", +)("fork unwind transport", () => { + it("recognizes only the exact process-owned tag identity", () => { + const tag = createForkUnwindTag(); + const other = createForkUnwindTag(); + const instance = new WebAssembly.Instance(throwingModule(), { + env: { [FORK_UNWIND_TAG_IMPORT_NAME]: tag }, + }); + + let thrown: unknown; + try { + (instance.exports.throw_unwind as () => void)(); + } catch (error) { + thrown = error; + } + expect(isForkUnwindException(thrown, tag)).toBe(true); + expect(isForkUnwindException(thrown, other)).toBe(false); + }); + + it("shares one identity across independently instantiated modules", () => { + const tag = createForkUnwindTag(); + const module = throwingModule(); + const imports = { env: { [FORK_UNWIND_TAG_IMPORT_NAME]: tag } }; + const first = new WebAssembly.Instance(module, imports); + const second = new WebAssembly.Instance(module, imports); + + for (const instance of [first, second]) { + expect(() => (instance.exports.throw_unwind as () => void)()).toThrow(); + try { + (instance.exports.throw_unwind as () => void)(); + } catch (error) { + expect(isForkUnwindException(error, tag)).toBe(true); + } + } + }); +}); diff --git a/host/test/fork-worker-import-exceptions.test.ts b/host/test/fork-worker-import-exceptions.test.ts new file mode 100644 index 0000000000..2071fe03fb --- /dev/null +++ b/host/test/fork-worker-import-exceptions.test.ts @@ -0,0 +1,454 @@ +import { Worker } from "node:worker_threads"; +import { describe, expect, it } from "vitest"; +import { + createForkExternrefImportMailbox, + defineForkExternrefImport, + type ForkExternrefImportBinding, + type ForkExternrefImportWake, + ForkExternrefImportOwnerCatalog, + ForkExternrefImportOwnerEndpoint, + ForkExternrefImportWorkerCaller, +} from "../src/fork-externref-import-mailbox"; +import { + ForkWorkerExceptionCapabilityOwner, + ForkWorkerLocalImportExceptionNormalizer, + FORK_WORKER_EXCEPTION_BEGIN_DESCRIPTOR, + FORK_WORKER_EXCEPTION_FORK_CAPTURE_ORDINAL, +} from "../src/fork-worker-import-exceptions"; +import { + isForkWorkerExceptionCapability, + unwrapForkWorkerExceptionCapability, +} from "../src/fork-worker-exception-capability"; +import { + ForkExternrefBroker, + type ForkExternrefGeneration, + ForkExternrefTokenCache, +} from "../src/fork-reference-broker"; + +class TestAuthority { + constructor( + readonly broker: ForkExternrefBroker, + readonly generation: ForkExternrefGeneration, + ) {} + + authorizeForWire( + pid: number, + generationId: number, + handle: number, + ): unknown { + this.assertBinding(pid, generationId); + return this.broker.authorize(this.generation, handle); + } + + registerForWire( + pid: number, + generationId: number, + value: unknown, + ): number { + this.assertBinding(pid, generationId); + return this.broker.register(this.generation, value); + } + + private assertBinding(pid: number, generationId: number): void { + expect(pid).toBe(this.generation.pid); + expect(generationId).toBe(this.generation.id); + } +} + +function harness() { + const broker = new ForkExternrefBroker(); + const generation = broker.createGeneration(301); + const tokens = new ForkExternrefTokenCache(generation.id); + const binding: ForkExternrefImportBinding = { + pid: generation.pid, + generationId: generation.id, + senderId: 41, + }; + const catalog = new ForkExternrefImportOwnerCatalog(); + const exceptionOwner = new ForkWorkerExceptionCapabilityOwner(); + exceptionOwner.install(catalog); + const mailbox = createForkExternrefImportMailbox( + catalog.mailboxCapacity, + ); + const endpoint = new ForkExternrefImportOwnerEndpoint( + mailbox, + binding, + catalog, + new TestAuthority(broker, generation), + { authorizeSender: () => {} }, + ); + const caller = new ForkExternrefImportWorkerCaller( + mailbox, + binding, + tokens, + (wake) => { + expect(endpoint.dispatch(wake, binding)).toBe(true); + }, + ); + const normalizer = new ForkWorkerLocalImportExceptionNormalizer( + caller, + tokens, + ); + return { + broker, + generation, + tokens, + binding, + exceptionOwner, + endpoint, + caller, + normalizer, + }; +} + +function thrownBy(fn: () => unknown): unknown { + let didThrow = false; + let thrown: unknown; + try { + fn(); + } catch (value) { + didThrow = true; + thrown = value; + } + expect(didThrow).toBe(true); + return thrown; +} + +function numberBits(value: number): bigint { + const bytes = new ArrayBuffer(8); + const view = new DataView(bytes); + view.setFloat64(0, value, true); + return view.getBigUint64(0, true); +} + +describe("Worker-local import exception normalization", () => { + it("preserves every primitive before fork and normalizes it exactly for a child", () => { + const state = harness(); + const customNanBytes = new ArrayBuffer(8); + const customNanView = new DataView(customNanBytes); + customNanView.setBigUint64(0, 0x7ff8_0000_0000_0042n, true); + const customNan = customNanView.getFloat64(0, true); + const longString = + `prefix-\ud800-${"reference-state-".repeat(20)}-\udfff-suffix`; + const hugeBigInt = (1n << 1000n) + 0x1234_5678_9abcn; + const globalSymbol = Symbol.for( + `kandelo-${"global-symbol-".repeat(12)}`, + ); + const localSymbol = Symbol(`local-${"symbol-".repeat(20)}`); + const values: unknown[] = [ + undefined, + null, + false, + true, + -0, + customNan, + 17.25, + hugeBigInt, + longString, + globalSymbol, + localSymbol, + ]; + + for (const [ordinal, original] of values.entries()) { + const wrapped = state.normalizer.wrap(ordinal, () => { + throw original; + }); + const importThrown = thrownBy(wrapped); + expect(Object.is(importThrown, original)).toBe(true); + + const token = + state.normalizer.normalizeUnclaimedForkException(importThrown); + const handle = state.tokens.encode(token); + expect(handle).not.toBeNull(); + const capability = state.broker.authorize( + state.generation, + handle!, + ); + expect(isForkWorkerExceptionCapability(capability)).toBe(true); + const boundary = unwrapForkWorkerExceptionCapability(capability); + if (typeof original === "number" && Number.isNaN(original)) { + expect(Number.isNaN(boundary)).toBe(true); + expect(numberBits(boundary as number)).toBe(numberBits(original)); + } else if (typeof original === "symbol") { + if (Symbol.keyFor(original) !== undefined) { + expect(Symbol.keyFor(boundary as symbol)).toBe( + Symbol.keyFor(original), + ); + } else { + expect((boundary as symbol).description).toBe(original.description); + } + } else { + expect(Object.is(boundary, original)).toBe(true); + } + } + expect(state.exceptionOwner.activeSessionCount).toBe(0); + }); + + it("preserves object/function/symbol rethrows and interns one child token", () => { + const state = harness(); + const values: unknown[] = [ + { workerOnly: true }, + function workerOnlyFunction() {}, + Symbol("worker-only-symbol"), + ]; + + for (const [ordinal, original] of values.entries()) { + const wrapped = state.normalizer.wrap(100 + ordinal, () => { + throw original; + }); + const first = thrownBy(wrapped); + const second = thrownBy(wrapped); + expect(first).toBe(original); + expect(second).toBe(original); + expect(state.tokens.encode(first)).toBeNull(); + + const firstToken = + state.normalizer.normalizeUnclaimedForkException(first); + const secondToken = + state.normalizer.normalizeUnclaimedForkException(second); + expect(secondToken).toBe(firstToken); + expect(state.tokens.encode(secondToken)).toBe( + state.tokens.encode(firstToken), + ); + + const capability = state.broker.authorize( + state.generation, + state.tokens.encode(firstToken)!, + ); + if (typeof original === "symbol") { + expect( + (unwrapForkWorkerExceptionCapability(capability) as symbol) + .description, + ).toBe(original.description); + } else { + expect(unwrapForkWorkerExceptionCapability(capability)).toBe( + capability, + ); + } + } + }); + + it("keeps complete Error name/message fields on the opaque capability", () => { + const state = harness(); + const error = new TypeError( + `bad-reference-${"payload-".repeat(30)}`, + ); + const wrapped = state.normalizer.wrap(207, () => { + throw error; + }); + const importThrown = thrownBy(wrapped); + expect(importThrown).toBe(error); + const token = + state.normalizer.normalizeUnclaimedForkException(importThrown); + const capability = state.broker.authorize( + state.generation, + state.tokens.encode(token)!, + ); + + expect(isForkWorkerExceptionCapability(capability)).toBe(true); + expect(capability).toMatchObject({ + sourceImportOrdinal: FORK_WORKER_EXCEPTION_FORK_CAPTURE_ORDINAL, + kind: "error", + name: "TypeError", + message: error.message, + }); + expect(unwrapForkWorkerExceptionCapability(capability)).toBe(capability); + }); + + it("preserves tagged catches before fork and normalizes only an unclaimed tag", () => { + const state = harness(); + const tag = new WebAssembly.Tag({ parameters: ["i32"] }); + const exception = new WebAssembly.Exception(tag, [37]); + + const ordinary = state.normalizer.wrap(208, () => { + throw exception; + }); + expect(thrownBy(ordinary)).toBe(exception); + + // ForkExceptionBroker calls this only after every activation-local exact + // tag codec has declined the exception. + const normalized = + state.normalizer.normalizeUnclaimedForkException(exception); + expect(normalized).not.toBe(exception); + const capability = state.broker.authorize( + state.generation, + state.tokens.encode(normalized)!, + ); + expect(capability).toMatchObject({ + sourceImportOrdinal: FORK_WORKER_EXCEPTION_FORK_CAPTURE_ORDINAL, + kind: "object", + }); + }); + + it("keeps nested Wasm traps fatal rather than turning them into JSTag values", () => { + const state = harness(); + const original = new WebAssembly.RuntimeError("nested Wasm trap"); + const wrapped = state.normalizer.wrap(210, () => { + throw original; + }); + + const trapped = thrownBy(wrapped); + expect(trapped).toBeInstanceOf(WebAssembly.RuntimeError); + expect(trapped).not.toBe(original); + expect(state.tokens.encode(trapped)).toBeNull(); + }); + + it("recreates a distinct child token for the same durable capability", () => { + const state = harness(); + const workerOnly = Object.freeze({ cannotClone: () => 1 }); + const parentToken = + state.normalizer.normalizeUnclaimedForkException(workerOnly); + const handle = state.tokens.encode(parentToken)!; + + const child = state.broker.createGeneration(302); + state.broker.acquireFork(state.generation, child, [handle]); + const childTokens = new ForkExternrefTokenCache(child.id); + const childToken = childTokens.materialize(handle); + + expect(childToken).not.toBe(parentToken); + expect(childTokens.encode(childToken)).toBe(handle); + expect(state.broker.authorize(child, handle)).toBe( + state.broker.authorize(state.generation, handle), + ); + }); + + it("drops incomplete scalar sessions when the exact Worker is torn down", () => { + const state = harness(); + state.caller.call(FORK_WORKER_EXCEPTION_BEGIN_DESCRIPTOR, [ + 1, + 9, + 6, + 0, + 0n, + 100, + 0, + ]); + expect(state.exceptionOwner.activeSessionCount).toBe(1); + + state.exceptionOwner.clearBinding(state.binding); + state.endpoint.close(); + expect(state.exceptionOwner.activeSessionCount).toBe(0); + }); + + it("replays a Worker-only opaque exception through a real fresh child Worker", async () => { + const broker = new ForkExternrefBroker(); + const parentGeneration = broker.createGeneration(501); + const parentBinding: ForkExternrefImportBinding = { + pid: parentGeneration.pid, + generationId: parentGeneration.id, + senderId: 51, + }; + const catalog = new ForkExternrefImportOwnerCatalog(); + const exceptionOwner = new ForkWorkerExceptionCapabilityOwner(); + exceptionOwner.install(catalog); + const echo = defineForkExternrefImport( + 77, + ["externref"], + ["externref"], + ); + catalog.register(echo, (_context, value) => value); + + const runWorker = async ( + mode: "parent" | "child", + generation: ForkExternrefGeneration, + binding: ForkExternrefImportBinding, + inheritedHandle?: number, + ): Promise => { + const mailbox = createForkExternrefImportMailbox( + catalog.mailboxCapacity, + ); + const endpoint = new ForkExternrefImportOwnerEndpoint( + mailbox, + binding, + catalog, + new TestAuthority(broker, generation), + { authorizeSender: () => {} }, + ); + const worker = new Worker( + new URL( + "./fixtures/fork-worker-import-exception-worker.ts", + import.meta.url, + ), + { + execArgv: ["--import", "tsx"], + workerData: { + mode, + binding, + inheritedHandle, + init: { + mailbox, + senderId: binding.senderId, + ownerImports: [{ + module: "host", + name: "echo", + descriptor: echo, + }], + }, + }, + }, + ); + try { + return await new Promise((resolve, reject) => { + const watchdog = setTimeout( + () => reject(new Error(`${mode} Worker watchdog expired`)), + 5_000, + ); + const settle = (action: () => void) => { + clearTimeout(watchdog); + action(); + }; + worker.on("message", (message: { + type: string; + wake?: ForkExternrefImportWake; + handle?: number; + message?: string; + }) => { + if (message.type === "wake") { + if (!endpoint.dispatch(message.wake!, binding)) { + settle(() => + reject(new Error(`${mode} Worker wake was not claimed`)) + ); + } + } else if (message.type === "complete") { + settle(() => resolve(message.handle!)); + } else if (message.type === "failed") { + settle(() => reject(new Error(message.message))); + } + }); + worker.once("error", (error) => settle(() => reject(error))); + }); + } finally { + exceptionOwner.clearBinding(binding); + endpoint.close(); + await worker.terminate(); + } + }; + + const parentHandle = await runWorker( + "parent", + parentGeneration, + parentBinding, + ); + const capability = broker.authorize(parentGeneration, parentHandle); + expect(isForkWorkerExceptionCapability(capability)).toBe(true); + expect(capability).toMatchObject({ + kind: "object", + sourceImportOrdinal: FORK_WORKER_EXCEPTION_FORK_CAPTURE_ORDINAL, + }); + + const childGeneration = broker.createGeneration(502); + broker.acquireFork(parentGeneration, childGeneration, [parentHandle]); + const childHandle = await runWorker( + "child", + childGeneration, + { + pid: childGeneration.pid, + generationId: childGeneration.id, + senderId: 52, + }, + parentHandle, + ); + expect(childHandle).toBe(parentHandle); + expect(broker.authorize(childGeneration, childHandle)).toBe(capability); + }, 12_000); +}); diff --git a/host/test/framebuffer-integration.test.ts b/host/test/framebuffer-integration.test.ts index 487f1a3f4f..cda924e191 100644 --- a/host/test/framebuffer-integration.test.ts +++ b/host/test/framebuffer-integration.test.ts @@ -24,7 +24,11 @@ import { NodePlatformIO } from "../src/platform/node"; import { NodeWorkerAdapter } from "../src/worker-adapter"; import { detectPtrWidth } from "../src/constants"; import { tryResolveBinary } from "../src/binary-resolver"; -import type { CentralizedWorkerInitMessage } from "../src/worker-protocol"; +import type { + CentralizedWorkerInitMessage, + WorkerToHostMessage, +} from "../src/worker-protocol"; +import { TestProcessReferenceOwners } from "./process-reference-owner-helper"; const fbtestBinary = tryResolveBinary("programs/fbtest.wasm") ?? ""; const kernelBinary = tryResolveBinary("kernel.wasm") ?? ""; @@ -55,15 +59,22 @@ describe.skipIf(!existsSync(fbtestBinary))("framebuffer integration", () => { const io = new NodePlatformIO(); const workerAdapter = new NodeWorkerAdapter(); - const workers = new Map>(); + const referenceOwners = new TestProcessReferenceOwners(); + const workers = new Map< + number, + ReturnType + >(); let pid = 0; let stdout = ""; + let stderr = ""; let stdoutResolved = false; let resolveOk: () => void; - const okPromise = new Promise((resolve) => { + let rejectOk: (reason: Error) => void; + const okPromise = new Promise((resolve, reject) => { resolveOk = resolve; + rejectOk = reject; }); let resolveExit: (status: number) => void; const exitPromise = new Promise((resolve) => { @@ -71,11 +82,17 @@ describe.skipIf(!existsSync(fbtestBinary))("framebuffer integration", () => { }); const kernel = new CentralizedKernelWorker( - { maxWorkers: 4, dataBufferSize: 65536, useSharedMemory: true, enableSyscallLog: false }, + { + maxWorkers: 4, + dataBufferSize: 65536, + useSharedMemory: true, + enableSyscallLog: !!process.env.KERNEL_SYSCALL_LOG, + }, io, { onExit: (exitPid, exitStatus) => { if (exitPid === pid) { + referenceOwners.release(exitPid); kernel.unregisterProcess(exitPid); const w = workers.get(exitPid); if (w) { @@ -96,7 +113,9 @@ describe.skipIf(!existsSync(fbtestBinary))("framebuffer integration", () => { resolveOk(); } }, - onStderr: () => {}, + onStderr: (data: Uint8Array) => { + stderr += new TextDecoder().decode(data); + }, }); await kernel.init(kernelWasmBytes); @@ -109,6 +128,7 @@ describe.skipIf(!existsSync(fbtestBinary))("framebuffer integration", () => { new Uint8Array(memory.buffer, channelOffset, CH_TOTAL_SIZE).fill(0); kernel.registerProcess(pid, memory, [channelOffset], { ptrWidth }); + const referenceInit = referenceOwners.start(pid); const initData: CentralizedWorkerInitMessage = { type: "centralized_init", @@ -119,9 +139,28 @@ describe.skipIf(!existsSync(fbtestBinary))("framebuffer integration", () => { argv: ["fbtest"], env: [], ptrWidth, + ...referenceInit, }; const mainWorker = workerAdapter.createWorker(initData); + referenceOwners.attach(pid, mainWorker); + mainWorker.on("error", (error) => rejectOk(error)); + mainWorker.on("message", (raw: unknown) => { + const message = raw as WorkerToHostMessage; + if (message.type === "error" && message.pid === pid) { + rejectOk(new Error(message.message)); + } + }); + mainWorker.on("exit", (code) => { + if (!stdoutResolved) { + rejectOk( + new Error( + `fbtest worker exited with status ${code} before readiness` + + (stderr ? `: ${stderr}` : ""), + ), + ); + } + }); workers.set(pid, mainWorker); try { @@ -129,7 +168,16 @@ describe.skipIf(!existsSync(fbtestBinary))("framebuffer integration", () => { await Promise.race([ okPromise, new Promise((_, reject) => - setTimeout(() => reject(new Error("fbtest didn't print 'ok' in 10s")), 10_000), + setTimeout( + () => + reject( + new Error( + "fbtest didn't print 'ok' in 10s" + + (stderr ? `: ${stderr}` : ""), + ), + ), + 10_000, + ), ), ]); @@ -154,7 +202,7 @@ describe.skipIf(!existsSync(fbtestBinary))("framebuffer integration", () => { // bit of `r << 16` ORs into the alpha byte; the test simply // recomputes the formula so it stays self-consistent. const expected = (r: number, c: number) => - ((0xff000000 | (r << 16) | c) >>> 0); + (0xff000000 | (r << 16) | c) >>> 0; expect(sample(0, 0)).toBe(expected(0, 0)); expect(sample(10, 20)).toBe(expected(10, 20)); expect(sample(255, 255)).toBe(expected(255, 255)); @@ -167,6 +215,7 @@ describe.skipIf(!existsSync(fbtestBinary))("framebuffer integration", () => { // test harness in main-thread mode. } finally { for (const [, w] of workers) await w.terminate().catch(() => {}); + referenceOwners.close(); // Avoid an unhandled-promise warning if the program never exits. void exitPromise.catch(() => {}); } diff --git a/host/test/gc-reference-state-fresh-worker.test.ts b/host/test/gc-reference-state-fresh-worker.test.ts new file mode 100644 index 0000000000..13aaa3cc08 --- /dev/null +++ b/host/test/gc-reference-state-fresh-worker.test.ts @@ -0,0 +1,58 @@ +import { execFileSync } from "node:child_process"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { runCentralizedProgram } from "./centralized-test-helper"; +import { + RAW_GC_REFERENCE_STATE_FRESH_WORKER_HEX, +} from "./fixtures/gc-reference-state-fresh-worker-bytes"; + +const testDir = dirname(fileURLToPath(import.meta.url)); +const fixtureSource = resolve( + testDir, + "fixtures/gc-reference-state-fresh-worker.wat", +); +const instrumenter = resolve( + testDir, + "../../tools/bin/wasm-fork-instrument", +); + +describe("Wasm GC reference state in a fresh process Worker", () => { + let workDir = ""; + let programPath = ""; + + beforeAll(() => { + workDir = mkdtempSync(join(tmpdir(), "kandelo-gc-reference-worker-")); + const rawPath = join(workDir, "gc-reference-state.raw.wasm"); + programPath = join(workDir, "gc-reference-state.wasm"); + // Keep the source path live in the test contract even though the checked + // byte fixture is required for WABT compatibility. + expect(fixtureSource).toMatch(/gc-reference-state-fresh-worker\.wat$/); + writeFileSync( + rawPath, + Buffer.from(RAW_GC_REFERENCE_STATE_FRESH_WORKER_HEX, "hex"), + ); + execFileSync(instrumenter, [rawPath, "-o", programPath]); + }); + + afterAll(() => { + if (workDir) rmSync(workDir, { recursive: true, force: true }); + }); + + it("preserves one cyclic identity across params, carryovers, globals, and tables", async () => { + const result = await runCentralizedProgram({ + programPath, + argv: ["gc-reference-state-fresh-worker"], + timeout: 30_000, + useDefaultRootfs: false, + }); + + expect( + result.exitCode, + `stdout:\n${result.stdout}\nstderr:\n${result.stderr}`, + ).toBe(0); + expect(result.stderr).toBe(""); + }); +}); diff --git a/host/test/generated-abi.test.ts b/host/test/generated-abi.test.ts index 2f0a7e46c0..5391ce188c 100644 --- a/host/test/generated-abi.test.ts +++ b/host/test/generated-abi.test.ts @@ -16,6 +16,8 @@ import { CH_DATA_SIZE, CH_ERRNO, CH_HEADER_SIZE, + CH_REQUEST_FLAGS, + CH_REQUEST_FLAG_DEFER_SIGNAL_DELIVERY, CH_RETURN, CH_SIG_BASE, CH_SIG_FLAGS, @@ -61,6 +63,13 @@ import { STRUCT_SIZE_WASM_STATFS, STRUCT_SIZE_WASM_TIMESPEC, SYSCALL_ARGS, + WPK_FORK_CAPABILITIES_SECTION, + WPK_FORK_CAPABILITIES_VERSION, + WPK_FORK_CAP_ACTIVATION_STATE_SAFE, + WPK_FORK_CAP_DYLINK_MAIN, + WPK_FORK_CAP_KNOWN_MASK, + WPK_FORK_CAP_REQUIRED_FLAGS, + WPK_FORK_CAP_SIDE_ENTRY, WPK_FORK_LINKED_FRAME_DESCRIPTOR_SIZE, WPK_FORK_LINKED_FRAME_FORMAT_MAGIC, WPK_FORK_LINKED_FRAME_FORMAT_SECTION, @@ -68,8 +77,53 @@ import { WPK_FORK_LINKED_FRAME_POINTER_WIDTHS, WPK_FORK_LINKED_FRAME_RECORD_ALIGNMENT, WPK_FORK_LINKED_FRAME_REQUIRED_FLAGS, + WPK_FORK_MODULE_STATE_ARENA_VERSION, + WPK_FORK_MODULE_STATE_CHUNK_FLAG_ROOT, + WPK_FORK_MODULE_STATE_CHUNK_FLAG_SEALED, + WPK_FORK_MODULE_STATE_CHUNK_KNOWN_FLAGS, + WPK_FORK_MODULE_STATE_CHUNK_MAGIC, + WPK_FORK_MODULE_STATE_DESCRIPTOR_SIZE, + WPK_FORK_MODULE_STATE_ELEMENT_SEGMENT_HEADER_SIZE, + WPK_FORK_MODULE_STATE_FLAG_EXPLICIT_OWNERS, + WPK_FORK_MODULE_STATE_FLAG_ROOT_PREFIX_POINTER, + WPK_FORK_MODULE_STATE_FLAG_SPARSE_TABLES, + WPK_FORK_MODULE_STATE_FORMAT_MAGIC, + WPK_FORK_MODULE_STATE_FORMAT_SECTION, + WPK_FORK_MODULE_STATE_FORMAT_VERSION, + WPK_FORK_MODULE_STATE_KNOWN_FLAGS, + WPK_FORK_MODULE_STATE_MAX_TABLE_PAGE_SHIFT, + WPK_FORK_MODULE_STATE_MIN_TABLE_PAGE_SHIFT, + WPK_FORK_MODULE_STATE_TABLE_PAGE_SHIFT, + WPK_FORK_MODULE_STATE_MODULE_RECORD_KNOWN_FLAGS, + WPK_FORK_MODULE_STATE_MODULE_RECORD_PAYLOAD_SIZE, + WPK_FORK_MODULE_STATE_MODULE_TEMPLATE_ID_SIZE, + WPK_FORK_MODULE_STATE_POINTER_WIDTHS, + WPK_FORK_MODULE_STATE_RECORD_ALIGNMENT, + WPK_FORK_MODULE_STATE_RECORD_HEADER_SIZE, + WPK_FORK_MODULE_STATE_RECORD_KINDS, + WPK_FORK_MODULE_STATE_RECORD_MAGIC, + WPK_FORK_MODULE_STATE_RECORD_VERSION, + WPK_FORK_MODULE_STATE_REQUIRED_FLAGS, + WPK_FORK_MODULE_STATE_ROOT_POINTER_WORD_OFFSET, + WPK_FORK_MODULE_STATE_TABLE_BASELINE_FINGERPRINT_SIZE, + WPK_FORK_MODULE_STATE_TABLE_DESCRIPTOR_PAYLOAD_SIZE, + WPK_FORK_MODULE_STATE_TABLE_FLAG_SPARSE_OVERRIDES, + WPK_FORK_MODULE_STATE_TABLE_KNOWN_FLAGS, + WPK_FORK_MODULE_STATE_TABLE_PAGE_HEADER_SIZE, + WPK_FORK_MODULE_STATE_TABLE_RUN_HEADER_SIZE, WPK_FORK_REQUIRED_EXPORTS, WPK_FORK_REQUIRED_IMPORTS, + WPK_FORK_REQUIRED_TABLE_IMPORTS, + WPK_FORK_STATIC_ROOT_CATALOG_EXPORT, + WPK_FORK_STATIC_ROOT_CATALOG_HEADER_SIZE, + WPK_FORK_STATIC_ROOT_CATALOG_MAGIC, + WPK_FORK_STATIC_ROOT_CATALOG_SECTION, + WPK_FORK_STATIC_ROOT_CATALOG_VERSION, + WPK_FORK_UNWIND_TAG_IMPORT_MODULE, + WPK_FORK_UNWIND_TAG_IMPORT_NAME, + WPK_FORK_UNWIND_TRANSPORT_PAYLOAD_ARITY, + WPK_FORK_UNWIND_TRANSPORT_SECTION, + WPK_FORK_UNWIND_TRANSPORT_VERSION, } from "../src/generated/abi"; const snapshot = JSON.parse( @@ -93,6 +147,14 @@ function statusNumber(name: string): number { return status.number; } +function requestFlag(name: string): number { + const flag = snapshot.channel_request_flags.find( + (entry: { name: string }) => entry.name === name, + ); + if (!flag) throw new Error(`missing channel_request_flags entry ${name}`); + return flag.bit; +} + function signalOffset(name: string): number { const slot = snapshot.channel_signal_area.slots.find((s: { name: string }) => s.name === name); if (!slot) throw new Error(`missing channel_signal_area slot ${name}`); @@ -112,13 +174,42 @@ function hostAdapterManifestField(name: string): { offset: number; size: number describe("generated host ABI bindings", () => { it("match the complete fork-artifact contract", () => { const fork = snapshot.program_artifact.fork_instrumentation; + const capabilities = fork.capabilities; const descriptor = fork.linked_frame_descriptor; + const staticRoots = fork.static_root_catalog; + const unwind = fork.unwind_transport; + expect(WPK_FORK_CAPABILITIES_SECTION).toBe(capabilities.section); + expect(WPK_FORK_CAPABILITIES_VERSION).toBe(capabilities.version); + expect(WPK_FORK_CAP_KNOWN_MASK).toBe(capabilities.known_mask); + expect(WPK_FORK_CAP_REQUIRED_FLAGS).toBe(capabilities.required_flags); + expect([ + { bit: WPK_FORK_CAP_SIDE_ENTRY, name: "side_entry" }, + { bit: WPK_FORK_CAP_DYLINK_MAIN, name: "dylink_main" }, + { + bit: WPK_FORK_CAP_ACTIVATION_STATE_SAFE, + name: "activation_state_safe", + }, + ]).toEqual(capabilities.flags); expect(WPK_FORK_LINKED_FRAME_FORMAT_SECTION).toBe(descriptor.section); expect(WPK_FORK_LINKED_FRAME_FORMAT_VERSION).toBe(descriptor.version); expect(WPK_FORK_LINKED_FRAME_FORMAT_MAGIC).toEqual(descriptor.magic_bytes); expect(WPK_FORK_LINKED_FRAME_DESCRIPTOR_SIZE).toBe(descriptor.descriptor_size); expect(WPK_FORK_LINKED_FRAME_RECORD_ALIGNMENT).toBe(descriptor.alignment); expect(WPK_FORK_LINKED_FRAME_REQUIRED_FLAGS).toBe(descriptor.required_flags); + expect(WPK_FORK_STATIC_ROOT_CATALOG_EXPORT).toBe(staticRoots.export); + expect(WPK_FORK_STATIC_ROOT_CATALOG_HEADER_SIZE).toBe( + staticRoots.header_size, + ); + expect(WPK_FORK_STATIC_ROOT_CATALOG_MAGIC).toEqual(staticRoots.magic_bytes); + expect(WPK_FORK_STATIC_ROOT_CATALOG_SECTION).toBe(staticRoots.section); + expect(WPK_FORK_STATIC_ROOT_CATALOG_VERSION).toBe(staticRoots.version); + expect(WPK_FORK_UNWIND_TAG_IMPORT_MODULE).toBe(unwind.import.module); + expect(WPK_FORK_UNWIND_TAG_IMPORT_NAME).toBe(unwind.import.name); + expect(WPK_FORK_UNWIND_TRANSPORT_PAYLOAD_ARITY).toBe( + unwind.payload_arity, + ); + expect(WPK_FORK_UNWIND_TRANSPORT_SECTION).toBe(unwind.section); + expect(WPK_FORK_UNWIND_TRANSPORT_VERSION).toBe(unwind.version); expect(WPK_FORK_LINKED_FRAME_POINTER_WIDTHS).toEqual( descriptor.pointer_widths.map( (format: { @@ -132,11 +223,129 @@ describe("generated host ABI bindings", () => { }), ), ); - expect(WPK_FORK_REQUIRED_IMPORTS).toEqual( - fork.required_imports.map(({ kind: _kind, ...requirement }: { kind: string }) => - requirement + + const moduleState = fork.module_state; + const moduleStateDescriptor = moduleState.descriptor; + expect(WPK_FORK_MODULE_STATE_FORMAT_SECTION).toBe(moduleStateDescriptor.section); + expect(WPK_FORK_MODULE_STATE_FORMAT_VERSION).toBe(moduleStateDescriptor.version); + expect(WPK_FORK_MODULE_STATE_FORMAT_MAGIC).toEqual(moduleStateDescriptor.magic_bytes); + expect(WPK_FORK_MODULE_STATE_DESCRIPTOR_SIZE).toBe( + moduleStateDescriptor.descriptor_size, + ); + expect(WPK_FORK_MODULE_STATE_RECORD_ALIGNMENT).toBe( + moduleStateDescriptor.alignment, + ); + expect(WPK_FORK_MODULE_STATE_KNOWN_FLAGS).toBe(moduleStateDescriptor.known_flags); + expect(WPK_FORK_MODULE_STATE_REQUIRED_FLAGS).toBe( + moduleStateDescriptor.required_flags, + ); + expect(WPK_FORK_MODULE_STATE_ROOT_POINTER_WORD_OFFSET).toBe( + moduleStateDescriptor.root_pointer_word_offset, + ); + expect([ + { + bit: WPK_FORK_MODULE_STATE_FLAG_ROOT_PREFIX_POINTER, + name: "root_prefix_pointer", + }, + { + bit: WPK_FORK_MODULE_STATE_FLAG_EXPLICIT_OWNERS, + name: "explicit_owners", + }, + { + bit: WPK_FORK_MODULE_STATE_FLAG_SPARSE_TABLES, + name: "sparse_tables", + }, + ]).toEqual(moduleStateDescriptor.flags); + + const moduleStateArena = moduleState.arena; + expect(WPK_FORK_MODULE_STATE_ARENA_VERSION).toBe(moduleStateArena.version); + expect(WPK_FORK_MODULE_STATE_CHUNK_MAGIC).toEqual( + moduleStateArena.chunk_magic_bytes, + ); + expect(WPK_FORK_MODULE_STATE_CHUNK_KNOWN_FLAGS).toBe( + moduleStateArena.known_chunk_flags, + ); + expect([ + { bit: WPK_FORK_MODULE_STATE_CHUNK_FLAG_ROOT, name: "root" }, + { bit: WPK_FORK_MODULE_STATE_CHUNK_FLAG_SEALED, name: "sealed" }, + ]).toEqual(moduleStateArena.chunk_flags); + expect(WPK_FORK_MODULE_STATE_POINTER_WIDTHS).toEqual( + moduleStateArena.pointer_widths.map( + (format: { bytes: number; chunk_header_size: number }) => ({ + bytes: format.bytes, + chunkHeaderSize: format.chunk_header_size, + }), ), ); + expect(WPK_FORK_MODULE_STATE_RECORD_VERSION).toBe( + moduleStateArena.record.version, + ); + expect(WPK_FORK_MODULE_STATE_RECORD_MAGIC).toEqual( + moduleStateArena.record.magic_bytes, + ); + expect(WPK_FORK_MODULE_STATE_RECORD_HEADER_SIZE).toBe( + moduleStateArena.record.header_size, + ); + expect(WPK_FORK_MODULE_STATE_RECORD_ALIGNMENT).toBe( + moduleStateArena.record.alignment, + ); + expect(WPK_FORK_MODULE_STATE_RECORD_KINDS).toEqual( + moduleStateArena.record.kinds, + ); + + const modulePayload = moduleState.record_payloads.module; + expect(WPK_FORK_MODULE_STATE_MODULE_TEMPLATE_ID_SIZE).toBe( + modulePayload.template_id_size, + ); + expect(WPK_FORK_MODULE_STATE_MODULE_RECORD_PAYLOAD_SIZE).toBe( + modulePayload.payload_size, + ); + expect(WPK_FORK_MODULE_STATE_MODULE_RECORD_KNOWN_FLAGS).toBe( + modulePayload.known_flags, + ); + const tablePayload = moduleState.record_payloads.table; + expect(WPK_FORK_MODULE_STATE_TABLE_BASELINE_FINGERPRINT_SIZE).toBe( + tablePayload.baseline_fingerprint_size, + ); + expect(WPK_FORK_MODULE_STATE_TABLE_DESCRIPTOR_PAYLOAD_SIZE).toBe( + tablePayload.descriptor_payload_size, + ); + expect(WPK_FORK_MODULE_STATE_TABLE_KNOWN_FLAGS).toBe(tablePayload.known_flags); + expect([ + { + bit: WPK_FORK_MODULE_STATE_TABLE_FLAG_SPARSE_OVERRIDES, + name: "sparse_overrides", + }, + ]).toEqual(tablePayload.flags); + expect(WPK_FORK_MODULE_STATE_TABLE_PAGE_HEADER_SIZE).toBe( + tablePayload.page_header_size, + ); + expect(WPK_FORK_MODULE_STATE_TABLE_RUN_HEADER_SIZE).toBe( + tablePayload.run_header_size, + ); + expect(WPK_FORK_MODULE_STATE_MIN_TABLE_PAGE_SHIFT).toBe( + tablePayload.min_page_shift, + ); + expect(WPK_FORK_MODULE_STATE_MAX_TABLE_PAGE_SHIFT).toBe( + tablePayload.max_page_shift, + ); + expect(WPK_FORK_MODULE_STATE_TABLE_PAGE_SHIFT).toBe( + tablePayload.page_shift, + ); + expect(WPK_FORK_MODULE_STATE_ELEMENT_SEGMENT_HEADER_SIZE).toBe( + moduleState.record_payloads.element_segments.header_size, + ); + + expect(WPK_FORK_REQUIRED_IMPORTS).toEqual( + fork.required_imports + .filter(({ kind }: { kind: string }) => kind === "func") + .map(({ kind: _kind, ...requirement }: { kind: string }) => requirement), + ); + expect(WPK_FORK_REQUIRED_TABLE_IMPORTS).toEqual( + fork.required_imports + .filter(({ kind }: { kind: string }) => kind === "table") + .map(({ kind: _kind, ...requirement }: { kind: string }) => requirement), + ); expect(WPK_FORK_REQUIRED_EXPORTS).toEqual( fork.required_exports.map(({ kind: _kind, ...requirement }: { kind: string }) => requirement @@ -150,6 +359,7 @@ describe("generated host ABI bindings", () => { it("match the ABI version and channel layout snapshot", () => { expect(ABI_VERSION).toBe(snapshot.abi_version); expect(snapshot.custom_sections).toContain(ABI_CUSTOM_SECTION); + expect(snapshot.custom_sections).toContain(WPK_FORK_MODULE_STATE_FORMAT_SECTION); expect(snapshot.kernel_exports.some((e: { name: string }) => e.name === ABI_KERNEL_EXPORT)).toBe(true); expect(CH_STATUS).toBe(fieldOffset("status")); @@ -157,6 +367,10 @@ describe("generated host ABI bindings", () => { expect(CH_ARGS).toBe(fieldOffset("args")); expect(CH_RETURN).toBe(fieldOffset("ret")); expect(CH_ERRNO).toBe(fieldOffset("errno")); + expect(CH_REQUEST_FLAGS).toBe(fieldOffset("request_flags")); + expect(CH_REQUEST_FLAG_DEFER_SIGNAL_DELIVERY).toBe( + requestFlag("defer_signal_delivery"), + ); expect(CH_ARG_SIZE).toBe(8); expect(CH_ARGS_COUNT).toBe(6); diff --git a/host/test/mouse-integration.test.ts b/host/test/mouse-integration.test.ts index c12e2b6e20..a9b266fa04 100644 --- a/host/test/mouse-integration.test.ts +++ b/host/test/mouse-integration.test.ts @@ -25,7 +25,11 @@ import { CAPTURED_STDIO, CentralizedKernelWorker } from "../src/kernel-worker"; import { NodePlatformIO } from "../src/platform/node"; import { NodeWorkerAdapter } from "../src/worker-adapter"; import { detectPtrWidth } from "../src/constants"; -import type { CentralizedWorkerInitMessage } from "../src/worker-protocol"; +import type { + CentralizedWorkerInitMessage, + WorkerToHostMessage, +} from "../src/worker-protocol"; +import { TestProcessReferenceOwners } from "./process-reference-owner-helper"; const __dirname = dirname(fileURLToPath(import.meta.url)); @@ -69,26 +73,40 @@ describe.skipIf(!existsSync(mousetestBinary))("mouse integration", () => { const io = new NodePlatformIO(); const workerAdapter = new NodeWorkerAdapter(); - const workers = new Map>(); + const referenceOwners = new TestProcessReferenceOwners(); + const workers = new Map< + number, + ReturnType + >(); let pid = 0; let stdout = ""; let resolveReady: () => void; - const readyPromise = new Promise((resolve) => { + let rejectReady: (reason: Error) => void; + const readyPromise = new Promise((resolve, reject) => { resolveReady = resolve; + rejectReady = reject; }); let resolveExit: (status: number) => void; - const exitPromise = new Promise((resolve) => { + let rejectExit: (reason: Error) => void; + const exitPromise = new Promise((resolve, reject) => { resolveExit = resolve; + rejectExit = reject; }); const kernel = new CentralizedKernelWorker( - { maxWorkers: 4, dataBufferSize: 65536, useSharedMemory: true, enableSyscallLog: false }, + { + maxWorkers: 4, + dataBufferSize: 65536, + useSharedMemory: true, + enableSyscallLog: false, + }, io, { onExit: (exitPid, exitStatus) => { if (exitPid === pid) { + referenceOwners.release(exitPid); kernel.unregisterProcess(exitPid); const w = workers.get(exitPid); if (w) { @@ -122,6 +140,7 @@ describe.skipIf(!existsSync(mousetestBinary))("mouse integration", () => { new Uint8Array(memory.buffer, channelOffset, CH_TOTAL_SIZE).fill(0); kernel.registerProcess(pid, memory, [channelOffset], { ptrWidth }); + const referenceInit = referenceOwners.start(pid); const initData: CentralizedWorkerInitMessage = { type: "centralized_init", @@ -132,16 +151,32 @@ describe.skipIf(!existsSync(mousetestBinary))("mouse integration", () => { argv: ["mousetest", "3"], env: [], ptrWidth, + ...referenceInit, }; const mainWorker = workerAdapter.createWorker(initData); + referenceOwners.attach(pid, mainWorker); + const rejectWorker = (error: Error): void => { + rejectReady(error); + rejectExit(error); + }; + mainWorker.on("error", rejectWorker); + mainWorker.on("message", (raw: unknown) => { + const message = raw as WorkerToHostMessage; + if (message.type === "error" && message.pid === pid) { + rejectWorker(new Error(message.message)); + } + }); workers.set(pid, mainWorker); try { await Promise.race([ readyPromise, new Promise((_, reject) => - setTimeout(() => reject(new Error("mousetest didn't print 'ready' in 10s")), 10_000), + setTimeout( + () => reject(new Error("mousetest didn't print 'ready' in 10s")), + 10_000, + ), ), ]); @@ -161,7 +196,11 @@ describe.skipIf(!existsSync(mousetestBinary))("mouse integration", () => { const exitCode = await Promise.race([ exitPromise, new Promise((_, reject) => - setTimeout(() => reject(new Error("mousetest didn't exit after 3 packets in 10s")), 10_000), + setTimeout( + () => + reject(new Error("mousetest didn't exit after 3 packets in 10s")), + 10_000, + ), ), ]); expect(exitCode).toBe(0); @@ -173,11 +212,15 @@ describe.skipIf(!existsSync(mousetestBinary))("mouse integration", () => { expect(lines.slice(1)).toHaveLength(3); for (let i = 0; i < 3; i++) { const ev = events[i]; - const b0 = expectedByte0(ev.dx, ev.dy, ev.buttons).toString(16).padStart(2, "0"); + const b0 = expectedByte0(ev.dx, ev.dy, ev.buttons) + .toString(16) + .padStart(2, "0"); expect(lines[1 + i]).toBe(`pkt ${b0} ${ev.dx} ${ev.dy}`); } } finally { for (const [, w] of workers) await w.terminate().catch(() => {}); + referenceOwners.close(); + void readyPromise.catch(() => {}); void exitPromise.catch(() => {}); } }, 30_000); diff --git a/host/test/patch-wasm-for-thread-gc.test.ts b/host/test/patch-wasm-for-thread-gc.test.ts new file mode 100644 index 0000000000..2fa2f2400f --- /dev/null +++ b/host/test/patch-wasm-for-thread-gc.test.ts @@ -0,0 +1,104 @@ +import { describe, expect, it } from "vitest"; +import { patchWasmForThread } from "../src/worker-main"; + +function uleb(value: number): number[] { + const bytes: number[] = []; + do { + let byte = value & 0x7f; + value >>>= 7; + if (value !== 0) byte |= 0x80; + bytes.push(byte); + } while (value !== 0); + return bytes; +} + +function name(value: string): number[] { + const bytes = new TextEncoder().encode(value); + return [...uleb(bytes.byteLength), ...bytes]; +} + +function section(id: number, payload: number[]): number[] { + return [id, ...uleb(payload.length), ...payload]; +} + +function body(instructions: number[], locals: number[] = []): number[] { + const payload = [ + ...uleb(locals.length === 0 ? 0 : 1), + ...(locals.length === 0 ? [] : [...uleb(1), ...locals]), + ...instructions, + 0x0b, + ]; + return [...uleb(payload.length), ...payload]; +} + +/** + * Current core-GC encoding, built directly because the repository's WABT + * release still emits an older experimental GC binary format. + */ +function gcThreadStartFixture(): ArrayBuffer { + const types = [ + ...uleb(1), // one recursive group + 0x4e, + ...uleb(4), + // type 0: (struct (field (mut (ref null 0)))) + 0x5f, ...uleb(1), 0x63, 0x00, 0x01, + // type 1: (array (mut i32)) + 0x5e, 0x7f, 0x01, + // type 2: (func) + 0x60, 0x00, 0x00, + // type 3: (func (param i32)), valid exception-tag type + 0x60, 0x01, 0x7f, 0x00, + ]; + const imports = [ + ...uleb(4), + ...name("env"), ...name("imported_function"), 0x00, ...uleb(2), + ...name("env"), ...name("gc_table"), 0x01, + 0x63, 0x00, // concrete nullable reference + 0x00, ...uleb(1), // limits: min 1 + ...name("env"), ...name("gc_global"), 0x03, + 0x63, 0x00, // concrete nullable reference + 0x01, // mutable + ...name("env"), ...name("event"), 0x04, + 0x00, ...uleb(3), // tag attribute + function type + ]; + const functions = [ + ...uleb(4), + ...uleb(2), // ctor + ...uleb(2), // __abi_version + ...uleb(2), // __get_channel_base_addr + ...uleb(2), // _start + ]; + const exports = [ + ...uleb(3), + ...name("__abi_version"), 0x00, ...uleb(2), + ...name("__get_channel_base_addr"), 0x00, ...uleb(3), + ...name("_start"), 0x00, ...uleb(4), + ]; + const code = [ + ...uleb(4), + ...body([]), + ...body([0x10, ...uleb(1)], [0x63, 0x00]), + ...body([0x10, ...uleb(1)], [0x63, 0x00]), + ...body([0x10, ...uleb(1)], [0x63, 0x00]), + ]; + return new Uint8Array([ + 0x00, 0x61, 0x73, 0x6d, + 0x01, 0x00, 0x00, 0x00, + ...section(1, types), + ...section(2, imports), + ...section(3, functions), + ...section(7, exports), + ...section(8, uleb(1)), + ...section(10, code), + ]).buffer; +} + +describe("patchWasmForThread GC/reference parsing", () => { + it("keeps section and function indexes aligned across concrete ref and tag imports", () => { + const source = gcThreadStartFixture(); + expect(WebAssembly.validate(source)).toBe(true); + const patched = patchWasmForThread(source); + expect(WebAssembly.validate(patched)).toBe(true); + expect(patched.byteLength).toBeLessThan(source.byteLength); + }); +}); diff --git a/host/test/plain-catch-payload-lifetime.test.ts b/host/test/plain-catch-payload-lifetime.test.ts index 666a52b08a..192308b90e 100644 --- a/host/test/plain-catch-payload-lifetime.test.ts +++ b/host/test/plain-catch-payload-lifetime.test.ts @@ -8,6 +8,11 @@ import { LinkedForkContinuation, readLinkedFrameFormat, } from "../src/fork-continuation"; +import { + ForkModuleStateArena, + readForkModuleStateRoot, +} from "../src/fork-module-state"; +import { SingleActivationForkRuntime } from "./fork-instrument-runtime-harness"; const FIRST_PAYLOAD = 0x1234; const SECOND_PAYLOAD = 0x5678; @@ -54,7 +59,8 @@ describe("plain-catch payload lifetime", () => { ); execFileSync(instrumenterPath, [rawPath, "-o", instrumentedPath]); - const module = new WebAssembly.Module(readFileSync(instrumentedPath)); + const instrumentedBytes = readFileSync(instrumentedPath); + const module = new WebAssembly.Module(instrumentedBytes); const memory = new WebAssembly.Memory({ initial: 4 }); const view = new DataView(memory.buffer); let instance: WebAssembly.Instance; @@ -70,6 +76,25 @@ describe("plain-catch payload lifetime", () => { () => {}, "plain-catch-lifetime", ); + let nextArenaAddress = 2 * 65_536; + const runtime = new SingleActivationForkRuntime({ + module, + moduleBytes: instrumentedBytes, + memory, + continuation, + newArena: () => new ForkModuleStateArena( + memory, + 4, + (size) => { + const address = nextArenaAddress; + nextArenaAddress += size; + return address; + }, + () => {}, + "plain-catch-lifetime module state", + ), + label: "plain-catch-lifetime", + }); const scratchIsSentinel = (base: number): boolean => view.getUint32(base + SCRATCH_ARM_OFFSET, true) === SENTINEL && @@ -82,19 +107,12 @@ describe("plain-catch payload lifetime", () => { instance = new WebAssembly.Instance(module, { env: { memory, - __wpk_fork_frame_reserve: (size: number) => - continuation.reserveFrame(size), - __wpk_fork_frame_commit: (payload: number) => - continuation.commitFrame(payload), - __wpk_fork_frame_next: (size: number) => - continuation.nextFrame(size), + ...runtime.envImports, }, kernel: { kernel_fork: () => { - const state = (instance.exports.wpk_fork_state as () => number)(); - if (state === 2) { - (instance.exports.wpk_fork_rewind_end as () => void)(); - continuation.finishReplayAndRelease(); + if (runtime.coordinator.phaseName() === "parent-replay") { + runtime.coordinator.finishReplay(); return normalForkCalls === 1 ? 7 : 11; } @@ -106,26 +124,21 @@ describe("plain-catch payload lifetime", () => { secondCaptureKeptReleasedMemory = scratchIsSentinel(moduleBuffer); } - moduleBuffer = Number(continuation.beginUnwind()); - (instance.exports.wpk_fork_unwind_begin as (addr: number) => void)( - moduleBuffer, - ); + runtime.beginCapture(); + moduleBuffer = runtime.coordinator.rootFor(0); return 0; }, }, }); + runtime.register(instance); const run = instance.exports.run as (payload: number) => number; fillScratch(0); - // The first pass drains frames and returns the function's result default. - expect(run(FIRST_PAYLOAD)).toBe(0); - (instance.exports.wpk_fork_unwind_end as () => void)(); - continuation.finishUnwind(); - continuation.beginReplay(); - (instance.exports.wpk_fork_rewind_begin as (addr: number) => void)( - moduleBuffer, - ); + // The first pass transports the private unwind to the worker boundary. + runtime.expectCaptureTransport(() => run(FIRST_PAYLOAD)); + runtime.coordinator.sealCapture(); + runtime.coordinator.beginParentReplay(); expect(run(FIRST_PAYLOAD)).toBe(FIRST_PAYLOAD + 7); // Once replay releases the continuation, its former bytes are no longer @@ -133,13 +146,9 @@ describe("plain-catch payload lifetime", () => { // mutate either that retired mapping or low memory before its fork call. fillScratch(0); fillScratch(moduleBuffer); - expect(run(SECOND_PAYLOAD)).toBe(0); - (instance.exports.wpk_fork_unwind_end as () => void)(); - continuation.finishUnwind(); - continuation.beginReplay(); - (instance.exports.wpk_fork_rewind_begin as (addr: number) => void)( - moduleBuffer, - ); + runtime.expectCaptureTransport(() => run(SECOND_PAYLOAD)); + runtime.coordinator.sealCapture(); + runtime.coordinator.beginParentReplay(); expect(run(SECOND_PAYLOAD)).toBe(SECOND_PAYLOAD + 11); expect({ @@ -212,11 +221,11 @@ describe("plain-catch payload lifetime", () => { ); execFileSync(instrumenterPath, [rawPath, "-o", instrumentedPath]); - const module = new WebAssembly.Module(readFileSync(instrumentedPath)); + const instrumentedBytes = readFileSync(instrumentedPath); + const module = new WebAssembly.Module(instrumentedBytes); const memory = new WebAssembly.Memory({ initial: 4 }); const view = new DataView(memory.buffer); let instance: WebAssembly.Instance; - let moduleBuffer = 0; let normalForkCalls = 0; let recursiveCaptureKeptLowMemory = false; const continuation = new LinkedForkContinuation( @@ -226,48 +235,49 @@ describe("plain-catch payload lifetime", () => { () => {}, "plain-catch-recursion", ); + const runtime = new SingleActivationForkRuntime({ + module, + moduleBytes: instrumentedBytes, + memory, + continuation, + newArena: () => new ForkModuleStateArena( + memory, + 4, + () => 2 * 65_536, + () => {}, + "plain-catch-recursion module state", + ), + label: "plain-catch-recursion", + }); instance = new WebAssembly.Instance(module, { env: { memory, - __wpk_fork_frame_reserve: (size: number) => - continuation.reserveFrame(size), - __wpk_fork_frame_commit: (payload: number) => - continuation.commitFrame(payload), - __wpk_fork_frame_next: (size: number) => - continuation.nextFrame(size), + ...runtime.envImports, }, kernel: { kernel_fork: () => { - const state = (instance.exports.wpk_fork_state as () => number)(); - if (state === 2) { - (instance.exports.wpk_fork_rewind_end as () => void)(); - continuation.finishReplayAndRelease(); + if (runtime.coordinator.phaseName() === "parent-replay") { + runtime.coordinator.finishReplay(); return 7; } normalForkCalls++; recursiveCaptureKeptLowMemory = view.getUint32(SCRATCH_ARM_OFFSET, true) === SENTINEL && view.getUint32(SCRATCH_PAYLOAD_OFFSET, true) === SENTINEL; - moduleBuffer = Number(continuation.beginUnwind()); - (instance.exports.wpk_fork_unwind_begin as (addr: number) => void)( - moduleBuffer, - ); + runtime.beginCapture(); return 0; }, }, }); + runtime.register(instance); const run = instance.exports.run as (depth: number) => number; view.setUint32(SCRATCH_ARM_OFFSET, SENTINEL, true); view.setUint32(SCRATCH_PAYLOAD_OFFSET, SENTINEL, true); - expect(run(2)).toBe(0); - (instance.exports.wpk_fork_unwind_end as () => void)(); - continuation.finishUnwind(); - continuation.beginReplay(); - (instance.exports.wpk_fork_rewind_begin as (addr: number) => void)( - moduleBuffer, - ); + runtime.expectCaptureTransport(() => run(2)); + runtime.coordinator.sealCapture(); + runtime.coordinator.beginParentReplay(); // WHY: all three calls execute the same static catch arm, but each // activation owns a distinct payload (102, 101, 100). The result slots @@ -293,7 +303,7 @@ describe("plain-catch payload lifetime", () => { } }); - it("distinguishes plain and catch_ref state in either reuse order", () => { + it("reconstructs mixed plain and catch_ref state in a fresh child instance", () => { const dir = mkdtempSync(join(tmpdir(), "kandelo-mixed-catch-lifetime-")); try { const watPath = join(dir, "mixed-catch-lifetime.wat"); @@ -303,12 +313,13 @@ describe("plain-catch payload lifetime", () => { (import "kernel" "kernel_fork" (func $fork (result i32))) (import "env" "memory" (memory 4)) (tag $plain (param i32)) - (tag $with_ref) + (tag $with_ref (param i32)) (func (export "run") (param $take_plain i32) (result i32) + (local $caught i32) (block $done (result i32) (block $plain_handler (result i32) - (block $ref_handler (result exnref) - (try_table (result exnref) + (block $ref_handler (result i32 exnref) + (try_table (result i32 exnref) (catch $plain $plain_handler) (catch_ref $with_ref $ref_handler) local.get $take_plain @@ -316,15 +327,19 @@ describe("plain-catch payload lifetime", () => { i32.const 41 throw $plain else + i32.const 42 throw $with_ref end unreachable)) drop + local.set $caught call $fork - i32.const 42 + local.get $caught i32.add br $done) + local.set $caught call $fork + local.get $caught i32.add br $done)))`); execFileSync("wat2wasm", [ @@ -337,62 +352,132 @@ describe("plain-catch payload lifetime", () => { new URL("../../tools/bin/wasm-fork-instrument", import.meta.url), ); execFileSync(instrumenterPath, [rawPath, "-o", instrumentedPath]); - const module = new WebAssembly.Module(readFileSync(instrumentedPath)); + const instrumentedBytes = readFileSync(instrumentedPath); + const module = new WebAssembly.Module(instrumentedBytes); const runOrder = (modes: readonly number[]): void => { - const memory = new WebAssembly.Memory({ initial: 4 }); - let instance: WebAssembly.Instance; + const parentMemory = new WebAssembly.Memory({ initial: 4 }); + let parentInstance: WebAssembly.Instance; let moduleBuffer = 0; - const continuation = new LinkedForkContinuation( - memory, + let nextParentArenaAddress = 2 * 65_536; + const parentContinuation = new LinkedForkContinuation( + parentMemory, readLinkedFrameFormat(module), () => 65_536, () => {}, - "mixed-catch-lifetime", + "mixed-catch-parent", ); - instance = new WebAssembly.Instance(module, { + const parentRuntime = new SingleActivationForkRuntime({ + module, + moduleBytes: instrumentedBytes, + memory: parentMemory, + continuation: parentContinuation, + newArena: () => new ForkModuleStateArena( + parentMemory, + 4, + (size) => { + const address = nextParentArenaAddress; + nextParentArenaAddress += size; + return address; + }, + () => {}, + "mixed-catch-parent module state", + ), + label: "mixed-catch-parent", + }); + parentInstance = new WebAssembly.Instance(module, { env: { - memory, - __wpk_fork_frame_reserve: (size: number) => - continuation.reserveFrame(size), - __wpk_fork_frame_commit: (payload: number) => - continuation.commitFrame(payload), - __wpk_fork_frame_next: (size: number) => - continuation.nextFrame(size), + memory: parentMemory, + ...parentRuntime.envImports, }, kernel: { kernel_fork: () => { - const state = (instance.exports.wpk_fork_state as () => number)(); - if (state === 2) { - (instance.exports.wpk_fork_rewind_end as () => void)(); - continuation.finishReplayAndRelease(); - return 7; - } - moduleBuffer = Number(continuation.beginUnwind()); - (instance.exports.wpk_fork_unwind_begin as (addr: number) => void)( - moduleBuffer, - ); + parentRuntime.beginCapture(); + moduleBuffer = parentRuntime.coordinator.rootFor(0); return 0; }, }, }); - const run = instance.exports.run as (takePlain: number) => number; + parentRuntime.register(parentInstance); + const parentRun = parentInstance.exports.run as (takePlain: number) => number; for (const mode of modes) { - expect(run(mode)).toBe(0); - (instance.exports.wpk_fork_unwind_end as () => void)(); - continuation.finishUnwind(); - continuation.beginReplay(); - (instance.exports.wpk_fork_rewind_begin as (addr: number) => void)( - moduleBuffer, + parentRuntime.expectCaptureTransport(() => parentRun(mode)); + parentRuntime.coordinator.sealCapture(); + + // Model the actual worker boundary: the child receives only copied + // linear memory and instantiates an otherwise fresh Wasm module. + const childMemory = new WebAssembly.Memory({ + initial: parentMemory.buffer.byteLength / 65_536, + }); + new Uint8Array(childMemory.buffer).set( + new Uint8Array(parentMemory.buffer), + ); + + const childContinuation = new LinkedForkContinuation( + childMemory, + readLinkedFrameFormat(module), + () => { + throw new Error("fresh child replay must not allocate a continuation"); + }, + () => {}, + "mixed-catch-child", + ); + const childRuntime = new SingleActivationForkRuntime({ + module, + moduleBytes: instrumentedBytes, + memory: childMemory, + continuation: childContinuation, + newArena: () => { + throw new Error( + "fresh child replay must attach copied module state", + ); + }, + label: "mixed-catch-child", + }); + let childInstance: WebAssembly.Instance; + childInstance = new WebAssembly.Instance(module, { + env: { + memory: childMemory, + ...childRuntime.envImports, + }, + kernel: { + kernel_fork: () => { + expect(childRuntime.coordinator.phaseName()).toBe( + "child-replay", + ); + childRuntime.coordinator.finishReplay(); + return 7; + }, + }, + }); + childRuntime.register(childInstance, { bootstrap: false }); + childRuntime.setCopiedProcessLaunchRoot(moduleBuffer); + const copiedArena = new ForkModuleStateArena( + childMemory, + 4, + () => { + throw new Error( + "fresh child replay must not allocate module-state storage", + ); + }, + () => {}, + "mixed-catch-child module state", + ); + copiedArena.attach( + readForkModuleStateRoot(childMemory, moduleBuffer, 4), ); - expect(run(mode)).toBe((mode ? 41 : 42) + 7); + childRuntime.coordinator.attachChild(copiedArena); + const childRun = childInstance.exports.run as (takePlain: number) => number; + expect(childRun(mode)).toBe((mode ? 41 : 42) + 7); + parentRuntime.coordinator.abort(); } }; - // WHY: the exnref table is intentionally reused. A stale non-null - // entry after catch_ref must not make the next plain activation take - // throw_ref, and a preceding plain catch must not suppress catch_ref. + // Both arm orders exercise one long-lived parent instance, while every + // child has empty module globals/tables. CatchRef can pass only if rewind + // restores the scalar tag payload and rethrows the tag to create a new + // instance-local exnref. runOrder([0, 1]); runOrder([1, 0]); } finally { diff --git a/host/test/process-reference-owner-helper.ts b/host/test/process-reference-owner-helper.ts new file mode 100644 index 0000000000..8c11e4d130 --- /dev/null +++ b/host/test/process-reference-owner-helper.ts @@ -0,0 +1,130 @@ +import { + ForkHostImportOwnerRuntime, + type ForkHostImportOwnerWorker, + type ForkHostImportWorkerInit, +} from "../src/fork-host-import-runtime"; +import { ForkExternrefProcessOwner } from "../src/fork-externref-process-owner"; +import type { ForkExternrefGeneration } from "../src/fork-reference-broker"; +import type { WorkerHandle } from "../src/worker-adapter"; +import type { WorkerToHostMessage } from "../src/worker-protocol"; + +export interface TestProcessReferenceInit { + readonly externrefGenerationId: number; + readonly forkHostImports: ForkHostImportWorkerInit; +} + +interface TestProcessReferenceRecord { + readonly generation: ForkExternrefGeneration; + readonly imports: ForkHostImportOwnerWorker; + worker?: WorkerHandle; + messageHandler?: (message: unknown) => void; +} + +/** + * Process-owned reference authority for tests that spawn process Workers + * directly instead of using NodeKernelHost or BrowserKernelHost. + */ +export class TestProcessReferenceOwners { + private readonly owner = new ForkExternrefProcessOwner(); + private readonly runtime = new ForkHostImportOwnerRuntime(this.owner); + private readonly records = new Map(); + + start(pid: number): TestProcessReferenceInit { + return this.install(this.owner.startGeneration(pid)); + } + + fork( + parentPid: number, + childPid: number, + memory: WebAssembly.Memory, + ptrWidth: 4 | 8, + moduleBufferAddress: number, + ): TestProcessReferenceInit { + const parent = this.records.get(parentPid); + if (!parent) { + throw new Error( + `missing test reference owner for fork parent ${parentPid}`, + ); + } + const child = this.owner.forkGenerationFromContinuation( + parent.generation, + childPid, + memory, + ptrWidth, + moduleBufferAddress, + `direct-worker test fork child pid=${childPid}`, + ).generation; + return this.install(child); + } + + attach(pid: number, worker: WorkerHandle): void { + const record = this.requireRecord(pid); + if (record.worker !== undefined) { + throw new Error( + `test reference owner for pid=${pid} is already attached`, + ); + } + record.worker = worker; + const messageHandler = (raw: unknown): void => { + const message = raw as WorkerToHostMessage; + if ( + message.type === "fork_host_import" && + this.records.get(pid) === record && + record.worker === worker + ) { + record.imports.dispatch(message.wake); + } + }; + record.messageHandler = messageHandler; + worker.on("message", messageHandler); + } + + release(pid: number): void { + const record = this.records.get(pid); + if (!record) return; + this.records.delete(pid); + if (record.worker && record.messageHandler) { + record.worker.off("message", record.messageHandler); + } + record.imports.close(); + this.owner.releaseGeneration(record.generation); + } + + close(): void { + for (const pid of [...this.records.keys()]) this.release(pid); + } + + private install( + generation: ForkExternrefGeneration, + ): TestProcessReferenceInit { + const pid = generation.pid; + if (this.records.has(pid)) { + this.owner.releaseGeneration(generation); + throw new Error(`duplicate test reference owner for pid=${pid}`); + } + let record: TestProcessReferenceRecord; + const imports = this.runtime.createWorker({ + pid, + generationId: generation.id, + authorizeSender: () => { + // WHY: numeric PID/generation/sender fields arrive through shared + // memory. The independently observed Worker object is the authority. + if (this.records.get(pid) !== record || record.worker === undefined) { + throw new Error(`stale direct-worker test sender for pid=${pid}`); + } + }, + }); + record = { generation, imports }; + this.records.set(pid, record); + return { + externrefGenerationId: generation.id, + forkHostImports: imports.init, + }; + } + + private requireRecord(pid: number): TestProcessReferenceRecord { + const record = this.records.get(pid); + if (!record) throw new Error(`missing test reference owner for pid=${pid}`); + return record; + } +} diff --git a/host/test/process-table-replication.test.ts b/host/test/process-table-replication.test.ts new file mode 100644 index 0000000000..78f5ffae70 --- /dev/null +++ b/host/test/process-table-replication.test.ts @@ -0,0 +1,192 @@ +import { describe, expect, it } from "vitest"; +import type { DlopenSupport } from "../src/worker-main"; +import { + __testCreateProcessTableReplicationOwner, +} from "../src/worker-main"; +import { + DylinkForkArchive, + type DylinkForkTablePatch, +} from "../src/dylink-fork-archive"; +import type { ForkActivationRegistry } from "../src/fork-activation-registry"; +import type { ForkModuleStateArena } from "../src/fork-module-state"; + +interface TestTableReplicationOwner { + beginMutation(): bigint; + commit( + activationId: number, + ownerId: number, + firstIndex: number | bigint, + length: number | bigint, + ): void; + reconcileNow(): number; +} + +function archiveFixture() { + const memory = new WebAssembly.Memory({ initial: 8, maximum: 8 }); + let head = 0; + let next = 4_096; + const archive = new DylinkForkArchive( + memory, + 4, + () => head, + (value) => { head = value; }, + (size) => { + const address = next; + next += Math.ceil(size / 8) * 8; + return { address, size }; + }, + () => {}, + "process table test archive", + ); + archive.sync({ nextHandle: 2, libraries: [] }); + return { archive, memory }; +} + +function dlopenFixture(archive: DylinkForkArchive): DlopenSupport { + let writerDepth = 0; + let readerDepth = 0; + let writerObserver = () => {}; + return { + imports: {}, + readForkState: () => archive.read(), + replayDlopens: () => {}, + resetForkChildLock: () => {}, + archive, + acquireArchiveWriter: () => { + if (writerDepth++ === 0) writerObserver(); + }, + releaseArchiveWriter: () => { + if (writerDepth <= 0) throw new Error("writer underflow"); + writerDepth--; + }, + acquireArchiveReader: () => { readerDepth++; }, + releaseArchiveReader: () => { + if (readerDepth <= 0) throw new Error("reader underflow"); + readerDepth--; + }, + withArchiveWriter: (operation: () => T): T => { + if (writerDepth++ === 0) writerObserver(); + try { + return operation(); + } finally { + writerDepth--; + } + }, + withArchiveReader: (operation: () => T): T => { + readerDepth++; + try { + return operation(); + } finally { + readerDepth--; + } + }, + writerOwned: () => writerDepth > 0, + setWriterAcquireObserver: (observer) => { writerObserver = observer; }, + setOperationAbortObserver: () => {}, + setCommitObserver: () => {}, + }; +} + +function arenaFixture(root: number): ForkModuleStateArena { + let active = false; + return { + begin: () => { + active = true; + return root; + }, + attach: () => { active = true; }, + release: () => { active = false; }, + hasActiveArena: () => active, + } as unknown as ForkModuleStateArena; +} + +function patch(generation?: number): DylinkForkTablePatch { + return { + ...(generation === undefined ? {} : { generation }), + activationId: 0, + ownerId: 1, + start: 0, + tableLength: 1, + runs: [{ + length: 1, + function: { activationId: 0, ordinal: 0 }, + }], + }; +} + +describe("process table replication publication", () => { + it("uses patches normally and transparently compacts at the journal bound", () => { + const { archive } = archiveFixture(); + const dlopen = dlopenFixture(archive); + let checkpoints = 0; + let typedFallback = false; + const registry = { + captureFuncrefTablePatch: () => typedFallback ? null : patch(), + captureTableState: () => { + checkpoints++; + return 512; + }, + restoreTableState: () => {}, + applyFuncrefTablePatch: () => {}, + } as unknown as ForkActivationRegistry; + const owner = __testCreateProcessTableReplicationOwner({ + generationAddress: 64, + registry, + dlopen, + newArena: () => arenaFixture(512), + materializeModules: () => {}, + restoreSnapshots: true, + label: "patch writer", + }) as TestTableReplicationOwner; + + for (let index = 0; index < 256; index++) { + owner.beginMutation(); + owner.commit(0, 1, 0, 1); + } + expect(archive.read().tablePatches).toHaveLength(256); + expect(checkpoints).toBe(0); + + owner.beginMutation(); + owner.commit(0, 1, 0, 1); + expect(checkpoints).toBe(1); + expect(archive.read()).toMatchObject({ + tableStateRoot: 512, + tablePatches: [], + }); + + typedFallback = true; + owner.beginMutation(); + owner.commit(0, 1, 0, 1); + expect(checkpoints).toBe(2); + expect(archive.read().tablePatches).toEqual([]); + }); + + it("skips only the fork child's copied baseline and applies later patches", () => { + const { archive } = archiveFixture(); + archive.publishTablePatch(patch()); + const applied: number[] = []; + const registry = { + applyFuncrefTablePatch: (value: DylinkForkTablePatch) => { + applied.push(value.generation!); + }, + restoreTableState: () => { + throw new Error("fork child must use its normal KFMS capture"); + }, + } as unknown as ForkActivationRegistry; + const child = __testCreateProcessTableReplicationOwner({ + generationAddress: 64, + registry, + dlopen: dlopenFixture(archive), + newArena: () => arenaFixture(512), + materializeModules: () => {}, + restoreSnapshots: false, + label: "fork child patch reader", + }) as TestTableReplicationOwner; + + child.reconcileNow(); + expect(applied).toEqual([]); + archive.publishTablePatch(patch()); + child.reconcileNow(); + expect(applied).toEqual([3]); + }); +}); diff --git a/host/test/signal-accept-livelock.test.ts b/host/test/signal-accept-livelock.test.ts index 1dd35fce10..b82c5771ab 100644 --- a/host/test/signal-accept-livelock.test.ts +++ b/host/test/signal-accept-livelock.test.ts @@ -23,7 +23,10 @@ import { CH_ARGS, CH_ARG_SIZE, CH_ERRNO, + CH_REQUEST_FLAGS, + CH_REQUEST_FLAG_DEFER_SIGNAL_DELIVERY, CH_RETURN, + CH_SIG_SIGNUM, CH_SYSCALL, } from "../src/generated/abi"; @@ -220,6 +223,38 @@ describe("signal delivery to a process blocked in accept()", () => { expect(dequeueSignal).toHaveBeenCalledWith(pid, pid, expect.any(Number)); }); + it("hands a deferred signal from a JavaScript completion to the next guest checkpoint", () => { + const worker = createWorkerHarness(); + const pid = 48; + const channel = createChannel(pid, 0); + worker.channelTids.set(`${pid}:${channel.channelOffset}`, pid); + const channelView = new DataView(channel.memory.buffer); + channelView.setUint32( + CH_REQUEST_FLAGS, + CH_REQUEST_FLAG_DEFER_SIGNAL_DELIVERY, + true, + ); + channelView.setUint32(CH_SIG_SIGNUM, 0, true); + const dequeueSignal = vi.fn(() => 10); + worker.kernelInstance.exports.kernel_dequeue_signal = dequeueSignal; + + expect(worker.dequeueSignalForDelivery(channel)).toBe(0); + expect(dequeueSignal).not.toHaveBeenCalled(); + expect(channelView.getUint32(CH_SIG_SIGNUM, true)).toBe(0); + + // Model libc's ordinary post-import checkpoint. Only that guest-owned + // completion may consume the signal into the trampoline's channel record. + channelView.setUint32(CH_REQUEST_FLAGS, 0, true); + new DataView(worker.kernelMemory.buffer).setUint32( + worker.scratchOffset + CH_SIG_SIGNUM, + 10, + true, + ); + expect(worker.dequeueSignalForDelivery(channel)).toBe(10); + expect(dequeueSignal).toHaveBeenCalledOnce(); + expect(channelView.getUint32(CH_SIG_SIGNUM, true)).toBe(10); + }); + it("fails closed when Rust rejects an exact signal dequeue task", () => { const worker = createWorkerHarness(); const pid = 48; diff --git a/host/test/sjlj-noexcept-boundary.test.ts b/host/test/sjlj-noexcept-boundary.test.ts index 53fe0be083..475b166054 100644 --- a/host/test/sjlj-noexcept-boundary.test.ts +++ b/host/test/sjlj-noexcept-boundary.test.ts @@ -13,7 +13,7 @@ const rawWasm64Fixture = join( repoRoot, "local-binaries/test-fixtures/wasm64/sjlj_noexcept_boundary.raw.wasm", ); -const instrumentedFixture = resolveBinary( +const instrumentedForkFixture = resolveBinary( "programs/sjlj_noexcept_boundary.wasm", ); const sigchldFixture = resolveBinary("programs/sigchld_sjlj.wasm"); @@ -22,20 +22,37 @@ const TERMINATED_BY_SIGABRT = 128 + 6; describe("LLVM Wasm SjLj across a noexcept boundary", () => { it("keeps the raw wasm32 control independent of fork instrumentation", () => { const rawModule = new WebAssembly.Module(readFileSync(rawWasm32Fixture)); - const instrumentedModule = new WebAssembly.Module( - readFileSync(instrumentedFixture), - ); const exportNames = (module: WebAssembly.Module) => WebAssembly.Module.exports(module).map(({ name }) => name); expect(exportNames(rawModule)).not.toContain("wpk_fork_state"); - expect(exportNames(instrumentedModule)).toContain("wpk_fork_state"); + }); + + it("admits the fork-bearing compiler output through ABI 43 instrumentation", () => { + const module = new WebAssembly.Module( + readFileSync(instrumentedForkFixture), + ); + const exportNames = WebAssembly.Module.exports(module) + .map(({ name }) => name); + + expect(exportNames).toContain("wpk_fork_state"); + }); + + it("forks through the instrumented compiler-EH artifact", async () => { + const result = await runCentralizedProgram({ + programPath: instrumentedForkFixture, + argv: ["sjlj_noexcept_boundary", "--fork-instrumentation-anchor"], + timeout: 10_000, + useDefaultRootfs: false, + }); + + expect(result.exitCode).toBe(0); }); it.each([ ["raw wasm32", rawWasm32Fixture], - ["fork-instrumented wasm32", instrumentedFixture], ["raw wasm64", rawWasm64Fixture], + ["instrumented wasm32", instrumentedForkFixture], ])("documents the pinned LLVM failure in the %s control", async (_, path) => { const result = await runCentralizedProgram({ programPath: path, @@ -50,9 +67,12 @@ describe("LLVM Wasm SjLj across a noexcept boundary", () => { expect(result.stdout).not.toContain("LANDING: siglongjmp resumed"); }); - it("resumes the same SjLj tag when it does not cross noexcept", async () => { + it.each([ + ["raw wasm32", rawWasm32Fixture], + ["instrumented wasm32", instrumentedForkFixture], + ])("resumes the same SjLj tag in the %s permissive boundary", async (_, path) => { const result = await runCentralizedProgram({ - programPath: instrumentedFixture, + programPath: path, argv: ["sjlj_noexcept_boundary", "--permissive"], timeout: 10_000, useDefaultRootfs: false, diff --git a/host/test/wasm-binary-parse.test.ts b/host/test/wasm-binary-parse.test.ts index d625dcce1d..9d14b2c90a 100644 --- a/host/test/wasm-binary-parse.test.ts +++ b/host/test/wasm-binary-parse.test.ts @@ -15,6 +15,30 @@ import { homedir } from "node:os"; import { join } from "node:path"; import { ABI_VERSION, + WPK_FORK_CAPABILITIES_SECTION, + WPK_FORK_CAPABILITIES_VERSION, + WPK_FORK_CAP_ACTIVATION_STATE_SAFE, + WPK_FORK_CAP_KNOWN_MASK, + WPK_FORK_EXCEPTION_CODEC_HEADER_SIZE, + WPK_FORK_EXCEPTION_CODEC_SECTION, + WPK_FORK_EXCEPTION_CODEC_TAG_RECORD_SIZE, + WPK_FORK_EXCEPTION_CODEC_VERSION, + WPK_FORK_EXCEPTION_CODEC_IMPORT_MODULE, + WPK_FORK_EXCEPTION_IMPORT_ACTIVATION, + WPK_FORK_GLOBAL_CATALOG_EXPORT_PREFIX, + WPK_FORK_IMPORTED_GLOBALS_FLAG_MUTABLE, + WPK_FORK_IMPORTED_GLOBALS_FLAG_SHARED, + WPK_FORK_IMPORTED_GLOBALS_HEADER_SIZE, + WPK_FORK_IMPORTED_GLOBALS_MAGIC, + WPK_FORK_IMPORTED_GLOBALS_RECORD_HEADER_SIZE, + WPK_FORK_IMPORTED_GLOBALS_SECTION, + WPK_FORK_IMPORTED_GLOBALS_VERSION, + WPK_FORK_IMPORTED_TABLES_FLAG_TABLE64, + WPK_FORK_IMPORTED_TABLES_HEADER_SIZE, + WPK_FORK_IMPORTED_TABLES_MAGIC, + WPK_FORK_IMPORTED_TABLES_RECORD_HEADER_SIZE, + WPK_FORK_IMPORTED_TABLES_SECTION, + WPK_FORK_IMPORTED_TABLES_VERSION, WPK_FORK_LINKED_FRAME_DESCRIPTOR_SIZE, WPK_FORK_LINKED_FRAME_FORMAT_MAGIC, WPK_FORK_LINKED_FRAME_FORMAT_SECTION, @@ -22,8 +46,27 @@ import { WPK_FORK_LINKED_FRAME_POINTER_WIDTHS, WPK_FORK_LINKED_FRAME_RECORD_ALIGNMENT, WPK_FORK_LINKED_FRAME_REQUIRED_FLAGS, + WPK_FORK_MODULE_STATE_ARENA_VERSION, + WPK_FORK_MODULE_STATE_DESCRIPTOR_SIZE, + WPK_FORK_MODULE_STATE_FORMAT_MAGIC, + WPK_FORK_MODULE_STATE_FORMAT_SECTION, + WPK_FORK_MODULE_STATE_FORMAT_VERSION, + WPK_FORK_MODULE_STATE_GLOBAL_TYPE_EXTERNREF, + WPK_FORK_MODULE_STATE_GLOBAL_TYPE_FUNCREF, + WPK_FORK_MODULE_STATE_GLOBAL_TYPE_I32, + WPK_FORK_MODULE_STATE_RECORD_ALIGNMENT, + WPK_FORK_MODULE_STATE_RECORD_VERSION, + WPK_FORK_MODULE_STATE_REQUIRED_FLAGS, + WPK_FORK_MODULE_STATE_ROOT_POINTER_WORD_OFFSET, WPK_FORK_REQUIRED_EXPORTS, WPK_FORK_REQUIRED_IMPORTS, + WPK_FORK_REQUIRED_TABLE_IMPORTS, + WPK_FORK_STATIC_ROOT_CATALOG_EXPORT, + WPK_FORK_STATIC_ROOT_CATALOG_HEADER_SIZE, + WPK_FORK_STATIC_ROOT_CATALOG_MAGIC, + WPK_FORK_STATIC_ROOT_CATALOG_SECTION, + WPK_FORK_STATIC_ROOT_CATALOG_VERSION, + WPK_FORK_TABLE_CATALOG_EXPORT_PREFIX, } from "../src/generated/abi"; import { describeWasmArtifactPolicyFailures, @@ -38,6 +81,13 @@ import { wasmHasCompleteForkInstrumentation, wasmImportsKernelFork, } from "../src/constants"; +import { + FORK_UNWIND_TAG_IMPORT_MODULE, + FORK_UNWIND_TAG_IMPORT_NAME, + FORK_UNWIND_TRANSPORT_PAYLOAD_ARITY, + FORK_UNWIND_TRANSPORT_SECTION, + FORK_UNWIND_TRANSPORT_VERSION, +} from "../src/fork-unwind-transport"; import { tryResolveBinary } from "../src/binary-resolver"; // --------------------------------------------------------------------------- @@ -92,20 +142,40 @@ function nameBytes(s: string): number[] { return [...uleb128(enc.length), ...enc]; } -interface GlobalImport { module: string; name: string; valType: 0x7F | 0x7E; mut: 0 | 1; } +interface GlobalImport { + module: string; + name: string; + valType: number; + mut: 0 | 1; + shared?: boolean; +} interface FuncImport { module: string; name: string; typeIdx: number; } +interface TableImport { + module: string; + name: string; + elementType: number; + table64: boolean; + minimum: number; + maximum: number | null; +} +type DefinedTable = Omit; +interface TagImport { module: string; name: string; typeIdx: number; } interface DefinedGlobal { valType: 0x7F | 0x7E; mut: 0 | 1; init: number[]; } interface ExportEntry { name: string; kind: 0 | 1 | 2 | 3; index: number; } interface FuncBody { locals: number[]; instructions: number[]; } function buildWasm(opts: { funcImports?: FuncImport[]; + tagImports?: TagImport[]; + tableImports?: TableImport[]; + tables?: DefinedTable[]; globalImports?: GlobalImport[]; types?: { params: number[]; results: number[] }[]; funcTypes?: number[]; // type index per defined function memoryPointerWidths?: Array<4 | 8>; globals?: DefinedGlobal[]; exports?: ExportEntry[]; + startFunctionIndex?: number; funcBodies?: FuncBody[]; customSections?: { name: string; data?: number[] }[]; }): ArrayBuffer { @@ -134,14 +204,45 @@ function buildWasm(opts: { // Import section (id=2) const fImps = opts.funcImports ?? []; + const tImps = opts.tagImports ?? []; + const tableImps = opts.tableImports ?? []; const gImps = opts.globalImports ?? []; - if (fImps.length + gImps.length > 0) { - const payload: number[] = [...uleb128(fImps.length + gImps.length)]; + if (fImps.length + tImps.length + tableImps.length + gImps.length > 0) { + const payload: number[] = [ + ...uleb128(fImps.length + tImps.length + tableImps.length + gImps.length), + ]; for (const fi of fImps) { payload.push(...nameBytes(fi.module), ...nameBytes(fi.name), 0x00, ...uleb128(fi.typeIdx)); } + for (const ti of tImps) { + payload.push( + ...nameBytes(ti.module), + ...nameBytes(ti.name), + 0x04, + 0x00, + ...uleb128(ti.typeIdx), + ); + } + for (const table of tableImps) { + const flags = (table.maximum === null ? 0 : 1) | (table.table64 ? 4 : 0); + payload.push( + ...nameBytes(table.module), + ...nameBytes(table.name), + 0x01, + table.elementType, + ...uleb128(flags), + ...uleb128(table.minimum), + ...(table.maximum === null ? [] : uleb128(table.maximum)), + ); + } for (const gi of gImps) { - payload.push(...nameBytes(gi.module), ...nameBytes(gi.name), 0x03, gi.valType, gi.mut); + payload.push( + ...nameBytes(gi.module), + ...nameBytes(gi.name), + 0x03, + gi.valType, + gi.mut | (gi.shared ? 0b10 : 0), + ); } bytes.push(...section(2, payload)); } @@ -154,6 +255,21 @@ function buildWasm(opts: { bytes.push(...section(3, payload)); } + const tables = opts.tables ?? []; + if (tables.length > 0) { + const payload: number[] = [...uleb128(tables.length)]; + for (const table of tables) { + const flags = (table.maximum === null ? 0 : 1) | (table.table64 ? 4 : 0); + payload.push( + table.elementType, + ...uleb128(flags), + ...uleb128(table.minimum), + ...(table.maximum === null ? [] : uleb128(table.maximum)), + ); + } + bytes.push(...section(4, payload)); + } + const memoryPointerWidths = opts.memoryPointerWidths ?? []; if (memoryPointerWidths.length > 0) { const payload = [...uleb128(memoryPointerWidths.length)]; @@ -183,6 +299,10 @@ function buildWasm(opts: { bytes.push(...section(7, payload)); } + if (opts.startFunctionIndex !== undefined) { + bytes.push(...section(8, uleb128(opts.startFunctionIndex))); + } + // Code section (id=10) const bodies = opts.funcBodies ?? []; if (bodies.length > 0) { @@ -219,42 +339,385 @@ function linkedFrameDescriptor(pointerWidth: 4 | 8): number[] { return [...bytes]; } +function moduleStateDescriptor(pointerWidth: 4 | 8): number[] { + const bytes = new Uint8Array(WPK_FORK_MODULE_STATE_DESCRIPTOR_SIZE); + bytes.set(WPK_FORK_MODULE_STATE_FORMAT_MAGIC, 0); + const view = new DataView(bytes.buffer); + view.setUint16(4, WPK_FORK_MODULE_STATE_FORMAT_VERSION, true); + view.setUint16(6, WPK_FORK_MODULE_STATE_DESCRIPTOR_SIZE, true); + view.setUint8(8, pointerWidth); + view.setUint8(9, WPK_FORK_MODULE_STATE_RECORD_ALIGNMENT); + view.setUint16(10, WPK_FORK_MODULE_STATE_REQUIRED_FLAGS, true); + view.setUint16(12, WPK_FORK_MODULE_STATE_ARENA_VERSION, true); + view.setUint16(14, WPK_FORK_MODULE_STATE_RECORD_VERSION, true); + view.setUint32(16, WPK_FORK_MODULE_STATE_ROOT_POINTER_WORD_OFFSET, true); + return [...bytes]; +} + +function exceptionCodecDescriptor( + tags: Array<{ + ordinal: number; + layoutId: number; + scalarByteLength: number; + referenceCount: number; + }> = [], +): number[] { + const bytes = new Uint8Array( + WPK_FORK_EXCEPTION_CODEC_HEADER_SIZE + + tags.length * WPK_FORK_EXCEPTION_CODEC_TAG_RECORD_SIZE, + ); + const view = new DataView(bytes.buffer); + view.setUint8(0, WPK_FORK_EXCEPTION_CODEC_VERSION); + view.setUint32(4, tags.length, true); + for (let index = 0; index < tags.length; index++) { + const tag = tags[index]!; + const offset = WPK_FORK_EXCEPTION_CODEC_HEADER_SIZE + + index * WPK_FORK_EXCEPTION_CODEC_TAG_RECORD_SIZE; + view.setUint32(offset, tag.ordinal, true); + view.setUint32(offset + 4, tag.layoutId, true); + view.setUint32(offset + 8, tag.scalarByteLength, true); + view.setUint32(offset + 12, tag.referenceCount, true); + } + return [...bytes]; +} + +function emptyImportedGlobalsDescriptor(): number[] { + return importedGlobalsDescriptor([]); +} + +const COMPLETE_FORK_SOURCE_TABLE_IMPORT_ORDINAL = + 1 + WPK_FORK_REQUIRED_IMPORTS.length + 1; +const COMPLETE_FORK_SOURCE_GLOBAL_IMPORT_ORDINAL = + COMPLETE_FORK_SOURCE_TABLE_IMPORT_ORDINAL + + WPK_FORK_REQUIRED_TABLE_IMPORTS.length; + +function importedGlobalsDescriptor( + records: Array<{ + owner: number; + typeCode: number; + flags: number; + module: string; + name: string; + importOrdinal?: number; + }>, +): number[] { + const encoded = records.map((record, index) => ({ + ...record, + importOrdinal: + record.importOrdinal ?? COMPLETE_FORK_SOURCE_GLOBAL_IMPORT_ORDINAL + index, + moduleBytes: new TextEncoder().encode(record.module), + nameBytes: new TextEncoder().encode(record.name), + })); + const byteLength = WPK_FORK_IMPORTED_GLOBALS_HEADER_SIZE + + encoded.reduce( + (total, record) => + total + + WPK_FORK_IMPORTED_GLOBALS_RECORD_HEADER_SIZE + + record.moduleBytes.length + + record.nameBytes.length, + 0, + ); + const bytes = new Uint8Array(byteLength); + bytes.set(WPK_FORK_IMPORTED_GLOBALS_MAGIC, 0); + const view = new DataView(bytes.buffer); + view.setUint16(4, WPK_FORK_IMPORTED_GLOBALS_VERSION, true); + view.setUint16(6, WPK_FORK_IMPORTED_GLOBALS_HEADER_SIZE, true); + view.setUint32(8, encoded.length, true); + let offset = WPK_FORK_IMPORTED_GLOBALS_HEADER_SIZE; + for (const record of encoded) { + const recordSize = WPK_FORK_IMPORTED_GLOBALS_RECORD_HEADER_SIZE + + record.moduleBytes.length + + record.nameBytes.length; + view.setUint32(offset, recordSize, true); + view.setUint32(offset + 4, record.owner, true); + view.setUint8(offset + 8, record.typeCode); + view.setUint8(offset + 9, record.flags); + view.setUint32(offset + 12, record.moduleBytes.length, true); + view.setUint32(offset + 16, record.nameBytes.length, true); + view.setUint32(offset + 20, record.importOrdinal, true); + bytes.set( + record.moduleBytes, + offset + WPK_FORK_IMPORTED_GLOBALS_RECORD_HEADER_SIZE, + ); + bytes.set( + record.nameBytes, + offset + + WPK_FORK_IMPORTED_GLOBALS_RECORD_HEADER_SIZE + + record.moduleBytes.length, + ); + offset += recordSize; + } + return [...bytes]; +} + +function importedTablesDescriptor( + records: Array<{ + owner: number; + typeCode: number; + flags: number; + module: string; + name: string; + importOrdinal?: number; + }>, +): number[] { + const encoded = records.map((record, index) => ({ + ...record, + importOrdinal: + record.importOrdinal ?? COMPLETE_FORK_SOURCE_TABLE_IMPORT_ORDINAL + index, + moduleBytes: new TextEncoder().encode(record.module), + nameBytes: new TextEncoder().encode(record.name), + })); + const byteLength = WPK_FORK_IMPORTED_TABLES_HEADER_SIZE + + encoded.reduce( + (total, record) => + total + + WPK_FORK_IMPORTED_TABLES_RECORD_HEADER_SIZE + + record.moduleBytes.length + + record.nameBytes.length, + 0, + ); + const bytes = new Uint8Array(byteLength); + bytes.set(WPK_FORK_IMPORTED_TABLES_MAGIC, 0); + const view = new DataView(bytes.buffer); + view.setUint16(4, WPK_FORK_IMPORTED_TABLES_VERSION, true); + view.setUint16(6, WPK_FORK_IMPORTED_TABLES_HEADER_SIZE, true); + view.setUint32(8, encoded.length, true); + let offset = WPK_FORK_IMPORTED_TABLES_HEADER_SIZE; + for (const record of encoded) { + const recordSize = WPK_FORK_IMPORTED_TABLES_RECORD_HEADER_SIZE + + record.moduleBytes.length + + record.nameBytes.length; + view.setUint32(offset, recordSize, true); + view.setUint32(offset + 4, record.owner, true); + view.setUint8(offset + 8, record.typeCode); + view.setUint8(offset + 9, record.flags); + view.setUint32(offset + 12, record.moduleBytes.length, true); + view.setUint32(offset + 16, record.nameBytes.length, true); + view.setUint32(offset + 20, record.importOrdinal, true); + bytes.set( + record.moduleBytes, + offset + WPK_FORK_IMPORTED_TABLES_RECORD_HEADER_SIZE, + ); + bytes.set( + record.nameBytes, + offset + + WPK_FORK_IMPORTED_TABLES_RECORD_HEADER_SIZE + + record.moduleBytes.length, + ); + offset += recordSize; + } + return [...bytes]; +} + +function staticRootCatalogDescriptor(count = 0): number[] { + const bytes = new Uint8Array(WPK_FORK_STATIC_ROOT_CATALOG_HEADER_SIZE); + bytes.set(WPK_FORK_STATIC_ROOT_CATALOG_MAGIC, 0); + const view = new DataView(bytes.buffer); + view.setUint16(4, WPK_FORK_STATIC_ROOT_CATALOG_VERSION, true); + view.setUint16(6, WPK_FORK_STATIC_ROOT_CATALOG_HEADER_SIZE, true); + view.setUint32(8, count, true); + return [...bytes]; +} + +type ForkArtifactValueType = + | "ptr" + | "i32" + | "i64" + | "anyref" + | "exnref" + | "externref" + | "funcref"; + +function wasmValueType( + value: ForkArtifactValueType, + pointerWidth: 4 | 8, +): number { + switch (value) { + case "ptr": + return pointerWidth === 8 ? I64 : I32; + case "i32": + return I32; + case "i64": + return I64; + case "anyref": + return 0x6e; + case "exnref": + return 0x69; + case "externref": + return 0x6f; + case "funcref": + return 0x70; + } +} + function completeForkWasm(options: { pointerWidth?: 4 | 8; memoryPointerWidth?: 4 | 8; exportPointerWidth?: 4 | 8; + capabilityFlags?: number | null; + capabilityPayloads?: number[][]; + unwindTransportPayloads?: number[][]; + moduleStatePayloads?: number[][]; + exceptionCodecPayloads?: number[][]; + importedGlobalsPayloads?: number[][]; + importedTablesPayloads?: number[][]; + includeActivationImport?: boolean; + sourceGlobalImports?: GlobalImport[]; + sourceTableImports?: TableImport[]; + includeGlobalCatalog?: boolean; + includeTableCatalog?: boolean; + includeResumeTable?: boolean; + staticRootPayloads?: number[][]; + includeStaticRootTable?: boolean; + staticRootCount?: number; + includeUnwindTag?: boolean; + includeLegacyDlopenImport?: boolean; + includeNativeStart?: boolean; + abiVersion?: number; + includeAbiMarker?: boolean; } = {}): ArrayBuffer { const pointerWidth = options.pointerWidth ?? 4; - const pointerType = pointerWidth === 8 ? I64 : I32; - const exportPointerType = (options.exportPointerWidth ?? pointerWidth) === 8 ? I64 : I32; - const types = [ - { params: [], results: [I32] }, - { params: [exportPointerType], results: [] }, - { params: [], results: [] }, - { params: [pointerType], results: [pointerType] }, - { params: [pointerType], results: [] }, - ]; + const exportPointerWidth = options.exportPointerWidth ?? pointerWidth; + const types: Array<{ params: number[]; results: number[] }> = []; + const typeIndices = new Map(); + const internType = ( + params: readonly ForkArtifactValueType[], + results: readonly ForkArtifactValueType[], + width: 4 | 8, + ): number => { + const type = { + params: params.map((value) => wasmValueType(value, width)), + results: results.map((value) => wasmValueType(value, width)), + }; + const key = `${type.params.join(",")}=>${type.results.join(",")}`; + const existing = typeIndices.get(key); + if (existing !== undefined) return existing; + const index = types.length; + types.push(type); + typeIndices.set(key, index); + return index; + }; + const kernelForkType = internType([], ["i32"], pointerWidth); + const emptyType = internType([], [], pointerWidth); const funcImports: FuncImport[] = [ - { module: "kernel", name: "kernel_fork", typeIdx: 0 }, + { module: "kernel", name: "kernel_fork", typeIdx: kernelForkType }, + ...(options.includeLegacyDlopenImport === true + ? [{ + module: "env", + name: "__wasm_dlopen", + typeIdx: internType( + ["ptr", "i32", "ptr", "i32", "i32"], + ["i32"], + pointerWidth, + ), + }] + : []), ...WPK_FORK_REQUIRED_IMPORTS.map((requirement) => ({ module: requirement.module, name: requirement.name, - typeIdx: requirement.results.length === 1 ? 3 : 4, + typeIdx: internType(requirement.params, requirement.results, pointerWidth), })), ]; - const forkTypeIndices = WPK_FORK_REQUIRED_EXPORTS.map((requirement) => { - if (requirement.results.length === 1) return 0; - return requirement.params.length === 1 ? 1 : 2; - }); + const forkTypeIndices = WPK_FORK_REQUIRED_EXPORTS.map((requirement) => + internType(requirement.params, requirement.results, exportPointerWidth) + ); + const abiType = internType([], ["i32"], pointerWidth); const firstDefinedFunction = funcImports.length; + const capabilityFlags = + options.capabilityFlags === undefined + ? WPK_FORK_CAP_ACTIVATION_STATE_SAFE + : options.capabilityFlags; + const capabilityPayloads = options.capabilityPayloads ?? + (capabilityFlags === null + ? [] + : [[WPK_FORK_CAPABILITIES_VERSION, capabilityFlags]]); + const sourceGlobalImports = options.sourceGlobalImports ?? []; + const sourceTableImports = options.sourceTableImports ?? []; + const activationGlobalImports: GlobalImport[] = + options.includeActivationImport === false ? [] : [{ + module: WPK_FORK_EXCEPTION_CODEC_IMPORT_MODULE, + name: WPK_FORK_EXCEPTION_IMPORT_ACTIVATION, + valType: I32, + mut: 0, + }]; + const requiredTableImports = options.includeResumeTable === false + ? [] + : WPK_FORK_REQUIRED_TABLE_IMPORTS.map((requirement) => ({ + module: requirement.module, + name: requirement.name, + elementType: wasmValueType(requirement.element, pointerWidth), + table64: requirement.table64, + minimum: requirement.minimum, + maximum: requirement.maximum, + })); + const staticRootCount = options.staticRootCount ?? 0; + const nativeStartLocalIndex = WPK_FORK_REQUIRED_EXPORTS.findIndex( + (requirement) => + requirement.params.length === 0 && requirement.results.length === 0, + ); + if (options.includeNativeStart === true && nativeStartLocalIndex < 0) { + throw new Error("fork fixture has no () -> () function for its native start"); + } return buildWasm({ - customSections: [{ - name: WPK_FORK_LINKED_FRAME_FORMAT_SECTION, - data: linkedFrameDescriptor(pointerWidth), - }], + customSections: [ + ...capabilityPayloads.map((data) => ({ + name: WPK_FORK_CAPABILITIES_SECTION, + data, + })), + { + name: WPK_FORK_LINKED_FRAME_FORMAT_SECTION, + data: linkedFrameDescriptor(pointerWidth), + }, + ...(options.exceptionCodecPayloads ?? + [exceptionCodecDescriptor()]).map((data) => ({ + name: WPK_FORK_EXCEPTION_CODEC_SECTION, + data, + })), + ...(options.importedGlobalsPayloads ?? + [emptyImportedGlobalsDescriptor()]).map((data) => ({ + name: WPK_FORK_IMPORTED_GLOBALS_SECTION, + data, + })), + ...(options.importedTablesPayloads ?? + [importedTablesDescriptor([])]).map((data) => ({ + name: WPK_FORK_IMPORTED_TABLES_SECTION, + data, + })), + ...(options.moduleStatePayloads ?? [moduleStateDescriptor(pointerWidth)]).map( + (data) => ({ + name: WPK_FORK_MODULE_STATE_FORMAT_SECTION, + data, + }), + ), + ...(options.staticRootPayloads ?? + [staticRootCatalogDescriptor(staticRootCount)]).map((data) => ({ + name: WPK_FORK_STATIC_ROOT_CATALOG_SECTION, + data, + })), + ...(options.unwindTransportPayloads ?? [[ + FORK_UNWIND_TRANSPORT_VERSION, + FORK_UNWIND_TRANSPORT_PAYLOAD_ARITY, + ]]).map((data) => ({ + name: FORK_UNWIND_TRANSPORT_SECTION, + data, + })), + ], types, funcImports, - funcTypes: [...forkTypeIndices, 0], + globalImports: [...sourceGlobalImports, ...activationGlobalImports], + tableImports: [...sourceTableImports, ...requiredTableImports], + tables: options.includeStaticRootTable === false ? [] : [{ + elementType: wasmValueType("anyref", pointerWidth), + table64: false, + minimum: staticRootCount, + maximum: staticRootCount, + }], + tagImports: options.includeUnwindTag === false ? [] : [{ + module: FORK_UNWIND_TAG_IMPORT_MODULE, + name: FORK_UNWIND_TAG_IMPORT_NAME, + typeIdx: emptyType, + }], + funcTypes: [...forkTypeIndices, abiType], memoryPointerWidths: [options.memoryPointerWidth ?? pointerWidth], exports: [ ...WPK_FORK_REQUIRED_EXPORTS.map((requirement, index) => ({ @@ -262,10 +725,48 @@ function completeForkWasm(options: { kind: 0 as const, index: firstDefinedFunction + index, })), - { + ...(options.includeAbiMarker === false ? [] : [{ name: "__abi_version", - kind: 0, + kind: 0 as const, index: firstDefinedFunction + forkTypeIndices.length, + }]), + ...(options.includeGlobalCatalog === false + ? [] + : sourceGlobalImports.map((_global, index) => ({ + name: `${WPK_FORK_GLOBAL_CATALOG_EXPORT_PREFIX}${index + 1}`, + kind: 3 as const, + index, + }))), + ...(options.includeTableCatalog === false + ? [] + : sourceTableImports.map((_table, index) => ({ + name: `${WPK_FORK_TABLE_CATALOG_EXPORT_PREFIX}${index + 1}`, + kind: 1 as const, + index, + }))), + ...(options.includeStaticRootTable === false ? [] : [{ + name: WPK_FORK_STATIC_ROOT_CATALOG_EXPORT, + kind: 1 as const, + index: sourceTableImports.length + requiredTableImports.length, + }]), + ], + ...(options.includeNativeStart === true + ? { startFunctionIndex: firstDefinedFunction + nativeStartLocalIndex } + : {}), + funcBodies: [ + ...WPK_FORK_REQUIRED_EXPORTS.map((requirement) => ({ + locals: [0], + instructions: requirement.results.length === 0 + ? [] + : requirement.results[0] === "i32" + ? [0x41, 0] + : requirement.results[0] === "i64" + ? [0x42, 0] + : [0x00], + })), + { + locals: [0], + instructions: [0x41, ...sleb128_i32(options.abiVersion ?? ABI_VERSION)], }, ], }); @@ -505,30 +1006,587 @@ describe("wasm artifact policy helpers", () => { expect(wasmHasCompleteForkInstrumentation(wasm)).toBe(false); const failures = describeWasmArtifactPolicyFailures(wasm, { expectedAbi: 12 }); - expect(failures).toContain( - "incomplete wasm-fork-instrument exports; missing wpk_fork_abort_begin, wpk_fork_abort_end, wpk_fork_rewind_begin, wpk_fork_rewind_end, wpk_fork_unwind_begin, wpk_fork_unwind_end", - ); + expect(failures.some((failure) => + failure.startsWith("incomplete wasm-fork-instrument exports; missing ") + && failure.includes("__wpk_fork_ref_decode_exnref") + && failure.includes("wpk_fork_unwind_end") + )).toBe(true); expect(failures).toContain( `missing required ${WPK_FORK_LINKED_FRAME_FORMAT_SECTION} descriptor`, ); - expect(failures).toContain( - "incomplete ABI 42 linked-frame imports; missing env.__wpk_fork_frame_commit, env.__wpk_fork_frame_next, env.__wpk_fork_frame_reserve", - ); + expect(failures.some((failure) => + failure.startsWith("incomplete ABI 43 fork-runtime imports; missing ") + && failure.includes("env.__wpk_fork_frame_commit") + && failure.includes("env.__wpk_fork_ref_exn_define") + )).toBe(true); }); - it("accepts the complete ABI 42 contract for wasm32 and wasm64", () => { + it("accepts the complete ABI 43 contract for wasm32 and wasm64", () => { for (const pointerWidth of [4, 8] as const) { const wasm = completeForkWasm({ pointerWidth }); expect(wasmHasCompleteForkInstrumentation(wasm)).toBe(true); - expect(describeWasmArtifactPolicyFailures(wasm, { expectedAbi: 12 })).toEqual([]); + expect(describeWasmArtifactPolicyFailures(wasm, { expectedAbi: ABI_VERSION })).toEqual([]); } }); + it("requires the exact private exception transport before accepting ABI 43 safety", () => { + const cases: Array<{ + label: string; + options: Parameters[0]; + diagnostic: string; + }> = [ + { + label: "missing tag", + options: { includeUnwindTag: false }, + diagnostic: "missing required private fork-unwind tag import", + }, + { + label: "missing descriptor", + options: { unwindTransportPayloads: [] }, + diagnostic: `missing required ${FORK_UNWIND_TRANSPORT_SECTION} descriptor`, + }, + { + label: "wrong descriptor", + options: { unwindTransportPayloads: [[FORK_UNWIND_TRANSPORT_VERSION, 1]] }, + diagnostic: `${FORK_UNWIND_TRANSPORT_SECTION} must be`, + }, + { + label: "duplicate descriptor", + options: { + unwindTransportPayloads: [ + [FORK_UNWIND_TRANSPORT_VERSION, FORK_UNWIND_TRANSPORT_PAYLOAD_ARITY], + [FORK_UNWIND_TRANSPORT_VERSION, FORK_UNWIND_TRANSPORT_PAYLOAD_ARITY], + ], + }, + diagnostic: "descriptors, expected exactly one", + }, + ]; + + for (const { label, options, diagnostic } of cases) { + const wasm = completeForkWasm(options); + expect(wasmHasCompleteForkInstrumentation(wasm), label).toBe(false); + expect(describeWasmArtifactPolicyFailures(wasm).join("\n"), label) + .toContain(diagnostic); + } + }); + + it("requires module-state ownership metadata in the same pointer-width epoch", () => { + const missing = completeForkWasm({ moduleStatePayloads: [] }); + expect(describeWasmArtifactPolicyFailures(missing).join("\n")) + .toContain(`missing required ${WPK_FORK_MODULE_STATE_FORMAT_SECTION} descriptor`); + + const mismatched = completeForkWasm({ + pointerWidth: 8, + moduleStatePayloads: [moduleStateDescriptor(4)], + }); + expect(describeWasmArtifactPolicyFailures(mismatched).join("\n")) + .toContain("pointer width 4 does not match linked frames 8"); + }); + + it("accepts shape-neutral exact-tag exception codec catalogs", () => { + const descriptor = exceptionCodecDescriptor([ + { + ordinal: 0, + layoutId: 17, + scalarByteLength: 40, + referenceCount: 0, + }, + { + ordinal: 1, + layoutId: 29, + scalarByteLength: 24, + referenceCount: 7, + }, + ]); + const wasm = completeForkWasm({ exceptionCodecPayloads: [descriptor] }); + + expect(wasmHasCompleteForkInstrumentation(wasm)).toBe(true); + expect(describeWasmArtifactPolicyFailures(wasm, { + expectedAbi: ABI_VERSION, + })).toEqual([]); + }); + + it("rejects malformed exception reconstruction metadata, not exception shapes", () => { + const noncanonical = exceptionCodecDescriptor([ + { + ordinal: 1, + layoutId: 4, + scalarByteLength: 0, + referenceCount: 0, + }, + ]); + const duplicateLayout = exceptionCodecDescriptor([ + { + ordinal: 0, + layoutId: 9, + scalarByteLength: 0, + referenceCount: 0, + }, + { + ordinal: 1, + layoutId: 9, + scalarByteLength: 0, + referenceCount: 2, + }, + ]); + const reserved = exceptionCodecDescriptor(); + reserved[1] = 1; + const cases = [ + { + label: "missing", + payloads: [] as number[][], + diagnostic: `missing required ${WPK_FORK_EXCEPTION_CODEC_SECTION} descriptor`, + }, + { + label: "duplicate", + payloads: [exceptionCodecDescriptor(), exceptionCodecDescriptor()], + diagnostic: "descriptors, expected exactly one", + }, + { + label: "truncated", + payloads: [[WPK_FORK_EXCEPTION_CODEC_VERSION]], + diagnostic: "descriptor is truncated", + }, + { + label: "reserved", + payloads: [reserved], + diagnostic: "reserved fields are nonzero", + }, + { + label: "noncanonical ordinal", + payloads: [noncanonical], + diagnostic: "tag ordinal 1 is noncanonical at 0", + }, + { + label: "duplicate layout", + payloads: [duplicateLayout], + diagnostic: "layout id 9 is invalid or duplicated", + }, + ]; + + for (const { label, payloads, diagnostic } of cases) { + const failures = describeWasmArtifactPolicyFailures( + completeForkWasm({ exceptionCodecPayloads: payloads }), + ); + expect(failures.join("\n"), label).toContain(diagnostic); + } + }); + + it("requires pre-instantiation global recipes and private codec bindings", () => { + const missingGlobals = completeForkWasm({ importedGlobalsPayloads: [] }); + expect(describeWasmArtifactPolicyFailures(missingGlobals).join("\n")) + .toContain(`missing required ${WPK_FORK_IMPORTED_GLOBALS_SECTION} descriptor`); + + const missingTables = completeForkWasm({ importedTablesPayloads: [] }); + expect(describeWasmArtifactPolicyFailures(missingTables).join("\n")) + .toContain(`missing required ${WPK_FORK_IMPORTED_TABLES_SECTION} descriptor`); + + const missingActivation = completeForkWasm({ includeActivationImport: false }); + expect(describeWasmArtifactPolicyFailures(missingActivation).join("\n")) + .toContain("missing required immutable exception-codec activation import"); + + const missingResumeTable = completeForkWasm({ includeResumeTable: false }); + expect(describeWasmArtifactPolicyFailures(missingResumeTable).join("\n")) + .toContain("missing required ABI 43 fork-runtime table import"); + }); + + it("binds duplicate and table64 import recipes one declaration at a time", () => { + const sourceTableImports: TableImport[] = [ + { + module: "env", + name: "dispatch", + elementType: 0x70, + table64: false, + minimum: 1, + maximum: 8, + }, + { + module: "env", + name: "dispatch", + elementType: 0x70, + table64: false, + minimum: 1, + maximum: 8, + }, + { + module: "state", + name: "objects", + elementType: 0x6f, + table64: true, + minimum: 0, + maximum: null, + }, + ]; + const descriptor = importedTablesDescriptor([ + { + owner: 1, + typeCode: WPK_FORK_MODULE_STATE_GLOBAL_TYPE_FUNCREF, + flags: 0, + module: "env", + name: "dispatch", + }, + { + owner: 2, + typeCode: WPK_FORK_MODULE_STATE_GLOBAL_TYPE_FUNCREF, + flags: 0, + module: "env", + name: "dispatch", + }, + { + owner: 3, + typeCode: WPK_FORK_MODULE_STATE_GLOBAL_TYPE_EXTERNREF, + flags: WPK_FORK_IMPORTED_TABLES_FLAG_TABLE64, + module: "state", + name: "objects", + }, + ]); + const wasm = completeForkWasm({ + sourceTableImports, + importedTablesPayloads: [descriptor], + }); + + expect(wasmHasCompleteForkInstrumentation(wasm)).toBe(true); + expect(describeWasmArtifactPolicyFailures(wasm, { + expectedAbi: ABI_VERSION, + })).toEqual([]); + }); + + it("rejects copied or incomplete imported-table ownership claims", () => { + const sourceTableImports: TableImport[] = [{ + module: "env", + name: "dispatch", + elementType: 0x70, + table64: false, + minimum: 1, + maximum: 8, + }]; + const correctRecord = { + owner: 1, + typeCode: WPK_FORK_MODULE_STATE_GLOBAL_TYPE_FUNCREF, + flags: 0, + module: "env", + name: "dispatch", + }; + const cases = [ + { + label: "empty copied descriptor", + options: { + sourceTableImports, + importedTablesPayloads: [importedTablesDescriptor([])], + }, + diagnostic: "omits imported table env.dispatch at index 0", + }, + { + label: "wrong declaration type", + options: { + sourceTableImports, + importedTablesPayloads: [importedTablesDescriptor([{ + ...correctRecord, + typeCode: WPK_FORK_MODULE_STATE_GLOBAL_TYPE_EXTERNREF, + }])], + }, + diagnostic: "owner 1 does not match its imported table declaration", + }, + { + label: "missing catalog owner", + options: { + sourceTableImports, + importedTablesPayloads: [ + importedTablesDescriptor([correctRecord]), + ], + includeTableCatalog: false, + }, + diagnostic: "owner 1 lacks exactly one table catalog export", + }, + ]; + + for (const { label, options, diagnostic } of cases) { + expect( + describeWasmArtifactPolicyFailures( + completeForkWasm(options), + ).join("\n"), + label, + ).toContain(diagnostic); + } + }); + + it("binds duplicate, reference, mutable, and shared global recipes one-for-one", () => { + const sourceGlobalImports: GlobalImport[] = [ + { + module: "env", + name: "callback", + valType: 0x6f, + mut: 0, + }, + { + module: "env", + name: "callback", + valType: 0x6f, + mut: 0, + }, + { + module: "state", + name: "epoch", + valType: I32, + mut: 1, + shared: true, + }, + { + // This fresh-child process binding is intentionally catalogued but + // reconstructed by the host before instantiation, not by KFIG. + module: "env", + name: "__channel_base", + valType: I32, + mut: 0, + }, + ]; + const descriptor = importedGlobalsDescriptor([ + { + owner: 1, + typeCode: WPK_FORK_MODULE_STATE_GLOBAL_TYPE_EXTERNREF, + flags: 0, + module: "env", + name: "callback", + }, + { + owner: 2, + typeCode: WPK_FORK_MODULE_STATE_GLOBAL_TYPE_EXTERNREF, + flags: 0, + module: "env", + name: "callback", + }, + { + owner: 3, + typeCode: WPK_FORK_MODULE_STATE_GLOBAL_TYPE_I32, + flags: + WPK_FORK_IMPORTED_GLOBALS_FLAG_MUTABLE + | WPK_FORK_IMPORTED_GLOBALS_FLAG_SHARED, + module: "state", + name: "epoch", + }, + ]); + const wasm = completeForkWasm({ + sourceGlobalImports, + importedGlobalsPayloads: [descriptor], + }); + + expect(wasmHasCompleteForkInstrumentation(wasm)).toBe(true); + expect(describeWasmArtifactPolicyFailures(wasm, { + expectedAbi: ABI_VERSION, + })).toEqual([]); + }); + + it("rejects copied or incomplete imported-global ownership claims", () => { + const sourceGlobalImports: GlobalImport[] = [{ + module: "env", + name: "callback", + valType: 0x6f, + mut: 1, + }]; + const correctRecord = { + owner: 1, + typeCode: WPK_FORK_MODULE_STATE_GLOBAL_TYPE_EXTERNREF, + flags: WPK_FORK_IMPORTED_GLOBALS_FLAG_MUTABLE, + module: "env", + name: "callback", + }; + const cases = [ + { + label: "empty copied descriptor", + options: { + sourceGlobalImports, + importedGlobalsPayloads: [emptyImportedGlobalsDescriptor()], + }, + diagnostic: "omits imported global env.callback at index 0", + }, + { + label: "wrong declaration type", + options: { + sourceGlobalImports, + importedGlobalsPayloads: [importedGlobalsDescriptor([{ + ...correctRecord, + typeCode: WPK_FORK_MODULE_STATE_GLOBAL_TYPE_I32, + }])], + }, + diagnostic: "owner 1 does not match its imported global declaration", + }, + { + label: "missing catalog owner", + options: { + sourceGlobalImports, + importedGlobalsPayloads: [ + importedGlobalsDescriptor([correctRecord]), + ], + includeGlobalCatalog: false, + }, + diagnostic: "owner 1 lacks exactly one global catalog export", + }, + ]; + + for (const { label, options, diagnostic } of cases) { + expect( + describeWasmArtifactPolicyFailures( + completeForkWasm(options), + ).join("\n"), + label, + ).toContain(diagnostic); + } + }); + + it("requires an exact fixed instance-local static-root catalog", () => { + const badMagic = staticRootCatalogDescriptor(); + badMagic[0] ^= 0xff; + const cases = [ + { + label: "missing descriptor", + options: { staticRootPayloads: [] }, + diagnostic: `missing required ${WPK_FORK_STATIC_ROOT_CATALOG_SECTION} descriptor`, + }, + { + label: "duplicate descriptor", + options: { + staticRootPayloads: [ + staticRootCatalogDescriptor(), + staticRootCatalogDescriptor(), + ], + }, + diagnostic: "descriptors, expected exactly one", + }, + { + label: "invalid magic", + options: { staticRootPayloads: [badMagic] }, + diagnostic: "has invalid magic", + }, + { + label: "missing table", + options: { includeStaticRootTable: false }, + diagnostic: "missing exactly one table export", + }, + { + label: "descriptor/table length drift", + options: { + staticRootCount: 2, + staticRootPayloads: [staticRootCatalogDescriptor(3)], + }, + diagnostic: "fixed table32 anyref catalog of length 3", + }, + ]; + + for (const { label, options, diagnostic } of cases) { + expect( + describeWasmArtifactPolicyFailures( + completeForkWasm(options), + ).join("\n"), + label, + ).toContain(diagnostic); + } + }); + + it("rejects every malformed or unsafe activation-state capability shape", () => { + const cases: Array<{ + label: string; + options: Parameters[0]; + diagnostic: string; + }> = [ + { + label: "missing", + options: { capabilityFlags: null }, + diagnostic: `missing required ${WPK_FORK_CAPABILITIES_SECTION} capability`, + }, + { + label: "unsafe flags", + options: { capabilityFlags: 0 }, + diagnostic: "omit required activation-state safety flags", + }, + { + label: "short payload", + options: { + capabilityPayloads: [[WPK_FORK_CAPABILITIES_VERSION]], + }, + diagnostic: "has 1 bytes, expected 2", + }, + { + label: "unsupported version", + options: { + capabilityPayloads: [[ + WPK_FORK_CAPABILITIES_VERSION + 1, + WPK_FORK_CAP_ACTIVATION_STATE_SAFE, + ]], + }, + diagnostic: "version 2 is unsupported", + }, + { + label: "unknown flags", + options: { + capabilityPayloads: [[ + WPK_FORK_CAPABILITIES_VERSION, + WPK_FORK_CAP_KNOWN_MASK | 0x80, + ]], + }, + diagnostic: "has unknown flags", + }, + { + label: "duplicate", + options: { + capabilityPayloads: [ + [WPK_FORK_CAPABILITIES_VERSION, WPK_FORK_CAP_ACTIVATION_STATE_SAFE], + [WPK_FORK_CAPABILITIES_VERSION, WPK_FORK_CAP_ACTIVATION_STATE_SAFE], + ], + }, + diagnostic: "sections, expected exactly one", + }, + ]; + + for (const { label, options, diagnostic } of cases) { + const wasm = completeForkWasm(options); + expect(wasmHasCompleteForkInstrumentation(wasm)).toBe(false); + expect( + describeWasmArtifactPolicyFailures(wasm).join("\n"), + label, + ).toContain(diagnostic); + } + }); + + it("does not let an ABI 42 artifact masquerade as ABI 43 with a copied capability", () => { + const wasm = completeForkWasm({ abiVersion: ABI_VERSION - 1 }); + expect(describeWasmArtifactPolicyFailures(wasm, { + expectedAbi: ABI_VERSION, + })).toContain(`ABI ${ABI_VERSION - 1}, expected ${ABI_VERSION}`); + }); + + it("does not let a reentrant legacy loader import carry the ABI 43 safety claim", () => { + const wasm = completeForkWasm({ includeLegacyDlopenImport: true }); + expect(wasmHasCompleteForkInstrumentation(wasm)).toBe(false); + expect(describeWasmArtifactPolicyFailures(wasm)).toContain( + "ABI 43 fork artifact retains reentrant env.__wasm_dlopen; " + + "rebuild and reinstrument it with the staged loader lowering", + ); + }); + + it("does not let a native start section carry the ABI 43 safety claim", () => { + const wasm = completeForkWasm({ includeNativeStart: true }); + expect(wasmHasCompleteForkInstrumentation(wasm)).toBe(false); + expect(describeWasmArtifactPolicyFailures(wasm)).toContain( + "ABI 43 fork artifact retains 1 native Wasm start section; rebuild and " + + "reinstrument it so initialization is owned by wpk_fork_module_bootstrap", + ); + }); + + it("does not accept a fork capability without an ABI epoch marker", () => { + const wasm = completeForkWasm({ includeAbiMarker: false }); + expect(describeWasmArtifactPolicyFailures(wasm, { + expectedAbi: ABI_VERSION, + })).toContain( + `ABI ${ABI_VERSION} fork artifact is missing __abi_version; ` + + "the activation-state capability epoch cannot be verified", + ); + }); + it("rejects descriptor and module-memory pointer-width drift", () => { const wasm = completeForkWasm({ pointerWidth: 8, memoryPointerWidth: 4 }); expect(wasmHasCompleteForkInstrumentation(wasm)).toBe(false); expect(describeWasmArtifactPolicyFailures(wasm)).toContain( - "ABI 42 linked-frame descriptor declares an 8-byte pointer but the module memory uses 4-byte addresses", + "ABI 43 linked-frame descriptor declares an 8-byte pointer but the module memory uses 4-byte addresses", ); }); @@ -536,7 +1594,7 @@ describe("wasm artifact policy helpers", () => { const wasm = completeForkWasm({ pointerWidth: 8, exportPointerWidth: 4 }); expect(wasmHasCompleteForkInstrumentation(wasm)).toBe(false); expect(describeWasmArtifactPolicyFailures(wasm)).toContain( - "ABI 42 wasm-fork-instrument export wpk_fork_abort_begin has the wrong signature; expected (i64) -> ()", + "ABI 43 wasm-fork-instrument export wpk_fork_abort_begin has the wrong signature; expected (i64) -> ()", ); }); @@ -627,7 +1685,7 @@ describe("wasm artifact policy helpers", () => { expectedAbi: 12, requireForkInstrumentation: false, forbidForkInstrumentation: true, - })).toContain("contains ABI 42 wasm-fork-instrument metadata, imports, or exports"); + })).toContain("contains ABI 43 wasm-fork-instrument metadata, imports, or exports"); }); }); @@ -662,6 +1720,21 @@ function hasWasmObjdump(): boolean { } } +function wasmObjdumpCanDecode(path: string): boolean { + try { + execFileSync("wasm-objdump", ["-j", "Global", "-x", path], { + stdio: "ignore", + }); + return true; + } catch { + // WHY: ABI 43 uses typed-reference and GC encodings that older WABT + // releases reject before they can associate __heap_base with its export. + // This optional parity probe must not mistake an obsolete external decoder + // for an invalid artifact; the mandatory parser cases above still run. + return false; + } +} + function objdumpHeapBase(path: string): bigint | null { const out = execFileSync("wasm-objdump", ["-j", "Global", "-x", path], { encoding: "utf-8" }); const m = out.match(/<__heap_base>\s*-\s*init\s+i(?:32|64)=(-?\d+)/); @@ -684,11 +1757,23 @@ function findCachedBinary(name: string, arch = "wasm32"): string | null { return null; } -const localDashBinary = tryResolveBinary("programs/dash.wasm"); +// This is optional cross-check coverage, not a package-index policy test. +// A concurrently edited source projection must skip the cached-binary probe +// without preventing the pure artifact-parser cases above from collecting. +let localDashBinary: string | null = null; +try { + localDashBinary = tryResolveBinary("programs/dash.wasm"); +} catch { + localDashBinary = null; +} const dashBinary = (localDashBinary && existsSync(localDashBinary)) ? localDashBinary : findCachedBinary("dash.wasm"); -const haveTooling = hasWasmObjdump() && !!dashBinary && existsSync(dashBinary); +const haveTooling = + hasWasmObjdump() + && !!dashBinary + && existsSync(dashBinary) + && wasmObjdumpCanDecode(dashBinary); describe.skipIf(!haveTooling)("extractHeapBase against cached binaries", () => { it("matches wasm-objdump for dash.wasm", () => { diff --git a/libc/glue/abi_constants.h b/libc/glue/abi_constants.h index 0ff0a85295..8adf6c8cae 100644 --- a/libc/glue/abi_constants.h +++ b/libc/glue/abi_constants.h @@ -4,7 +4,7 @@ #define WASM_POSIX_ABI_CONSTANTS_H /* Mirrors wasm_posix_shared::ABI_VERSION. */ -#define WASM_POSIX_ABI_VERSION 42u +#define WASM_POSIX_ABI_VERSION 43u /* Default process-wasm pthread slot declaration. */ #define WASM_POSIX_THREAD_SLOT_DECL_DEFAULT -1 diff --git a/libc/glue/channel_syscall.c b/libc/glue/channel_syscall.c index 08b8d8c464..34da18261f 100644 --- a/libc/glue/channel_syscall.c +++ b/libc/glue/channel_syscall.c @@ -12,7 +12,7 @@ * 8 48B arguments (6 x i64) * 56 8B return value (i64) * 64 4B errno (i32) - * 68 4B reserved/pad + * 68 4B request flags * 72 64KB data transfer buffer * * Each thread has its own channel region within the process's shared @@ -85,6 +85,7 @@ int *__errno_location(void); #define CH_ARG_SIZE 8 #define CH_RETURN 56 #define CH_ERRNO 64 +#define CH_REQUEST_FLAGS 68 #define CH_DATA 72 #define CH_DATA_SIZE 65536 @@ -160,6 +161,7 @@ uintptr_t __get_channel_base_addr(void) { /* SYS_EXIT needs special handling */ #define SYS_EXIT 34 +#define SYS_GETPID 28 /* SYS_FORK/VFORK — kernel_fork import is the fork-continuation boundary. * wasm-fork-instrument rewrites the call graph around kernel.kernel_fork, enabling @@ -182,6 +184,31 @@ int32_t kernel_fork(void); __attribute__((import_module("kernel"), import_name("kernel_exit"))) _Noreturn void kernel_exit(int32_t status); +static long __do_syscall(long n, long long a1, long long a2, long long a3, + long long a4, long long a5, long long a6); + +/* + * Complete one ordinary guest-owned channel request after a host import that + * performed channel work in JavaScript. Those host-owned completions leave + * caught signals kernel-pending because they cannot invoke this file's signal + * trampoline. GETPID is side-effect-free and gives the pending signal an exact + * libc-owned completion without introducing a host-to-Wasm callback. + */ +/* + * The fork instrumenter uses this stable local entry when it lowers the + * historical monolithic __wasm_dlopen import to ABI 43's staged protocol. + * Exporting it lets the generated adapter hand deferred signal delivery back + * to libc after each host-owned loader request, without a host-to-Wasm + * callback or a second signal implementation in the instrumenter. + */ +__attribute__((used)) +__attribute__((retain)) +__attribute__((export_name("__wasm_posix_signal_checkpoint"))) +void __wasm_posix_signal_checkpoint(void) +{ + (void)__do_syscall(SYS_GETPID, 0, 0, 0, 0, 0, 0); +} + /* Direct fork/vfork/_Fork — call kernel_fork without going through the * general syscall dispatcher. This ensures fork instrumentation only covers * fork callers, not every function that makes any syscall. */ @@ -203,13 +230,22 @@ __attribute__((noinline)) int _Fork(void) { long ret = (long)kernel_fork(); + if (ret == 0) { + __wasm_posix_after_fork_child(); + } else { + /* + * WHY: fork transaction allocation and cleanup are consumed by the + * process Worker rather than this libc trampoline, so the host leaves + * caught signals kernel-pending. Re-enter through one ordinary channel + * completion before returning to user code; this invokes any handler + * without a reentrant host-to-Wasm call. + */ + __wasm_posix_signal_checkpoint(); + } if (ret < 0) { *__errno_location() = (int)(-ret); return -1; } - if (ret == 0) { - __wasm_posix_after_fork_child(); - } return (int)ret; } @@ -232,9 +268,6 @@ int vfork(void) /* Signal delivery — invoked after each syscall if a signal is pending */ /* ------------------------------------------------------------------ */ -/* Forward declaration */ -static long __do_syscall(long n, long long a1, long long a2, long long a3, - long long a4, long long a5, long long a6); extern long __syscall_cp_check(long r); extern int __syscall_cp_cancel_pending_disabled(void); @@ -458,6 +491,7 @@ static long __do_syscall_impl(long n, long long a1, long long a2, long long a3, *(int64_t *)(uintptr_t)(base + CH_ARGS + 3 * CH_ARG_SIZE) = (int64_t)a4; *(int64_t *)(uintptr_t)(base + CH_ARGS + 4 * CH_ARG_SIZE) = (int64_t)a5; *(int64_t *)(uintptr_t)(base + CH_ARGS + 5 * CH_ARG_SIZE) = (int64_t)a6; + *(uint32_t *)(uintptr_t)(base + CH_REQUEST_FLAGS) = 0; /* Set status to PENDING and wake the kernel worker. * Use inline asm to read __channel_base directly from the wasm global, diff --git a/libc/glue/dlopen.c b/libc/glue/dlopen.c index 4c73bd3b28..47a25cf37f 100644 --- a/libc/glue/dlopen.c +++ b/libc/glue/dlopen.c @@ -7,12 +7,14 @@ * * Flow: * 1. dlopen() reads the .so file via normal open/read/close syscalls - * 2. Calls __wasm_dlopen() host import with the bytes in memory - * 3. Host compiles the Wasm side module, instantiates it into the - * process's memory/table space, returns a handle - * 4. dlsym() calls __wasm_dlsym() host import to look up symbols - * 5. For functions: returns the table index (== C function pointer) - * 6. For data: returns the relocated memory address + * 2. Calls __wasm_dlopen_prepare() with the bytes in memory + * 3. Host returns a private transaction token without entering guest code + * 4. Each next step performs host-only compilation/instantiation as needed, + * then libc calls the returned initialization entry through the process + * table until the host atomically returns the public handle + * 5. dlsym() calls __wasm_dlsym() host import to look up symbols + * 6. For functions: returns the table index (== C function pointer) + * 7. For data: returns the relocated memory address */ #include @@ -21,13 +23,17 @@ #include #include #include +#include /* Host imports — implemented in worker-main.ts */ -extern int __wasm_dlopen(const void *bytes, int len, - const char *name, int name_len); +extern int __wasm_dlopen_main(void); +extern int __wasm_dlopen_prepare(const void *bytes, int len, + const char *name, int name_len, int flags); +extern int __wasm_dlopen_next(int transaction, int *handle); extern int __wasm_dlsym(int handle, const char *name, int name_len); extern int __wasm_dlclose(int handle); extern int __wasm_dlerror(char *buf, int buf_max); +extern void __wasm_posix_signal_checkpoint(void); /* RTLD flags (match musl dlfcn.h) */ #ifndef RTLD_LAZY @@ -50,12 +56,10 @@ static void set_dl_error(const char *msg) { } void *dlopen(const char *path, int flags) { - (void)flags; - if (!path) { /* An empty host request returns an opaque handle for the main * program's global symbol scope. */ - int handle = __wasm_dlopen(NULL, 0, NULL, 0); + int handle = __wasm_dlopen_main(); if (handle <= 0) { int elen = __wasm_dlerror(dl_error_buf, (int)sizeof(dl_error_buf) - 1); if (elen > 0) { @@ -113,11 +117,19 @@ void *dlopen(const char *path, int flags) { return NULL; } - /* Call host to compile + instantiate the Wasm side module */ - int handle = __wasm_dlopen(buf, (int)st.st_size, path, (int)strlen(path)); + /* + * The host owns module compilation and instance construction, but it must + * not call back into Wasm while this import frame is active. Each returned + * table entry has the canonical void(void) shape, so bootstrap, + * relocations, constructors, and any fork continuation beneath them remain + * an ordinary Wasm-to-Wasm call chain. + */ + int transaction = __wasm_dlopen_prepare( + buf, (int)st.st_size, path, (int)strlen(path), flags); + __wasm_posix_signal_checkpoint(); free(buf); - if (handle <= 0) { + if (transaction <= 0) { /* Get detailed error from host */ int elen = __wasm_dlerror(dl_error_buf, (int)sizeof(dl_error_buf) - 1); if (elen > 0) { @@ -129,6 +141,42 @@ void *dlopen(const char *path, int flags) { return NULL; } + int handle = 0; + for (;;) { + int entry = __wasm_dlopen_next(transaction, &handle); + /* + * WHY: the host may have used mmap or VFS channel requests while + * advancing this transaction. Their completions are JavaScript-owned, + * so deliver any caught signal only now, after the import returned and + * before entering a guest initializer. + */ + __wasm_posix_signal_checkpoint(); + if (entry < 0) { + int elen = __wasm_dlerror( + dl_error_buf, (int)sizeof(dl_error_buf) - 1); + if (elen > 0) { + dl_error_buf[elen] = '\0'; + dl_error_set = 1; + } else { + set_dl_error("wasm initialization failed"); + } + return NULL; + } + if (entry == 0) break; + ((void (*)(void))(uintptr_t)(unsigned int)entry)(); + } + + if (handle <= 0) { + int elen = __wasm_dlerror(dl_error_buf, (int)sizeof(dl_error_buf) - 1); + if (elen > 0) { + dl_error_buf[elen] = '\0'; + dl_error_set = 1; + } else { + set_dl_error("wasm loader commit failed"); + } + return NULL; + } + dl_error_set = 0; return (void *)(long)handle; } @@ -163,6 +211,7 @@ int dlclose(void *handle) { if (!handle) return 0; int h = (int)(long)handle; int ret = __wasm_dlclose(h); + __wasm_posix_signal_checkpoint(); if (ret != 0) { set_dl_error("dlclose failed"); } else { diff --git a/libc/musl-overlay/src/thread/wasm32posix/clone.c b/libc/musl-overlay/src/thread/wasm32posix/clone.c index 6e1475e48a..691ff5a501 100644 --- a/libc/musl-overlay/src/thread/wasm32posix/clone.c +++ b/libc/musl-overlay/src/thread/wasm32posix/clone.c @@ -17,6 +17,7 @@ extern int32_t kernel_clone(uint32_t fn_ptr, uint32_t stack_ptr, uint32_t flags, uint32_t arg, uint32_t ptid_ptr, uint32_t tls_ptr, uint32_t ctid_ptr); +extern void __wasm_posix_signal_checkpoint(void); int __clone(int (*fn)(void *), void *stack, int flags, void *arg, ...) { @@ -35,7 +36,7 @@ int __clone(int (*fn)(void *), void *stack, int flags, void *arg, ...) */ uintptr_t stack_ptr = (uintptr_t)stack & ~(uintptr_t)15; - return kernel_clone( + int result = kernel_clone( (uint32_t)(uintptr_t)fn, (uint32_t)stack_ptr, (uint32_t)flags, @@ -44,4 +45,11 @@ int __clone(int (*fn)(void *), void *stack, int flags, void *arg, ...) (uint32_t)(uintptr_t)tls, (uint32_t)(uintptr_t)ctid ); + /* + * kernel_clone's channel completion is consumed by process-worker + * JavaScript. Run caught handlers only after that import has returned, + * through the same libc-owned checkpoint used by fork and staged dlopen. + */ + __wasm_posix_signal_checkpoint(); + return result; } diff --git a/packages/registry/program-packages.json b/packages/registry/program-packages.json index 5f587af03b..f02f6f8e95 100644 --- a/packages/registry/program-packages.json +++ b/packages/registry/program-packages.json @@ -4,344 +4,344 @@ "bash": { "manifestSha256": "6478060f28d430d18a6ebe7c603392d35a41d9bcf6bc74322351d1450b9c5335", "cacheKeys": { - "wasm32": "1d813d7f9db4979fc5c0eef6b37e213cb73a42395757cc92ef8ebeee2e62913d", - "wasm64": "84099d7abfb7265061070b6782d0094b21372afd94058c16d1f82467d9ab8da8" + "wasm32": "1be72f8540c1231d0b21201b92e591f1370837e2acd7d0521f3c94a26d172e80", + "wasm64": "ff84106f5d3708575b0d997477edcdd4a9ff4a92c3f051377a3c6001545d1108" } }, "bc": { "manifestSha256": "a65661463bb7047b91ff00153bd99fd962c96e571934ab4e923f5215fb6ecfbd", "cacheKeys": { - "wasm32": "2e7bb9ff2a2cd8dd3b28d46c7d733f873d764820ff023a736a41cc1e16745207", - "wasm64": "42504f670c8c9ad8b6ddfcfbe2a9be6dc2110b4c5d976ff2d5858adeb7c510d5" + "wasm32": "26c9f859aad63a3476ce9172e505ebcdb60ecdfe9343bad67f8ce43be57caa99", + "wasm64": "af45d9b9e7b4949d142b6357fae4ea99c4d19bf2daa09a4bb33e36e667cf3994" } }, "bzip2": { "manifestSha256": "59e1e53f5675e7c9148634933ed0f4c7192743727025cae4e843b6924c489abf", "cacheKeys": { - "wasm32": "bd59c26ea552d64a009bebb2d0150b0e2003e227f0bdfe9f0a47a3ae97f9a336", - "wasm64": "ee810512cf9779fa0d7235a25dbe64cb0180b125f1747568cf22cb627eb8f8f8" + "wasm32": "1834481836a740e656f731164abb5ac9903153551c601c8db6d980b207aabb7f", + "wasm64": "98cd0de2317a6f49e0b06ea2a18a62b61f009ab5d343d29bc4fbbc0dd8480fcc" } }, "coreutils": { "manifestSha256": "b8baabc9af9283434e0d80f7fdc0aaef242ffa31092ab0fc39a0603a12b3cca7", "cacheKeys": { - "wasm32": "33ca2b6c08b61b6ba3b013b13271c98b9b63d75f0a5c47d6d265c27c10c50458", - "wasm64": "67ffde57ec940c7107d901231ed2d1c886957edc34de1c60c08094d1260ae797" + "wasm32": "640ffdd14b7ecbe96a0632eeb7dd47c2606cd195bdca24ba8a1b2395e51e2c93", + "wasm64": "afb0ed522f8e36ff024d7fc0833da00cb10e538718dd8179ab2223340dd59efa" } }, "cpython": { "manifestSha256": "7dd4f446697a73941ec940c2ccba4d53be73fe6947f1e701031a4d0aa964c4ff", "cacheKeys": { - "wasm32": "3395181397427732850ece33d0fd3edf875903acb625e59a348577f2613d70f6", - "wasm64": "31b01d999dfbde5bce591d3511418fc84f6ed7042b9d73a1a2e0b4fe8a69920f" + "wasm32": "6bb296c95c4b5f1ad47d5fae5b79bdbdd283b17136893c61d8fd8d8a9121dcc1", + "wasm64": "4d64ae206b7f68cbad2300ed2efabe4a62e760a9e3f7b9d63e842274c1660c5a" } }, "curl": { "manifestSha256": "55523d50261f46dc4aaaff458d1cd87c6f96eaecd687a7540ead35c96906366e", "cacheKeys": { - "wasm32": "44aa318c3c067dc7183b20535ac525637df817431351c244f6bc37b029792a84", - "wasm64": "ca4d46d318391dbb76e16b1a75c7eb3a5aa034b3214b936560e1857a6c4f4e2a" + "wasm32": "d55e6c3dd174664839a42e00604aa69a10bc07439a005bf30f01fc3755381e07", + "wasm64": "3f4bcd161a84d609ec236407117350f549aed3010c4c5dd1b1ffc2839e8951a8" } }, "dash": { "manifestSha256": "10ce1bb611fefd78a6c9ec9038f6113e29a7062b1a73e08eccf9b33004cea9a8", "cacheKeys": { - "wasm32": "2ec3ac4e8baa4bfb4ced00ff411feb0ae28e2acf3b5709ac7c042c5a57295aa6", - "wasm64": "55133070386497aa605c431cf06777d717e4c4ececf2983cc507b858c4b529cd" + "wasm32": "e6a122a8bb1ea383cf8917b15c78dc4e148af24a391f61634117ec2c8d13deb9", + "wasm64": "52d1a589d726fcf1382bdac637206cb94a553338aad8e90648615cd9e32cd688" } }, "diffutils": { "manifestSha256": "3a78f0a46bae43ce5ea235c6638c2cbd1d8559b0b62b1fb42443b35968c1aca3", "cacheKeys": { - "wasm32": "5fd58502ff1d28c11cc7941a2f21d5ab2d9f40bfbe113f766d274361ab5e0a88", - "wasm64": "f10a466f469db4ac17f9be69e0441b238c649c27f0d2869a6df23d0196d3ba64" + "wasm32": "b6423d111d2b4d56085317bac4eba82478e6be1519ed7a6b639b914f4dfc9980", + "wasm64": "d52ac3b62404fc34138aa218c570d707346b09fd2c6336ff088af0ae0cbcad90" } }, "dinit": { "manifestSha256": "44e125e1337503e27f8cff531cdbd79c4f406fd2d282b916a7746cf3a14057bf", "cacheKeys": { - "wasm32": "43651e927c2672e0a4f749a5cce47db470dd19d72736ea4c6b6a6b3d55d9f7ab", - "wasm64": "bbcd2d163560d0271c38581dae6f292cc4a119fc8f86af85552310157bef511a" + "wasm32": "459ad0cd459ea1873d4dce5c19279d1c5797457dfd0ddfaf2b575e25170fea80", + "wasm64": "b8645ab412a8456dc389eab0eaa657ba592ab415abce32e2e6f3a7aade9ff32b" } }, "erlang": { "manifestSha256": "475d6037e73a0f1e2f4381067194b2422751206979ea17209e10efa5a2a93bbb", "cacheKeys": { - "wasm32": "ff0aaf3f73b5c6186a6b72544d245bf4b4021021266430803ce781c2d3128856", - "wasm64": "ab7b04e55de36ba5515ce2b4cf6d831ed8d6dcedabeb3c52177b3042f08b107b" + "wasm32": "526bc87726501edb49e1e69f91779e64a00997091a3269bbbbd46cd7a85bcb84", + "wasm64": "7e8e10566e2d542794faf7436ee6f8298e462054cb0d4c8815e10b02299b9eec" } }, "erlang-vfs": { "manifestSha256": "15efb74f1825e2b85bedfeb8d98c19fa648c4be39cb9defb65330a8f21eb9141", "cacheKeys": { - "wasm32": "af8826989737cc530b222a7049f2ec7dcb628adfc5b8bff543afe04b21f9bae7", - "wasm64": "58c34283c08d722d67737f4392c3151aee964cda18c06fe034839daace38feb1" + "wasm32": "51bc36813ccbc4333cb04924f8cf80ecdaad53b56d259777845ae270d9eba3a8", + "wasm64": "17cda02a067491fcd3ac6c9b860d68e21891cf2e6d311be0cff6b16959d1c4dd" } }, "fbdoom": { "manifestSha256": "a00e0d9c84fcdbb3bd95f296cb3422d60b86dcff4c40734eea1bb0bec4c7d902", "cacheKeys": { - "wasm32": "7ec10e23738b06c8fcd6b1953b252476522a01c5b9cea19c0b16bea88217a712", - "wasm64": "9acd1242e676f6a897861d740aa0b14136e50ac611ba1a40dc9edb63d89c86e1" + "wasm32": "78caa3cd3a94566b4fc860ac8a2813b3fd571be4ce8cb56783ab9534ae9d6fa6", + "wasm64": "2d804db34afac83ba5e6d9b50626f982ba972bc9ee50d46688514d97acd5d4bd" } }, "file": { "manifestSha256": "7874c1affcbbf2c8ab8c8d4beb087f275af57b5caae6a8947bf5a63a181552b2", "cacheKeys": { - "wasm32": "4c75af58729ffafa2d5ff73b3522aacb97465e1a85e7163274ff0994d30017c3", - "wasm64": "cd5c07e821687325b602fecad1d92eaef7b68a0018c9329bd3c0d753dba83a68" + "wasm32": "039f051da9fd7c4111b2cbf731e9a9d950f9d7816927e245cdf2f9f1a6b7f8a3", + "wasm64": "abd8263c0c9e09a77c2f85dfd0434056ea6c0a1e4b8aa814732766ac5ab17d0f" } }, "findutils": { "manifestSha256": "cceedf52aea67fbb0da03cfce6f2e9b1a7b5561fa1656c2762b43c651a625d02", "cacheKeys": { - "wasm32": "e9894ce98cd2055fa27987b544eb72df96fc824ce7c9372d1db4294879985ac1", - "wasm64": "69651dede1cf19f3503a8e4a5f3f14c09e1909bd36aab8229795fcd3ddc4a520" + "wasm32": "1cf00e9e4542eb4cbe7dc5dc378a11b8e7f8d8087dfa041386cf674f045ea408", + "wasm64": "1218555af95690a865a0544c394e263bf1d16dd2ac4c8ae64764b4add025f662" } }, "gawk": { "manifestSha256": "a2567b8b0778805e385e1d3a41ac8b5d74aaf9d293b222001b17d09b11e5a0ba", "cacheKeys": { - "wasm32": "85ef6c0fdac5fa9ac1ab10bb1a057038e6d08015e571b8b36eb2c89505e1917f", - "wasm64": "4e109db7864e9c1b5f5497d1a71d6a34ec921969c5853d2dfbc7a7e38718cab6" + "wasm32": "57a33bb378e1e653901be62db5cd273c9d2393ca90339b57538f95286ee79f6e", + "wasm64": "f7c1f63002a0c9a7f51f4cfaba37e8f2f516a8e3087bee4dd65e9d8e7ba66e13" } }, "git": { "manifestSha256": "c2bc79e62c9a4e840ae94a16af0f41070460eaf3976a990dc60d2db977bee952", "cacheKeys": { - "wasm32": "1f940f813db52db5777a114926c9a143eda473ed9d232bcbd3cd196f3e3ff0b4", - "wasm64": "c2596e6e748741ef33ac5ed340be7ebe2bb3c21c419b95ad2dd6e548ef9ce63f" + "wasm32": "86f7980e416d9ad1872abad3e51d640048e64d3ae1b6d1ab6ddb933f91eea711", + "wasm64": "72401517545a7fb338f7b6573e712ca5d92497d09174864dd0657202b10e9d9f" } }, "grep": { "manifestSha256": "59270d3bfb33167b32d13246bf855b49308f8c9c0a66d9635c804f702421fbc6", "cacheKeys": { - "wasm32": "ccfc4ba22252a0d57c55ed8ada09dff22883fb35a37a7ea1efa395bcb405a748", - "wasm64": "15300a1a26b74ff6ee98c84a2c64a2797f61fd7b3fd5e5357ddab7ed4a491db3" + "wasm32": "6dcbd57aee61506f8e1939b13b7260b2a8c4f2bcc8c7f2a36a14e4799c5f7db5", + "wasm64": "ed5392b64beeaedb99e7967865092765fdcb60627a668e2fc45c22f56d2e4d49" } }, "gzip": { "manifestSha256": "33853ebe2301caf2979b667830e6c5afc5256f8717137624f08079e6a27f370a", "cacheKeys": { - "wasm32": "46f2e656c573c0311dd0e820c58632e59d29f80745bd6bd9a1f28cc082c470df", - "wasm64": "b49e8812bbd6c08fbac02a82c7837d6697eccc2934c25adf5d221bfc40f6bca3" + "wasm32": "e9487ab0a57b8450881474e661225fbb7b50f94c0e61698a265e07189a564159", + "wasm64": "1f37bf89160c4c525b24269e1c39a5d9677835c0636d14bb55bc5849ffb7a605" } }, "homebrew-bootstrap": { "manifestSha256": "b171060c86cb6642e8cadef9d2698e671c6bf188868c7d835b3e8ecf3a3b054a", "cacheKeys": { - "wasm32": "8230aaca4bfc1f7bff80d2830534151afff6edb2c03f640af4afa385d145426d", - "wasm64": "fa191c56e67b1e59d9f9b0b56ff1914c2dc584a1a39a8763cc4cb82bd890a038" + "wasm32": "3ca8b809f79e6cf3432a36a881e33098fff06edb64e1a1c1ce16c24f0d52115b", + "wasm64": "a922d5047129f1f0ffeb42f777bfb95ad51faaa6ad2de8163d3ef245913cf542" } }, "icu": { "manifestSha256": "f5b1f02d169ec1108fc32d9da55be53c3efb96e3a7aeba6a03aec62e0175f0dc", "cacheKeys": { - "wasm32": "48a3ee79c8510a1fe933281741cb0fb56f2a67fa7dba9d2cd18c0085cf49f375", - "wasm64": "39dcb39e822ec17e87b3bcf2426f155568221d9fd8eace1c55645edd1e366247" + "wasm32": "3785cb4f88b3800ceb98a05d72cf21a82221711b31b413b6d0a18a8962db522f", + "wasm64": "7b52084242113beb9a3839c36bb5e9b15cc32de0c1bb277c581094fdd14f0f2c" } }, "kandelo-sdk": { "manifestSha256": "c081879f1becd855917cff96f17429bcf2b9fd61bf95211eca9ff8fb625bb2eb", "cacheKeys": { - "wasm32": "c14c5b31bdea4e86f612a88b34102f831be3f09a189faebaf12b0777920ed647", - "wasm64": "71923a448f3c26d321943d9479462e6587b10c47d083c7b36c9cdc528b1094b2" + "wasm32": "7948a22dc0761233217e578404ca791db8d8b6a2280e37b9bc4a98f8bb5d4209", + "wasm64": "e37ad378159e2f5e498057c5fd62c32a728de3177354c3ff6c395e780ffdd1e0" } }, "kernel": { "manifestSha256": "db1d66db8575562ac7b3e720dbaa06d7dd5eef577d80220ea4167dc2d69b863b", "cacheKeys": { - "wasm32": "f864c4947733ce7a9c1bd6782038da97680db641c03b7696db690dac065e6436", - "wasm64": "593c2103aafab94813266312952e11752195ab635711e6e94681cd0da90983ec" + "wasm32": "8b523b3920ab2566957d0bb1f14639e8555990218cf8dffe5388175ab4732518", + "wasm64": "a7d1ca37050055d2f21d9e53cb3425bb425c1bbe4f18ead949b4ec3dff64b5ed" } }, "lamp": { "manifestSha256": "250b64635d64f8537178188fb5488dbfd2980d79a1c2ffbf346af67008088ba0", "cacheKeys": { - "wasm32": "b7b5b9e0bb1298d84322990bc7122693b42f07200ba95d289e35f6cf25d35171", - "wasm64": "30c7fb0cd279a07e679e72657428dff187c7cde67f9f0f6ef44df48c0f1c9e20" + "wasm32": "8a003a6fafcc1c2df9e688db670be1962c9dcf05206e4dae30ec01475a65b309", + "wasm64": "c336c48fe599a4609c5631f57b339f4db3c6a33a8288d882bcbcc5771e9e2ee8" } }, "less": { "manifestSha256": "996d61545cfe83dcb663a33e8763e0edbd4db4a605fd69a91b243cf79b1f9b17", "cacheKeys": { - "wasm32": "9a509fe274611ce618bb65afde230b2e526adfed9f908dd286d63749bf924547", - "wasm64": "a5fe1e95f40aac2c65b24ab6acb1459974888d69d9caf7d61fcb3dbfe8f7422c" + "wasm32": "e6f4f1d981f02ac3682fe5785aaab591194d957514da0d335d62521fc9876a0e", + "wasm64": "d41bd83e69b944c4dd00a9d83b37d6eda76d7ceafbb139b7be3720868f2efe77" } }, "libcurl": { "manifestSha256": "c771e6cdc83b43840db4dd1fdce4b6189ba4f281596f7485bcda9c65e2ecda00", "cacheKeys": { - "wasm32": "8273da44c60d3edb2032e3566040ed8fd07c2d3d5e43bfd4bbe94ff7ec0b95c6", - "wasm64": "3b0cd009fe52f374ad5f09968d2bd19701fadafcf74eed404f3cd5f7bea12e4d" + "wasm32": "2e9bfe5431ec06c510c8f31c6a936d95f604294cb6f7cb6b5a666fa46e3b5202", + "wasm64": "b30274c3e9faa715b3de7067e030332ea134435aec8b4fdcdcb2ffbe90f27cd5" } }, "libcxx": { "manifestSha256": "6e1ebfa6914043770eae2a6c156b915acf6a56fe763f07d215ffe6f6f2305007", "cacheKeys": { - "wasm32": "53612e1c46d4ce4e8d07b911f753fda620b8b8597c38c3775fc1769f8a63ba89", - "wasm64": "986b49777db0287089ed8b493606cb6ef82f97992ab3d11d5f054d8fb239707f" + "wasm32": "0b9e4ae7831b78deebd709b0e12cd30cca8381d6840664543ec11ba7e11a88ce", + "wasm64": "b1e809860de54f8bede2f335b88451d798b046cf1e7909ae47f47ebb38677994" } }, "libiconv": { "manifestSha256": "fa60b386f4ba0cd5bbbd595f67477cbfb0cc401a14bbd4769630c502bb3f24b1", "cacheKeys": { - "wasm32": "b494e26a4bd579392ae78f894f37f1d7b7ce87f0401f9d53edbf982b0753f76b", - "wasm64": "aa8b571ab96d3178deb1658b5c7e0aec78f44504728087ad69eebd78e5f41c50" + "wasm32": "2ae7c3ce68bb2b52fb722df897584d2501ac4146946692f3f8b08901ed4e2cef", + "wasm64": "72be2fdc4796e9566580a204c40b05f8ebce6078709475b5776f1f0cf5476fba" } }, "libpng": { "manifestSha256": "79ed5c5c072a0267cce5c7a2fe37d8494571b17e44b952f619e04ec1c4a9db10", "cacheKeys": { - "wasm32": "e31da77ef2ff8fee85f28978d0356ebb799f3ae71458f0814fd9bf5c9d708af3", - "wasm64": "21c6a7748c748ad86fb9183872f9815924f0c19433b978bd0f2b594126396562" + "wasm32": "00b34f15486aefd49b0c87e606e84de6df8388d923678a9b99675a5469aa643b", + "wasm64": "7ee92c22b53fc3fda173a8b634c6834dd35a8a80094cf025d667506f210c5559" } }, "libxml2": { "manifestSha256": "908490efcdc7783c76ce36af31d1db24bfcc7ba5f1e7b9b5298cc81cb110256c", "cacheKeys": { - "wasm32": "f8e7709dc6d6606b0d5ad24b1bb2f7aece6df4e171f92dfe7ea344713c14d4e8", - "wasm64": "7b775c7e413c81eca44fc5ec1d68a836ecfedeabde97b640bfc4cb4e7b99b1d1" + "wasm32": "4c2d44bb43f74f3160efe3089232b173c6425c1a996b29b41f2e61df26d1df25", + "wasm64": "350200a2ae01d3469aed5bffaeb1dc8844e549c1c7bf73c7b8d7cc4d919f0acc" } }, "libzip": { "manifestSha256": "83c65d63a6416e79436bf4e792759a4701d568df0e8c010394a9732cf659f0cc", "cacheKeys": { - "wasm32": "0e020e8d1c342ccf5dac59079aab2b2ec35b083b03107ac172cc0f9ac97a13ec", - "wasm64": "87ade5abc15b17b2119681a85d1b1728ce018ff8a6df14975c249f34ec6308b0" + "wasm32": "592acc36c12b140841b44149254872b4a6915c5e8ec65592f58bfc7cd90a1ec9", + "wasm64": "0535e56fbb3a0ccb31c1182f4e099003b3e56807c482ce8ba1d5a6868d9c7696" } }, "lsof": { "manifestSha256": "cfd199bbe082435f7a2e703e2738a368af1eeb5b76b6e93c376054f21ce94419", "cacheKeys": { - "wasm32": "3660780917e27528d76a83fb1ae76f3a547fe348cd1ba979bc9f015a03c1213b", - "wasm64": "3cd130d749e8f4010848fada35b7043715e048faa8584a4f11b2b3d14bc83700" + "wasm32": "9fd429eb76997413c78204f5814b80a5a67388b5db80a21277f0fc1bf74b09c2", + "wasm64": "e74e031bf915745883e6cd2b06bab11afea49d46d8f2958a15bc393bdf0122d3" } }, "m4": { "manifestSha256": "2c6582d99d6eabfb9da52badf49b899e9fdcba76a728536b25bc3741f3331543", "cacheKeys": { - "wasm32": "a7063dcf8ce5bfe288b1520beac1eaf1d19f736fbdd683f344d83d74dcbab727", - "wasm64": "11d165cf45a57702bf92d4e48cf7e3df92b0e68888a345b56a6c5de0302764d5" + "wasm32": "0b20308410ffac3629eacbc4c68603033ae1fce51bfaf356965672779cd5639f", + "wasm64": "ed5551ae7eaf64f78a7c4e153621507e237679689e1a27ebf5bd4564cb885a4b" } }, "make": { "manifestSha256": "f878d2d730f36a4c6dfe1fff1b4ccc704757ea22d8c4c8ccb95b11cd641e5fed", "cacheKeys": { - "wasm32": "49a0f581bbc70db8db660c144bbaf5683ea30f36f305c434c5e4d772c334723e", - "wasm64": "ba775a7a716319c326836f8eda62ec97b5407ec7fb64b24630070444af6b2aad" + "wasm32": "d30ffaed8cdf6c3d2840c726ac592a55c8314162e955a536898b0ae54d3fdfb5", + "wasm64": "134c7b334b255c0dd6c056aa9c0abb1434e3865278c88bb97ca02bb07cab4218" } }, "mariadb": { "manifestSha256": "aeb221be233e4b57bccb4b3375020f94c3ec593dac2734631f00e3b1aec9f48c", "cacheKeys": { - "wasm32": "769572acd89a8fa7738058f6c2f48e5270868d1cc1422c4a0610b100cf851f77", - "wasm64": "815139d5343320f5a8ca5aeaff9421abb2ff6e568b13a90c2e5cc341e2ff5aa5" + "wasm32": "0ab88ecb5225970388e87bfc6ad3eb683947ca6372961af75e5e77d990a15468", + "wasm64": "d36411979372f848892e912f13d057157b9cc438647cc7f3ca16abce3a056369" } }, "mariadb-test": { "manifestSha256": "d4ccce161d659a5a87c80fa1117054bbccea870e0d4a46bd8a070d3930ad0e3f", "cacheKeys": { - "wasm32": "b70268f40e5f5e45c040132373266ddee6ad38fab897a60afcc47ea12ad0cf2a", - "wasm64": "e89252da9da6afc71d2f3082719b62725c8d21ab5f307e8f2d83c99054d76fa6" + "wasm32": "52fddeeb4fd4afd17733bfbcd3d50f6dfefdc3f505d45f91a91ab36d3fb3ff1f", + "wasm64": "651a04b18a70c70058126d1307c8f29c911b37950c858e2e3d0bea4d78d263ac" } }, "mariadb-vfs": { "manifestSha256": "26e74ac84d89b2a839ed72783437061a23022ef2f88ad0f52b6aacd0442ee1cf", "cacheKeys": { - "wasm32": "314fcb39160d1792c099471f02523816ec0f539f2085d425a04b4862b4515d43", - "wasm64": "30f2b70fb885278907dfc2a47d287d5ebe4ef61663f46c9ae6546401bedebe03" + "wasm32": "efa6b13177fed5b6bbc7dfa03097ca734745e1f323f96071161538a76ad06923", + "wasm64": "e63163f3e0082080070ddeff98d6d2b80778e26c3b35268f452ad19b0f014522" } }, "modeset": { "manifestSha256": "36a8683c1c7309701d361c5f77e543a6c1531397b26c72dbb864528c59c42954", "cacheKeys": { - "wasm32": "e61f328544530bf45ff0a339e0ed552c7c235bab8fa104e79c297ff9cea0d96e", - "wasm64": "a2a9f75ce10229a41ed2d3ada84a82f3bcd7caba4d5553a7ffb70572a3d12ed6" + "wasm32": "b1d459dd4751d91c6ffe8d51584eb8ed1e80fe4d0fd3b8a968b1ae705d1d504a", + "wasm64": "7ba4d906cf59e6a39915a2631e958ebf6e8599e9e7cb9a393f878220845b56e5" } }, "msmtpd": { "manifestSha256": "09de04a422ddf631a29a259d816e081f8d317d1077808ede55d8c500baae748f", "cacheKeys": { - "wasm32": "82b32c29b05f3854bebfb414875d603fe66829ab79b0ea02b4a1d00e09fe3a52", - "wasm64": "a4d308062b45985c514b5c87d6d7c682c0ceef539f36436d63bbfc0292ae80c3" + "wasm32": "1676f17c26dd7d6bee804d444aaf37b2d43a65a0f7d8c985e75d965aaa7566ca", + "wasm64": "72726539e83850d628a13d819a7a26792abff16f68d0870f5f02f7a943005220" } }, "nano": { "manifestSha256": "1ba6d340c95581319982257afd6a3554b333b880a7e69991b8c573da883f86c4", "cacheKeys": { - "wasm32": "ee0b8eacc8659f5d690f5b3ee419584ae24403fe6b8e955e01e5192e6a73b635", - "wasm64": "48c7c4a66eb08ac6b77a31ab124b427b3bc0ffde398c9c05c40357379544ca5e" + "wasm32": "5fb7f6651d97a643ce8c0bb06e1fb7ce45ee670ca988cc46dc15d830f6da150d", + "wasm64": "d9772589d0746a8e3a11c76e3be42a7ffb40392b786258bde86e304bfe342ad1" } }, "ncurses": { "manifestSha256": "0a1c180eeade627e204aa47065dbfebfcdc9359d25e2db436932c7fa3a86fc1e", "cacheKeys": { - "wasm32": "1d43a87fe208ed807f1e71640e4fcf7c4417699c318780e49a56c58d37e54ebb", - "wasm64": "a9ac8bb376e464cc09e0f916efe67c17d5c6ebba4aa08d8e49c30d26638c02a8" + "wasm32": "2a8d1a03af1e6866e6da9368cbf163ae6fbed7a0f641f9005fa2baa118b97422", + "wasm64": "8087d0b85d82f9ef056355d8df7eca5b5b26c8609200314827737ef0129d88b6" } }, "netcat": { "manifestSha256": "4cf33cd1ab768b3ad0108da8b68c2ff72e98469cd86a0bf2d0da54a7d4f6fa16", "cacheKeys": { - "wasm32": "e1f8dbb4e078921b75b109603c6b67a8035b3301659af937960841e563db2151", - "wasm64": "b013d1080a30b93e86e639c3c8ac0c6d63cc58be72bba3052140ce4e0c0881fa" + "wasm32": "edcf82d4ca910e21da6b53ea843635207e965f86c684da632ebd832dc616eafd", + "wasm64": "809c20ef237bba59f60635288b52c3285f15191128215300d0d1c279f04a4a84" } }, "nethack": { "manifestSha256": "1a2f12ec2770bd8d40006d6c878a3493b97c71c2bb63ad08c7f6ad99bfd53504", "cacheKeys": { - "wasm32": "0acaa903f47da0a7730a75b9296217ce4723c5d4a53bff772ae18a1f0acc190f", - "wasm64": "2dd42758fee3163011477d852d7b335db98c181744f7fa0db5129846a2bc27f4" + "wasm32": "0262e86e8a61a49c25bbfe1c49cdc3ea0d068e5914ea9843aa0f5561acb15186", + "wasm64": "c31f0eea291ba184aa57614c9e487c2c4fe216ae23b90adcf9b5f81008878124" } }, "nethack-browser-bundle": { "manifestSha256": "b8294981da7ce4299dfb271d4aecb90ababa97a4d1acc9b716ae05bbe52beb53", "cacheKeys": { - "wasm32": "a02b8fc8563203eab00795dfd2474d85769857ece24d9ce0e818c7bd8c56bca5", - "wasm64": "ef4a488fb6c78ebd5b79fbc43f2f208d148c4942e69295372cc99420cb487300" + "wasm32": "bd33eda7a841d8fcb82ba527a529cab87f7316cdae428c13381b7687be85a744", + "wasm64": "929d679fff052967747a49e8fede41dc7e3018b75472f8eeb848315a3e984ea9" } }, "nginx": { "manifestSha256": "41b0aaf05fdd0d8da42642703f9e60fd7b9545d8029896918b87c771c359147b", "cacheKeys": { - "wasm32": "6a92877569b9bf8dabd5333d3f0d5e96388379e28ec00f4d53e0ee675cb58eae", - "wasm64": "f55b140ba6604b5eab3a374514325e46dbd53edd0db3e512ed2255713e06d403" + "wasm32": "afde4ab291d7b9ed1014f24d6c7ca300951a290c8458c4fb0bfa0fd88270853c", + "wasm64": "0e6c4ac578a8d814a270d8ab00da24d679d9c347a0832941a24d9e465c1cc0b0" } }, "nginx-php-vfs": { "manifestSha256": "97976410cb02f8ba710d856b4ac904bcf976d677b50eefdee39fb64176070d4b", "cacheKeys": { - "wasm32": "772ed9fc96057a9c9d80840c4983aed19e0557a8e40003ba4fe96e5a93808e2e", - "wasm64": "c8220ef3538d7a5439c49752e8b51d55ce7e14f558e02285522b8e81556129c4" + "wasm32": "7c5622d5665de78cdc2dcc528963a56685a441c19d201bd05f8a3f9b853c92f8", + "wasm64": "6b03a4592de37078fad63e0ba30abc67f1b0b419648bb7d4b0a12689d0243d20" } }, "nginx-vfs": { "manifestSha256": "46aa2d3250ac5cd0f102a85c3a36a086c7a50ba1f9e05fa1c2aae3d07ad16f10", "cacheKeys": { - "wasm32": "40b4988e6e280e717ba3feb8353f76bbc6992598ce2e8cbf0bf53f14b9cdb4cf", - "wasm64": "87c465fe920c874d628e5bc2e2e39862b32b1bf41955488c7dadbc94086980c3" + "wasm32": "b3d0854b1f0ef76b68068120cdc6212ce2e440a6fb040ca6e27645416ccfd92e", + "wasm64": "707ca898297296213105e5d70316e82988fd2d6d1cc0b898b7d1c80d7e000ad4" } }, "node": { "manifestSha256": "2131a24dfda8587860e57fc65b573086477d376b065b3b9ca4e1dfe4d79f5790", "cacheKeys": { - "wasm32": "62a1b6d2516b3676a069b611e6b0af454ffe26013151b8c71e7168ec31a7fa5d", - "wasm64": "05622518a0cb61bd985d2858bb15d8679d5a3e8be5846ac69a017b43647cfaa6" + "wasm32": "eefc841b11ba61a0a126322c01f777575bf1d5d1bdf9a4527460c6c5a60b4472", + "wasm64": "9af6a72680df3b811265d97000267950a9d0bd08efb26e2481320eb47fb64430" } }, "node-vfs": { "manifestSha256": "33315fb1b3030a4c187ae075eac08f717de8d7ab017b86c6458778ac9070eece", "cacheKeys": { - "wasm32": "e7b886d3361bfa9a2628c648633cce9960faab811cfcc1829e85ba2a61897adc", - "wasm64": "37cde0a666c0f6caff5f891121d77d1db634aaa0bf8498fdbb89d2fb70d8f7c2" + "wasm32": "e688564dbefe46d48eaca9d9cb8cf9de7c3723704e1abb47b858f36aecd2560c", + "wasm64": "dc4f6b7b7c94a516f627257c7f10474e74e97a583f644760edb82cd925121c9b" } }, "openssl": { "manifestSha256": "374f5dce6b2691b630b55cb82955b33d2305384e4bd64746c581621bb0793993", "cacheKeys": { - "wasm32": "bdef227990c15092dfb24dc2abfcb8cd2699b409bef100c9d69a62c770b3b950", - "wasm64": "11532ba6511df310a86990312eab22eecc7f96b9803dfa8a520f3a146b237ffd" + "wasm32": "d497113aa10b49502c0a3eda046a2800793279477bbb5dc4b0897f5a862d06d7", + "wasm64": "e622094dcdffd0fe90c813e9deb1d8dbecbcc6a822e27348b29bee0d00f5c11d" } }, "pcre2-source": { @@ -354,197 +354,197 @@ "perl": { "manifestSha256": "6cdc4dbc54d0e4008cff41f82ec8918c0e44b4aef23903539c1bc0f538fe3ad2", "cacheKeys": { - "wasm32": "2d820d399d95ba7f9f3c8935fba1d869142fbfc09667b8149dcbb33363aa4785", - "wasm64": "335451ce5e2093c95e758fb808e94150a2817af7c039ce2312659ff741810a0a" + "wasm32": "0d664e5463e94a64001765aba8a432d631e85d88e3bbb5a94e1a6d1cc0e2f3c7", + "wasm64": "630107f582d33772b5e326140eb2281f337ff4e2704ec51411fbd989c2a82c70" } }, "perl-vfs": { "manifestSha256": "1345cc102bc8c24a6bc276410c00b4fcc99ba5f6adc9b031f882aaa35d53c8bc", "cacheKeys": { - "wasm32": "41870476e5efbf57f60c4fdcc190e178360fa0610adf2bf6ecf06df24aa393f7", - "wasm64": "b307e174f1a075e9f880052c7126672b7c0f7096c74b8a53a399dda2006d70d4" + "wasm32": "be145f3ba65afd8ba907de3cf51221a7f67fcc3136b81101b13cce0390b7c09a", + "wasm64": "5192ff5072f222f363620286e3c8bccee90c6b730242b4675ecac497c3b45575" } }, "php": { "manifestSha256": "fcd9d7915ed8935b05418e148edcb312adf0b1e4e627fe4aab75444b35b33bbc", "cacheKeys": { - "wasm32": "c5e70ff8e8ab519b31810e4dac16a7500033f549395152f7b00b239d6170acf6", - "wasm64": "5485a281b368a3c7dab477a3bcf14a41999b0df42ee16e2cb3d88746632e1c0d" + "wasm32": "e0bae805a3ed026870ab3dd18f0dca768894bc36d9764ac55f4fc3aa83caaea3", + "wasm64": "38d99c2ee917b3e36f2db9339f5e0292893a4ebe4af2b71c740f1b73e0f6ea86" } }, "posix-utils-lite": { "manifestSha256": "8fd7190b2848ef80143adc7b9268c79e13cd4db3430f7105faf49a4b4407a06f", "cacheKeys": { - "wasm32": "2308c7f1868ea3e052209430626f792e07a168286c45893724195d4709c60220", - "wasm64": "93ed9c8befec4c18904838a92cfca14c8eee7c403652a2379edaa453ceef6c62" + "wasm32": "7446b2dc9dec882ec9b99433dbc10917b9773b615b45b39faec6060b9bc0dc5b", + "wasm64": "e2188c181113778e4a5d03ead6cad1f5be7bcfe1d01cf69f0066b62310ec4533" } }, "python-vfs": { "manifestSha256": "d1aa99feae65f06c0ca8cf52c2176534b2723fd4ee6eba6e72c205245e1c9c26", "cacheKeys": { - "wasm32": "492e7349b34a9b8d9ae873902fd6b4b750607e999d059fb011715dde52144c2c", - "wasm64": "571d1515e8def92afcbd6b937b08633b1945ad9ce9961d8af30c1c864641fcde" + "wasm32": "43cac1e1fb84c1c13838f785d2ee054a9704e83a161bfcdae5a297ea905aaa43", + "wasm64": "5b97f23b01576680915daa942defe170ea9ce0bf8e641906314f14ceda0599fd" } }, "redis": { "manifestSha256": "151fb507de953ba2ba94b8f881bc7d66b4c741c1359a2f590cc07f9a4cd368a3", "cacheKeys": { - "wasm32": "da7392e43962a0288ff0a9958190215ee01476e1d235f4275b790507ce5e4715", - "wasm64": "2b89d7853c8cbb8041daa1f7766dd88ce8bd4f2f08d71a33344cd653f7a8480d" + "wasm32": "1fa5f605040b656ea3270f8807f3e38460a9c331cb1624b5d8bb5491786ffa64", + "wasm64": "27dd08ce51f14c31ddd6a078b6c79af453b593b3e85891be4739d740cfbce43f" } }, "redis-vfs": { "manifestSha256": "234c45c40f94e2a89f98295313b82cb9fce15a412ac829d9e87394e0dc731813", "cacheKeys": { - "wasm32": "480f32952a8f7be451762b6012f9312ca4b1dad0b04d93c387f52427d660daa5", - "wasm64": "1502729cd03b57cc1643f4901d0b9ab9e36910751ceb51ff8adceeef90ccd25b" + "wasm32": "4178da00cb2ea56b789da8cefe6edf33e5f0b524f64134c3fd9829ead4cd8e1a", + "wasm64": "45e8a7a5a58e005747c4c2869c44432909bd2532fe236050f1f83042828a325d" } }, "rootfs": { "manifestSha256": "0420201025170f48c32949ac62707d4577cdc13a53ad1f01f59d318ec07c8d6e", "cacheKeys": { - "wasm32": "4feb895e5854b8d3f1e3fd9e29af4f6f001ffaf458d311df1a6451f259134368", - "wasm64": "6028bdf295de0e9c2c302a5a96df54dc3ea963f73f673fddfb6b9a0b4c219cea" + "wasm32": "5526de953f3a2e515a2625a7d2348429bf644310695ea563edd2e5ba2f57e10f", + "wasm64": "7986cb568ff6df4e3dbd9f5536e31b64755857daa7b3862fee98f2fe03141544" } }, "ruby": { "manifestSha256": "6ea67a246c29cd927a19decbb36ae4c4d478ff6642d2c77e08a6b2cfeb8251d2", "cacheKeys": { - "wasm32": "fd5e4efd09f5f8fbb9b7fe6671ef1c9253f3d78f3709783b5002a3a1983163ab", - "wasm64": "c0cb2beb1218c89bfdf674375e350d33efb7dc53636ea3a3c52e374b31eb55ca" + "wasm32": "a7d611f70473dbfc1e89312fde7aa8502373cdd2c15e10f4b7b6d67dd19ffc5c", + "wasm64": "326c829580ce5d14205047a5ad5f732893bb54e0c35c0359bf3d751f918628d8" } }, "sed": { "manifestSha256": "2a08e9c5dacc5facc8983c1ff35ff2a72db328405e9a1f464cc7ad155c4d08af", "cacheKeys": { - "wasm32": "7ba13641a1723f061e59a41a5db4e713fa7ebc37db885087217c7cb2e1661c19", - "wasm64": "3151db4c0b7ed9433fd0835bacf7e63512d6ae09a39cdf149b407b9615c1dd82" + "wasm32": "d5f738b9a42966519b9dcfab117f041d2d3a38d608d483166b1342d3b1e67c68", + "wasm64": "aa635cb41e444e0253c4dda365b7da4226f32ab80f27f023d1717ac33911e1d9" } }, "shell": { "manifestSha256": "107adeee07fd0a10d9f2bb40a0cd84a6144bff91e7505b5ae9afeabaa000beb9", "cacheKeys": { - "wasm32": "5be1bf60e0954662e5973417c42baa3ba028c4260f41e8e690a927be981fa0f1", - "wasm64": "31ce326602fd0b40b4b745b7f557176402840ae0a1585f99f11606aa2d5ec944" + "wasm32": "53498ffe143171264f70b0c60a20e6811a144d9ef5d39bb6c7b92184b3e5e6af", + "wasm64": "17bc5f7fe3059fce30609db6e6c65952fe84667ecf7a5ef7eec19b576243bed8" } }, "spidermonkey": { "manifestSha256": "c72ef4e43ee2a4fc74390d03a239fce3b545a6d3b64f02ccc4d44972cec626bd", "cacheKeys": { - "wasm32": "2c2b1f5959b20978e10725bc42cf1d3e0c48a7c699f3b3d564d843c71b8b1ea7", - "wasm64": "a052012cc83b6ddaa1e2c6d5a4d7e275817f3e5843ff53b68fd47fd760e42013" + "wasm32": "313e60b92a277a65ebba3d4e33831978477a4618baed284467b485bca9baa689", + "wasm64": "dc8c861b2feaea56ab3dd944ce0bd3b04de4cdca80de4443699fd42050c107f9" } }, "spidermonkey-node": { "manifestSha256": "f156c4c5044d2a9447324e7a2a35f8c3e6fa367b3e44f674dcc30ab6b946fae7", "cacheKeys": { - "wasm32": "1fc4b5fe1002e77e10e531ecd916c99a766c69583716fca36d5fbba88fa389da", - "wasm64": "f2076505c95522a1394b79695c4edf13974ed05f68fbca8a2cfb9884be1fe615" + "wasm32": "f9ac0d7dab5982c814401c7b7dad5eb8efe5e54b2235ee3d94a2835ba212b0e7", + "wasm64": "23fdfcad37128586e1fb7716c02aa1f3a34017c14dad23e3c6c9937ef4a2faf5" } }, "sqlite": { "manifestSha256": "ab9560b4eab53445034e2a87a27ccee9d05ddb0db73fee28f5000325b0b1cba5", "cacheKeys": { - "wasm32": "a21f509748a9f5a086db9ae7bb2341a3c57171c954b31a3982b2b47a98794694", - "wasm64": "1f7a45fb1bdab0c6e32713f5ddba158f0d4e6268110288b8d4f148e0eb990dc8" + "wasm32": "35544fa8d8d50cd23c78606836caf96f7636cf20360d1cdcf503fcdc1f06c400", + "wasm64": "ff0ea759e162bb2bda08776e4e5ae01f734221276f05a445e3827166b79e5d5c" } }, "sqlite-cli": { "manifestSha256": "1cebb768f9473dc58fc6e72c9b24565760aedaec4cbdb29ad43b22a0a5fd7030", "cacheKeys": { - "wasm32": "1302bc16ff5d928390e99d8eda54447da06d6d601d2e93d3d77aa7a877973835", - "wasm64": "e2caa70802da2cf6cc091e7cf3bbec04e1cdfaa97dd7e0dfc853d278b39660f3" + "wasm32": "49cbbc5d5ab7347c4baf94cd98af99d4f7e9cff2fc91c3f677e06f17bcb368ec", + "wasm64": "96ddd87b243733d424e05ef0bc578adb5acd983c6c11938efd74e953b81b9a1b" } }, "tar": { "manifestSha256": "08fa090c122d3105c735d74560bdd7b8b083a6f9ca8bdec17de0b4993e1be7fd", "cacheKeys": { - "wasm32": "7db0ac25cdc6534a25f0f563bd5431e5af1664284cf53b213e9868e52fa6d072", - "wasm64": "614058a11e3563ddefbc3853baf1dd46efe944fc38d299073eb78b3c5151c069" + "wasm32": "d712156abc4f50607933e6c58eae2c452c53f4a376224aaa1af0d37b32abf823", + "wasm64": "7460cce0488f3a138a6c375927932e080f834db3ba9ac2a02870b87d899cfe7a" } }, "tcl": { "manifestSha256": "67253d47de7df9184e68ec3c49835455746941835a26db9e588aa8fbee7d4636", "cacheKeys": { - "wasm32": "0256f1aedd61bdae5546348dd188638aa2192a8ba5cdd013584b1701f42e2133", - "wasm64": "e99d1a35e80148841c32edbcc34f0e485cc7d745a63a7ce9e318cf0cf5e6c892" + "wasm32": "3c78c9269ceb90fa1077bc1538d812d5e7b313c66f7586d401f0c89e7c49d747", + "wasm64": "95baefabee9ad45258ee478820c17e09208837e39a95cf56490909e0b0a125b9" } }, "texlive": { "manifestSha256": "028ada023f8a8f9966c8a28575242a339aaaf0e8870fe4a221986133b381c3c0", "cacheKeys": { - "wasm32": "40b7b146f334e95fbc826ab23999bfb3dc2032705667c9ffca2a53a48f4613d3", - "wasm64": "4f560a5bdede5c76835874c5ac82e9bf15bf69ba55e2a7619fb11581086b68f8" + "wasm32": "a2d188b024d8082fc1765476929035c18ab305961de478854141e811f163e57f", + "wasm64": "710f9e2eea5b05d72fda86b1fef2854e714e01fdb7e310195482a447e7bfe44b" } }, "unzip": { "manifestSha256": "ef4e5e34258827ab3cbc1930da3fd9514f08f2d7d56c7c3e0d5c495a4d3a20e9", "cacheKeys": { - "wasm32": "d058df9f05e8d6b84e4cab72005e03c5de8968f4d72f6ef3ceceb8815d03532d", - "wasm64": "a396cc7e9ba4a8f2205f8e46f6b9735f439abc872b70c2dfd47163189a601629" + "wasm32": "662e66598d4333078259cc3daca5b44ecc19fe5eb6297451be1189a7f606ae20", + "wasm64": "bfd46c16d3c7ef79146e81a118fac0deedf0786a4758bcb7e23a7a6d367cb808" } }, "userspace": { "manifestSha256": "221176f2a096dee19bbefd89da5c7f50138ecd92d37ec9d7d6b35dbb25e86924", "cacheKeys": { - "wasm32": "3a42c75d397d02825da5b6ea7eeb510f9b776a359e0ca1283d4109a0c4dde404", - "wasm64": "0551507fd8bda0e30814b11a0b75022246a58966abdd358bb5ac3c2158738e52" + "wasm32": "67450243ea8c4f3ff51dc66f865d7ae4f80a555ec001e603949ec7a6d0d9b2d8", + "wasm64": "bae82ec3fde3d8104cb0ef41c6fc4684c3f432d4e4b8bd014bef4c5172469ae7" } }, "vim": { "manifestSha256": "c211660ef41d01f95f3fbdf675b74891e897e3f304e04f1bf5586f326007382c", "cacheKeys": { - "wasm32": "e0a4f2dccc1020ab2c5fde09301251c7da82dc9cb3baa24729f490294cc8373a", - "wasm64": "331c796ec4e4ae5781ee395770b852c7a7da0d5ba4ab58e1a7afdcd329fc43c6" + "wasm32": "b2d332ff56123c0576814b63090fa7d405b439021659c8846f34056580bcb531", + "wasm64": "eb7b7dc7cd7ebe588868b3c5e97d46024fbd95da65fbca61756c884ff2a3cb0c" } }, "vim-browser-bundle": { "manifestSha256": "e31d84057db49c3534381604d0c8f3c7d765e33ede560a7ea7b01a5a8f1aa0d3", "cacheKeys": { - "wasm32": "fcfbb7dc5ba2daeea207eac0b7e991938d8cf3d778e7328f03cf31eb381e976d", - "wasm64": "dbed5eacbc44782c763216feb896d056fc02fde50a66e585589008fb6de9ab61" + "wasm32": "4e098ad47f6ebf65458ea5b59f4b23920dc7c463dfc839ef62647c828a08fd52", + "wasm64": "f10ebb12c19e38067ab4af3a69844adaa5c88153cfd60ddcb437b889a31809ed" } }, "wget": { "manifestSha256": "d3c7ba9bc1ae708b99850a6eb2cae521c85bf1010c90135a935f5dfae57a53a0", "cacheKeys": { - "wasm32": "bdf72a4f1ea11c510f3d1644802df33786cd9c8ee158561ba72c8004436e3a56", - "wasm64": "ce87453bd578d4377ee8299a3707f948223412b7b9276d08fd065316a36e1b5c" + "wasm32": "8947f2fb28c765360635ed05745d13f2dbd214531c1a137d8cc206cf1b73ada0", + "wasm64": "728f42262a9ac9195c4032b3cbc09f50638c368b1511cdcff0b0a55a6a50c92b" } }, "wordpress": { "manifestSha256": "36465e0596a06e855a8524eecfd87da98643bc13b37ee12939f5aab44bf33418", "cacheKeys": { - "wasm32": "3273c155264665eaa8369f4ae19d674317c66d5f04603006a1aad357c39c9e37", - "wasm64": "32b94e1a478fa81c644d64febc579f0aff1332150f65c03bb7ab84d355df6d8d" + "wasm32": "0a75a06c7a970ec83f3ad17b6b58f1c01e0134309b466b50caef33bf0cdf0ffc", + "wasm64": "dea13a33897cfa5ffca0dc0bba8b99ad1fb17b34f3d1c0b40def31c854dc237f" } }, "xz": { "manifestSha256": "8707d35b8647b1af8fa165dcb6ce7b2a6a007e19ab2daca2680c39395864a737", "cacheKeys": { - "wasm32": "261ecf6d602b685b0977854cc97ac51f13a0bf27c28753fcf35a6384714d6c9c", - "wasm64": "ae3820dd71871c6bfc3f3e584b45146752edac0cc93005e58d7e59b8ec37b9c7" + "wasm32": "ab7ba59e47ab9682297ed6104348b28f11a98eba83a9c2240062d7559b79694a", + "wasm64": "29387b078e0e711b7a4ec42febee8a1ea3b3a9712005bcf65418811610e295f4" } }, "zip": { "manifestSha256": "9c9cfa8220aaeaea26736b55f7cd0a7517b8853c07f9f0d241ad67dd43592e36", "cacheKeys": { - "wasm32": "7adafa87971ab6e08449b0958b55364989c06297b9027c1acb06f72959d43cfd", - "wasm64": "7dc135d2ede863f05633fac8f056fb4eea57d071310744127bfb21710e002be9" + "wasm32": "af083f1df4648a11c46f20c8db7804bf455a117ae12819eb857dd46256fbac39", + "wasm64": "98b9d3ef9c67dd18bbbaea882183c4586989a0d94ffb2431f6ef387e7ce11316" } }, "zlib": { "manifestSha256": "d4f4701c0c30dc843ebe2111ed3e2284836cdfc4bc05a130c8fab4f31b8e6357", "cacheKeys": { - "wasm32": "a826c5a1662265850d13e48202fda10c240400126b9fcc0ca33ee0f57e5969f1", - "wasm64": "09bd3336c2aea228832343eafe49f021f347fcb5dce9a827ccbeb11761dc0411" + "wasm32": "f372595b0d4dc5d079fde6f5a3d0401d8b0e8e35a87221df9f51a607efd15fdb", + "wasm64": "5717b46eb9f24dd3b0563336bc74bddc37996b9cb5db9a4976672e9622b9a68f" } }, "zstd": { "manifestSha256": "89f266938c2c253700a838e6352c9de66c976c84883325aaa979f72b732357e4", "cacheKeys": { - "wasm32": "47bf7eb3741ccdb3d7e0e320c7440c2106aa997e476830cbc2d63532240c0a38", - "wasm64": "1941bcb45970fb0966394cb4bcddfa4b1dfce88b5ff4927f1580191c8f8e01eb" + "wasm32": "694f316b82bef68ed79efc2905fcc1134867d38e8e25ba737f8893b859575ce1", + "wasm64": "a0a3184bb54b7d8dc86a931515463eebe60f0eb4836da76f19a5fc62f091b32c" } } }, @@ -555,14 +555,14 @@ "wasm32" ], "cacheKeys": { - "wasm32": "1d813d7f9db4979fc5c0eef6b37e213cb73a42395757cc92ef8ebeee2e62913d" + "wasm32": "1be72f8540c1231d0b21201b92e591f1370837e2acd7d0521f3c94a26d172e80" }, "dependencyClosures": { "wasm32": [ { "packageName": "ncurses", "manifestSha256": "0a1c180eeade627e204aa47065dbfebfcdc9359d25e2db436932c7fa3a86fc1e", - "cacheKey": "1d43a87fe208ed807f1e71640e4fcf7c4417699c318780e49a56c58d37e54ebb" + "cacheKey": "2a8d1a03af1e6866e6da9368cbf163ae6fbed7a0f641f9005fa2baa118b97422" } ] }, @@ -582,7 +582,7 @@ "wasm32" ], "cacheKeys": { - "wasm32": "2e7bb9ff2a2cd8dd3b28d46c7d733f873d764820ff023a736a41cc1e16745207" + "wasm32": "26c9f859aad63a3476ce9172e505ebcdb60ecdfe9343bad67f8ce43be57caa99" }, "dependencyClosures": { "wasm32": [] @@ -603,7 +603,7 @@ "wasm32" ], "cacheKeys": { - "wasm32": "bd59c26ea552d64a009bebb2d0150b0e2003e227f0bdfe9f0a47a3ae97f9a336" + "wasm32": "1834481836a740e656f731164abb5ac9903153551c601c8db6d980b207aabb7f" }, "dependencyClosures": { "wasm32": [] @@ -624,7 +624,7 @@ "wasm32" ], "cacheKeys": { - "wasm32": "33ca2b6c08b61b6ba3b013b13271c98b9b63d75f0a5c47d6d265c27c10c50458" + "wasm32": "640ffdd14b7ecbe96a0632eeb7dd47c2606cd195bdca24ba8a1b2395e51e2c93" }, "dependencyClosures": { "wasm32": [] @@ -645,14 +645,14 @@ "wasm32" ], "cacheKeys": { - "wasm32": "3395181397427732850ece33d0fd3edf875903acb625e59a348577f2613d70f6" + "wasm32": "6bb296c95c4b5f1ad47d5fae5b79bdbdd283b17136893c61d8fd8d8a9121dcc1" }, "dependencyClosures": { "wasm32": [ { "packageName": "zlib", "manifestSha256": "d4f4701c0c30dc843ebe2111ed3e2284836cdfc4bc05a130c8fab4f31b8e6357", - "cacheKey": "a826c5a1662265850d13e48202fda10c240400126b9fcc0ca33ee0f57e5969f1" + "cacheKey": "f372595b0d4dc5d079fde6f5a3d0401d8b0e8e35a87221df9f51a607efd15fdb" } ] }, @@ -679,19 +679,19 @@ "wasm32" ], "cacheKeys": { - "wasm32": "44aa318c3c067dc7183b20535ac525637df817431351c244f6bc37b029792a84" + "wasm32": "d55e6c3dd174664839a42e00604aa69a10bc07439a005bf30f01fc3755381e07" }, "dependencyClosures": { "wasm32": [ { "packageName": "openssl", "manifestSha256": "374f5dce6b2691b630b55cb82955b33d2305384e4bd64746c581621bb0793993", - "cacheKey": "bdef227990c15092dfb24dc2abfcb8cd2699b409bef100c9d69a62c770b3b950" + "cacheKey": "d497113aa10b49502c0a3eda046a2800793279477bbb5dc4b0897f5a862d06d7" }, { "packageName": "zlib", "manifestSha256": "d4f4701c0c30dc843ebe2111ed3e2284836cdfc4bc05a130c8fab4f31b8e6357", - "cacheKey": "a826c5a1662265850d13e48202fda10c240400126b9fcc0ca33ee0f57e5969f1" + "cacheKey": "f372595b0d4dc5d079fde6f5a3d0401d8b0e8e35a87221df9f51a607efd15fdb" } ] }, @@ -711,7 +711,7 @@ "wasm32" ], "cacheKeys": { - "wasm32": "2ec3ac4e8baa4bfb4ced00ff411feb0ae28e2acf3b5709ac7c042c5a57295aa6" + "wasm32": "e6a122a8bb1ea383cf8917b15c78dc4e148af24a391f61634117ec2c8d13deb9" }, "dependencyClosures": { "wasm32": [] @@ -732,7 +732,7 @@ "wasm32" ], "cacheKeys": { - "wasm32": "5fd58502ff1d28c11cc7941a2f21d5ab2d9f40bfbe113f766d274361ab5e0a88" + "wasm32": "b6423d111d2b4d56085317bac4eba82478e6be1519ed7a6b639b914f4dfc9980" }, "dependencyClosures": { "wasm32": [] @@ -774,14 +774,14 @@ "wasm32" ], "cacheKeys": { - "wasm32": "43651e927c2672e0a4f749a5cce47db470dd19d72736ea4c6b6a6b3d55d9f7ab" + "wasm32": "459ad0cd459ea1873d4dce5c19279d1c5797457dfd0ddfaf2b575e25170fea80" }, "dependencyClosures": { "wasm32": [ { "packageName": "libcxx", "manifestSha256": "6e1ebfa6914043770eae2a6c156b915acf6a56fe763f07d215ffe6f6f2305007", - "cacheKey": "53612e1c46d4ce4e8d07b911f753fda620b8b8597c38c3775fc1769f8a63ba89" + "cacheKey": "0b9e4ae7831b78deebd709b0e12cd30cca8381d6840664543ec11ba7e11a88ce" } ] }, @@ -815,7 +815,7 @@ "wasm32" ], "cacheKeys": { - "wasm32": "ff0aaf3f73b5c6186a6b72544d245bf4b4021021266430803ce781c2d3128856" + "wasm32": "526bc87726501edb49e1e69f91779e64a00997091a3269bbbbd46cd7a85bcb84" }, "dependencyClosures": { "wasm32": [] @@ -843,14 +843,14 @@ "wasm32" ], "cacheKeys": { - "wasm32": "af8826989737cc530b222a7049f2ec7dcb628adfc5b8bff543afe04b21f9bae7" + "wasm32": "51bc36813ccbc4333cb04924f8cf80ecdaad53b56d259777845ae270d9eba3a8" }, "dependencyClosures": { "wasm32": [ { "packageName": "erlang", "manifestSha256": "475d6037e73a0f1e2f4381067194b2422751206979ea17209e10efa5a2a93bbb", - "cacheKey": "ff0aaf3f73b5c6186a6b72544d245bf4b4021021266430803ce781c2d3128856" + "cacheKey": "526bc87726501edb49e1e69f91779e64a00997091a3269bbbbd46cd7a85bcb84" } ] }, @@ -870,7 +870,7 @@ "wasm32" ], "cacheKeys": { - "wasm32": "7ec10e23738b06c8fcd6b1953b252476522a01c5b9cea19c0b16bea88217a712" + "wasm32": "78caa3cd3a94566b4fc860ac8a2813b3fd571be4ce8cb56783ab9534ae9d6fa6" }, "dependencyClosures": { "wasm32": [] @@ -891,7 +891,7 @@ "wasm32" ], "cacheKeys": { - "wasm32": "4c75af58729ffafa2d5ff73b3522aacb97465e1a85e7163274ff0994d30017c3" + "wasm32": "039f051da9fd7c4111b2cbf731e9a9d950f9d7816927e245cdf2f9f1a6b7f8a3" }, "dependencyClosures": { "wasm32": [] @@ -919,7 +919,7 @@ "wasm32" ], "cacheKeys": { - "wasm32": "e9894ce98cd2055fa27987b544eb72df96fc824ce7c9372d1db4294879985ac1" + "wasm32": "1cf00e9e4542eb4cbe7dc5dc378a11b8e7f8d8087dfa041386cf674f045ea408" }, "dependencyClosures": { "wasm32": [] @@ -947,7 +947,7 @@ "wasm32" ], "cacheKeys": { - "wasm32": "85ef6c0fdac5fa9ac1ab10bb1a057038e6d08015e571b8b36eb2c89505e1917f" + "wasm32": "57a33bb378e1e653901be62db5cd273c9d2393ca90339b57538f95286ee79f6e" }, "dependencyClosures": { "wasm32": [] @@ -968,7 +968,7 @@ "wasm32" ], "cacheKeys": { - "wasm32": "1f940f813db52db5777a114926c9a143eda473ed9d232bcbd3cd196f3e3ff0b4" + "wasm32": "86f7980e416d9ad1872abad3e51d640048e64d3ae1b6d1ab6ddb933f91eea711" }, "dependencyClosures": { "wasm32": [] @@ -996,7 +996,7 @@ "wasm32" ], "cacheKeys": { - "wasm32": "ccfc4ba22252a0d57c55ed8ada09dff22883fb35a37a7ea1efa395bcb405a748" + "wasm32": "6dcbd57aee61506f8e1939b13b7260b2a8c4f2bcc8c7f2a36a14e4799c5f7db5" }, "dependencyClosures": { "wasm32": [] @@ -1017,7 +1017,7 @@ "wasm32" ], "cacheKeys": { - "wasm32": "46f2e656c573c0311dd0e820c58632e59d29f80745bd6bd9a1f28cc082c470df" + "wasm32": "e9487ab0a57b8450881474e661225fbb7b50f94c0e61698a265e07189a564159" }, "dependencyClosures": { "wasm32": [] @@ -1038,7 +1038,7 @@ "wasm32" ], "cacheKeys": { - "wasm32": "8230aaca4bfc1f7bff80d2830534151afff6edb2c03f640af4afa385d145426d" + "wasm32": "3ca8b809f79e6cf3432a36a881e33098fff06edb64e1a1c1ce16c24f0d52115b" }, "dependencyClosures": { "wasm32": [] @@ -1066,14 +1066,14 @@ "wasm32" ], "cacheKeys": { - "wasm32": "c14c5b31bdea4e86f612a88b34102f831be3f09a189faebaf12b0777920ed647" + "wasm32": "7948a22dc0761233217e578404ca791db8d8b6a2280e37b9bc4a98f8bb5d4209" }, "dependencyClosures": { "wasm32": [ { "packageName": "libcxx", "manifestSha256": "6e1ebfa6914043770eae2a6c156b915acf6a56fe763f07d215ffe6f6f2305007", - "cacheKey": "53612e1c46d4ce4e8d07b911f753fda620b8b8597c38c3775fc1769f8a63ba89" + "cacheKey": "0b9e4ae7831b78deebd709b0e12cd30cca8381d6840664543ec11ba7e11a88ce" } ] }, @@ -1093,69 +1093,69 @@ "wasm32" ], "cacheKeys": { - "wasm32": "b7b5b9e0bb1298d84322990bc7122693b42f07200ba95d289e35f6cf25d35171" + "wasm32": "8a003a6fafcc1c2df9e688db670be1962c9dcf05206e4dae30ec01475a65b309" }, "dependencyClosures": { "wasm32": [ { "packageName": "dinit", "manifestSha256": "44e125e1337503e27f8cff531cdbd79c4f406fd2d282b916a7746cf3a14057bf", - "cacheKey": "43651e927c2672e0a4f749a5cce47db470dd19d72736ea4c6b6a6b3d55d9f7ab" + "cacheKey": "459ad0cd459ea1873d4dce5c19279d1c5797457dfd0ddfaf2b575e25170fea80" }, { "packageName": "homebrew-bootstrap", "manifestSha256": "b171060c86cb6642e8cadef9d2698e671c6bf188868c7d835b3e8ecf3a3b054a", - "cacheKey": "8230aaca4bfc1f7bff80d2830534151afff6edb2c03f640af4afa385d145426d" + "cacheKey": "3ca8b809f79e6cf3432a36a881e33098fff06edb64e1a1c1ce16c24f0d52115b" }, { "packageName": "icu", "manifestSha256": "f5b1f02d169ec1108fc32d9da55be53c3efb96e3a7aeba6a03aec62e0175f0dc", - "cacheKey": "48a3ee79c8510a1fe933281741cb0fb56f2a67fa7dba9d2cd18c0085cf49f375" + "cacheKey": "3785cb4f88b3800ceb98a05d72cf21a82221711b31b413b6d0a18a8962db522f" }, { "packageName": "libcurl", "manifestSha256": "c771e6cdc83b43840db4dd1fdce4b6189ba4f281596f7485bcda9c65e2ecda00", - "cacheKey": "8273da44c60d3edb2032e3566040ed8fd07c2d3d5e43bfd4bbe94ff7ec0b95c6" + "cacheKey": "2e9bfe5431ec06c510c8f31c6a936d95f604294cb6f7cb6b5a666fa46e3b5202" }, { "packageName": "libcxx", "manifestSha256": "6e1ebfa6914043770eae2a6c156b915acf6a56fe763f07d215ffe6f6f2305007", - "cacheKey": "53612e1c46d4ce4e8d07b911f753fda620b8b8597c38c3775fc1769f8a63ba89" + "cacheKey": "0b9e4ae7831b78deebd709b0e12cd30cca8381d6840664543ec11ba7e11a88ce" }, { "packageName": "libiconv", "manifestSha256": "fa60b386f4ba0cd5bbbd595f67477cbfb0cc401a14bbd4769630c502bb3f24b1", - "cacheKey": "b494e26a4bd579392ae78f894f37f1d7b7ce87f0401f9d53edbf982b0753f76b" + "cacheKey": "2ae7c3ce68bb2b52fb722df897584d2501ac4146946692f3f8b08901ed4e2cef" }, { "packageName": "libxml2", "manifestSha256": "908490efcdc7783c76ce36af31d1db24bfcc7ba5f1e7b9b5298cc81cb110256c", - "cacheKey": "f8e7709dc6d6606b0d5ad24b1bb2f7aece6df4e171f92dfe7ea344713c14d4e8" + "cacheKey": "4c2d44bb43f74f3160efe3089232b173c6425c1a996b29b41f2e61df26d1df25" }, { "packageName": "libzip", "manifestSha256": "83c65d63a6416e79436bf4e792759a4701d568df0e8c010394a9732cf659f0cc", - "cacheKey": "0e020e8d1c342ccf5dac59079aab2b2ec35b083b03107ac172cc0f9ac97a13ec" + "cacheKey": "592acc36c12b140841b44149254872b4a6915c5e8ec65592f58bfc7cd90a1ec9" }, { "packageName": "mariadb", "manifestSha256": "aeb221be233e4b57bccb4b3375020f94c3ec593dac2734631f00e3b1aec9f48c", - "cacheKey": "769572acd89a8fa7738058f6c2f48e5270868d1cc1422c4a0610b100cf851f77" + "cacheKey": "0ab88ecb5225970388e87bfc6ad3eb683947ca6372961af75e5e77d990a15468" }, { "packageName": "msmtpd", "manifestSha256": "09de04a422ddf631a29a259d816e081f8d317d1077808ede55d8c500baae748f", - "cacheKey": "82b32c29b05f3854bebfb414875d603fe66829ab79b0ea02b4a1d00e09fe3a52" + "cacheKey": "1676f17c26dd7d6bee804d444aaf37b2d43a65a0f7d8c985e75d965aaa7566ca" }, { "packageName": "nginx", "manifestSha256": "41b0aaf05fdd0d8da42642703f9e60fd7b9545d8029896918b87c771c359147b", - "cacheKey": "6a92877569b9bf8dabd5333d3f0d5e96388379e28ec00f4d53e0ee675cb58eae" + "cacheKey": "afde4ab291d7b9ed1014f24d6c7ca300951a290c8458c4fb0bfa0fd88270853c" }, { "packageName": "openssl", "manifestSha256": "374f5dce6b2691b630b55cb82955b33d2305384e4bd64746c581621bb0793993", - "cacheKey": "bdef227990c15092dfb24dc2abfcb8cd2699b409bef100c9d69a62c770b3b950" + "cacheKey": "d497113aa10b49502c0a3eda046a2800793279477bbb5dc4b0897f5a862d06d7" }, { "packageName": "pcre2-source", @@ -1165,22 +1165,22 @@ { "packageName": "php", "manifestSha256": "fcd9d7915ed8935b05418e148edcb312adf0b1e4e627fe4aab75444b35b33bbc", - "cacheKey": "c5e70ff8e8ab519b31810e4dac16a7500033f549395152f7b00b239d6170acf6" + "cacheKey": "e0bae805a3ed026870ab3dd18f0dca768894bc36d9764ac55f4fc3aa83caaea3" }, { "packageName": "shell", "manifestSha256": "107adeee07fd0a10d9f2bb40a0cd84a6144bff91e7505b5ae9afeabaa000beb9", - "cacheKey": "5be1bf60e0954662e5973417c42baa3ba028c4260f41e8e690a927be981fa0f1" + "cacheKey": "53498ffe143171264f70b0c60a20e6811a144d9ef5d39bb6c7b92184b3e5e6af" }, { "packageName": "sqlite", "manifestSha256": "ab9560b4eab53445034e2a87a27ccee9d05ddb0db73fee28f5000325b0b1cba5", - "cacheKey": "a21f509748a9f5a086db9ae7bb2341a3c57171c954b31a3982b2b47a98794694" + "cacheKey": "35544fa8d8d50cd23c78606836caf96f7636cf20360d1cdcf503fcdc1f06c400" }, { "packageName": "zlib", "manifestSha256": "d4f4701c0c30dc843ebe2111ed3e2284836cdfc4bc05a130c8fab4f31b8e6357", - "cacheKey": "a826c5a1662265850d13e48202fda10c240400126b9fcc0ca33ee0f57e5969f1" + "cacheKey": "f372595b0d4dc5d079fde6f5a3d0401d8b0e8e35a87221df9f51a607efd15fdb" } ] }, @@ -1200,7 +1200,7 @@ "wasm32" ], "cacheKeys": { - "wasm32": "9a509fe274611ce618bb65afde230b2e526adfed9f908dd286d63749bf924547" + "wasm32": "e6f4f1d981f02ac3682fe5785aaab591194d957514da0d335d62521fc9876a0e" }, "dependencyClosures": { "wasm32": [] @@ -1221,7 +1221,7 @@ "wasm32" ], "cacheKeys": { - "wasm32": "3660780917e27528d76a83fb1ae76f3a547fe348cd1ba979bc9f015a03c1213b" + "wasm32": "9fd429eb76997413c78204f5814b80a5a67388b5db80a21277f0fc1bf74b09c2" }, "dependencyClosures": { "wasm32": [] @@ -1242,7 +1242,7 @@ "wasm32" ], "cacheKeys": { - "wasm32": "a7063dcf8ce5bfe288b1520beac1eaf1d19f736fbdd683f344d83d74dcbab727" + "wasm32": "0b20308410ffac3629eacbc4c68603033ae1fce51bfaf356965672779cd5639f" }, "dependencyClosures": { "wasm32": [] @@ -1263,7 +1263,7 @@ "wasm32" ], "cacheKeys": { - "wasm32": "49a0f581bbc70db8db660c144bbaf5683ea30f36f305c434c5e4d772c334723e" + "wasm32": "d30ffaed8cdf6c3d2840c726ac592a55c8314162e955a536898b0ae54d3fdfb5" }, "dependencyClosures": { "wasm32": [] @@ -1285,15 +1285,15 @@ "wasm64" ], "cacheKeys": { - "wasm32": "769572acd89a8fa7738058f6c2f48e5270868d1cc1422c4a0610b100cf851f77", - "wasm64": "815139d5343320f5a8ca5aeaff9421abb2ff6e568b13a90c2e5cc341e2ff5aa5" + "wasm32": "0ab88ecb5225970388e87bfc6ad3eb683947ca6372961af75e5e77d990a15468", + "wasm64": "d36411979372f848892e912f13d057157b9cc438647cc7f3ca16abce3a056369" }, "dependencyClosures": { "wasm32": [ { "packageName": "libcxx", "manifestSha256": "6e1ebfa6914043770eae2a6c156b915acf6a56fe763f07d215ffe6f6f2305007", - "cacheKey": "53612e1c46d4ce4e8d07b911f753fda620b8b8597c38c3775fc1769f8a63ba89" + "cacheKey": "0b9e4ae7831b78deebd709b0e12cd30cca8381d6840664543ec11ba7e11a88ce" }, { "packageName": "pcre2-source", @@ -1305,7 +1305,7 @@ { "packageName": "libcxx", "manifestSha256": "6e1ebfa6914043770eae2a6c156b915acf6a56fe763f07d215ffe6f6f2305007", - "cacheKey": "986b49777db0287089ed8b493606cb6ef82f97992ab3d11d5f054d8fb239707f" + "cacheKey": "b1e809860de54f8bede2f335b88451d798b046cf1e7909ae47f47ebb38677994" }, { "packageName": "pcre2-source", @@ -1337,34 +1337,34 @@ "wasm32" ], "cacheKeys": { - "wasm32": "b70268f40e5f5e45c040132373266ddee6ad38fab897a60afcc47ea12ad0cf2a" + "wasm32": "52fddeeb4fd4afd17733bfbcd3d50f6dfefdc3f505d45f91a91ab36d3fb3ff1f" }, "dependencyClosures": { "wasm32": [ { "packageName": "coreutils", "manifestSha256": "b8baabc9af9283434e0d80f7fdc0aaef242ffa31092ab0fc39a0603a12b3cca7", - "cacheKey": "33ca2b6c08b61b6ba3b013b13271c98b9b63d75f0a5c47d6d265c27c10c50458" + "cacheKey": "640ffdd14b7ecbe96a0632eeb7dd47c2606cd195bdca24ba8a1b2395e51e2c93" }, { "packageName": "dash", "manifestSha256": "10ce1bb611fefd78a6c9ec9038f6113e29a7062b1a73e08eccf9b33004cea9a8", - "cacheKey": "2ec3ac4e8baa4bfb4ced00ff411feb0ae28e2acf3b5709ac7c042c5a57295aa6" + "cacheKey": "e6a122a8bb1ea383cf8917b15c78dc4e148af24a391f61634117ec2c8d13deb9" }, { "packageName": "dinit", "manifestSha256": "44e125e1337503e27f8cff531cdbd79c4f406fd2d282b916a7746cf3a14057bf", - "cacheKey": "43651e927c2672e0a4f749a5cce47db470dd19d72736ea4c6b6a6b3d55d9f7ab" + "cacheKey": "459ad0cd459ea1873d4dce5c19279d1c5797457dfd0ddfaf2b575e25170fea80" }, { "packageName": "libcxx", "manifestSha256": "6e1ebfa6914043770eae2a6c156b915acf6a56fe763f07d215ffe6f6f2305007", - "cacheKey": "53612e1c46d4ce4e8d07b911f753fda620b8b8597c38c3775fc1769f8a63ba89" + "cacheKey": "0b9e4ae7831b78deebd709b0e12cd30cca8381d6840664543ec11ba7e11a88ce" }, { "packageName": "mariadb", "manifestSha256": "aeb221be233e4b57bccb4b3375020f94c3ec593dac2734631f00e3b1aec9f48c", - "cacheKey": "769572acd89a8fa7738058f6c2f48e5270868d1cc1422c4a0610b100cf851f77" + "cacheKey": "0ab88ecb5225970388e87bfc6ad3eb683947ca6372961af75e5e77d990a15468" }, { "packageName": "pcre2-source", @@ -1390,35 +1390,35 @@ "wasm64" ], "cacheKeys": { - "wasm32": "314fcb39160d1792c099471f02523816ec0f539f2085d425a04b4862b4515d43", - "wasm64": "30f2b70fb885278907dfc2a47d287d5ebe4ef61663f46c9ae6546401bedebe03" + "wasm32": "efa6b13177fed5b6bbc7dfa03097ca734745e1f323f96071161538a76ad06923", + "wasm64": "e63163f3e0082080070ddeff98d6d2b80778e26c3b35268f452ad19b0f014522" }, "dependencyClosures": { "wasm32": [ { "packageName": "coreutils", "manifestSha256": "b8baabc9af9283434e0d80f7fdc0aaef242ffa31092ab0fc39a0603a12b3cca7", - "cacheKey": "33ca2b6c08b61b6ba3b013b13271c98b9b63d75f0a5c47d6d265c27c10c50458" + "cacheKey": "640ffdd14b7ecbe96a0632eeb7dd47c2606cd195bdca24ba8a1b2395e51e2c93" }, { "packageName": "dash", "manifestSha256": "10ce1bb611fefd78a6c9ec9038f6113e29a7062b1a73e08eccf9b33004cea9a8", - "cacheKey": "2ec3ac4e8baa4bfb4ced00ff411feb0ae28e2acf3b5709ac7c042c5a57295aa6" + "cacheKey": "e6a122a8bb1ea383cf8917b15c78dc4e148af24a391f61634117ec2c8d13deb9" }, { "packageName": "dinit", "manifestSha256": "44e125e1337503e27f8cff531cdbd79c4f406fd2d282b916a7746cf3a14057bf", - "cacheKey": "43651e927c2672e0a4f749a5cce47db470dd19d72736ea4c6b6a6b3d55d9f7ab" + "cacheKey": "459ad0cd459ea1873d4dce5c19279d1c5797457dfd0ddfaf2b575e25170fea80" }, { "packageName": "libcxx", "manifestSha256": "6e1ebfa6914043770eae2a6c156b915acf6a56fe763f07d215ffe6f6f2305007", - "cacheKey": "53612e1c46d4ce4e8d07b911f753fda620b8b8597c38c3775fc1769f8a63ba89" + "cacheKey": "0b9e4ae7831b78deebd709b0e12cd30cca8381d6840664543ec11ba7e11a88ce" }, { "packageName": "mariadb", "manifestSha256": "aeb221be233e4b57bccb4b3375020f94c3ec593dac2734631f00e3b1aec9f48c", - "cacheKey": "769572acd89a8fa7738058f6c2f48e5270868d1cc1422c4a0610b100cf851f77" + "cacheKey": "0ab88ecb5225970388e87bfc6ad3eb683947ca6372961af75e5e77d990a15468" }, { "packageName": "pcre2-source", @@ -1430,27 +1430,27 @@ { "packageName": "coreutils", "manifestSha256": "b8baabc9af9283434e0d80f7fdc0aaef242ffa31092ab0fc39a0603a12b3cca7", - "cacheKey": "67ffde57ec940c7107d901231ed2d1c886957edc34de1c60c08094d1260ae797" + "cacheKey": "afb0ed522f8e36ff024d7fc0833da00cb10e538718dd8179ab2223340dd59efa" }, { "packageName": "dash", "manifestSha256": "10ce1bb611fefd78a6c9ec9038f6113e29a7062b1a73e08eccf9b33004cea9a8", - "cacheKey": "55133070386497aa605c431cf06777d717e4c4ececf2983cc507b858c4b529cd" + "cacheKey": "52d1a589d726fcf1382bdac637206cb94a553338aad8e90648615cd9e32cd688" }, { "packageName": "dinit", "manifestSha256": "44e125e1337503e27f8cff531cdbd79c4f406fd2d282b916a7746cf3a14057bf", - "cacheKey": "bbcd2d163560d0271c38581dae6f292cc4a119fc8f86af85552310157bef511a" + "cacheKey": "b8645ab412a8456dc389eab0eaa657ba592ab415abce32e2e6f3a7aade9ff32b" }, { "packageName": "libcxx", "manifestSha256": "6e1ebfa6914043770eae2a6c156b915acf6a56fe763f07d215ffe6f6f2305007", - "cacheKey": "986b49777db0287089ed8b493606cb6ef82f97992ab3d11d5f054d8fb239707f" + "cacheKey": "b1e809860de54f8bede2f335b88451d798b046cf1e7909ae47f47ebb38677994" }, { "packageName": "mariadb", "manifestSha256": "aeb221be233e4b57bccb4b3375020f94c3ec593dac2734631f00e3b1aec9f48c", - "cacheKey": "815139d5343320f5a8ca5aeaff9421abb2ff6e568b13a90c2e5cc341e2ff5aa5" + "cacheKey": "d36411979372f848892e912f13d057157b9cc438647cc7f3ca16abce3a056369" }, { "packageName": "pcre2-source", @@ -1475,7 +1475,7 @@ "wasm32" ], "cacheKeys": { - "wasm32": "e61f328544530bf45ff0a339e0ed552c7c235bab8fa104e79c297ff9cea0d96e" + "wasm32": "b1d459dd4751d91c6ffe8d51584eb8ed1e80fe4d0fd3b8a968b1ae705d1d504a" }, "dependencyClosures": { "wasm32": [] @@ -1496,7 +1496,7 @@ "wasm32" ], "cacheKeys": { - "wasm32": "82b32c29b05f3854bebfb414875d603fe66829ab79b0ea02b4a1d00e09fe3a52" + "wasm32": "1676f17c26dd7d6bee804d444aaf37b2d43a65a0f7d8c985e75d965aaa7566ca" }, "dependencyClosures": { "wasm32": [] @@ -1517,7 +1517,7 @@ "wasm32" ], "cacheKeys": { - "wasm32": "ee0b8eacc8659f5d690f5b3ee419584ae24403fe6b8e955e01e5192e6a73b635" + "wasm32": "5fb7f6651d97a643ce8c0bb06e1fb7ce45ee670ca988cc46dc15d830f6da150d" }, "dependencyClosures": { "wasm32": [] @@ -1538,7 +1538,7 @@ "wasm32" ], "cacheKeys": { - "wasm32": "1d43a87fe208ed807f1e71640e4fcf7c4417699c318780e49a56c58d37e54ebb" + "wasm32": "2a8d1a03af1e6866e6da9368cbf163ae6fbed7a0f641f9005fa2baa118b97422" }, "dependencyClosures": { "wasm32": [] @@ -1622,7 +1622,7 @@ "wasm32" ], "cacheKeys": { - "wasm32": "e1f8dbb4e078921b75b109603c6b67a8035b3301659af937960841e563db2151" + "wasm32": "edcf82d4ca910e21da6b53ea843635207e965f86c684da632ebd832dc616eafd" }, "dependencyClosures": { "wasm32": [] @@ -1643,14 +1643,14 @@ "wasm32" ], "cacheKeys": { - "wasm32": "0acaa903f47da0a7730a75b9296217ce4723c5d4a53bff772ae18a1f0acc190f" + "wasm32": "0262e86e8a61a49c25bbfe1c49cdc3ea0d068e5914ea9843aa0f5561acb15186" }, "dependencyClosures": { "wasm32": [ { "packageName": "ncurses", "manifestSha256": "0a1c180eeade627e204aa47065dbfebfcdc9359d25e2db436932c7fa3a86fc1e", - "cacheKey": "1d43a87fe208ed807f1e71640e4fcf7c4417699c318780e49a56c58d37e54ebb" + "cacheKey": "2a8d1a03af1e6866e6da9368cbf163ae6fbed7a0f641f9005fa2baa118b97422" } ] }, @@ -1670,19 +1670,19 @@ "wasm32" ], "cacheKeys": { - "wasm32": "a02b8fc8563203eab00795dfd2474d85769857ece24d9ce0e818c7bd8c56bca5" + "wasm32": "bd33eda7a841d8fcb82ba527a529cab87f7316cdae428c13381b7687be85a744" }, "dependencyClosures": { "wasm32": [ { "packageName": "ncurses", "manifestSha256": "0a1c180eeade627e204aa47065dbfebfcdc9359d25e2db436932c7fa3a86fc1e", - "cacheKey": "1d43a87fe208ed807f1e71640e4fcf7c4417699c318780e49a56c58d37e54ebb" + "cacheKey": "2a8d1a03af1e6866e6da9368cbf163ae6fbed7a0f641f9005fa2baa118b97422" }, { "packageName": "nethack", "manifestSha256": "1a2f12ec2770bd8d40006d6c878a3493b97c71c2bb63ad08c7f6ad99bfd53504", - "cacheKey": "0acaa903f47da0a7730a75b9296217ce4723c5d4a53bff772ae18a1f0acc190f" + "cacheKey": "0262e86e8a61a49c25bbfe1c49cdc3ea0d068e5914ea9843aa0f5561acb15186" } ] }, @@ -1702,7 +1702,7 @@ "wasm32" ], "cacheKeys": { - "wasm32": "6a92877569b9bf8dabd5333d3f0d5e96388379e28ec00f4d53e0ee675cb58eae" + "wasm32": "afde4ab291d7b9ed1014f24d6c7ca300951a290c8458c4fb0bfa0fd88270853c" }, "dependencyClosures": { "wasm32": [] @@ -1723,84 +1723,84 @@ "wasm32" ], "cacheKeys": { - "wasm32": "772ed9fc96057a9c9d80840c4983aed19e0557a8e40003ba4fe96e5a93808e2e" + "wasm32": "7c5622d5665de78cdc2dcc528963a56685a441c19d201bd05f8a3f9b853c92f8" }, "dependencyClosures": { "wasm32": [ { "packageName": "dinit", "manifestSha256": "44e125e1337503e27f8cff531cdbd79c4f406fd2d282b916a7746cf3a14057bf", - "cacheKey": "43651e927c2672e0a4f749a5cce47db470dd19d72736ea4c6b6a6b3d55d9f7ab" + "cacheKey": "459ad0cd459ea1873d4dce5c19279d1c5797457dfd0ddfaf2b575e25170fea80" }, { "packageName": "homebrew-bootstrap", "manifestSha256": "b171060c86cb6642e8cadef9d2698e671c6bf188868c7d835b3e8ecf3a3b054a", - "cacheKey": "8230aaca4bfc1f7bff80d2830534151afff6edb2c03f640af4afa385d145426d" + "cacheKey": "3ca8b809f79e6cf3432a36a881e33098fff06edb64e1a1c1ce16c24f0d52115b" }, { "packageName": "icu", "manifestSha256": "f5b1f02d169ec1108fc32d9da55be53c3efb96e3a7aeba6a03aec62e0175f0dc", - "cacheKey": "48a3ee79c8510a1fe933281741cb0fb56f2a67fa7dba9d2cd18c0085cf49f375" + "cacheKey": "3785cb4f88b3800ceb98a05d72cf21a82221711b31b413b6d0a18a8962db522f" }, { "packageName": "kernel", "manifestSha256": "db1d66db8575562ac7b3e720dbaa06d7dd5eef577d80220ea4167dc2d69b863b", - "cacheKey": "f864c4947733ce7a9c1bd6782038da97680db641c03b7696db690dac065e6436" + "cacheKey": "8b523b3920ab2566957d0bb1f14639e8555990218cf8dffe5388175ab4732518" }, { "packageName": "libcurl", "manifestSha256": "c771e6cdc83b43840db4dd1fdce4b6189ba4f281596f7485bcda9c65e2ecda00", - "cacheKey": "8273da44c60d3edb2032e3566040ed8fd07c2d3d5e43bfd4bbe94ff7ec0b95c6" + "cacheKey": "2e9bfe5431ec06c510c8f31c6a936d95f604294cb6f7cb6b5a666fa46e3b5202" }, { "packageName": "libcxx", "manifestSha256": "6e1ebfa6914043770eae2a6c156b915acf6a56fe763f07d215ffe6f6f2305007", - "cacheKey": "53612e1c46d4ce4e8d07b911f753fda620b8b8597c38c3775fc1769f8a63ba89" + "cacheKey": "0b9e4ae7831b78deebd709b0e12cd30cca8381d6840664543ec11ba7e11a88ce" }, { "packageName": "libiconv", "manifestSha256": "fa60b386f4ba0cd5bbbd595f67477cbfb0cc401a14bbd4769630c502bb3f24b1", - "cacheKey": "b494e26a4bd579392ae78f894f37f1d7b7ce87f0401f9d53edbf982b0753f76b" + "cacheKey": "2ae7c3ce68bb2b52fb722df897584d2501ac4146946692f3f8b08901ed4e2cef" }, { "packageName": "libxml2", "manifestSha256": "908490efcdc7783c76ce36af31d1db24bfcc7ba5f1e7b9b5298cc81cb110256c", - "cacheKey": "f8e7709dc6d6606b0d5ad24b1bb2f7aece6df4e171f92dfe7ea344713c14d4e8" + "cacheKey": "4c2d44bb43f74f3160efe3089232b173c6425c1a996b29b41f2e61df26d1df25" }, { "packageName": "libzip", "manifestSha256": "83c65d63a6416e79436bf4e792759a4701d568df0e8c010394a9732cf659f0cc", - "cacheKey": "0e020e8d1c342ccf5dac59079aab2b2ec35b083b03107ac172cc0f9ac97a13ec" + "cacheKey": "592acc36c12b140841b44149254872b4a6915c5e8ec65592f58bfc7cd90a1ec9" }, { "packageName": "nginx", "manifestSha256": "41b0aaf05fdd0d8da42642703f9e60fd7b9545d8029896918b87c771c359147b", - "cacheKey": "6a92877569b9bf8dabd5333d3f0d5e96388379e28ec00f4d53e0ee675cb58eae" + "cacheKey": "afde4ab291d7b9ed1014f24d6c7ca300951a290c8458c4fb0bfa0fd88270853c" }, { "packageName": "openssl", "manifestSha256": "374f5dce6b2691b630b55cb82955b33d2305384e4bd64746c581621bb0793993", - "cacheKey": "bdef227990c15092dfb24dc2abfcb8cd2699b409bef100c9d69a62c770b3b950" + "cacheKey": "d497113aa10b49502c0a3eda046a2800793279477bbb5dc4b0897f5a862d06d7" }, { "packageName": "php", "manifestSha256": "fcd9d7915ed8935b05418e148edcb312adf0b1e4e627fe4aab75444b35b33bbc", - "cacheKey": "c5e70ff8e8ab519b31810e4dac16a7500033f549395152f7b00b239d6170acf6" + "cacheKey": "e0bae805a3ed026870ab3dd18f0dca768894bc36d9764ac55f4fc3aa83caaea3" }, { "packageName": "shell", "manifestSha256": "107adeee07fd0a10d9f2bb40a0cd84a6144bff91e7505b5ae9afeabaa000beb9", - "cacheKey": "5be1bf60e0954662e5973417c42baa3ba028c4260f41e8e690a927be981fa0f1" + "cacheKey": "53498ffe143171264f70b0c60a20e6811a144d9ef5d39bb6c7b92184b3e5e6af" }, { "packageName": "sqlite", "manifestSha256": "ab9560b4eab53445034e2a87a27ccee9d05ddb0db73fee28f5000325b0b1cba5", - "cacheKey": "a21f509748a9f5a086db9ae7bb2341a3c57171c954b31a3982b2b47a98794694" + "cacheKey": "35544fa8d8d50cd23c78606836caf96f7636cf20360d1cdcf503fcdc1f06c400" }, { "packageName": "zlib", "manifestSha256": "d4f4701c0c30dc843ebe2111ed3e2284836cdfc4bc05a130c8fab4f31b8e6357", - "cacheKey": "a826c5a1662265850d13e48202fda10c240400126b9fcc0ca33ee0f57e5969f1" + "cacheKey": "f372595b0d4dc5d079fde6f5a3d0401d8b0e8e35a87221df9f51a607efd15fdb" } ] }, @@ -1820,34 +1820,34 @@ "wasm32" ], "cacheKeys": { - "wasm32": "40b4988e6e280e717ba3feb8353f76bbc6992598ce2e8cbf0bf53f14b9cdb4cf" + "wasm32": "b3d0854b1f0ef76b68068120cdc6212ce2e440a6fb040ca6e27645416ccfd92e" }, "dependencyClosures": { "wasm32": [ { "packageName": "dinit", "manifestSha256": "44e125e1337503e27f8cff531cdbd79c4f406fd2d282b916a7746cf3a14057bf", - "cacheKey": "43651e927c2672e0a4f749a5cce47db470dd19d72736ea4c6b6a6b3d55d9f7ab" + "cacheKey": "459ad0cd459ea1873d4dce5c19279d1c5797457dfd0ddfaf2b575e25170fea80" }, { "packageName": "homebrew-bootstrap", "manifestSha256": "b171060c86cb6642e8cadef9d2698e671c6bf188868c7d835b3e8ecf3a3b054a", - "cacheKey": "8230aaca4bfc1f7bff80d2830534151afff6edb2c03f640af4afa385d145426d" + "cacheKey": "3ca8b809f79e6cf3432a36a881e33098fff06edb64e1a1c1ce16c24f0d52115b" }, { "packageName": "libcxx", "manifestSha256": "6e1ebfa6914043770eae2a6c156b915acf6a56fe763f07d215ffe6f6f2305007", - "cacheKey": "53612e1c46d4ce4e8d07b911f753fda620b8b8597c38c3775fc1769f8a63ba89" + "cacheKey": "0b9e4ae7831b78deebd709b0e12cd30cca8381d6840664543ec11ba7e11a88ce" }, { "packageName": "nginx", "manifestSha256": "41b0aaf05fdd0d8da42642703f9e60fd7b9545d8029896918b87c771c359147b", - "cacheKey": "6a92877569b9bf8dabd5333d3f0d5e96388379e28ec00f4d53e0ee675cb58eae" + "cacheKey": "afde4ab291d7b9ed1014f24d6c7ca300951a290c8458c4fb0bfa0fd88270853c" }, { "packageName": "shell", "manifestSha256": "107adeee07fd0a10d9f2bb40a0cd84a6144bff91e7505b5ae9afeabaa000beb9", - "cacheKey": "5be1bf60e0954662e5973417c42baa3ba028c4260f41e8e690a927be981fa0f1" + "cacheKey": "53498ffe143171264f70b0c60a20e6811a144d9ef5d39bb6c7b92184b3e5e6af" } ] }, @@ -1867,29 +1867,29 @@ "wasm32" ], "cacheKeys": { - "wasm32": "62a1b6d2516b3676a069b611e6b0af454ffe26013151b8c71e7168ec31a7fa5d" + "wasm32": "eefc841b11ba61a0a126322c01f777575bf1d5d1bdf9a4527460c6c5a60b4472" }, "dependencyClosures": { "wasm32": [ { "packageName": "libcxx", "manifestSha256": "6e1ebfa6914043770eae2a6c156b915acf6a56fe763f07d215ffe6f6f2305007", - "cacheKey": "53612e1c46d4ce4e8d07b911f753fda620b8b8597c38c3775fc1769f8a63ba89" + "cacheKey": "0b9e4ae7831b78deebd709b0e12cd30cca8381d6840664543ec11ba7e11a88ce" }, { "packageName": "openssl", "manifestSha256": "374f5dce6b2691b630b55cb82955b33d2305384e4bd64746c581621bb0793993", - "cacheKey": "bdef227990c15092dfb24dc2abfcb8cd2699b409bef100c9d69a62c770b3b950" + "cacheKey": "d497113aa10b49502c0a3eda046a2800793279477bbb5dc4b0897f5a862d06d7" }, { "packageName": "spidermonkey", "manifestSha256": "c72ef4e43ee2a4fc74390d03a239fce3b545a6d3b64f02ccc4d44972cec626bd", - "cacheKey": "2c2b1f5959b20978e10725bc42cf1d3e0c48a7c699f3b3d564d843c71b8b1ea7" + "cacheKey": "313e60b92a277a65ebba3d4e33831978477a4618baed284467b485bca9baa689" }, { "packageName": "zlib", "manifestSha256": "d4f4701c0c30dc843ebe2111ed3e2284836cdfc4bc05a130c8fab4f31b8e6357", - "cacheKey": "a826c5a1662265850d13e48202fda10c240400126b9fcc0ca33ee0f57e5969f1" + "cacheKey": "f372595b0d4dc5d079fde6f5a3d0401d8b0e8e35a87221df9f51a607efd15fdb" } ] }, @@ -1909,44 +1909,44 @@ "wasm32" ], "cacheKeys": { - "wasm32": "e7b886d3361bfa9a2628c648633cce9960faab811cfcc1829e85ba2a61897adc" + "wasm32": "e688564dbefe46d48eaca9d9cb8cf9de7c3723704e1abb47b858f36aecd2560c" }, "dependencyClosures": { "wasm32": [ { "packageName": "homebrew-bootstrap", "manifestSha256": "b171060c86cb6642e8cadef9d2698e671c6bf188868c7d835b3e8ecf3a3b054a", - "cacheKey": "8230aaca4bfc1f7bff80d2830534151afff6edb2c03f640af4afa385d145426d" + "cacheKey": "3ca8b809f79e6cf3432a36a881e33098fff06edb64e1a1c1ce16c24f0d52115b" }, { "packageName": "libcxx", "manifestSha256": "6e1ebfa6914043770eae2a6c156b915acf6a56fe763f07d215ffe6f6f2305007", - "cacheKey": "53612e1c46d4ce4e8d07b911f753fda620b8b8597c38c3775fc1769f8a63ba89" + "cacheKey": "0b9e4ae7831b78deebd709b0e12cd30cca8381d6840664543ec11ba7e11a88ce" }, { "packageName": "node", "manifestSha256": "2131a24dfda8587860e57fc65b573086477d376b065b3b9ca4e1dfe4d79f5790", - "cacheKey": "62a1b6d2516b3676a069b611e6b0af454ffe26013151b8c71e7168ec31a7fa5d" + "cacheKey": "eefc841b11ba61a0a126322c01f777575bf1d5d1bdf9a4527460c6c5a60b4472" }, { "packageName": "openssl", "manifestSha256": "374f5dce6b2691b630b55cb82955b33d2305384e4bd64746c581621bb0793993", - "cacheKey": "bdef227990c15092dfb24dc2abfcb8cd2699b409bef100c9d69a62c770b3b950" + "cacheKey": "d497113aa10b49502c0a3eda046a2800793279477bbb5dc4b0897f5a862d06d7" }, { "packageName": "shell", "manifestSha256": "107adeee07fd0a10d9f2bb40a0cd84a6144bff91e7505b5ae9afeabaa000beb9", - "cacheKey": "5be1bf60e0954662e5973417c42baa3ba028c4260f41e8e690a927be981fa0f1" + "cacheKey": "53498ffe143171264f70b0c60a20e6811a144d9ef5d39bb6c7b92184b3e5e6af" }, { "packageName": "spidermonkey", "manifestSha256": "c72ef4e43ee2a4fc74390d03a239fce3b545a6d3b64f02ccc4d44972cec626bd", - "cacheKey": "2c2b1f5959b20978e10725bc42cf1d3e0c48a7c699f3b3d564d843c71b8b1ea7" + "cacheKey": "313e60b92a277a65ebba3d4e33831978477a4618baed284467b485bca9baa689" }, { "packageName": "zlib", "manifestSha256": "d4f4701c0c30dc843ebe2111ed3e2284836cdfc4bc05a130c8fab4f31b8e6357", - "cacheKey": "a826c5a1662265850d13e48202fda10c240400126b9fcc0ca33ee0f57e5969f1" + "cacheKey": "f372595b0d4dc5d079fde6f5a3d0401d8b0e8e35a87221df9f51a607efd15fdb" } ] }, @@ -1966,7 +1966,7 @@ "wasm32" ], "cacheKeys": { - "wasm32": "2d820d399d95ba7f9f3c8935fba1d869142fbfc09667b8149dcbb33363aa4785" + "wasm32": "0d664e5463e94a64001765aba8a432d631e85d88e3bbb5a94e1a6d1cc0e2f3c7" }, "dependencyClosures": { "wasm32": [] @@ -1987,14 +1987,14 @@ "wasm32" ], "cacheKeys": { - "wasm32": "41870476e5efbf57f60c4fdcc190e178360fa0610adf2bf6ecf06df24aa393f7" + "wasm32": "be145f3ba65afd8ba907de3cf51221a7f67fcc3136b81101b13cce0390b7c09a" }, "dependencyClosures": { "wasm32": [ { "packageName": "perl", "manifestSha256": "6cdc4dbc54d0e4008cff41f82ec8918c0e44b4aef23903539c1bc0f538fe3ad2", - "cacheKey": "2d820d399d95ba7f9f3c8935fba1d869142fbfc09667b8149dcbb33363aa4785" + "cacheKey": "0d664e5463e94a64001765aba8a432d631e85d88e3bbb5a94e1a6d1cc0e2f3c7" } ] }, @@ -2014,54 +2014,54 @@ "wasm32" ], "cacheKeys": { - "wasm32": "c5e70ff8e8ab519b31810e4dac16a7500033f549395152f7b00b239d6170acf6" + "wasm32": "e0bae805a3ed026870ab3dd18f0dca768894bc36d9764ac55f4fc3aa83caaea3" }, "dependencyClosures": { "wasm32": [ { "packageName": "icu", "manifestSha256": "f5b1f02d169ec1108fc32d9da55be53c3efb96e3a7aeba6a03aec62e0175f0dc", - "cacheKey": "48a3ee79c8510a1fe933281741cb0fb56f2a67fa7dba9d2cd18c0085cf49f375" + "cacheKey": "3785cb4f88b3800ceb98a05d72cf21a82221711b31b413b6d0a18a8962db522f" }, { "packageName": "libcurl", "manifestSha256": "c771e6cdc83b43840db4dd1fdce4b6189ba4f281596f7485bcda9c65e2ecda00", - "cacheKey": "8273da44c60d3edb2032e3566040ed8fd07c2d3d5e43bfd4bbe94ff7ec0b95c6" + "cacheKey": "2e9bfe5431ec06c510c8f31c6a936d95f604294cb6f7cb6b5a666fa46e3b5202" }, { "packageName": "libcxx", "manifestSha256": "6e1ebfa6914043770eae2a6c156b915acf6a56fe763f07d215ffe6f6f2305007", - "cacheKey": "53612e1c46d4ce4e8d07b911f753fda620b8b8597c38c3775fc1769f8a63ba89" + "cacheKey": "0b9e4ae7831b78deebd709b0e12cd30cca8381d6840664543ec11ba7e11a88ce" }, { "packageName": "libiconv", "manifestSha256": "fa60b386f4ba0cd5bbbd595f67477cbfb0cc401a14bbd4769630c502bb3f24b1", - "cacheKey": "b494e26a4bd579392ae78f894f37f1d7b7ce87f0401f9d53edbf982b0753f76b" + "cacheKey": "2ae7c3ce68bb2b52fb722df897584d2501ac4146946692f3f8b08901ed4e2cef" }, { "packageName": "libxml2", "manifestSha256": "908490efcdc7783c76ce36af31d1db24bfcc7ba5f1e7b9b5298cc81cb110256c", - "cacheKey": "f8e7709dc6d6606b0d5ad24b1bb2f7aece6df4e171f92dfe7ea344713c14d4e8" + "cacheKey": "4c2d44bb43f74f3160efe3089232b173c6425c1a996b29b41f2e61df26d1df25" }, { "packageName": "libzip", "manifestSha256": "83c65d63a6416e79436bf4e792759a4701d568df0e8c010394a9732cf659f0cc", - "cacheKey": "0e020e8d1c342ccf5dac59079aab2b2ec35b083b03107ac172cc0f9ac97a13ec" + "cacheKey": "592acc36c12b140841b44149254872b4a6915c5e8ec65592f58bfc7cd90a1ec9" }, { "packageName": "openssl", "manifestSha256": "374f5dce6b2691b630b55cb82955b33d2305384e4bd64746c581621bb0793993", - "cacheKey": "bdef227990c15092dfb24dc2abfcb8cd2699b409bef100c9d69a62c770b3b950" + "cacheKey": "d497113aa10b49502c0a3eda046a2800793279477bbb5dc4b0897f5a862d06d7" }, { "packageName": "sqlite", "manifestSha256": "ab9560b4eab53445034e2a87a27ccee9d05ddb0db73fee28f5000325b0b1cba5", - "cacheKey": "a21f509748a9f5a086db9ae7bb2341a3c57171c954b31a3982b2b47a98794694" + "cacheKey": "35544fa8d8d50cd23c78606836caf96f7636cf20360d1cdcf503fcdc1f06c400" }, { "packageName": "zlib", "manifestSha256": "d4f4701c0c30dc843ebe2111ed3e2284836cdfc4bc05a130c8fab4f31b8e6357", - "cacheKey": "a826c5a1662265850d13e48202fda10c240400126b9fcc0ca33ee0f57e5969f1" + "cacheKey": "f372595b0d4dc5d079fde6f5a3d0401d8b0e8e35a87221df9f51a607efd15fdb" } ] }, @@ -2137,7 +2137,7 @@ "wasm32" ], "cacheKeys": { - "wasm32": "2308c7f1868ea3e052209430626f792e07a168286c45893724195d4709c60220" + "wasm32": "7446b2dc9dec882ec9b99433dbc10917b9773b615b45b39faec6060b9bc0dc5b" }, "dependencyClosures": { "wasm32": [] @@ -2410,19 +2410,19 @@ "wasm32" ], "cacheKeys": { - "wasm32": "492e7349b34a9b8d9ae873902fd6b4b750607e999d059fb011715dde52144c2c" + "wasm32": "43cac1e1fb84c1c13838f785d2ee054a9704e83a161bfcdae5a297ea905aaa43" }, "dependencyClosures": { "wasm32": [ { "packageName": "cpython", "manifestSha256": "7dd4f446697a73941ec940c2ccba4d53be73fe6947f1e701031a4d0aa964c4ff", - "cacheKey": "3395181397427732850ece33d0fd3edf875903acb625e59a348577f2613d70f6" + "cacheKey": "6bb296c95c4b5f1ad47d5fae5b79bdbdd283b17136893c61d8fd8d8a9121dcc1" }, { "packageName": "zlib", "manifestSha256": "d4f4701c0c30dc843ebe2111ed3e2284836cdfc4bc05a130c8fab4f31b8e6357", - "cacheKey": "a826c5a1662265850d13e48202fda10c240400126b9fcc0ca33ee0f57e5969f1" + "cacheKey": "f372595b0d4dc5d079fde6f5a3d0401d8b0e8e35a87221df9f51a607efd15fdb" } ] }, @@ -2442,7 +2442,7 @@ "wasm32" ], "cacheKeys": { - "wasm32": "da7392e43962a0288ff0a9958190215ee01476e1d235f4275b790507ce5e4715" + "wasm32": "1fa5f605040b656ea3270f8807f3e38460a9c331cb1624b5d8bb5491786ffa64" }, "dependencyClosures": { "wasm32": [] @@ -2470,24 +2470,24 @@ "wasm32" ], "cacheKeys": { - "wasm32": "480f32952a8f7be451762b6012f9312ca4b1dad0b04d93c387f52427d660daa5" + "wasm32": "4178da00cb2ea56b789da8cefe6edf33e5f0b524f64134c3fd9829ead4cd8e1a" }, "dependencyClosures": { "wasm32": [ { "packageName": "dinit", "manifestSha256": "44e125e1337503e27f8cff531cdbd79c4f406fd2d282b916a7746cf3a14057bf", - "cacheKey": "43651e927c2672e0a4f749a5cce47db470dd19d72736ea4c6b6a6b3d55d9f7ab" + "cacheKey": "459ad0cd459ea1873d4dce5c19279d1c5797457dfd0ddfaf2b575e25170fea80" }, { "packageName": "libcxx", "manifestSha256": "6e1ebfa6914043770eae2a6c156b915acf6a56fe763f07d215ffe6f6f2305007", - "cacheKey": "53612e1c46d4ce4e8d07b911f753fda620b8b8597c38c3775fc1769f8a63ba89" + "cacheKey": "0b9e4ae7831b78deebd709b0e12cd30cca8381d6840664543ec11ba7e11a88ce" }, { "packageName": "redis", "manifestSha256": "151fb507de953ba2ba94b8f881bc7d66b4c741c1359a2f590cc07f9a4cd368a3", - "cacheKey": "da7392e43962a0288ff0a9958190215ee01476e1d235f4275b790507ce5e4715" + "cacheKey": "1fa5f605040b656ea3270f8807f3e38460a9c331cb1624b5d8bb5491786ffa64" } ] }, @@ -2507,79 +2507,79 @@ "wasm32" ], "cacheKeys": { - "wasm32": "4feb895e5854b8d3f1e3fd9e29af4f6f001ffaf458d311df1a6451f259134368" + "wasm32": "5526de953f3a2e515a2625a7d2348429bf644310695ea563edd2e5ba2f57e10f" }, "dependencyClosures": { "wasm32": [ { "packageName": "bash", "manifestSha256": "6478060f28d430d18a6ebe7c603392d35a41d9bcf6bc74322351d1450b9c5335", - "cacheKey": "1d813d7f9db4979fc5c0eef6b37e213cb73a42395757cc92ef8ebeee2e62913d" + "cacheKey": "1be72f8540c1231d0b21201b92e591f1370837e2acd7d0521f3c94a26d172e80" }, { "packageName": "bc", "manifestSha256": "a65661463bb7047b91ff00153bd99fd962c96e571934ab4e923f5215fb6ecfbd", - "cacheKey": "2e7bb9ff2a2cd8dd3b28d46c7d733f873d764820ff023a736a41cc1e16745207" + "cacheKey": "26c9f859aad63a3476ce9172e505ebcdb60ecdfe9343bad67f8ce43be57caa99" }, { "packageName": "coreutils", "manifestSha256": "b8baabc9af9283434e0d80f7fdc0aaef242ffa31092ab0fc39a0603a12b3cca7", - "cacheKey": "33ca2b6c08b61b6ba3b013b13271c98b9b63d75f0a5c47d6d265c27c10c50458" + "cacheKey": "640ffdd14b7ecbe96a0632eeb7dd47c2606cd195bdca24ba8a1b2395e51e2c93" }, { "packageName": "dash", "manifestSha256": "10ce1bb611fefd78a6c9ec9038f6113e29a7062b1a73e08eccf9b33004cea9a8", - "cacheKey": "2ec3ac4e8baa4bfb4ced00ff411feb0ae28e2acf3b5709ac7c042c5a57295aa6" + "cacheKey": "e6a122a8bb1ea383cf8917b15c78dc4e148af24a391f61634117ec2c8d13deb9" }, { "packageName": "diffutils", "manifestSha256": "3a78f0a46bae43ce5ea235c6638c2cbd1d8559b0b62b1fb42443b35968c1aca3", - "cacheKey": "5fd58502ff1d28c11cc7941a2f21d5ab2d9f40bfbe113f766d274361ab5e0a88" + "cacheKey": "b6423d111d2b4d56085317bac4eba82478e6be1519ed7a6b639b914f4dfc9980" }, { "packageName": "file", "manifestSha256": "7874c1affcbbf2c8ab8c8d4beb087f275af57b5caae6a8947bf5a63a181552b2", - "cacheKey": "4c75af58729ffafa2d5ff73b3522aacb97465e1a85e7163274ff0994d30017c3" + "cacheKey": "039f051da9fd7c4111b2cbf731e9a9d950f9d7816927e245cdf2f9f1a6b7f8a3" }, { "packageName": "findutils", "manifestSha256": "cceedf52aea67fbb0da03cfce6f2e9b1a7b5561fa1656c2762b43c651a625d02", - "cacheKey": "e9894ce98cd2055fa27987b544eb72df96fc824ce7c9372d1db4294879985ac1" + "cacheKey": "1cf00e9e4542eb4cbe7dc5dc378a11b8e7f8d8087dfa041386cf674f045ea408" }, { "packageName": "gawk", "manifestSha256": "a2567b8b0778805e385e1d3a41ac8b5d74aaf9d293b222001b17d09b11e5a0ba", - "cacheKey": "85ef6c0fdac5fa9ac1ab10bb1a057038e6d08015e571b8b36eb2c89505e1917f" + "cacheKey": "57a33bb378e1e653901be62db5cd273c9d2393ca90339b57538f95286ee79f6e" }, { "packageName": "grep", "manifestSha256": "59270d3bfb33167b32d13246bf855b49308f8c9c0a66d9635c804f702421fbc6", - "cacheKey": "ccfc4ba22252a0d57c55ed8ada09dff22883fb35a37a7ea1efa395bcb405a748" + "cacheKey": "6dcbd57aee61506f8e1939b13b7260b2a8c4f2bcc8c7f2a36a14e4799c5f7db5" }, { "packageName": "m4", "manifestSha256": "2c6582d99d6eabfb9da52badf49b899e9fdcba76a728536b25bc3741f3331543", - "cacheKey": "a7063dcf8ce5bfe288b1520beac1eaf1d19f736fbdd683f344d83d74dcbab727" + "cacheKey": "0b20308410ffac3629eacbc4c68603033ae1fce51bfaf356965672779cd5639f" }, { "packageName": "make", "manifestSha256": "f878d2d730f36a4c6dfe1fff1b4ccc704757ea22d8c4c8ccb95b11cd641e5fed", - "cacheKey": "49a0f581bbc70db8db660c144bbaf5683ea30f36f305c434c5e4d772c334723e" + "cacheKey": "d30ffaed8cdf6c3d2840c726ac592a55c8314162e955a536898b0ae54d3fdfb5" }, { "packageName": "ncurses", "manifestSha256": "0a1c180eeade627e204aa47065dbfebfcdc9359d25e2db436932c7fa3a86fc1e", - "cacheKey": "1d43a87fe208ed807f1e71640e4fcf7c4417699c318780e49a56c58d37e54ebb" + "cacheKey": "2a8d1a03af1e6866e6da9368cbf163ae6fbed7a0f641f9005fa2baa118b97422" }, { "packageName": "posix-utils-lite", "manifestSha256": "8fd7190b2848ef80143adc7b9268c79e13cd4db3430f7105faf49a4b4407a06f", - "cacheKey": "2308c7f1868ea3e052209430626f792e07a168286c45893724195d4709c60220" + "cacheKey": "7446b2dc9dec882ec9b99433dbc10917b9773b615b45b39faec6060b9bc0dc5b" }, { "packageName": "sed", "manifestSha256": "2a08e9c5dacc5facc8983c1ff35ff2a72db328405e9a1f464cc7ad155c4d08af", - "cacheKey": "7ba13641a1723f061e59a41a5db4e713fa7ebc37db885087217c7cb2e1661c19" + "cacheKey": "d5f738b9a42966519b9dcfab117f041d2d3a38d608d483166b1342d3b1e67c68" } ] }, @@ -2599,14 +2599,14 @@ "wasm32" ], "cacheKeys": { - "wasm32": "fd5e4efd09f5f8fbb9b7fe6671ef1c9253f3d78f3709783b5002a3a1983163ab" + "wasm32": "a7d611f70473dbfc1e89312fde7aa8502373cdd2c15e10f4b7b6d67dd19ffc5c" }, "dependencyClosures": { "wasm32": [ { "packageName": "zlib", "manifestSha256": "d4f4701c0c30dc843ebe2111ed3e2284836cdfc4bc05a130c8fab4f31b8e6357", - "cacheKey": "a826c5a1662265850d13e48202fda10c240400126b9fcc0ca33ee0f57e5969f1" + "cacheKey": "f372595b0d4dc5d079fde6f5a3d0401d8b0e8e35a87221df9f51a607efd15fdb" } ] }, @@ -2633,7 +2633,7 @@ "wasm32" ], "cacheKeys": { - "wasm32": "7ba13641a1723f061e59a41a5db4e713fa7ebc37db885087217c7cb2e1661c19" + "wasm32": "d5f738b9a42966519b9dcfab117f041d2d3a38d608d483166b1342d3b1e67c68" }, "dependencyClosures": { "wasm32": [] @@ -2654,14 +2654,14 @@ "wasm32" ], "cacheKeys": { - "wasm32": "5be1bf60e0954662e5973417c42baa3ba028c4260f41e8e690a927be981fa0f1" + "wasm32": "53498ffe143171264f70b0c60a20e6811a144d9ef5d39bb6c7b92184b3e5e6af" }, "dependencyClosures": { "wasm32": [ { "packageName": "homebrew-bootstrap", "manifestSha256": "b171060c86cb6642e8cadef9d2698e671c6bf188868c7d835b3e8ecf3a3b054a", - "cacheKey": "8230aaca4bfc1f7bff80d2830534151afff6edb2c03f640af4afa385d145426d" + "cacheKey": "3ca8b809f79e6cf3432a36a881e33098fff06edb64e1a1c1ce16c24f0d52115b" } ] }, @@ -2681,24 +2681,24 @@ "wasm32" ], "cacheKeys": { - "wasm32": "2c2b1f5959b20978e10725bc42cf1d3e0c48a7c699f3b3d564d843c71b8b1ea7" + "wasm32": "313e60b92a277a65ebba3d4e33831978477a4618baed284467b485bca9baa689" }, "dependencyClosures": { "wasm32": [ { "packageName": "libcxx", "manifestSha256": "6e1ebfa6914043770eae2a6c156b915acf6a56fe763f07d215ffe6f6f2305007", - "cacheKey": "53612e1c46d4ce4e8d07b911f753fda620b8b8597c38c3775fc1769f8a63ba89" + "cacheKey": "0b9e4ae7831b78deebd709b0e12cd30cca8381d6840664543ec11ba7e11a88ce" }, { "packageName": "openssl", "manifestSha256": "374f5dce6b2691b630b55cb82955b33d2305384e4bd64746c581621bb0793993", - "cacheKey": "bdef227990c15092dfb24dc2abfcb8cd2699b409bef100c9d69a62c770b3b950" + "cacheKey": "d497113aa10b49502c0a3eda046a2800793279477bbb5dc4b0897f5a862d06d7" }, { "packageName": "zlib", "manifestSha256": "d4f4701c0c30dc843ebe2111ed3e2284836cdfc4bc05a130c8fab4f31b8e6357", - "cacheKey": "a826c5a1662265850d13e48202fda10c240400126b9fcc0ca33ee0f57e5969f1" + "cacheKey": "f372595b0d4dc5d079fde6f5a3d0401d8b0e8e35a87221df9f51a607efd15fdb" } ] }, @@ -2718,29 +2718,29 @@ "wasm32" ], "cacheKeys": { - "wasm32": "1fc4b5fe1002e77e10e531ecd916c99a766c69583716fca36d5fbba88fa389da" + "wasm32": "f9ac0d7dab5982c814401c7b7dad5eb8efe5e54b2235ee3d94a2835ba212b0e7" }, "dependencyClosures": { "wasm32": [ { "packageName": "libcxx", "manifestSha256": "6e1ebfa6914043770eae2a6c156b915acf6a56fe763f07d215ffe6f6f2305007", - "cacheKey": "53612e1c46d4ce4e8d07b911f753fda620b8b8597c38c3775fc1769f8a63ba89" + "cacheKey": "0b9e4ae7831b78deebd709b0e12cd30cca8381d6840664543ec11ba7e11a88ce" }, { "packageName": "openssl", "manifestSha256": "374f5dce6b2691b630b55cb82955b33d2305384e4bd64746c581621bb0793993", - "cacheKey": "bdef227990c15092dfb24dc2abfcb8cd2699b409bef100c9d69a62c770b3b950" + "cacheKey": "d497113aa10b49502c0a3eda046a2800793279477bbb5dc4b0897f5a862d06d7" }, { "packageName": "spidermonkey", "manifestSha256": "c72ef4e43ee2a4fc74390d03a239fce3b545a6d3b64f02ccc4d44972cec626bd", - "cacheKey": "2c2b1f5959b20978e10725bc42cf1d3e0c48a7c699f3b3d564d843c71b8b1ea7" + "cacheKey": "313e60b92a277a65ebba3d4e33831978477a4618baed284467b485bca9baa689" }, { "packageName": "zlib", "manifestSha256": "d4f4701c0c30dc843ebe2111ed3e2284836cdfc4bc05a130c8fab4f31b8e6357", - "cacheKey": "a826c5a1662265850d13e48202fda10c240400126b9fcc0ca33ee0f57e5969f1" + "cacheKey": "f372595b0d4dc5d079fde6f5a3d0401d8b0e8e35a87221df9f51a607efd15fdb" } ] }, @@ -2760,7 +2760,7 @@ "wasm32" ], "cacheKeys": { - "wasm32": "1302bc16ff5d928390e99d8eda54447da06d6d601d2e93d3d77aa7a877973835" + "wasm32": "49cbbc5d5ab7347c4baf94cd98af99d4f7e9cff2fc91c3f677e06f17bcb368ec" }, "dependencyClosures": { "wasm32": [] @@ -2781,7 +2781,7 @@ "wasm32" ], "cacheKeys": { - "wasm32": "7db0ac25cdc6534a25f0f563bd5431e5af1664284cf53b213e9868e52fa6d072" + "wasm32": "d712156abc4f50607933e6c58eae2c452c53f4a376224aaa1af0d37b32abf823" }, "dependencyClosures": { "wasm32": [] @@ -2802,7 +2802,7 @@ "wasm32" ], "cacheKeys": { - "wasm32": "0256f1aedd61bdae5546348dd188638aa2192a8ba5cdd013584b1701f42e2133" + "wasm32": "3c78c9269ceb90fa1077bc1538d812d5e7b313c66f7586d401f0c89e7c49d747" }, "dependencyClosures": { "wasm32": [] @@ -2823,19 +2823,19 @@ "wasm32" ], "cacheKeys": { - "wasm32": "40b7b146f334e95fbc826ab23999bfb3dc2032705667c9ffca2a53a48f4613d3" + "wasm32": "a2d188b024d8082fc1765476929035c18ab305961de478854141e811f163e57f" }, "dependencyClosures": { "wasm32": [ { "packageName": "libpng", "manifestSha256": "79ed5c5c072a0267cce5c7a2fe37d8494571b17e44b952f619e04ec1c4a9db10", - "cacheKey": "e31da77ef2ff8fee85f28978d0356ebb799f3ae71458f0814fd9bf5c9d708af3" + "cacheKey": "00b34f15486aefd49b0c87e606e84de6df8388d923678a9b99675a5469aa643b" }, { "packageName": "zlib", "manifestSha256": "d4f4701c0c30dc843ebe2111ed3e2284836cdfc4bc05a130c8fab4f31b8e6357", - "cacheKey": "a826c5a1662265850d13e48202fda10c240400126b9fcc0ca33ee0f57e5969f1" + "cacheKey": "f372595b0d4dc5d079fde6f5a3d0401d8b0e8e35a87221df9f51a607efd15fdb" } ] }, @@ -2862,7 +2862,7 @@ "wasm32" ], "cacheKeys": { - "wasm32": "d058df9f05e8d6b84e4cab72005e03c5de8968f4d72f6ef3ceceb8815d03532d" + "wasm32": "662e66598d4333078259cc3daca5b44ecc19fe5eb6297451be1189a7f606ae20" }, "dependencyClosures": { "wasm32": [] @@ -2883,7 +2883,7 @@ "wasm32" ], "cacheKeys": { - "wasm32": "e0a4f2dccc1020ab2c5fde09301251c7da82dc9cb3baa24729f490294cc8373a" + "wasm32": "b2d332ff56123c0576814b63090fa7d405b439021659c8846f34056580bcb531" }, "dependencyClosures": { "wasm32": [] @@ -2904,14 +2904,14 @@ "wasm32" ], "cacheKeys": { - "wasm32": "fcfbb7dc5ba2daeea207eac0b7e991938d8cf3d778e7328f03cf31eb381e976d" + "wasm32": "4e098ad47f6ebf65458ea5b59f4b23920dc7c463dfc839ef62647c828a08fd52" }, "dependencyClosures": { "wasm32": [ { "packageName": "vim", "manifestSha256": "c211660ef41d01f95f3fbdf675b74891e897e3f304e04f1bf5586f326007382c", - "cacheKey": "e0a4f2dccc1020ab2c5fde09301251c7da82dc9cb3baa24729f490294cc8373a" + "cacheKey": "b2d332ff56123c0576814b63090fa7d405b439021659c8846f34056580bcb531" } ] }, @@ -2931,7 +2931,7 @@ "wasm32" ], "cacheKeys": { - "wasm32": "bdf72a4f1ea11c510f3d1644802df33786cd9c8ee158561ba72c8004436e3a56" + "wasm32": "8947f2fb28c765360635ed05745d13f2dbd214531c1a137d8cc206cf1b73ada0" }, "dependencyClosures": { "wasm32": [] @@ -2952,84 +2952,84 @@ "wasm32" ], "cacheKeys": { - "wasm32": "3273c155264665eaa8369f4ae19d674317c66d5f04603006a1aad357c39c9e37" + "wasm32": "0a75a06c7a970ec83f3ad17b6b58f1c01e0134309b466b50caef33bf0cdf0ffc" }, "dependencyClosures": { "wasm32": [ { "packageName": "dinit", "manifestSha256": "44e125e1337503e27f8cff531cdbd79c4f406fd2d282b916a7746cf3a14057bf", - "cacheKey": "43651e927c2672e0a4f749a5cce47db470dd19d72736ea4c6b6a6b3d55d9f7ab" + "cacheKey": "459ad0cd459ea1873d4dce5c19279d1c5797457dfd0ddfaf2b575e25170fea80" }, { "packageName": "homebrew-bootstrap", "manifestSha256": "b171060c86cb6642e8cadef9d2698e671c6bf188868c7d835b3e8ecf3a3b054a", - "cacheKey": "8230aaca4bfc1f7bff80d2830534151afff6edb2c03f640af4afa385d145426d" + "cacheKey": "3ca8b809f79e6cf3432a36a881e33098fff06edb64e1a1c1ce16c24f0d52115b" }, { "packageName": "icu", "manifestSha256": "f5b1f02d169ec1108fc32d9da55be53c3efb96e3a7aeba6a03aec62e0175f0dc", - "cacheKey": "48a3ee79c8510a1fe933281741cb0fb56f2a67fa7dba9d2cd18c0085cf49f375" + "cacheKey": "3785cb4f88b3800ceb98a05d72cf21a82221711b31b413b6d0a18a8962db522f" }, { "packageName": "libcurl", "manifestSha256": "c771e6cdc83b43840db4dd1fdce4b6189ba4f281596f7485bcda9c65e2ecda00", - "cacheKey": "8273da44c60d3edb2032e3566040ed8fd07c2d3d5e43bfd4bbe94ff7ec0b95c6" + "cacheKey": "2e9bfe5431ec06c510c8f31c6a936d95f604294cb6f7cb6b5a666fa46e3b5202" }, { "packageName": "libcxx", "manifestSha256": "6e1ebfa6914043770eae2a6c156b915acf6a56fe763f07d215ffe6f6f2305007", - "cacheKey": "53612e1c46d4ce4e8d07b911f753fda620b8b8597c38c3775fc1769f8a63ba89" + "cacheKey": "0b9e4ae7831b78deebd709b0e12cd30cca8381d6840664543ec11ba7e11a88ce" }, { "packageName": "libiconv", "manifestSha256": "fa60b386f4ba0cd5bbbd595f67477cbfb0cc401a14bbd4769630c502bb3f24b1", - "cacheKey": "b494e26a4bd579392ae78f894f37f1d7b7ce87f0401f9d53edbf982b0753f76b" + "cacheKey": "2ae7c3ce68bb2b52fb722df897584d2501ac4146946692f3f8b08901ed4e2cef" }, { "packageName": "libxml2", "manifestSha256": "908490efcdc7783c76ce36af31d1db24bfcc7ba5f1e7b9b5298cc81cb110256c", - "cacheKey": "f8e7709dc6d6606b0d5ad24b1bb2f7aece6df4e171f92dfe7ea344713c14d4e8" + "cacheKey": "4c2d44bb43f74f3160efe3089232b173c6425c1a996b29b41f2e61df26d1df25" }, { "packageName": "libzip", "manifestSha256": "83c65d63a6416e79436bf4e792759a4701d568df0e8c010394a9732cf659f0cc", - "cacheKey": "0e020e8d1c342ccf5dac59079aab2b2ec35b083b03107ac172cc0f9ac97a13ec" + "cacheKey": "592acc36c12b140841b44149254872b4a6915c5e8ec65592f58bfc7cd90a1ec9" }, { "packageName": "msmtpd", "manifestSha256": "09de04a422ddf631a29a259d816e081f8d317d1077808ede55d8c500baae748f", - "cacheKey": "82b32c29b05f3854bebfb414875d603fe66829ab79b0ea02b4a1d00e09fe3a52" + "cacheKey": "1676f17c26dd7d6bee804d444aaf37b2d43a65a0f7d8c985e75d965aaa7566ca" }, { "packageName": "nginx", "manifestSha256": "41b0aaf05fdd0d8da42642703f9e60fd7b9545d8029896918b87c771c359147b", - "cacheKey": "6a92877569b9bf8dabd5333d3f0d5e96388379e28ec00f4d53e0ee675cb58eae" + "cacheKey": "afde4ab291d7b9ed1014f24d6c7ca300951a290c8458c4fb0bfa0fd88270853c" }, { "packageName": "openssl", "manifestSha256": "374f5dce6b2691b630b55cb82955b33d2305384e4bd64746c581621bb0793993", - "cacheKey": "bdef227990c15092dfb24dc2abfcb8cd2699b409bef100c9d69a62c770b3b950" + "cacheKey": "d497113aa10b49502c0a3eda046a2800793279477bbb5dc4b0897f5a862d06d7" }, { "packageName": "php", "manifestSha256": "fcd9d7915ed8935b05418e148edcb312adf0b1e4e627fe4aab75444b35b33bbc", - "cacheKey": "c5e70ff8e8ab519b31810e4dac16a7500033f549395152f7b00b239d6170acf6" + "cacheKey": "e0bae805a3ed026870ab3dd18f0dca768894bc36d9764ac55f4fc3aa83caaea3" }, { "packageName": "shell", "manifestSha256": "107adeee07fd0a10d9f2bb40a0cd84a6144bff91e7505b5ae9afeabaa000beb9", - "cacheKey": "5be1bf60e0954662e5973417c42baa3ba028c4260f41e8e690a927be981fa0f1" + "cacheKey": "53498ffe143171264f70b0c60a20e6811a144d9ef5d39bb6c7b92184b3e5e6af" }, { "packageName": "sqlite", "manifestSha256": "ab9560b4eab53445034e2a87a27ccee9d05ddb0db73fee28f5000325b0b1cba5", - "cacheKey": "a21f509748a9f5a086db9ae7bb2341a3c57171c954b31a3982b2b47a98794694" + "cacheKey": "35544fa8d8d50cd23c78606836caf96f7636cf20360d1cdcf503fcdc1f06c400" }, { "packageName": "zlib", "manifestSha256": "d4f4701c0c30dc843ebe2111ed3e2284836cdfc4bc05a130c8fab4f31b8e6357", - "cacheKey": "a826c5a1662265850d13e48202fda10c240400126b9fcc0ca33ee0f57e5969f1" + "cacheKey": "f372595b0d4dc5d079fde6f5a3d0401d8b0e8e35a87221df9f51a607efd15fdb" } ] }, @@ -3049,7 +3049,7 @@ "wasm32" ], "cacheKeys": { - "wasm32": "261ecf6d602b685b0977854cc97ac51f13a0bf27c28753fcf35a6384714d6c9c" + "wasm32": "ab7ba59e47ab9682297ed6104348b28f11a98eba83a9c2240062d7559b79694a" }, "dependencyClosures": { "wasm32": [] @@ -3070,7 +3070,7 @@ "wasm32" ], "cacheKeys": { - "wasm32": "7adafa87971ab6e08449b0958b55364989c06297b9027c1acb06f72959d43cfd" + "wasm32": "af083f1df4648a11c46f20c8db7804bf455a117ae12819eb857dd46256fbac39" }, "dependencyClosures": { "wasm32": [] @@ -3091,7 +3091,7 @@ "wasm32" ], "cacheKeys": { - "wasm32": "47bf7eb3741ccdb3d7e0e320c7440c2106aa997e476830cbc2d63532240c0a38" + "wasm32": "694f316b82bef68ed79efc2905fcc1134867d38e8e25ba737f8893b859575ce1" }, "dependencyClosures": { "wasm32": [] diff --git a/programs/f_03_wasm_gc_anyref.c b/programs/f_03_wasm_gc_anyref.c index 3937013a85..8c2486a710 100644 --- a/programs/f_03_wasm_gc_anyref.c +++ b/programs/f_03_wasm_gc_anyref.c @@ -2,19 +2,14 @@ // type (anyref / eqref) on the fork path (A5). // // Coverage matrix: docs/plans/2026-05-13-fork-instrument-megaPR-eliminate-guard-dispatch-and-modern-EH-plan.md -// Stub: wasm-GC reference types have no C-source surface. The fixture -// needs a hand-written WAT module containing an `anyref` or `eqref` -// local on the fork path; the test driver invokes `wasm-fork-instrument` -// directly and asserts it exits non-zero with a clear error message -// naming the function and ref type (the existing classify_ref panic -// is the current mechanism). -// -// Replace this stub with the WAT fixture + driver harness when the -// test is wired up in the commit that documents the accepted limit. +// wasm-GC reference types have no C-source surface, so the authoritative +// rejection coverage lives in crates/fork-instrument/tests/coverage_wat.rs. +// That hand-written WAT invokes the instrumenter directly and requires a +// precise, non-panicking unsupported-reference diagnostic. #include int main(void) { - printf("STUB: F-03 anyref accepted limit (WAT + driver pending)\n"); - return 1; // Intentional FAIL — test driver marks this it.fails. + printf("STUB: F-03 anyref accepted limit (covered by WAT)\n"); + return 1; // Intentional FAIL — runtime coverage skips this C-only stub. } diff --git a/programs/p_11_fork_continuation_enomem.c b/programs/p_11_fork_continuation_enomem.c index 416792f528..91236563e4 100644 --- a/programs/p_11_fork_continuation_enomem.c +++ b/programs/p_11_fork_continuation_enomem.c @@ -100,8 +100,8 @@ int main(void) { release_fillers(filler_count); return 1; } - if (filler_count == 0) { - printf("FAIL: no filler mapping was available\n"); + if (filler_count < 2) { + printf("FAIL: fewer than two filler mappings were available\n"); return 1; } @@ -161,15 +161,21 @@ int main(void) { return 1; } - // WHY: one free page lets beginUnwind allocate its root chunk. The deep - // call chain then needs another chunk, so failure occurs after frames have - // been committed and exercises ABORT_UNWINDING rather than the simpler - // root-allocation error path. - filler_count--; - if (munmap(filler_mappings[filler_count], WASM_PAGE_BYTES) != 0) { - printf("FAIL: could not make one continuation page available errno=%d\n", errno); - release_fillers(filler_count); - return 1; + // WHY: ABI 43 owns process/module/reference metadata separately from the + // linked stack. Two free pages let capture allocate the metadata arena and + // the continuation root. The deep call chain then needs a third page, so + // failure occurs after frames have committed and exercises + // ABORT_UNWINDING rather than the simpler root-allocation error path. + for (int i = 0; i < 2; i++) { + filler_count--; + if (munmap(filler_mappings[filler_count], WASM_PAGE_BYTES) != 0) { + printf( + "FAIL: could not make fork transaction page available errno=%d\n", + errno + ); + release_fillers(filler_count); + return 1; + } } errno = 0; @@ -201,25 +207,34 @@ int main(void) { } printf("NO_PHANTOM_CHILD: ok\n"); - // The abort replay must unmap its partial chain. Prove that the one free - // page is reusable before relying on it for the recovery fork. - void *probe = mmap( - NULL, - WASM_PAGE_BYTES, - PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, - -1, - 0 - ); - if (probe == MAP_FAILED) { - printf("FAIL: continuation allocation leaked errno=%d\n", errno); - release_fillers(filler_count); - return 1; + // Abort replay must unmap both transaction roots and the partial linked + // chain. Hold two probe mappings concurrently to prove both pages are + // reusable before relying on them for the recovery fork. + void *probes[2] = {MAP_FAILED, MAP_FAILED}; + for (int i = 0; i < 2; i++) { + probes[i] = mmap( + NULL, + WASM_PAGE_BYTES, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, + -1, + 0 + ); + if (probes[i] == MAP_FAILED) { + printf("FAIL: fork transaction allocation leaked errno=%d\n", errno); + for (int j = 0; j < i; j++) { + munmap(probes[j], WASM_PAGE_BYTES); + } + release_fillers(filler_count); + return 1; + } } - if (munmap(probe, WASM_PAGE_BYTES) != 0) { - printf("FAIL: probe cleanup errno=%d\n", errno); - release_fillers(filler_count); - return 1; + for (int i = 0; i < 2; i++) { + if (munmap(probes[i], WASM_PAGE_BYTES) != 0) { + printf("FAIL: probe cleanup errno=%d\n", errno); + release_fillers(filler_count); + return 1; + } } printf("CONTINUATION_PAGE_REUSED: ok\n"); diff --git a/scripts/build-programs.sh b/scripts/build-programs.sh index 95348488cb..2027a4f524 100755 --- a/scripts/build-programs.sh +++ b/scripts/build-programs.sh @@ -103,6 +103,8 @@ build_program() { local name name=$(basename "$src" .c) local wasm="$out_dir/${name}.wasm" + local raw_wasm="$out_dir/${name}.raw.wasm" + local next_wasm="$out_dir/${name}.next.wasm" # Auto-append GL stubs when the source pulls in EGL/GLES headers. # Static linking won't pick symbols out of libEGL.a / libGLESv2.a @@ -118,6 +120,9 @@ build_program() { fi echo " Compiling $name..." + # WHY: a failed compile or instrumentation pass must not leave a raw or + # stale-ABI module at the resolver-visible final path. + rm -f "$wasm" "$raw_wasm" "$next_wasm" # Bash 3.2 (macOS system bash) under `set -u` treats expansion of # an empty array as unbound; the `${arr[@]+...}` guard suppresses # that when extra_libs is empty. @@ -125,15 +130,16 @@ build_program() { "${LINK_PRE_LIBS[@]}" \ ${extra_libs[@]+"${extra_libs[@]}"} \ "${LINK_POST_LIBS[@]}" \ - -o "$wasm" + -o "$raw_wasm" # Apply fork instrumentation if the program uses fork. The tool is a # no-op for modules without `kernel.kernel_fork`, so it's safe to run # unconditionally on every program. Programs without fork stay # byte-identical except for a small ABI metadata section the tool # always emits (see runtime::inject_runtime). - "$FORK_INSTRUMENT" "$wasm" -o "$wasm.instr" - mv "$wasm.instr" "$wasm" + "$FORK_INSTRUMENT" "$raw_wasm" -o "$next_wasm" + mv "$next_wasm" "$wasm" + rm -f "$raw_wasm" } # Build a C++ program via the SDK's wasm32posix-c++ wrapper. The SDK @@ -148,8 +154,11 @@ build_cpp_program() { local name name=$(basename "$src" .cpp) local wasm="$out_dir/${name}.wasm" + local raw_wasm="$out_dir/${name}.raw.wasm" + local next_wasm="$out_dir/${name}.next.wasm" echo " Compiling $name (C++)..." + rm -f "$wasm" "$raw_wasm" "$next_wasm" # -fwasm-exceptions is required for clang to lower C++ try/catch # to wasm-EH `try`/`catch` instructions. Without it clang emits # `__cxa_throw; unreachable` and DCEs the catch handlers, so the @@ -160,24 +169,26 @@ build_cpp_program() { -fwasm-exceptions \ "$src" \ -lc++ -lc++abi \ - -o "$wasm" + -o "$raw_wasm" - # Preserve a real pre-instrumentation control for issue #918. The source - # contains an unreachable-at-test-time fork branch solely so the normal - # output is transformed below. A raw module with kernel_fork but without - # wpk_fork_* exports is test evidence, not a distributable program, so it - # lives outside the resolver's programs tree. + # Preserve a raw no-fork control for issue #918 independently of the + # normally instrumented fork-bearing program. if [ "$name" = "sjlj_noexcept_boundary" ]; then mkdir -p "$TEST_FIXTURE_DIR/wasm32" - cp "$wasm" "$TEST_FIXTURE_DIR/wasm32/${name}.raw.wasm" + wasm32posix-c++ \ + -O2 \ + -fwasm-exceptions \ + -DKANDELO_SJLJ_NO_FORK_ANCHOR \ + "$src" \ + -lc++ -lc++abi \ + -o "$TEST_FIXTURE_DIR/wasm32/${name}.raw.wasm" fi - # Phase 7: fork support comes from wasm-fork-instrument. The tool is - # a no-op for modules without `kernel.kernel_fork`, so it's safe to - # run unconditionally — programs without fork stay byte-identical - # except for the ABI metadata section. - "$FORK_INSTRUMENT" "$wasm" -o "$wasm.instr" - mv "$wasm.instr" "$wasm" + # Publish the resolver-visible path only after instrumentation and its + # complete ABI 43 artifact contract succeed. + "$FORK_INSTRUMENT" "$raw_wasm" -o "$next_wasm" + mv "$next_wasm" "$wasm" + rm -f "$raw_wasm" } ensure_libcxx_in_sysroot() { diff --git a/scripts/check-abi-version.sh b/scripts/check-abi-version.sh index 373b785a4f..4c3ed1e9e8 100755 --- a/scripts/check-abi-version.sh +++ b/scripts/check-abi-version.sh @@ -87,8 +87,11 @@ version_bumped=0 snapshot_changed=0 if git rev-parse --verify --quiet "$base_ref" >/dev/null ; then if ! git diff --quiet "$base_ref" -- crates/shared/src/lib.rs 2>/dev/null ; then + # Do not use `grep -q` here: with pipefail, an early match can close + # the pipe while a large ABI diff is still being written, turning + # git's SIGPIPE into a false "version was not bumped" result. if git diff "$base_ref" -- crates/shared/src/lib.rs \ - | grep -qE '^\+pub const ABI_VERSION: u32 = ' ; then + | grep -E '^\+pub const ABI_VERSION: u32 = ' >/dev/null ; then version_bumped=1 fi fi diff --git a/scripts/check-dev-shell-tools.sh b/scripts/check-dev-shell-tools.sh index 641fe16155..a0c1e82d3e 100755 --- a/scripts/check-dev-shell-tools.sh +++ b/scripts/check-dev-shell-tools.sh @@ -24,3 +24,14 @@ for tool in cmake make; do esac "$tool" --version >/dev/null done + +for tool_path in "${AR:-}" "${RANLIB:-}"; do + case "$tool_path" in + "$nix_store"/*/bin/llvm-ar | "$nix_store"/*/bin/llvm-ranlib) ;; + *) + echo "ERROR: archive tool resolved outside the declared LLVM tool set: ${tool_path:-}" >&2 + exit 1 + ;; + esac + "$tool_path" --version >/dev/null +done diff --git a/scripts/ci-run-test-suite.sh b/scripts/ci-run-test-suite.sh index 32f804dc10..6150c0a92a 100755 --- a/scripts/ci-run-test-suite.sh +++ b/scripts/ci-run-test-suite.sh @@ -16,6 +16,13 @@ host_target() { rustc -vV | awk '/^host/ {print $2}' } +suite="${1:-}" +if [ -z "$suite" ]; then + echo "usage: $0 [group]" >&2 + exit 2 +fi +group="${2:-${TEST_GROUP:-all}}" + # Prepared CI workspaces transport fetched programs as relative links into a # repo-local copy of the exact content-addressed cache generations. Point both # the Rust and TypeScript resolvers at that identity before any suite can read @@ -23,23 +30,25 @@ host_target() { portable_cache="$REPO_ROOT/.ci-test-binary-cache" if [ -d "$portable_cache/programs" ]; then export WASM_POSIX_BINARY_CACHE_ROOT="$portable_cache" - prepared_xtask="$REPO_ROOT/target/$(host_target)/release/xtask" - if [ ! -f "$prepared_xtask" ] || [ ! -x "$prepared_xtask" ]; then - echo "ci-run-test-suite: missing executable prepared package checker: $prepared_xtask" >&2 - exit 1 - fi - # WHY: each conformance case starts a fresh Node resolver under a short - # timeout. Without the packed checker path, every process may start Cargo - # preparation and leave later cases waiting on its build lock. - export WASM_POSIX_XTASK_BIN="$prepared_xtask" fi -suite="${1:-}" -if [ -z "$suite" ]; then - echo "usage: $0 [group]" >&2 - exit 2 -fi -group="${2:-${TEST_GROUP:-all}}" +# WHY: every conformance case starts a fresh Node resolver under a short +# timeout. Prepare one exact worktree-local checker before parallel cases +# begin; each process still executes the source-freshness check, but none +# starts a competing Cargo build or waits on Cargo's target-directory lock. +case "$suite" in + vitest|browser|libc|posix|sortix) + prepared_xtask="$REPO_ROOT/target/$(host_target)/release/xtask" + if [ ! -d "$portable_cache/programs" ]; then + cargo build --release -p xtask --target "$(host_target)" --quiet + fi + if [ ! -f "$prepared_xtask" ] || [ ! -x "$prepared_xtask" ]; then + echo "ci-run-test-suite: missing executable prepared package checker: $prepared_xtask" >&2 + exit 1 + fi + export WASM_POSIX_XTASK_BIN="$prepared_xtask" + ;; +esac invalid_group() { echo "unknown $suite test group: $group" >&2 @@ -119,6 +128,7 @@ case "$suite" in npx playwright test \ test/coi.spec.ts \ test/package-deferred-tree-browser.spec.ts \ + test/wasm-gc-reference-transport.spec.ts \ test/wasm-trap-signal.spec.ts \ --project=chromium --project=firefox --project=webkit ) diff --git a/scripts/homebrew-validate-wasm-executable.sh b/scripts/homebrew-validate-wasm-executable.sh index c73edd7f36..435a347978 100644 --- a/scripts/homebrew-validate-wasm-executable.sh +++ b/scripts/homebrew-validate-wasm-executable.sh @@ -31,46 +31,119 @@ fi wasm_require_no_legacy_asyncify "$wasm_path" -relocatable_status=0 -wasm_is_relocatable_object "$wasm_path" || relocatable_status=$? -case "$relocatable_status" in - 0) +artifact_identity="" +identity_status=0 +used_artifact_identity=0 +artifact_imports_kernel_fork=0 +artifact_has_fork_exports=0 +artifact_identity="$(wasm_artifact_identity "$wasm_path")" || identity_status=$? +if [ "$identity_status" -eq 0 ]; then + # WHY: ABI 43 helpers use Wasm reference/exception proposals that older + # WABT releases cannot disassemble. The production wasmparser tool owns one + # bounded structural request for object kind, memory width, and the exact + # constant ABI export; source-only environments retain the compatibility + # path below. + relocatable="" + memory_count="" + memory64_count="" + abi_state="" + artifact_abi="" + artifact_imports_kernel_fork="" + artifact_has_fork_exports="" + extra="" + IFS=$'\t' read -r relocatable memory_count memory64_count abi_state artifact_abi \ + artifact_imports_kernel_fork artifact_has_fork_exports extra <<<"$artifact_identity" + if [ -n "$extra" ] || + [[ ! "$relocatable" =~ ^[01]$ ]] || + [[ ! "$memory_count" =~ ^[0-9]+$ ]] || + [[ ! "$memory64_count" =~ ^[0-9]+$ ]] || + [[ ! "$artifact_imports_kernel_fork" =~ ^[01]$ ]] || + [[ ! "$artifact_has_fork_exports" =~ ^[01]$ ]]; then + echo "homebrew-validate-wasm-executable.sh: cannot inspect Wasm object kind: $wasm_path" >&2 + exit 1 + fi + used_artifact_identity=1 + if [ "$relocatable" = 1 ]; then echo "homebrew-validate-wasm-executable.sh: executable is a relocatable Wasm object: $wasm_path" >&2 exit 1 - ;; - 1) ;; - *) - echo "homebrew-validate-wasm-executable.sh: cannot inspect Wasm object kind: $wasm_path" >&2 + fi + if [ "$memory_count" != 1 ] || + { [ "$memory64_count" != 0 ] && [ "$memory64_count" != 1 ]; }; then + echo "homebrew-validate-wasm-executable.sh: executable must define or import exactly one inspectable memory: $wasm_path" >&2 exit 1 - ;; -esac + fi + if [ "$memory64_count" = 1 ]; then + artifact_arch=wasm64 + else + artifact_arch=wasm32 + fi + case "$abi_state" in + present) + if [[ ! "$artifact_abi" =~ ^[0-9]+$ ]]; then + echo "homebrew-validate-wasm-executable.sh: cannot validate __abi_version: $wasm_path" >&2 + exit 1 + fi + ;; + missing) + echo "homebrew-validate-wasm-executable.sh: executable lacks __abi_version: $wasm_path" >&2 + exit 1 + ;; + invalid) + echo "homebrew-validate-wasm-executable.sh: cannot validate __abi_version: $wasm_path" >&2 + exit 1 + ;; + *) + echo "homebrew-validate-wasm-executable.sh: cannot validate __abi_version: $wasm_path" >&2 + exit 1 + ;; + esac +elif [ "$identity_status" -eq 127 ]; then + relocatable_status=0 + wasm_is_relocatable_object "$wasm_path" || relocatable_status=$? + case "$relocatable_status" in + 0) + echo "homebrew-validate-wasm-executable.sh: executable is a relocatable Wasm object: $wasm_path" >&2 + exit 1 + ;; + 1) ;; + *) + echo "homebrew-validate-wasm-executable.sh: cannot inspect Wasm object kind: $wasm_path" >&2 + exit 1 + ;; + esac -artifact_arch="" -arch_status=0 -artifact_arch="$(wasm_memory_arch "$wasm_path")" || arch_status=$? -if [ "$arch_status" -ne 0 ]; then - echo "homebrew-validate-wasm-executable.sh: executable must define or import exactly one inspectable memory: $wasm_path" >&2 + artifact_arch="" + arch_status=0 + artifact_arch="$(wasm_memory_arch "$wasm_path")" || arch_status=$? + if [ "$arch_status" -ne 0 ]; then + echo "homebrew-validate-wasm-executable.sh: executable must define or import exactly one inspectable memory: $wasm_path" >&2 + exit 1 + fi + + artifact_abi="" + abi_status=0 + artifact_abi="$(wasm_extract_abi_version "$wasm_path")" || abi_status=$? + case "$abi_status" in + 0) ;; + 1) + echo "homebrew-validate-wasm-executable.sh: executable lacks __abi_version: $wasm_path" >&2 + exit 1 + ;; + *) + echo "homebrew-validate-wasm-executable.sh: cannot validate __abi_version: $wasm_path" >&2 + exit 1 + ;; + esac +else + echo "homebrew-validate-wasm-executable.sh: cannot inspect Wasm object kind: $wasm_path" >&2 exit 1 fi + if [ "$artifact_arch" != "$expected_arch" ]; then echo "homebrew-validate-wasm-executable.sh: executable architecture $artifact_arch does not match expected architecture $expected_arch: $wasm_path" >&2 exit 1 fi -artifact_abi="" -abi_status=0 -artifact_abi="$(wasm_extract_abi_version "$wasm_path")" || abi_status=$? -case "$abi_status" in - 0) ;; - 1) - echo "homebrew-validate-wasm-executable.sh: executable lacks __abi_version: $wasm_path" >&2 - exit 1 - ;; - *) - echo "homebrew-validate-wasm-executable.sh: cannot validate __abi_version: $wasm_path" >&2 - exit 1 - ;; -esac if [ "$artifact_abi" != "$expected_abi" ]; then echo "homebrew-validate-wasm-executable.sh: executable ABI $artifact_abi does not match expected ABI $expected_abi: $wasm_path" >&2 exit 1 @@ -79,26 +152,33 @@ fi wasm_require_fork_instrumentation_if_needed "$wasm_path" fork_required=0 -predicate_status=0 -wasm_imports_kernel_fork "$wasm_path" || predicate_status=$? -case "$predicate_status" in - 0) fork_required=1 ;; - 1) ;; - *) - echo "homebrew-validate-wasm-executable.sh: cannot inspect kernel fork import: $wasm_path" >&2 - exit 1 - ;; -esac -predicate_status=0 -wasm_has_any_wpk_fork_export "$wasm_path" || predicate_status=$? -case "$predicate_status" in - 0) fork_required=1 ;; - 1) ;; - *) - echo "homebrew-validate-wasm-executable.sh: cannot inspect fork exports: $wasm_path" >&2 - exit 1 - ;; -esac +if [ "$used_artifact_identity" -eq 1 ]; then + if [ "$artifact_imports_kernel_fork" = 1 ] || + [ "$artifact_has_fork_exports" = 1 ]; then + fork_required=1 + fi +else + predicate_status=0 + wasm_imports_kernel_fork "$wasm_path" || predicate_status=$? + case "$predicate_status" in + 0) fork_required=1 ;; + 1) ;; + *) + echo "homebrew-validate-wasm-executable.sh: cannot inspect kernel fork import: $wasm_path" >&2 + exit 1 + ;; + esac + predicate_status=0 + wasm_has_any_wpk_fork_export "$wasm_path" || predicate_status=$? + case "$predicate_status" in + 0) fork_required=1 ;; + 1) ;; + *) + echo "homebrew-validate-wasm-executable.sh: cannot inspect fork exports: $wasm_path" >&2 + exit 1 + ;; + esac +fi if [ "$fork_required" -eq 1 ]; then printf 'required\n' diff --git a/scripts/resolve-binary.bundle.mjs b/scripts/resolve-binary.bundle.mjs index 7858f40b77..3b55d91a51 100644 --- a/scripts/resolve-binary.bundle.mjs +++ b/scripts/resolve-binary.bundle.mjs @@ -1,13 +1,13 @@ // Generated by scripts/build-resolve-binary-bundle.sh; do not edit. Third-party notices: resolve-binary.bundle.LICENSES.txt -var hs=Object.defineProperty;var bn=(r,e,t)=>()=>{if(t)throw t[0];try{return r&&(e=r(r=0)),e}catch(n){throw t=[n],n}};var xr=(r,e)=>{for(var t in e)hs(r,t,{get:e[t],enumerable:!0})};import{createRequire as ho}from"module";function gi(r,e){return yi(r,{i:2},e&&e.out,e&&e.dictionary)}var yo,ft,go,po,J,ct,mo,ai,ci,wo,li,ft,fi,vo,ui,Eo,_c,Zn,be,M,Rt,Bt,M,M,M,M,di,M,So,zo,Gn,ge,Wn,hi,tn,bo,fe,yi,ko,xo,lt,pi,Io,Ao,Hn=bn(()=>{yo=ho("/");try{ft=yo("worker_threads"),go=ft.Worker,po=ft.isMarkedAsUntransferable}catch{}J=Uint8Array,ct=Uint16Array,mo=Int32Array,ai=new J([0,0,0,0,0,0,0,0,1,1,1,1,2,2,2,2,3,3,3,3,4,4,4,4,5,5,5,5,0,0,0,0]),ci=new J([0,0,0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,8,8,9,9,10,10,11,11,12,12,13,13,0,0]),wo=new J([16,17,18,0,8,7,9,6,10,5,11,4,12,3,13,2,14,1,15]),li=function(r,e){for(var t=new ct(31),n=0;n<31;++n)t[n]=e+=1<>1|(M&21845)<<1,be=(be&52428)>>2|(be&13107)<<2,be=(be&61680)>>4|(be&3855)<<4,Zn[M]=((be&65280)>>8|(be&255)<<8)>>1;Rt=(function(r,e,t){for(var n=r.length,i=0,o=new ct(e);i>c]=l}else for(a=new ct(n),i=0;i>15-r[i]);return a}),Bt=new J(288);for(M=0;M<144;++M)Bt[M]=8;for(M=144;M<256;++M)Bt[M]=9;for(M=256;M<280;++M)Bt[M]=7;for(M=280;M<288;++M)Bt[M]=8;di=new J(32);for(M=0;M<32;++M)di[M]=5;So=Rt(Bt,9,1),zo=Rt(di,5,1),Gn=function(r){for(var e=r[0],t=1;te&&(e=r[t]);return e},ge=function(r,e,t){var n=e/8|0;return(r[n]|r[n+1]<<8)>>(e&7)&t},Wn=function(r,e){var t=e/8|0;return(r[t]|r[t+1]<<8|r[t+2]<<16)>>(e&7)},hi=function(r){return(r+7)/8|0},tn=function(r,e,t){return(e==null||e<0)&&(e=0),(t==null||t>r.length)&&(t=r.length),new J(r.subarray(e,t))},bo=["unexpected EOF","invalid block type","invalid length/literal","invalid distance","stream finished","no stream handler",,"no callback","invalid UTF-8 data","extra field too long","date not in range 1980-2099","filename too long","stream finishing","invalid zip data"],fe=function(r,e,t){var n=new Error(e||bo[r]);if(n.code=r,Error.captureStackTrace&&Error.captureStackTrace(n,fe),!t)throw n;return n},yi=function(r,e,t,n){var i=r.length,o=n?n.length:0;if(!i||e.f&&!e.l)return t||new J(0);var s=!t,a=s||e.i!=2,c=e.i;s&&(t=new J(i*3));var l=function(_e){var Le=t.length;if(_e>Le){var Gt=new J(Math.max(Le*2,_e));Gt.set(t),t=Gt}},h=e.f||0,f=e.p||0,d=e.b||0,y=e.l,g=e.d,p=e.m,w=e.n,u=i*8;do{if(!y){h=ge(r,f,1);var m=ge(r,f+1,3);if(f+=3,m)if(m==1)y=So,g=zo,p=9,w=5;else if(m==2){var S=ge(r,f,31)+257,k=ge(r,f+10,15)+4,I=S+ge(r,f+5,31)+1;f+=14;for(var A=new J(I),B=new J(19),N=0;N>4;if(v<16)A[N++]=v;else{var _=0,Z=0;for(v==16?(Z=3+ge(r,f,3),f+=2,_=A[N-1]):v==17?(Z=3+ge(r,f,7),f+=3):v==18&&(Z=11+ge(r,f,127),f+=7);Z--;)A[N++]=_}}var Me=A.subarray(0,S),re=A.subarray(S);p=Gn(Me),w=Gn(re),y=Rt(Me,p,1),g=Rt(re,w,1)}else fe(1);else{var v=hi(f)+4,E=r[v-4]|r[v-3]<<8,z=v+E;if(z>i){c&&fe(0);break}a&&l(d+E),t.set(r.subarray(v,z),d),e.b=d+=E,e.p=f=z*8,e.f=h;continue}if(f>u){c&&fe(0);break}}a&&l(d+131072);for(var gt=(1<>4;if(f+=_&15,f>u){c&&fe(0);break}if(_||fe(2),Ee<256)t[d++]=Ee;else if(Ee==256){Fe=f,y=null;break}else{var pt=Ee-254;if(Ee>264){var N=Ee-257,Ie=ai[N];pt=ge(r,f,(1<>4;je||fe(3),f+=je&15;var re=Eo[he];if(he>3){var Ie=ci[he];re+=Wn(r,f)&(1<u){c&&fe(0);break}a&&l(d+131072);var Ae=d+pt;if(d>3&1)+(e>>4&1);n>0;n-=!r[t++]);return t+(e&2)},lt=(function(){function r(e,t){typeof e=="function"&&(t=e,e={}),this.ondata=t;var n=e&&e.dictionary&&e.dictionary.subarray(-32768);this.s={i:0,b:n?n.length:0},this.o=new J(32768),this.p=new J(0),n&&this.o.set(n)}return r.prototype.e=function(e){if(this.ondata||fe(5),this.d&&fe(4),!this.p.length)this.p=e;else if(e.length){var t=new J(this.p.length+e.length);t.set(this.p),t.set(e,this.p.length),this.p=t}},r.prototype.c=function(e){this.s.i=+(this.d=e||!1);var t=this.s.b,n=yi(this.p,this.s,this.o);this.ondata(tn(n,t,this.s.b),this.d),this.o=tn(n,this.s.b-32768),this.s.b=this.o.length,this.p=tn(this.p,this.s.p/8|0),this.s.p&=7},r.prototype.push=function(e,t){this.e(e),this.c(t)},r})();pi=(function(){function r(e,t){this.v=1,this.r=0,lt.call(this,e,t)}return r.prototype.push=function(e,t){if(lt.prototype.e.call(this,e),this.r+=e.length,this.v){var n=this.p.subarray(this.v-1),i=n.length>3?xo(n):4;if(i>n.length){if(!t)return}else this.v>1&&this.onmember&&this.onmember(this.r-n.length);this.p=n.subarray(i),this.v=0}lt.prototype.c.call(this,0),this.s.f&&!this.s.l?(this.v=hi(this.s.p)+9,this.s={i:0},this.o=new J(0),this.push(new J(0),t)):t&<.prototype.c.call(this,t)},r})(),Io=typeof TextDecoder<"u"&&new TextDecoder,Ao=0;try{Io.decode(ko,{stream:!0}),Ao=1}catch{}});var jn={};xr(jn,{extractZipEntry:()=>No,extractZipEntryBounded:()=>Co,fetchZipCentralDirectory:()=>Fo,parseZipCentralDirectory:()=>Nt});function zi(r){let e=new DataView(r.buffer,r.byteOffset,r.byteLength),t=Math.max(0,r.length-vi);for(let n=r.length-Po;n>=t;n--)if(e.getUint32(n,!0)===_o)return n;throw new Error("Zip EOCD record not found")}function Nt(r){let e=new DataView(r.buffer,r.byteOffset,r.byteLength),t=zi(r),n=e.getUint16(t+10,!0),i=e.getUint32(t+16,!0),o=[],s=i;for(let a=0;a>8,E;v===mi?E=p>>16&65535:m.startsWith("bin/")||m.startsWith("sbin/")||m.includes("/bin/")||m.includes("/sbin/")?E=493:E=420;let z=m.endsWith("/"),S=v===mi&&(E&To)===Oo;o.push({fileName:m,fileNameBytes:u,compressedSize:h,uncompressedSize:f,compressionMethod:l,localHeaderOffset:w,mode:E,isDirectory:z,isSymlink:S,externalAttrs:p,creatorOS:v}),s+=Vn+d+y+g}return o}function bi(r,e){if(r.byteLength!==e.byteLength)return!1;for(let t=0;t{if(a.byteLength>t-o)throw new Error(`ZIP member ${e.fileName} expands beyond ${t} bytes`);i.set(a,o),o+=a.byteLength}).push(n,!0),o!==t)throw new Error(`ZIP member ${e.fileName} expanded ${o} bytes, expected ${t}`);return i}function Mo(r,e){let t=new DataView(r.buffer,r.byteOffset,r.byteLength),n=e.localHeaderOffset;if(n<0||n>r.byteLength-qn||t.getUint32(n,!0)!==wi)throw new Error(`Invalid local file header signature at offset ${n}`);let i=t.getUint16(n+8,!0),o=t.getUint16(n+26,!0),s=t.getUint16(n+28,!0),a=n+qn,c=a+o+s,l=c+e.compressedSize;if(i!==e.compressionMethod||cr.byteLength||!bi(r.subarray(a,a+o),e.fileNameBytes))throw new Error(`ZIP member ${e.fileName} has inconsistent local metadata`);return r.subarray(c,l)}async function Fo(r){let e=await fetch(r,{method:"HEAD"});if(!e.ok)throw new Error(`HEAD request failed: ${e.status} ${e.statusText}`);let t=parseInt(e.headers.get("content-length")||"0",10),n=e.headers.get("accept-ranges");if(!t||n!=="bytes"){let u=await fetch(r);if(!u.ok)throw new Error(`Fetch failed: ${u.status} ${u.statusText}`);let m=new Uint8Array(await u.arrayBuffer());return{entries:Nt(m),totalSize:m.length}}let i=Math.min(t,vi),o=t-i,s=await fetch(r,{headers:{Range:`bytes=${o}-${t-1}`}});if(s.status!==206){let u=await fetch(r);if(!u.ok)throw new Error(`Fetch failed: ${u.status} ${u.statusText}`);let m=new Uint8Array(await u.arrayBuffer());return{entries:Nt(m),totalSize:m.length}}let a=new Uint8Array(await s.arrayBuffer()),c=new DataView(a.buffer,a.byteOffset,a.byteLength),l=zi(a),h=c.getUint32(l+12,!0),f=c.getUint32(l+16,!0);if(f>=o){let u=t,m=new Uint8Array(u);return m.set(a,o),{entries:Nt(m),totalSize:u}}let d=f+h-1,y=await fetch(r,{headers:{Range:`bytes=${f}-${d}`}});if(y.status!==206)throw new Error(`Range request for CD failed: ${y.status}`);let g=new Uint8Array(await y.arrayBuffer()),p=t,w=new Uint8Array(p);return w.set(g,f),w.set(a,o),{entries:Nt(w),totalSize:p}}var _o,Lo,wi,vi,Po,Vn,qn,Ei,Si,mi,Oo,To,Ro,Bo,Yn=bn(()=>{"use strict";Hn();_o=101010256,Lo=33639248,wi=67324752,vi=65557,Po=22,Vn=46,qn=30,Ei=0,Si=8,mi=3,Oo=40960,To=61440,Ro=new TextDecoder("utf-8",{fatal:!0,ignoreBOM:!0}),Bo=new TextEncoder});var Pi={};xr(Pi,{DEFAULT_TAR_GZIP_LIMITS:()=>Li,TarParseError:()=>L,parseTarGzip:()=>Ko});function Ko(r,e={}){let t=e.label??"TAR gzip archive",n=Wo(e.limits,t);if(r.byteLength===0||r.byteLength>n.maxCompressedBytes)throw new L(`${t}: compressed byte count ${r.byteLength} is outside 1..${n.maxCompressedBytes}`);let i=Zo(r,t);if(i===0||i>n.maxUncompressedBytes)throw new L(`${t}: declared uncompressed byte count ${i} is outside 1..${n.maxUncompressedBytes}`);let o=Ho(r,t,i);if(o.byteLength!==i)throw new L(`${t}: gzip expanded to ${o.byteLength} bytes, expected ${i}`);let s=new DataView(r.buffer,r.byteOffset,r.byteLength).getUint32(r.byteLength-8,!0);if(Vo(o)!==s)throw new L(`${t}: gzip CRC32 mismatch`);return Go(o,t,n)}function Go(r,e,t){if(r.byteLength%ke!==0)throw new L(`${e}: TAR byte count is not block-aligned`);let n=[],i=0,o=0,s=0,a=null,c={},l=!1;for(;i+ke<=r.byteLength;){let h=r.subarray(i,i+ke);if(i+=ke,Jn(h)){if(i+ke>r.byteLength)throw new L(`${e}: TAR end marker is truncated`);let z=r.subarray(i,i+ke);if(!Jn(z))throw new L(`${e}: TAR has only one zero end block`);if(i+=ke,!Jn(r.subarray(i)))throw new L(`${e}: TAR has nonzero data after its end marker`);l=!0;break}Xo(h,e);let f=Ct(h,156,1,e)||"0",d=er(h,124,12,`${e}: TAR entry size`),y=er(h,100,8,`${e}: TAR entry mode`)&$o,g=Jo(h,e,t.maxPathBytes),p=Ct(h,157,100,e);if(f==="x"||f==="g"){if(s+=1,s>t.maxEntries+1)throw new L(`${e}: TAR extension header count exceeds ${t.maxEntries+1}`);let z=xi(r,i,d,e);i=Ii(i,d,r.byteLength,e);let S=jo(z,e,t);f==="x"?a=S:c={...c,...S};continue}if(o+=1,o>t.maxEntries)throw new L(`${e}: TAR entry count exceeds ${t.maxEntries}`);let w={...c,...a??{}};a=null;let u=w.size===void 0?d:Yo(w.size,`${e}: PAX entry size`),m=xi(r,i,u,e);i=Ii(i,u,r.byteLength,e);let v=Qn(w.path??g,e,t.maxPathBytes),E=w.linkpath??p;switch(f){case"0":case"\0":n.push({path:v,type:"file",mode:y,data:m});break;case"5":Xn(u,e,"directory",v),n.push({path:v,type:"directory",mode:y});break;case"2":Xn(u,e,"symlink",v),Ai(E,e,v,t.maxLinkBytes,!1),n.push({path:v,type:"symlink",mode:y,linkName:E});break;case"1":Xn(u,e,"hardlink",v),Ai(E,e,v,t.maxLinkBytes,!0),n.push({path:v,type:"hardlink",mode:y,linkName:Qn(E,`${e}: hardlink target`,t.maxPathBytes)});break;case"3":case"4":case"6":throw new L(`${e}: unsupported TAR device/FIFO entry ${v}`);default:throw new L(`${e}: unsupported TAR entry type ${JSON.stringify(f)} for ${v}`)}}if(!l)throw new L(`${e}: TAR is missing its two-block end marker`);if(a!==null)throw new L(`${e}: local PAX header has no following entry`);return n}function Wo(r,e){let t={...Li,...r};for(let[n,i]of Object.entries(t))if(!Number.isSafeInteger(i)||i<=0)throw new L(`${e}: ${n} must be a positive safe integer`);return t}function Zo(r,e){if(r.byteLength<18||r[0]!==31||r[1]!==139||r[2]!==8)throw new L(`${e}: invalid gzip header`);return new DataView(r.buffer,r.byteOffset,r.byteLength).getUint32(r.byteLength-4,!0)}function Ho(r,e,t){let n=new Uint8Array(t),i=0,o=!1,s=new pi(a=>{if(a.byteLength>t-i)throw new L(`${e}: gzip expansion exceeds its declared ${t} bytes`);n.set(a,i),i+=a.byteLength});s.onmember=()=>{throw o=!0,new L(`${e}: concatenated gzip members are unsupported`)};try{s.push(r,!0)}catch(a){throw a instanceof L?a:new L(`${e}: cannot gunzip archive: ${ea(a)}`)}if(o)throw new L(`${e}: concatenated gzip members are unsupported`);return n.subarray(0,i)}function Vo(r){let e=4294967295;for(let t of r)e=Uo[(e^t)&255]^e>>>8;return(e^4294967295)>>>0}function qo(){let r=new Uint32Array(256);for(let e=0;e>>1^((t&1)===0?0:3988292384);r[e]=t>>>0}return r}function xi(r,e,t,n){if(t>r.byteLength-e)throw new L(`${n}: TAR entry is truncated`);return r.subarray(e,e+t)}function Ii(r,e,t,n){let o=Math.ceil(e/ke)*ke;if(!Number.isSafeInteger(o)||o>t-r)throw new L(`${n}: TAR entry padding is truncated`);return r+o}function jo(r,e,t){let n={},i=0;for(;i9)throw new L(`${e}: invalid PAX record length`);if(s=s*10+p,!Number.isSafeInteger(s))throw new L(`${e}: invalid PAX record length`)}let a=i+s;if(s<=o-i+2||a>r.byteLength||r[a-1]!==10)throw new L(`${e}: truncated PAX record`);let c=o+1;for(;c=a-1)throw new L(`${e}: invalid PAX record`);let l=r.subarray(o+1,c);if(l.byteLength>256)throw new L(`${e}: PAX record key is too long`);let h=tr(l,`${e}: PAX record key`),f=r.subarray(c+1,a-1),d=h==="path"?t.maxPathBytes:h==="linkpath"?t.maxLinkBytes:h==="size"?32:0;if(d===0){i=a;continue}if(f.byteLength>d)throw new L(`${e}: PAX ${h} value is too long`);let y=tr(f,`${e}: PAX record value`);n[h]=y,i=a}return n}function Yo(r,e){if(!/^(0|[1-9][0-9]*)$/.test(r))throw new L(`${e} is invalid`);let t=Number(r);if(!Number.isSafeInteger(t)||t<0)throw new L(`${e} is invalid`);return t}function Xo(r,e){let t=er(r,148,8,`${e}: TAR checksum`),n=0;for(let i=0;i=148&&i<156?32:r[i];if(t!==n)throw new L(`${e}: TAR checksum mismatch`)}function Jo(r,e,t){let n=Ct(r,0,100,e),i=Ct(r,345,155,e);return Qn(i?`${i}/${n}`:n,e,t)}function Qn(r,e,t){let n=r;for(;n.startsWith("./");)n=n.slice(2);return n=n.replace(/\/+$/g,""),Qo(n,`${e}: TAR path`,t),n}function Ct(r,e,t,n){let i=e,o=e+t;for(;in||r.includes("\0"))throw new L(`${e}: link target for ${t} is invalid`);if(i&&r.includes("\\"))throw new L(`${e}: hardlink target for ${t} is invalid`)}function Qo(r,e,t){if(r.length===0||r.startsWith("/")||r.includes("\0")||r.includes("\\")||_i.encode(r).byteLength>t)throw new L(`${e} ${JSON.stringify(r)} must be a bounded relative POSIX path`);for(let n of r.split("/"))if(n.length===0||n==="."||n==="..")throw new L(`${e} ${JSON.stringify(r)} contains an unsafe path segment`)}function Jn(r){for(let e of r)if(e!==0)return!1;return!0}function tr(r,e){try{return Do.decode(r)}catch{throw new L(`${e} contains non-UTF-8 text`)}}function ea(r){return r instanceof Error?r.message:String(r)}var ke,$o,ki,Do,_i,Uo,Li,L,Oi=bn(()=>{"use strict";Hn();ke=512,$o=4095,ki=1024*1024,Do=new TextDecoder("utf-8",{fatal:!0,ignoreBOM:!0}),_i=new TextEncoder,Uo=qo(),Li=Object.freeze({maxCompressedBytes:256*ki,maxUncompressedBytes:512*ki,maxEntries:1e5,maxPathBytes:4096,maxLinkBytes:65536}),L=class extends Error{constructor(e){super(e),this.name="TarParseError"}}});import{existsSync as $t,lstatSync as En,readdirSync as Ra,readFileSync as Ze,realpathSync as me,statSync as He}from"node:fs";import{createHash as rs}from"node:crypto";import{spawnSync as wr}from"node:child_process";import{basename as Ba,dirname as Ut,isAbsolute as Sn,join as $,relative as Na,resolve as pe,sep as Ca}from"node:path";import{fileURLToPath as Ma}from"node:url";var mt="kandelo.wpk_fork.linked_frames";var Ir=[75,76,67,70],wt=24,Ar=8,kn=3,_r=[{bytes:4,chunkHeaderSize:32,nodeHeaderSize:24},{bytes:8,chunkHeaderSize:56,nodeHeaderSize:32}],Xe=[{module:"env",name:"__wpk_fork_frame_commit",params:["ptr"],results:[]},{module:"env",name:"__wpk_fork_frame_next",params:["ptr"],results:["ptr"]},{module:"env",name:"__wpk_fork_frame_reserve",params:["ptr"],results:["ptr"]}],vt=[{name:"wpk_fork_abort_begin",params:["ptr"],results:[]},{name:"wpk_fork_abort_end",params:[],results:[]},{name:"wpk_fork_rewind_begin",params:["ptr"],results:[]},{name:"wpk_fork_rewind_end",params:[],results:[]},{name:"wpk_fork_state",params:[],results:["i32"]},{name:"wpk_fork_unwind_begin",params:["ptr"],results:[]},{name:"wpk_fork_unwind_end",params:[],results:[]}];var Lr=["__abi_version","kernel_alloc_scratch","kernel_create_process","kernel_create_process_with_stdio","kernel_dequeue_signal","kernel_exec_prepare","kernel_exec_setup_for_thread","kernel_fork_process","kernel_get_parent_pid","kernel_get_process_exit_signal","kernel_get_process_state","kernel_handle_channel","kernel_has_sa_nocldstop","kernel_host_adapter_manifest_len","kernel_host_adapter_manifest_ptr","kernel_ipc_shmat_for_process","kernel_ipc_shmat_for_task","kernel_ipc_shmdt_for_process","kernel_ipc_shmdt_for_task","kernel_mark_process_signaled","kernel_pipe_has_readers","kernel_posix_timer_fire","kernel_prepare_write_operation","kernel_reap_exited_child","kernel_remove_process","kernel_set_current_tid","kernel_spawn_process","kernel_thread_exit","kernel_validate_task","kernel_wait_child_poll"];var G={LINK_MAX:0,MAX_CANON:1,MAX_INPUT:2,NAME_MAX:3,PATH_MAX:4,PIPE_BUF:5,CHOWN_RESTRICTED:6,NO_TRUNC:7,VDISABLE:8,SYNC_IO:9,ASYNC_IO:10,PRIO_IO:11,SOCK_MAXBUF:12,FILESIZEBITS:13,REC_INCR_XFER_SIZE:14,REC_MAX_XFER_SIZE:15,REC_MIN_XFER_SIZE:16,REC_XFER_ALIGN:17,ALLOC_SIZE_MIN:18,SYMLINK_MAX:19,POSIX2_SYMLINKS:20,FALLOC:21,TEXTDOMAIN_MAX:22,TIMESTAMP_RESOLUTION:23};function x(r,e){let t=0,n=0,i=e;for(;;){let o=r[i++];if(t|=(o&127)<=21&&n<=34?Et(e,t):n===84||n>=92&&n<=99||n>=112&&n<=123||n>=124&&n<=131||n>=156&&n<=159?t+1:t:r===254?n===0||n===1||n===2?Et(e,t):n===3?t:n>=16&&n<=79?Et(e,t):null:null}function ws(r,e,t){let[n,i]=x(r,e);e+=i+n;let[o,s]=x(r,e);e+=s+o;let a=r[e++];if(a===0){t.funcImports++;let[,c]=x(r,e);e+=c}else if(a===1){e++;let c=r[e++],[,l]=x(r,e);if(e+=l,c&1){let[,h]=x(r,e);e+=h}}else if(a===2){let c=r[e++],[,l]=x(r,e);if(e+=l,c&1){let[,h]=x(r,e);e+=h}}else a===3&&(t.globalImports++,e+=2);return e}function Wt(r){return r.length>=8&&r[0]===0&&r[1]===97&&r[2]===115&&r[3]===109}function Pe(r,e){let[t,n]=x(r,e);return e+=n,[new TextDecoder().decode(r.subarray(e,e+t)),e+t]}function vs(r,e){if(e.length===0)return!0;let t=new TextEncoder().encode(e);e:for(let n=0;n<=r.length-t.length;n++){for(let i=0;ir);function Pr(r,e,t){if(!t)throw new Error(`function ${e} refers to an unknown type`);let n=r.get(e)??[];n.push(t),r.set(e,n)}function xn(r,e){let[t,n]=x(r,e);e+=n;let[,i]=x(r,e);if(e+=i,(t&1)!==0){let[,o]=x(r,e);e+=o}return{flags:t,next:e}}function Ss(r){let e=new Uint8Array(r);if(!Wt(e))throw new Error("not a wasm binary");let t=[],n=[],i=[],o={functionImports:new Map,functionExports:new Map,memoryPointerWidths:[],linkedFrameDescriptors:[],importsKernelFork:!1},s=8;for(;se.length)throw new Error("wasm section exceeds file size");let d=h,y=!1;if(a===0){let[g,p]=Pe(e,d);g===mt&&o.linkedFrameDescriptors.push(e.slice(p,f))}else if(a===1){y=!0;let[g,p]=x(e,d);d+=p;for(let w=0;wr[c]===a))throw new Error("linked-frame descriptor has invalid magic");let e=new DataView(r.buffer,r.byteOffset,r.byteLength),t=e.getUint16(4,!0);if(t!==1)throw new Error(`linked-frame descriptor version ${t} is unsupported`);let n=e.getUint16(6,!0);if(n!==wt)throw new Error(`linked-frame descriptor declares size ${n}, expected ${wt}`);let i=e.getUint8(8),o=_r.find(({bytes:a})=>a===i);if(!o)throw new Error(`linked-frame descriptor pointer width ${i} is unsupported`);if(e.getUint8(9)!==Ar)throw new Error(`linked-frame descriptor alignment ${e.getUint8(9)} is unsupported`);let s=e.getUint16(10,!0);if(s!==kn)throw new Error(`linked-frame descriptor flags 0x${s.toString(16)} do not equal required flags 0x${kn.toString(16)}`);if(e.getUint32(12,!0)!==o.chunkHeaderSize||e.getUint32(16,!0)!==o.nodeHeaderSize)throw new Error(`linked-frame descriptor header sizes do not match its ${i}-byte pointer width`);return o.bytes}function Or(r,e){return r==="i32"?127:e===8?126:127}function Tr(r,e,t,n){return r.params.length===e.length&&r.results.length===t.length&&r.params.every((i,o)=>i===Or(e[o],n))&&r.results.every((i,o)=>i===Or(t[o],n))}function Rr(r,e,t){let n=i=>i==="ptr"&&t===8?"i64":"i32";return`(${r.map(n).join(", ")}) -> (${e.map(n).join(", ")})`}function bs(r){let e=[];for(let s of vt){let a=r.functionExports.get(s.name);a&&a.length!==1&&e.push(`duplicate ABI 42 wasm-fork-instrument export ${s.name}`)}let t=vt.filter(({name:s})=>!r.functionExports.has(s)).map(({name:s})=>s);t.length>0&&e.push(`incomplete wasm-fork-instrument exports; missing ${t.join(", ")}`);let n=null;if(r.linkedFrameDescriptors.length===0)e.push(`missing required ${mt} descriptor`);else if(r.linkedFrameDescriptors.length!==1)e.push(`has ${r.linkedFrameDescriptors.length} ${mt} descriptors, expected exactly one`);else try{n=zs(r.linkedFrameDescriptors[0])}catch(s){e.push(s instanceof Error?s.message:String(s))}let i=Xe.filter(({module:s,name:a})=>r.functionImports.has(`${s}.${a}`)),o=r.importsKernelFork||i.length>0;if(o){let s=Xe.filter(({module:a,name:c})=>!r.functionImports.has(`${a}.${c}`)).map(({module:a,name:c})=>`${a}.${c}`);s.length>0&&e.push(`incomplete ABI 42 linked-frame imports; missing ${s.join(", ")}`);for(let a of Xe){let c=`${a.module}.${a.name}`,l=r.functionImports.get(c);l&&l.length!==1&&e.push(`duplicate ABI 42 linked-frame import ${c}`)}}if(n!==null){if(r.memoryPointerWidths.length!==1)e.push(`ABI 42 fork instrumentation requires exactly one module memory, found ${r.memoryPointerWidths.length}`);else if(r.memoryPointerWidths[0]!==n){let s=n===8?"an":"a";e.push(`ABI 42 linked-frame descriptor declares ${s} ${n}-byte pointer but the module memory uses ${r.memoryPointerWidths[0]}-byte addresses`)}for(let s of vt){let a=r.functionExports.get(s.name);a?.length===1&&!Tr(a[0],s.params,s.results,n)&&e.push(`ABI 42 wasm-fork-instrument export ${s.name} has the wrong signature; expected ${Rr(s.params,s.results,n)}`)}if(o)for(let s of Xe){let a=`${s.module}.${s.name}`,c=r.functionImports.get(a);c?.length===1&&!Tr(c[0],s.params,s.results,n)&&e.push(`ABI 42 linked-frame import ${a} has the wrong signature; expected ${Rr(s.params,s.results,n)}`)}}return e}function ks(r){let e=new Uint8Array(r);if(!Wt(e))return[];let t=[],n=8;for(;nt.startsWith("reloc."))}function Nr(r,e={}){let t=[];if(Is(r)&&t.push("contains asyncify_"),e.expectedAbi!==void 0&&e.expectedAbi!==null){let f=Ls(r);f!==null&&f!==e.expectedAbi&&t.push(`ABI ${f}, expected ${e.expectedAbi}`)}let n=new Set(xs(r));if(e.requiredExports){let f=e.requiredExports.filter(d=>!n.has(d));f.length>0&&t.push(`missing required exports: ${f.join(", ")}`)}let i=Es.filter(f=>n.has(f)),o=ks(r),s=Br(r),a=Xe.filter(({module:f,name:d})=>o.includes(`${f}.${d}`)),c=s.filter(f=>f===mt).length,l=i.length>0||a.length>0||c>0;if(e.forbidForkInstrumentation&&l&&t.push("contains ABI 42 wasm-fork-instrument metadata, imports, or exports"),(e.requireForkInstrumentation??!As(r))&&(l||o.includes("kernel.kernel_fork")))try{t.push(...bs(Ss(r)))}catch(f){t.push(`cannot validate ABI 42 fork-artifact contract: ${f instanceof Error?f.message:String(f)}`)}return t}function _s(r,e){let t=new Uint8Array(r);if(t.length<8)return null;let n=0,i=null,o=null,s=8;for(;s=c)return null;let p=a;for(let m=0;m=g)return null;let[p,w]=x(t,y);y+=w;for(let u=0;ug)return null}return y}function d(y,g=0){if(g>4)return null;let p=h(y);if(!p)return null;let w=f(p.start,p.end);if(w===null)return null;let u=w,m=p.end;for(;u=32&&v<=38||v===208){let[,E]=x(t,u);u+=E}else if(v>=40&&v<=62)u=Et(t,u);else if(v===63||v===64)u++;else if(v===66){let[,E]=gs(t,u);u+=E}else if(v===67)u+=4;else if(v===68)u+=8;else if(v===252||v===253||v===254){let E=ms(v,t,u);if(E===null)return null;u=E}}return null}return d(i)}function Ls(r){return _s(r,"__abi_version")}var Ps=ArrayBuffer,H=Uint8Array,Zt=Uint16Array,Os=Int16Array;var Ht=Int32Array,An=function(r,e,t){if(H.prototype.slice)return H.prototype.slice.call(r,e,t);(e==null||e<0)&&(e=0),(t==null||t>r.length)&&(t=r.length);var n=new H(t-e);return n.set(r.subarray(e,t)),n},zt=function(r,e,t,n){if(H.prototype.fill)return H.prototype.fill.call(r,e,t,n);for((t==null||t<0)&&(t=0),(n==null||n>r.length)&&(n=r.length);tr.length)&&(n=r.length);t2046MB)","invalid block type","FSE accuracy too high","match distance too far back","unexpected EOF"],V=function(r,e,t){var n=new Error(e||Rs[r]);if(n.code=r,Error.captureStackTrace&&Error.captureStackTrace(n,V),!t)throw n;return n},Cr=function(r,e,t){for(var n=0,i=0;n>>0},Ns=function(r,e){var t=r[0]|r[1]<<8|r[2]<<16;if(t==3126568&&r[3]==253){var n=r[4],i=n>>5&1,o=n>>2&1,s=n&3,a=n>>6;n&8&&V(0);var c=6-i,l=s==3?4:s,h=Cr(r,c,l);c+=l;var f=a?1<>3);y=g+(g>>3)*(r[5]&7)}y>2145386496&&V(1);var p=new H((e==1?d||y:e?0:y)+12);return p[0]=1,p[4]=4,p[8]=8,{b:c+f,y:0,l:0,d:h,w:e&&e!=1?e:p.subarray(12),e:y,o:new Ht(p.buffer,0,3),u:d,c:o,m:Math.min(131072,y)}}else if((t>>4|r[3]<<20)==25481893)return Bs(r,4)+8;V(0)},De=function(r){for(var e=0;1<t&&V(3);for(var o=1<0;){var m=De(s+1),v=n>>3,E=(1<>(n&7)&E,S=(1<S&&(z-=k)),d[++a]=--z,z==-1?(s+=z,w[--h]=a):s-=z,!z)do{var A=n>>3;c=(r[A]|r[A+1]<<8)>>(n&7)&3,n+=2,a+=c}while(c==3)}(a>255||s)&&V(0);for(var B=0,N=(o>>1)+(o>>3)+3,te=o-1,j=0;j<=a;++j){var R=d[j];if(R<1){y[j]=-R;continue}for(l=0;l=h)}}for(B&&V(0),l=0;l>3,{b:i,s:w,n:u,t:g}]},Cs=function(r,e){var t=0,n=-1,i=new H(292),o=r[e],s=i.subarray(0,256),a=i.subarray(256,268),c=new Zt(i.buffer,268);if(o<128){var l=bt(r,e+1,6),h=l[0],f=l[1];e+=o;var d=h<<3,y=r[e];y||V(0);for(var g=0,p=0,w=f.b,u=w,m=(++e<<3)-8+De(y);m-=w,!(m>3;if(g+=(r[v]|r[v+1]<<8)>>(m&7)&(1<>3,p+=(r[v]|r[v+1]<<8)>>(m&7)&(1<255&&V(0)}else{for(n=o-127;t>4,s[t+1]=E&15}++e}var z=0;for(t=0;t11&&V(0),z+=S&&1<0;--t){var j=c[t];zt(te,t,j,c[t-1]=j+a[t]*(1<a&&f>3,y=(r[d]|r[d+1]<<8|r[d+2]<<16)>>(h&7);c=(c<>2,s=o<<1,a=o+s;St(r.subarray(n,n+=r[0]|r[1]<<8),e.subarray(0,o),t),St(r.subarray(n,n+=r[2]|r[3]<<8),e.subarray(o,s),t),St(r.subarray(n,n+=r[4]|r[5]<<8),e.subarray(s,a),t),St(r.subarray(n),e.subarray(a),t)},Gs=function(r,e,t){var n,i=e.b,o=r[i],s=o>>1&3;e.l=o&1;var a=o>>3|r[i+1]<<5|r[i+2]<<13,c=(i+=3)+a;if(s==1)return i>=r.length?void 0:(e.b=i+1,t?(zt(t,r[i],e.y,e.y+=a),t):zt(new H(a),r[i]));if(!(c>r.length)){if(s==0)return e.b=c,t?(t.set(r.subarray(i,c),e.y),e.y+=a,t):An(r,i,c);if(s==2){var l=r[i],h=l&3,f=l>>2&3,d=l>>4,y=0,g=0;h<2?f&1?d|=r[++i]<<4|(f&2&&r[++i]<<12):d=l>>3:(g=f,f<2?(d|=(r[++i]&63)<<4,y=r[i]>>6|r[++i]<<2):f==2?(d|=r[++i]<<4|(r[++i]&3)<<12,y=r[i]>>2|r[++i]<<6):(d|=r[++i]<<4|(r[++i]&63)<<12,y=r[i]>>6|r[++i]<<2|r[++i]<<10)),++i;var p=t?t.subarray(e.y,e.y+e.m):new H(e.m),w=p.length-d;if(h==0)p.set(r.subarray(i,i+=d),w);else if(h==1)zt(p,r[i++],w);else{var u=e.h;if(h==2){var m=Cs(r,i);y+=i-(i=m[0]),e.h=u=m[1]}else u||V(0);(g?Ks:St)(r.subarray(i,i+=y),p.subarray(w),u)}var v=r[i++];if(v){v==255?v=(r[i++]|r[i++]<<8)+32512:v>127&&(v=v-128<<8|r[i++]);var E=r[i++];E&3&&V(0);for(var z=[Fs,$s,Ms],S=2;S>-1;--S){var k=E>>(S<<1)+2&3;if(k==1){var I=new H([0,0,r[i++]]);z[S]={s:I.subarray(2,3),n:I.subarray(0,1),t:new Zt(I.buffer,0,1),b:0}}else k==2?(n=bt(r,i,9-(S&1)),i=n[0],z[S]=n[1]):k==3&&(e.t||V(0),z[S]=e.t[S])}var A=e.t=z,B=A[0],N=A[1],te=A[2],j=r[c-1];j||V(0);var R=(c<<3)-8+De(j)-te.b,P=R>>3,_=0,Z=(r[P]|r[P+1]<<8)>>(R&7)&(1<>3;var Me=(r[P]|r[P+1]<<8)>>(R&7)&(1<>3;var re=(r[P]|r[P+1]<<8)>>(R&7)&(1<>3;var je=1<>>(R&7)&je-1);P=(R-=Ln[Fe])>>3;var Ae=Us[Fe]+((r[P]|r[P+1]<<8|r[P+2]<<16)>>(R&7)&(1<>3;var $e=Ds[gt]+((r[P]|r[P+1]<<8|r[P+2]<<16)>>(R&7)&(1<<_n[gt])-1);if(P=(R-=Kt)>>3,Z=te.t[Z]+((r[P]|r[P+1]<<8)>>(R&7)&(1<>3,re=B.t[re]+((r[P]|r[P+1]<<8)>>(R&7)&(1<>3,Me=N.t[Me]+((r[P]|r[P+1]<<8)>>(R&7)&(1<3)e.o[2]=e.o[1],e.o[1]=e.o[0],e.o[0]=he-=3;else{var Ye=he-($e!=0);Ye?(he=Ye==3?e.o[0]-1:e.o[Ye],Ye>1&&(e.o[2]=e.o[1]),e.o[1]=e.o[0],e.o[0]=he):he=e.o[0]}for(var S=0;S<$e;++S)p[_+S]=p[w+S];_+=$e,w+=$e;var _e=_-he;if(_e<0){var Le=-_e,Gt=e.e+_e;Le>Ae&&(Le=Ae);for(var S=0;S=i){let I=(y+1)*4096;try{e.grow(I)}catch{throw new b(X)}if(i=Math.floor(e.byteLength/4096),y>=i)throw new b(X)}new Uint8Array(e).fill(0);let g=new r(e);g.w32(Bn,Tn),g.w32(Nn,Rn),g.w32(jt,4096),g.w32(Qe,i),g.w32(Oe,s),g.w32(Ue,h),g.w32(Xt,f),g.w32(Ur,d),g.w32(Jt,y),g.w32(eo,a),g.w32(to,c),g.w32(no,l),g.w32(It,o),g.w32(Kr,256);let p=f*4096;for(let I=0;I>2)+(I>>5);g.i32[A]|=1<<(I&31)}let w=i-y;Atomics.store(g.i32,et>>2,w),g.blockAllocHint=y;let u=h*4096;g.i32[u>>2]|=3,Atomics.store(g.i32,Yt>>2,s-2),g.inodeAllocHint=2;let m=g.inodeOffset(1);g.w32(m+C,U|493),g.w32(m+D,2),g.w64(m+oe,1);let v=g.blockAlloc();if(v<0)throw new b(X);g.w32(m+Y,v);let E=v*4096,z=Re(O+1),S=Re(O+2);g.w32(E,1),g.view.setUint16(E+4,z,!0),g.view.setUint16(E+6,1,!0),g.u8[E+O]=46;let k=E+z;return g.w32(k,1),g.view.setUint16(k+4,S,!0),g.view.setUint16(k+6,2,!0),g.u8[k+O]=46,g.u8[k+O+1]=46,g.w64(m+T,z+S),Atomics.store(g.i32,Cn>>2,1),g}static inspectImageCapacity(e){if(e.byteLengththis.snapshotBytesUnlocked(e))}snapshotState(e){return this.withNamespaceLock(()=>({bytes:this.snapshotBytesUnlocked(e),identities:this.collectIdentityStateUnlocked()}))}identityState(){return this.withNamespaceLock(()=>this.collectIdentityStateUnlocked())}snapshotBytesUnlocked(e){let t=e?.normalizeTimestampsMs;if(t!==void 0&&(!Number.isSafeInteger(t)||t<0))throw new b(W,"Snapshot timestamp must be a non-negative safe integer in milliseconds");let n=t===void 0?void 0:BigInt(t);for(let a=0;a>2)!==0)throw new b(Fn,"Cannot save a VFS image with open descriptors")}let i=this.r32(Oe);for(let a=0;a=1&&this.inodeIsAllocated(a)?n:0n;s.setBigUint64(c+At,l,!0),s.setBigUint64(c+ie,l,!0),s.setBigUint64(c+q,l,!0)}}return o}collectIdentityStateUnlocked(){let e=new Map,t=[{ino:1,path:"/"}],n=new Set;for(;t.length>0;){let i=t.pop();if(n.has(i.ino))throw new b(F);n.add(i.ino);let o=this.inodeOffset(i.ino);if((this.r32(o+C)&K)!==U)throw new b(F);let s=this.r64(o+T),a=0;for(;a>2)>>>0,paths:[]},e.set(S,k)),k.paths.push(v),(this.r32(E+C)&K)===U&&t.push({ino:p,path:v})}}y+=w}a+=d}}return e}statfs(){let e=this.r32(jt),t=this.r32(Qe),n=this.r32(It),i=typeof this.buffer.maxByteLength=="number"?this.buffer.maxByteLength:this.buffer.byteLength,o=Math.floor(i/e),s=Math.max(t,Math.min(n,o)),a=Atomics.load(this.i32,et>>2),c=Math.max(0,s-t);return{blockSize:e,totalBlocks:s,freeBlocks:a+c,totalInodes:this.r32(Oe),freeInodes:Atomics.load(this.i32,Yt>>2),maxName:255}}r32(e){return this.view.getUint32(e,!0)}w32(e,t){this.view.setUint32(e,t,!0)}r64(e){return Number(this.view.getBigUint64(e,!0))}w64(e,t){this.view.setBigUint64(e,BigInt(t),!0)}waitForAtomicChange(e,t){if(this.atomicsWaitAllowed!==!1)try{Atomics.wait(this.i32,e,t),this.atomicsWaitAllowed=!0;return}catch(n){if(!(n instanceof TypeError))throw n;this.atomicsWaitAllowed=!1}for(;Atomics.load(this.i32,e)===t;);}resetAllocationHints(){this.blockAllocHint=this.findNextFreeBlockHint(),this.inodeAllocHint=this.findNextFreeInodeHint()}findNextFreeBlockHint(){let e=this.r32(Qe),t=this.r32(Jt),n=this.r32(Xt)*4096;for(let i=t;i>2)+(i>>5),s=i&31;if((Atomics.load(this.i32,o)&1<>2)+(n>>5),o=n&31;if((Atomics.load(this.i32,i)&1<>2;for(;;){if(Atomics.compareExchange(this.i32,e,0,1)===0)return;this.waitForAtomicChange(e,1)}}sbUnlock(){let e=Qt>>2;Atomics.store(this.i32,e,0),Atomics.notify(this.i32,e,1/0)}namespaceLock(){let e=en>>2;for(;;){if(Atomics.compareExchange(this.i32,e,0,1)===0)return;this.waitForAtomicChange(e,1)}}namespaceUnlock(){let e=en>>2;Atomics.store(this.i32,e,0),Atomics.notify(this.i32,e,1/0)}withNamespaceLock(e){this.namespaceLock();try{return e()}finally{this.namespaceUnlock()}}resetRestoredRuntimeState(){Atomics.store(this.i32,Qt>>2,0),Atomics.store(this.i32,en>>2,0),this.u8.fill(0,256,4096);let e=this.r32(Oe),t=this.r32(Ue)*4096;for(let n=0;n>5)*4)&1<<(n&31))===0||this.r32(i+D)!==0)continue;let s=this.r32(i+C),a=this.r64(i+T);(s&K)===xt&&a<=40?(this.u8.fill(0,i+Y,i+Y+40),this.w64(i+T,0)):this.inodeTruncate(n,0),this.inodeFree(n)}}blockAlloc(){let e=this.r32(Qe),t=this.r32(Xt)*4096,n=this.r32(Jt),i=this.blockAllocHint>=n&&this.blockAllocHint>2)+(a>>5),l=a&31,h=Atomics.load(this.i32,c);if(h&1<>2,1),this.blockAllocHint=a+1>2)+(e>>5),i=e&31;for(;;){let o=Atomics.load(this.i32,n),s=o&~(1<>2,1),e>=this.r32(Jt)&&e>2)>0)return 0;let e=this.r32(Qe),t=this.r32(It),n=this.r32(Kr),i=e+n;if(i>t&&(i=t,n=i-e,n===0))return X;let o=i*4096;if(this.buffer.byteLength>2,n),Atomics.add(this.i32,Cn>>2,1),this.blockAllocHint=e,0}finally{this.sbUnlock()}}inodeOffset(e){let n=this.r32(Ur)+Math.floor(e/32),i=e%32*128;return n*4096+i}inodeAlloc(){let e=this.r32(Oe),t=this.r32(Ue)*4096,n=this.inodeAllocHint>=2&&this.inodeAllocHint>2)+(s>>5),c=s&31,l=Atomics.load(this.i32,a);if(l&1<>2,1),this.inodeAllocHint=s+1>2,1)+1}inodeFree(e){let n=(this.r32(Ue)*4096>>2)+(e>>5),i=e&31;for(;;){let o=Atomics.load(this.i32,n);if((o&1<>2,1),e>=2&&e0&&this.w32(n+Te,i-1),i<=1&&this.r32(n+D)===0&&(this.inodeTruncate(e,0),t=!0)}finally{this.inodeWriteUnlock(e)}t&&this.inodeFree(e)}inodeDropLinkRefLocked(e){let t=this.inodeOffset(e),n=this.r32(t+D);return n>1?(this.w32(t+D,n-1),this.w64(t+q,Date.now()),!1):this.inodeOrphanLocked(e)}inodeOrphanLocked(e){let t=this.inodeOffset(e);if(this.w32(t+D,0),this.w64(t+q,Date.now()),this.r32(t+Te)>0)return!1;let n=this.r32(t+C),i=this.r64(t+T);return(n&K)===xt&&i<=40?(this.u8.fill(0,t+Y,t+Y+40),this.w64(t+T,0)):this.inodeTruncate(e,0),!0}inodeReadLock(e){let t=this.inodeOffset(e)+tt>>2;for(;;){let n=Atomics.load(this.i32,t);if(n&jr){this.waitForAtomicChange(t,n);continue}if(Atomics.compareExchange(this.i32,t,n,n+1)===n)return}}inodeReadUnlock(e){let t=this.inodeOffset(e)+tt>>2;(Atomics.sub(this.i32,t,1)&ro)===1&&Atomics.notify(this.i32,t,1)}inodeWriteLock(e){let t=this.inodeOffset(e)+tt>>2;for(;;){let n=Atomics.load(this.i32,t);if(n!==0){this.waitForAtomicChange(t,n);continue}if(Atomics.compareExchange(this.i32,t,0,jr)===0)return}}inodeWriteUnlock(e){let t=this.inodeOffset(e)+tt>>2;Atomics.store(this.i32,t,0),Atomics.notify(this.i32,t,1/0)}inodeBlockMap(e,t,n){let i=this.inodeOffset(e);if(t<10){let o=this.r32(i+Y+t*4);if(o!==0)return o;if(!n)return 0;let s=this.blockAllocWithGrow();return s<0||this.w32(i+Y+t*4,s),s}if(t-=10,t<1024){let o=this.r32(i+_t),s=!1;if(o===0){if(!n)return 0;if(o=this.blockAllocWithGrow(),o<0)return o;this.w32(i+_t,o),s=!0}let a=o*4096+t*4,c=this.r32(a);if(c!==0)return c;if(!n)return 0;let l=this.blockAllocWithGrow();return l<0?(s&&(this.w32(i+_t,0),this.blockFree(o)),l):(this.w32(a,l),l)}if(t-=1024,t<1024*1024){let o=Math.floor(t/1024),s=t%1024,a=this.r32(i+nt),c=!1;if(a===0){if(!n)return 0;if(a=this.blockAllocWithGrow(),a<0)return a;this.w32(i+nt,a),c=!0}let l=a*4096+o*4,h=this.r32(l),f=!1;if(h===0){if(!n)return 0;if(h=this.blockAllocWithGrow(),h<0)return c&&(this.w32(i+nt,0),this.blockFree(a)),h;this.w32(l,h),f=!0}let d=h*4096+s*4,y=this.r32(d);if(y!==0)return y;if(!n)return 0;let g=this.blockAllocWithGrow();return g<0?(f&&(this.w32(l,0),this.blockFree(h)),c&&(this.w32(i+nt,0),this.blockFree(a)),g):(this.w32(d,g),g)}return W}inodeReadData(e,t,n,i){let o=this.inodeOffset(e),s=this.r64(o+T);if(t>=s)return 0;t+i>s&&(i=s-t);let a=0,c=0;for(;i>0;){let l=Math.floor(t/4096),h=t%4096,f=4096-h;f>i&&(f=i);let d=this.inodeBlockMap(e,l,!1);if(d<=0)n.fill(0,c,c+f);else{let y=d*4096+h;n.set(this.u8.subarray(y,y+f),c)}c+=f,t+=f,i-=f,a+=f}return a}inodeWriteData(e,t,n,i){let o=this.inodeOffset(e),s=this.r64(o+T);t>s&&this.zeroOldEofTail(e,s);let a=0,c=0;for(;i>0;){let l=Math.floor(t/4096),h=t%4096,f=4096-h;f>i&&(f=i);let d=this.inodeBlockMap(e,l,!0);if(d<0){if(a===0)return d;break}let y=d*4096+h;this.u8.set(n.subarray(c,c+f),y),c+=f,t+=f,i-=f,a+=f}if(a>0&&t>this.r64(o+T)&&this.w64(o+T,t),a>0){let l=Date.now();this.w64(o+ie,l),this.w64(o+q,l),Atomics.add(this.i32,o+ae>>2,1)}return a}zeroInodeRange(e,t,n){for(;t0){let c=a*4096+o;this.u8.fill(0,c,c+s)}t+=s}}zeroOldEofTail(e,t){let n=t%4096;if(n===0)return;let i=Math.floor(t/4096),o=this.inodeBlockMap(e,i,!1);if(o<=0)return;let s=o*4096+n;this.u8.fill(0,s,o*4096+4096)}freeBlocksFrom(e,t){let n=this.inodeOffset(e);for(let s=t;s<10;s++){let a=this.r32(n+Y+s*4);a&&(this.blockFree(a),this.w32(n+Y+s*4,0))}let i=this.r32(n+_t);if(i){let s=t>10?t-10:0;for(let a=s;a<1024;a++){let c=i*4096+a*4,l=this.r32(c);l&&(this.blockFree(l),this.w32(c,0))}s===0&&(this.blockFree(i),this.w32(n+_t,0))}let o=this.r32(n+nt);if(o){let s=t>1034?t-10-1024:0,a=Math.floor(s/1024);for(let c=a;c<1024;c++){let l=o*4096+c*4,h=this.r32(l);if(!h)continue;let f=c===a?s%1024:0;for(let d=f;d<1024;d++){let y=h*4096+d*4,g=this.r32(y);g&&(this.blockFree(g),this.w32(y,0))}f===0&&(this.blockFree(h),this.w32(l,0))}a===0&&(this.blockFree(o),this.w32(n+nt,0))}}inodeTruncate(e,t,n=!1){let i=this.inodeOffset(e),o=this.r64(i+T),s=t!==o;if(t>=o){if(t>o&&this.zeroOldEofTail(e,o),this.w64(i+T,t),s||n){let c=Date.now();this.w64(i+ie,c),this.w64(i+q,c),Atomics.add(this.i32,i+ae>>2,1)}return}t%4096!==0&&this.zeroInodeRange(e,t,Math.ceil(t/4096)*4096);let a=Math.ceil(t/4096);if(this.freeBlocksFrom(e,a),this.w64(i+T,t),s||n){let c=Date.now();this.w64(i+ie,c),this.w64(i+q,c),Atomics.add(this.i32,i+ae>>2,1)}}validateFileSize(e){if(!Number.isSafeInteger(e)||e<0)throw new b(W);if(e>rt)throw new b(Lt)}validateSeekPosition(e){if(!Number.isSafeInteger(e))throw new b(Qr);if(e<0)throw new b(W);if(e>rt)throw new b(Lt)}touchDirectoryMutation(e){let t=this.inodeOffset(e),n=Date.now();this.w64(t+ie,n),this.w64(t+q,n);let i=Atomics.add(this.i32,t+Zr>>2,1)+1>>>0,o=this.dirIndexes.get(e);o&&(o.mutationSequence=i,o.size=this.r64(t+T))}dirNameKey(e){return it(e)}dirEntryNameMatches(e,t){if(this.view.getUint16(e+6,!0)!==t.length)return!1;for(let i=0;i=O&&n%4===0&&e+n<=t&&i<=n-O}inodeIsAllocated(e){let t=this.r32(Oe);if(e<=0||e>=t)return!1;let n=this.r32(Ue)*4096;return(Atomics.load(this.i32,(n>>2)+(e>>5))&1<<(e&31))!==0}rebuildDirIndex(e,t,n,i){let o=new Map,s=[],a=0;for(;a4096-h&&(y=4096-h);let g=h;for(;g=O&&s.push({abs:p,recLen:u});g+=u}a+=y}let c={generation:t,mutationSequence:n,size:i,entries:o,free:s};return this.dirIndexes.set(e,c),c}getDirIndex(e){let t=this.inodeOffset(e),n=this.r64(t+T),i=this.r64(t+oe),o=Atomics.load(this.i32,t+Zr>>2)>>>0,s=this.dirIndexes.get(e);return s&&s.generation===i&&s.mutationSequence===o&&s.size===n?s:(s&&this.dirIndexes.delete(e),n=0;s--){let a=e.free[s];if(!(a.recLen4096-c&&(f=4096-c);let d=c;for(;dn)return-1;a=c,s+=l}return s===n?a:-1}dirAppendEntry(e,t,n,i=-1){let o=this.inodeOffset(e),s=this.r64(o+T),a=Re(O+t.length),c=s,l=Math.floor(c/4096),h=c%4096,f=0;if(h!==0&&h+a>4096){let g=4096-h,p=0;if(g>=O){if(p=this.inodeBlockMap(e,l,!1),p<=0)return F}else if(i<0&&(i=this.findLastDirEntryInBlock(e,l,h)),i<0)return F;if(f=this.inodeBlockMap(e,l+1,!0),f<0)return f;if(g>=O){let w=p*4096+h;this.w32(w,0),this.view.setUint16(w+4,g,!0),this.view.setUint16(w+6,0,!0)}else{let u=this.view.getUint16(i+4,!0)+g;this.view.setUint16(i+4,u,!0),this.updateDirIndexRecLen(e,i,u)}c=(l+1)*4096,l++,h=0}let d;if(h===0){if(d=f||this.inodeBlockMap(e,l,!0),d<0)return d}else if(d=this.inodeBlockMap(e,l,!1),d<=0)return F;let y=d*4096+h;return this.w32(y,n),this.view.setUint16(y+4,a,!0),this.view.setUint16(y+6,t.length,!0),this.u8.set(t,y+O),this.w64(o+T,c+a),this.touchDirectoryMutation(e),this.updateDirIndexAdd(e,t,n,y,a),0}dirAddEntry(e,t,n){let i=this.getDirIndex(e);if(typeof i=="number")return i;if(i)return this.useDirIndexFreeSlot(i,e,t,n)?0:this.dirAppendEntry(e,t,n);let o=this.inodeOffset(e),s=this.r64(o+T),a=Re(O+t.length),c=-1,l=0;for(;l4096-f&&(g=4096-f);let p=f;for(;pf+g||v>m-O)return F;if(u===0&&m>=a)return this.w32(w,n),this.view.setUint16(w+6,t.length,!0),this.u8.set(t,w+O),this.touchDirectoryMutation(e),this.updateDirIndexAdd(e,t,n,w,m),0;let E=Re(O+v),z=m-E;if(u!==0&&z>=a){this.view.setUint16(w+4,E,!0);let S=w+E;return this.w32(S,n),this.view.setUint16(S+4,z,!0),this.view.setUint16(S+6,t.length,!0),this.u8.set(t,S+O),this.touchDirectoryMutation(e),this.updateDirIndexAdd(e,t,n,S,z),0}c=w,p+=m}l+=g}return this.dirAppendEntry(e,t,n,c)}dirRemoveEntry(e,t){let n=this.getDirIndex(e);if(typeof n=="number")return n;if(n){let a=this.dirNameKey(t),c=n.entries.get(a);if(!c)return ye;if(this.r32(c.abs)===c.ino&&this.view.getUint16(c.abs+4,!0)===c.recLen&&this.view.getUint16(c.abs+6,!0)===c.nameLen&&this.dirEntryNameMatches(c.abs,t))return this.w32(c.abs,0),n.entries.delete(a),n.free.push({abs:c.abs,recLen:c.recLen}),this.touchDirectoryMutation(e),0;n.entries.delete(a)}let i=this.inodeOffset(e),o=this.r64(i+T),s=0;for(;s4096-c&&(f=4096-c);let d=c;for(;d4096-l&&(d=4096-l);let y=l;for(;y4096-s&&(l=4096-s);let h=s;for(;hs+l||g>y-O)throw new b(F);if(d!==0){if(g===1&&this.u8[f+O]===46){h+=y;continue}if(g===2&&this.u8[f+O]===46&&this.u8[f+O+1]===46){h+=y;continue}return!1}h+=y}i+=l}return!0}dirIsAncestor(e,t){let n=t;for(let i=0;i<8*1024;i++){if(n===e)return!0;if(n===1)return!1;let o=this.dirLookup(n,Yr);if(o<0||o===n)throw new b(F);n=o}throw new b(F)}pathResolve(e,t){if(!e.startsWith("/"))return ye;let n=1,i=e.split("/").filter(s=>s.length>0),o=0;for(let s=0;s255)return $n;let c=ce.encode(a),l;this.inodeReadLock(n);try{let d=this.inodeOffset(n);if((this.r32(d+C)&K)!==U)return Se;l=this.dirLookup(n,c)}finally{this.inodeReadUnlock(n)}if(l<0)return l;let h=this.inodeOffset(l);if((this.r32(h+C)&K)===xt&&(!(s===i.length-1)||t)){if(++o>8)return Jr;let y=this.r64(h+T),g;if(y<=40)g=it(this.u8.subarray(h+Y,h+Y+y));else{let p=new Uint8Array(y);this.inodeReadData(l,0,p,y),g=Ot.decode(p)}if(g.startsWith("/")){n=1;let p=g.split("/").filter(u=>u.length>0),w=i.slice(s+1);i.length=0,i.push(...p,...w),s=-1}else{let p=g.split("/").filter(u=>u.length>0),w=i.slice(s+1);i.length=s,i.push(...p,...w),s--}continue}n=l}return n}pathResolveParent(e){if(!e.startsWith("/"))throw new b(W,"Path must be absolute");let t=e.split("/").filter(c=>c.length>0);if(t.length===0)throw new b(W,"Cannot operate on /");let n=t.pop();if(n.length>255)throw new b($n);let i="/"+t.join("/"),o=this.pathResolve(i,!0);if(o<0)throw new b(o);let s=this.inodeOffset(o);if((this.r32(s+C)&K)!==U)throw new b(Se);return{parentIno:o,name:n}}fdAlloc(e,t,n){for(let i=0;i>2;if(Atomics.compareExchange(this.i32,s,0,1)===0)return this.w32(o+Hr,e),this.w64(o+Ke,0),this.w32(o+Vr,t),this.w32(o+qr,n?1:0),this.inodeAddOpenRef(e)?i:(Atomics.store(this.i32,s,0),ye)}return Xr}fdGet(e){if(e<0||e>=Vt)return null;let t=256+e*24;return Atomics.load(this.i32,t>>2)?{base:t,ino:this.r32(t+Hr),offset:this.r64(t+Ke),flags:this.r32(t+Vr),isDir:this.r32(t+qr)!==0}:null}fdFree(e){if(e>=0&&e>2,0)}}buildStat(e){let t=this.inodeOffset(e);return{ino:e,generation:this.r64(t+oe),dataSequence:this.r32(t+ae),mode:this.r32(t+C),linkCount:this.r32(t+D),size:this.r64(t+T),mtime:this.r64(t+ie),ctime:this.r64(t+q),atime:this.r64(t+At),uid:this.r32(t+Gr),gid:this.r32(t+Wr)}}namespaceEntryIdentity(e){let t=this.inodeOffset(e);return{ino:e,generation:this.r64(t+oe),linkCount:this.r32(t+D),mode:this.r32(t+C)}}open(e,t,n=420){return this.withNamespaceLock(()=>this.openUnlocked(e,t,n))}createLazyStub(e,t){return this.withNamespaceLock(()=>{let n=this.openUnlocked(e,Dr|Pt,t);try{let i=this.fdGet(n);if(!i)throw new b(Q);this.inodeWriteLock(i.ino);try{return this.inodeTruncate(i.ino,0,!0),this.buildStat(i.ino)}finally{this.inodeWriteUnlock(i.ino)}}finally{this.closeUnlocked(n)}})}replaceIfIdentity(e,t,n,i,o){return this.withNamespaceLock(()=>{let s=this.pathResolve(e,!0);if(s<0||s!==t)return!1;let a=this.inodeOffset(s);if(this.r64(a+oe)!==n||this.r32(a+ae)!==i||(this.r32(a+C)&K)!==kt)return!1;this.validateFileSize(o.byteLength),this.inodeWriteLock(s);try{if(this.r64(a+oe)!==n||this.r32(a+ae)!==i||this.r64(a+T)!==0)return!1;let c=this.r64(a+ie),l=this.r64(a+q);this.inodeTruncate(s,0,!0);let h=o.byteLength>0?this.inodeWriteData(s,0,o,o.byteLength):0;if(h!==o.byteLength)throw this.inodeTruncate(s,0,!0),Atomics.store(this.i32,a+ae>>2,i),this.w64(a+ie,c),this.w64(a+q,l),new b(h<0?h:X);return!0}finally{this.inodeWriteUnlock(s)}})}replaceManyIfIdentities(e){return e.length===0?!0:this.withNamespaceLock(()=>{let t=[],n=new Set;for(let o of e){this.validateFileSize(o.data.byteLength);let s=-1;for(let a of o.paths){let c=this.pathResolve(a,!0);if(c!==o.expectedIno)continue;let l=this.inodeOffset(c);if(this.r64(l+oe)===o.expectedGeneration&&this.r32(l+ae)===o.expectedDataSequence&&(this.r32(l+C)&K)===kt&&this.r64(l+T)===0){s=c;break}}if(s<0)return!1;if(n.has(s))throw new b(W,"duplicate conditional replacement inode");n.add(s),t.push({...o,ino:s})}let i=[...n].sort((o,s)=>o-s);for(let o of i)this.inodeWriteLock(o);try{for(let a of t){let c=this.inodeOffset(a.ino);if(this.r64(c+oe)!==a.expectedGeneration||this.r32(c+ae)!==a.expectedDataSequence||(this.r32(c+C)&K)!==kt||this.r64(c+T)!==0)return!1}let o=t.map(a=>{let c=this.inodeOffset(a.ino);return{ino:a.ino,dataSequence:this.r32(c+ae),mtime:this.r64(c+ie),ctime:this.r64(c+q)}}),s=0;try{for(let a of t){s++,this.inodeTruncate(a.ino,0,!0);let c=a.data.byteLength>0?this.inodeWriteData(a.ino,0,a.data,a.data.byteLength):0;if(c!==a.data.byteLength)throw new b(c<0?c:X)}}catch(a){for(let c=s-1;c>=0;c--){let l=o[c],h=this.inodeOffset(l.ino);this.inodeTruncate(l.ino,0,!0),Atomics.store(this.i32,h+ae>>2,l.dataSequence),this.w64(h+ie,l.mtime),this.w64(h+q,l.ctime)}throw a}return!0}finally{for(let o=i.length-1;o>=0;o--)this.inodeWriteUnlock(i[o])}})}openUnlocked(e,t,n=420){let i=t&qt,o=(t&Pt)!==0,s=(t&Un)!==0;if(o&&s){let f=this.pathResolve(e,!1);if(f>=0)throw new b(st);if(f!==ye)throw new b(f)}let a=this.pathResolve(e,!0);if(a<0&&a===ye&&o){let{parentIno:f,name:d}=this.pathResolveParent(e);this.inodeWriteLock(f);try{let y=ce.encode(d),g=this.dirLookup(f,y);if(g>=0){if(s)throw new b(st);a=g}else{let p=this.inodeAlloc();if(p<0)throw new b(X);let w=this.inodeOffset(p);this.w32(w+C,kt|n&4095),this.w32(w+D,1),this.w64(w+T,0);let u=Date.now();this.w64(w+At,u),this.w64(w+ie,u),this.w64(w+q,u);let m=this.dirAddEntry(f,y,p);if(m<0)throw this.inodeFree(p),new b(m);a=p}}finally{this.inodeWriteUnlock(f)}}if(a<0)throw new b(a);let c=this.inodeOffset(a),l=this.r32(c+C);if((l&K)===U&&i!==Je)throw new b(Ge);if(t&js&&(l&K)!==U)throw new b(Se);if(t&Tt){if((l&K)===U)throw new b(Ge);this.inodeWriteLock(a),this.inodeTruncate(a,0,!0),this.inodeWriteUnlock(a)}let h=this.fdAlloc(a,t,!1);if(h<0)throw new b(h);return h}close(e){this.withNamespaceLock(()=>this.closeUnlocked(e))}closeUnlocked(e){let t=this.fdGet(e);if(!t)throw new b(Q);this.fdFree(e),this.inodeDropOpenRef(t.ino)}read(e,t){let n=this.fdGet(e);if(!n)throw new b(Q);let i=this.inodeOffset(n.ino);if((this.r32(i+C)&K)===U)throw new b(Ge);this.inodeReadLock(n.ino);try{let s=this.inodeReadData(n.ino,n.offset,t,t.length),a=256+e*24;return this.w64(a+Ke,n.offset+s),s}finally{this.inodeReadUnlock(n.ino)}}readAt(e,t,n){let i=this.fdGet(e);if(!i)throw new b(Q);let o=this.inodeOffset(i.ino);if((this.r32(o+C)&K)===U)throw new b(Ge);this.validateSeekPosition(n),this.inodeReadLock(i.ino);try{return this.inodeReadData(i.ino,n,t,t.length)}finally{this.inodeReadUnlock(i.ino)}}write(e,t){let n=this.fdGet(e);if(!n)throw new b(Q);if((n.flags&qt)===Je)throw new b(Q);this.inodeWriteLock(n.ino);try{let o=n.offset;if(n.flags&qs){let c=this.inodeOffset(n.ino);o=this.r64(c+T)}if(!Number.isSafeInteger(o)||o<0)throw new b(W);if(o>rt||t.length>rt-o)throw new b(Lt);let s=this.inodeWriteData(n.ino,o,t,t.length);if(s<0)return s;let a=256+e*24;return this.w64(a+Ke,o+s),s}finally{this.inodeWriteUnlock(n.ino)}}writeAt(e,t,n){let i=this.fdGet(e);if(!i)throw new b(Q);if((i.flags&qt)===Je)throw new b(Q);this.validateSeekPosition(n),this.inodeWriteLock(i.ino);try{if(n>rt||t.length>rt-n)throw new b(Lt);return this.inodeWriteData(i.ino,n,t,t.length)}finally{this.inodeWriteUnlock(i.ino)}}lseek(e,t,n){let i=this.fdGet(e);if(!i)throw new b(Q);let o;if(n===Ys)o=t;else if(n===Xs)o=i.offset+t;else if(n===Js){let a=this.inodeOffset(i.ino);o=this.r64(a+T)+t}else throw new b(W);this.validateSeekPosition(o);let s=256+e*24;return this.w64(s+Ke,o),o}ftruncate(e,t){let n=this.fdGet(e);if(!n)throw new b(Q);if((n.flags&qt)===Je)throw new b(Q);this.validateFileSize(t),this.inodeWriteLock(n.ino);try{this.inodeTruncate(n.ino,t,!0)}finally{this.inodeWriteUnlock(n.ino)}}fstat(e){let t=this.fdGet(e);if(!t)throw new b(Q);this.inodeReadLock(t.ino);try{return this.buildStat(t.ino)}finally{this.inodeReadUnlock(t.ino)}}stat(e){return this.withNamespaceLock(()=>this.statUnlocked(e))}statUnlocked(e){let t=this.pathResolve(e,!0);if(t<0)throw new b(t);this.inodeReadLock(t);try{return this.buildStat(t)}finally{this.inodeReadUnlock(t)}}lstat(e){return this.withNamespaceLock(()=>this.lstatUnlocked(e))}lstatUnlocked(e){let t=this.pathResolve(e,!1);if(t<0)throw new b(t);this.inodeReadLock(t);try{return this.buildStat(t)}finally{this.inodeReadUnlock(t)}}unlink(e){return this.withNamespaceLock(()=>this.unlinkUnlocked(e))}unlinkUnlocked(e){let{parentIno:t,name:n}=this.pathResolveParent(e),i=ce.encode(n),o=e.length>1&&e.endsWith("/");this.inodeWriteLock(t);try{let s=this.dirLookup(t,i);if(s<0)throw new b(s);let a=this.inodeOffset(s),c=this.r32(a+C);if(o&&(c&K)!==U)throw new b(Se);if((c&K)===U)throw new b(Ge);let l=this.namespaceEntryIdentity(s),h=this.dirRemoveEntry(t,i);if(h<0)throw new b(h);let f=!1;this.inodeWriteLock(s);try{f=this.inodeDropLinkRefLocked(s)}finally{this.inodeWriteUnlock(s)}return f&&this.inodeFree(s),l}finally{this.inodeWriteUnlock(t)}}rename(e,t){return this.withNamespaceLock(()=>this.renameUnlocked(e,t))}renameUnlocked(e,t){let{parentIno:n,name:i}=this.pathResolveParent(e),{parentIno:o,name:s}=this.pathResolveParent(t);if(Mn(i)||Mn(s))throw new b(W);let a=ce.encode(i),c=ce.encode(s),l=e.length>1&&e.endsWith("/"),h=t.length>1&&t.endsWith("/"),f=Math.min(n,o),d=Math.max(n,o);this.inodeWriteLock(f),f!==d&&this.inodeWriteLock(d);try{let y=this.dirLookup(n,a);if(y<0)throw new b(y);let g=this.inodeOffset(y),w=this.r32(g+C)&K,u=this.namespaceEntryIdentity(y);if((l||h)&&w!==U)throw new b(Se);if(w===U&&this.dirIsAncestor(y,o))throw new b(W);let m=this.dirLookup(o,c),v=!1,E;if(m>=0){if(m===y)return{source:u,replaced:u};E=this.namespaceEntryIdentity(m);let S=this.inodeOffset(m),I=this.r32(S+C)&K;if(w===U&&I!==U)throw new b(Se);if(w!==U&&I===U)throw new b(Ge);let A=!1,B=m===n||m===o;B||this.inodeWriteLock(m);try{if(I===U&&!this.dirIsEmpty(m))throw new b(Dn);let N=this.dirReplaceEntryIno(o,c,y);if(N<0)throw new b(N);A=I===U?this.inodeOrphanLocked(m):this.inodeDropLinkRefLocked(m)}finally{B||this.inodeWriteUnlock(m)}A&&this.inodeFree(m),v=I===U}else{let S=this.dirAddEntry(o,c,y);if(S<0)throw new b(S)}let z=this.dirRemoveEntry(n,a);if(z<0)throw new b(z);if(w===U){if(n!==o){let S=this.inodeOffset(n);this.w32(S+D,this.r32(S+D)-1);let k=this.inodeOffset(o);this.w32(k+D,this.r32(k+D)+1),this.inodeWriteLock(y);try{let I=this.dirReplaceEntryIno(y,Yr,o);if(I<0)throw new b(I);this.w64(g+q,Date.now())}finally{this.inodeWriteUnlock(y)}}if(v){let S=this.inodeOffset(o);this.w32(S+D,this.r32(S+D)-1)}}else if(v){let S=this.inodeOffset(o);this.w32(S+D,this.r32(S+D)-1)}return{source:u,replaced:E}}finally{f!==d&&this.inodeWriteUnlock(d),this.inodeWriteUnlock(f)}}mkdir(e,t=493){this.withNamespaceLock(()=>this.mkdirUnlocked(e,t))}mkdirUnlocked(e,t=493){let{parentIno:n,name:i}=this.pathResolveParent(e),o=ce.encode(i);this.inodeWriteLock(n);try{if(this.dirLookup(n,o)>=0)throw new b(st);let a=this.inodeAlloc();if(a<0)throw new b(X);let c=this.inodeOffset(a);this.w32(c+C,U|t),this.w32(c+D,2),this.w64(c+T,0);let l=Date.now();this.w64(c+At,l),this.w64(c+ie,l),this.w64(c+q,l);let h=this.blockAllocWithGrow();if(h<0)throw this.inodeFree(a),new b(X);this.w32(c+Y,h);let f=h*4096,d=Re(O+1),y=Re(O+2);this.w32(f,a),this.view.setUint16(f+4,d,!0),this.view.setUint16(f+6,1,!0),this.u8[f+O]=46;let g=f+d;this.w32(g,n),this.view.setUint16(g+4,y,!0),this.view.setUint16(g+6,2,!0),this.u8[g+O]=46,this.u8[g+O+1]=46,this.w64(c+T,d+y);let p=this.dirAddEntry(n,o,a);if(p<0)throw this.blockFree(h),this.inodeFree(a),new b(p);let w=this.inodeOffset(n);this.w32(w+D,this.r32(w+D)+1)}finally{this.inodeWriteUnlock(n)}}rmdir(e){this.withNamespaceLock(()=>this.rmdirUnlocked(e))}rmdirUnlocked(e){let{parentIno:t,name:n}=this.pathResolveParent(e);if(Mn(n))throw new b(W);let i=ce.encode(n);this.inodeWriteLock(t);try{let o=this.dirLookup(t,i);if(o<0)throw new b(o);let s=this.inodeOffset(o);if((this.r32(s+C)&K)!==U)throw new b(Se);let c=!1;this.inodeWriteLock(o);try{if(!this.dirIsEmpty(o))throw new b(Dn);let h=this.dirRemoveEntry(t,i);if(h<0)throw new b(h);c=this.inodeOrphanLocked(o)}finally{this.inodeWriteUnlock(o)}c&&this.inodeFree(o);let l=this.inodeOffset(t);this.w32(l+D,this.r32(l+D)-1)}finally{this.inodeWriteUnlock(t)}}symlink(e,t){this.withNamespaceLock(()=>this.symlinkUnlocked(e,t))}symlinkUnlocked(e,t){let{parentIno:n,name:i}=this.pathResolveParent(t),o=ce.encode(i),s=ce.encode(e);this.inodeWriteLock(n);try{if(this.dirLookup(n,o)>=0)throw new b(st);let c=this.inodeAlloc();if(c<0)throw new b(X);let l=this.inodeOffset(c);if(this.w32(l+C,xt|511),this.w32(l+D,1),s.length<=40)this.u8.set(s,l+Y),this.w64(l+T,s.length);else{this.w64(l+T,0);let f=this.inodeWriteData(c,0,s,s.length);if(f!==s.length)throw f>0&&this.inodeTruncate(c,0),this.inodeFree(c),new b(f<0?f:X)}let h=this.dirAddEntry(n,o,c);if(h<0)throw s.length<=40?(this.u8.fill(0,l+Y,l+Y+40),this.w64(l+T,0)):this.inodeTruncate(c,0),this.inodeFree(c),new b(h)}finally{this.inodeWriteUnlock(n)}}chmod(e,t){this.withNamespaceLock(()=>this.chmodUnlocked(e,t))}chmodUnlocked(e,t){let n=this.pathResolve(e,!0);if(n<0)throw new b(n);this.inodeWriteLock(n);try{let i=this.inodeOffset(n),o=this.r32(i+C);this.w32(i+C,o&K|t&4095),this.w64(i+q,Date.now())}finally{this.inodeWriteUnlock(n)}}fchmod(e,t){let n=this.fdGet(e);if(!n)throw new b(Q);this.inodeWriteLock(n.ino);try{let i=this.inodeOffset(n.ino),o=this.r32(i+C);this.w32(i+C,o&K|t&4095),this.w64(i+q,Date.now())}finally{this.inodeWriteUnlock(n.ino)}}chown(e,t,n){this.withNamespaceLock(()=>this.chownUnlocked(e,t,n))}chownUnlocked(e,t,n){let i=this.pathResolve(e,!0);if(i<0)throw new b(i);this.inodeWriteLock(i);try{this.chownInodeUnlocked(i,t,n)}finally{this.inodeWriteUnlock(i)}}fchown(e,t,n){let i=this.fdGet(e);if(!i)throw new b(Q);this.inodeWriteLock(i.ino);try{this.chownInodeUnlocked(i.ino,t,n)}finally{this.inodeWriteUnlock(i.ino)}}lchown(e,t,n){this.withNamespaceLock(()=>this.lchownUnlocked(e,t,n))}lchownUnlocked(e,t,n){let i=this.pathResolve(e,!1);if(i<0)throw new b(i);this.inodeWriteLock(i);try{this.chownInodeUnlocked(i,t,n)}finally{this.inodeWriteUnlock(i)}}chownInodeUnlocked(e,t,n){let i=this.inodeOffset(e);t!==$r&&this.w32(i+Gr,t),n!==$r&&this.w32(i+Wr,n);let o=this.r32(i+C);(o&K)===kt&&(o&Vs)!==0&&this.w32(i+C,o&~(Zs|Hs)),this.w64(i+q,Date.now())}utimens(e,t,n,i,o){this.withNamespaceLock(()=>this.utimensUnlocked(e,t,n,i,o))}utimensUnlocked(e,t,n,i,o){let s=this.pathResolve(e,!0);if(s<0)throw new b(s);this.inodeWriteLock(s);try{let a=this.inodeOffset(s),c=1073741823,l=1073741822,h=Date.now();if(n!==l){let f=n===c?h:t*1e3+Math.floor(n/1e6);this.w64(a+At,f)}if(o!==l){let f=o===c?h:i*1e3+Math.floor(o/1e6);this.w64(a+ie,f)}this.w64(a+q,h)}finally{this.inodeWriteUnlock(s)}}link(e,t){return this.withNamespaceLock(()=>this.linkUnlocked(e,t))}linkUnlocked(e,t){let n=this.pathResolve(e,!1);if(n<0)throw new b(n);let i=this.inodeOffset(n);if((this.r32(i+C)&K)===U)throw new b(Qs);let{parentIno:s,name:a}=this.pathResolveParent(t),c=ce.encode(a);this.inodeWriteLock(s);try{if(this.dirLookup(s,c)>=0)throw new b(st);let h=this.dirAddEntry(s,c,n);if(h<0)throw new b(h);this.inodeWriteLock(n);try{let f=this.r32(i+D);this.w32(i+D,f+1),this.w64(i+q,Date.now())}finally{this.inodeWriteUnlock(n)}return{...this.namespaceEntryIdentity(n),linkCount:this.r32(i+D)}}finally{this.inodeWriteUnlock(s)}}readlink(e){return this.withNamespaceLock(()=>this.readlinkUnlocked(e))}readlinkUnlocked(e){let t=this.pathResolve(e,!1);if(t<0)throw new b(t);let n=this.inodeOffset(t);if((this.r32(n+C)&K)!==xt)throw new b(W);let o=this.r64(n+T);if(o<=40)return it(this.u8.subarray(n+Y,n+Y+o));this.inodeReadLock(t);try{let s=new Uint8Array(o);return this.inodeReadData(t,0,s,o),Ot.decode(s)}finally{this.inodeReadUnlock(t)}}opendir(e){return this.withNamespaceLock(()=>this.opendirUnlocked(e))}opendirUnlocked(e){let t=this.pathResolve(e,!0);if(t<0)throw new b(t);let n=this.inodeOffset(t);if((this.r32(n+C)&K)!==U)throw new b(Se);let o=this.fdAlloc(t,Je,!0);if(o<0)throw new b(o);return o}readdirEntry(e){return this.withNamespaceLock(()=>this.readdirEntryUnlocked(e))}readdirEntryUnlocked(e){let t=this.fdGet(e);if(!t||!t.isDir)throw new b(Q);let n=this.inodeOffset(t.ino),i=this.r64(n+T);for(;t.offset=this.r32(Oe))throw new b(F);let p=this.r32(Ue)*4096;if((this.r32(p+(h>>5)*4)&1<<(h&31))===0)throw new b(F);let u=it(this.u8.subarray(l+O,l+O+d)),m=this.buildStat(h);return this.w64(g+Ke,y),t.offset=y,{name:u,stat:m}}return null}closedir(e){this.close(e)}readdir(e){let t=this.opendir(e),n=[];try{let i;for(;(i=this.readdirEntry(t))!==null;)i.name!=="."&&i.name!==".."&&n.push(i.name)}finally{this.closedir(t)}return n}writeFile(e,t){let n=typeof t=="string"?ce.encode(t):t,i=this.open(e,Dr|Pt|Tt);try{this.write(i,n)}finally{this.close(i)}}readFile(e){let t=this.open(e,Je);try{let n=this.fstat(t),i=new Uint8Array(n.size);return this.read(t,i),i}finally{this.close(t)}}readFileText(e){return Ot.decode(this.readFile(e))}};function ei(r,e){let t=new Map,n=new Map;for(let s of r){if(t.has(s.path))throw new Error(`${e} duplicates path ${s.path}`);if(t.set(s.path,s),s.type==="file"){if(!s.inodeGroup)throw new Error(`${e} file ${s.path} has no inode group`);if(n.has(s.inodeGroup))throw new Error(`${e} inode group ${s.inodeGroup} has multiple files`);n.set(s.inodeGroup,s)}}let i=new Set,o=new Map;for(let s of r){if(s.type!=="hardlink"||o.has(s.path))continue;let a=[],c=s,l;for(;c.type==="hardlink";){let f=o.get(c.path);if(f){l=f;break}if(i.has(c.path))throw new Error(`${e} hardlink cycle reaches ${c.path}`);if(i.add(c.path),a.push(c),!c.target)throw new Error(`${e} hardlink ${c.path} has no target`);let d=t.get(c.target);if(!d)throw new Error(`${e} hardlink ${c.path} target ${c.target} is missing`);if(d.type!=="file"&&d.type!=="hardlink"||!c.inodeGroup||d.inodeGroup!==c.inodeGroup||d.size!==c.size||d.mode!==c.mode)throw new Error(`${e} hardlink ${c.path} has an invalid target`);c=d}l??=c.type==="file"?c:void 0;let h=n.get(s.inodeGroup??"");if(!l||l!==h)throw new Error(`${e} hardlink ${s.path} does not resolve to its inode`);for(let f=a.length-1;f>=0;f-=1){let d=a[f];if(n.get(d.inodeGroup??"")!==l)throw new Error(`${e} hardlink ${d.path} does not resolve to its inode`);i.delete(d.path),o.set(d.path,l)}}return{canonicalByGroup:n,canonicalTargetByPath:o}}var le={maxArchiveBytes:268435456,maxExpandedBytes:268435456,maxPayloadBytes:268435456,maxEntries:1e5,maxPathBytes:4096,maxSymlinkTargetBytes:65536,maxStringBytes:8192,maxTransportsPerTree:8,maxActivationCapabilities:32,maxActivationRoots:64,maxActivationCapabilityBytes:255},we={maxArchiveBytes:512*1024*1024,maxExpandedBytes:512*1024*1024,maxPayloadBytes:512*1024*1024,maxEntries:1e5,maxGroups:512};function ti(r,e="Deferred tree collection"){for(let[t,n]of Object.entries(r))if(!Number.isSafeInteger(n)||n<0)throw new Error(`${e} ${t} usage is invalid`);if(r.groups>we.maxGroups)throw new Error(`${e} exceeds the ${we.maxGroups}-group cap`);if(r.archiveBytes>we.maxArchiveBytes)throw new Error(`${e} exceeds the archive-byte cap`);if(r.expandedBytes>we.maxExpandedBytes)throw new Error(`${e} exceeds the expansion cap`);if(r.payloadBytes>we.maxPayloadBytes)throw new Error(`${e} exceeds the payload-byte cap`);if(r.entries>we.maxEntries)throw new Error(`${e} exceeds the entry-count cap`)}var ot="/home/linuxbrew/.linuxbrew",ii=[["@@HOMEBREW_PREFIX@@",ot],["@@HOMEBREW_CELLAR@@",`${ot}/Cellar`],["@@HOMEBREW_REPOSITORY@@",ot],["@@HOMEBREW_LIBRARY@@",`${ot}/Library`],["@@HOMEBREW_PERL@@",`${ot}/opt/perl/bin/perl`]],Kn="@@HOMEBREW_JAVA@@",oo=/^openjdk(?:@\d+(?:\.\d+)*)?/,at=new TextEncoder,ao=[...ii.map(([r])=>r),Kn].map(r=>({placeholder:r,bytes:at.encode(r)}));function si(r){let e;try{e=JSON.parse(new TextDecoder("utf-8",{fatal:!0}).decode(r))}catch(a){throw new Error("INSTALL_RECEIPT.json is not valid UTF-8 JSON: "+uo(a))}if(typeof e!="object"||e===null||Array.isArray(e))throw new Error("INSTALL_RECEIPT.json must contain an object");let t=e,n=t.changed_files;if(n!=null&&!Array.isArray(n))throw new Error("INSTALL_RECEIPT.json changed_files must be an array or null when present");let i=Array.isArray(n)?n:[];if(i.length>1e5)throw new Error(`INSTALL_RECEIPT.json declares ${i.length} changed files, limit 100000`);let o=[],s=new Set;for(let[a,c]of i.entries()){if(typeof c!="string")throw new Error(`INSTALL_RECEIPT.json changed_files[${a}] is not a string`);if(lo(c,"Homebrew changed file"),s.has(c))throw new Error(`INSTALL_RECEIPT.json repeats changed file ${c}`);s.add(c),o.push(c)}return{changedFiles:o,runtimeDependencies:t.runtime_dependencies}}function oi(r,e,t){let n=r;for(let[s,a]of ii)n=ri(n,at.encode(s),at.encode(a));let i=at.encode(Kn);if(ni(n,i)){let s=co(e.runtimeDependencies);if(s===void 0)throw new Error(`Homebrew changed file ${t} uses ${Kn} without exactly one OpenJDK runtime dependency`);n=ri(n,i,at.encode(s))}let o=ao.find(({bytes:s})=>ni(n,s));if(o!==void 0)throw new Error(`Homebrew changed file ${t} retains ${o.placeholder}`);return n}function co(r){if(!Array.isArray(r))return;let e=[];for(let n of r){if(typeof n!="object"||n===null||Array.isArray(n))continue;let i=n,o=typeof i.full_name=="string"?i.full_name.split("/").at(-1):typeof i.name=="string"?i.name.split("/").at(-1):void 0,s=o===void 0?null:oo.exec(o);o!==void 0&&s?.[0]===o&&e.push(o)}let t=[...new Set(e)];return t.length===1?`${ot}/opt/${t[0]}/libexec`:void 0}function lo(r,e){if(r.length===0||r.startsWith("/")||r.includes("\\")||r.includes("\0")||fo(r)||at.encode(r).byteLength>4096||r.split("/").some(t=>t===""||t==="."||t===".."))throw new Error(`${e} has an unsafe path segment: ${r}`)}function fo(r){for(let e=0;e57343)){if(t<=56319&&e+1=56320&&r.charCodeAt(e+1)<=57343){e+=1;continue}return!0}}return!1}function ni(r,e){if(e.byteLength===0||e.byteLength>r.byteLength)return!1;e:for(let t=0;t<=r.byteLength-e.byteLength;t+=1){for(let n=0;ngn||r.includes("\0")||r.includes("\\"))throw new Error(`Lazy archive mount prefix must be an absolute POSIX path: ${JSON.stringify(r)}`);let e=r.replace(/\/+$/,"");if(e==="")return"/";if(e.slice(1).split("/").some(n=>n===""||n==="."||n===".."))throw new Error(`Lazy archive mount prefix is not canonical: ${JSON.stringify(r)}`);return e}function ga(r,e,t,n){let i=pn(t),o=new Map,s=e.map(a=>{let c=a.fileName,l=`Lazy archive ${JSON.stringify(r)} member ${JSON.stringify(c)}`;if(c.length===0)throw new Error(`${l} has an empty path`);if(c.includes("\0"))throw new Error(`${l} contains a NUL byte`);if(c.includes("\\"))throw new Error(`${l} contains a backslash`);if(c.startsWith("/")||/^[A-Za-z]:\//.test(c))throw new Error(`${l} must be relative, not absolute`);if(a.isDirectory&&a.isSymlink)throw new Error(`${l} has conflicting directory and symlink types`);if(a.isDirectory!==c.endsWith("/"))throw new Error(`${l} has inconsistent directory metadata`);let h=a.isDirectory?c.slice(0,-1):c,f=h.split("/");if(h.length===0||f.some(d=>d===""||d==="."||d===".."))throw new Error(`${l} is not a canonical relative POSIX path`);if(o.has(h))throw new Error(`${l} collides with another member at ${JSON.stringify(h)}`);if(a.isSymlink&&!n?.has(c))throw new Error(`Lazy archive symlink target was not provided: ${c}`);return o.set(h,a),{entry:a,archivePath:h,vfsPath:i==="/"?`/${h}`:`${i}/${h}`}});for(let{archivePath:a}of s){let c=a.split("/");for(let l=1;ldt)throw new Error(`VFS image metadata exceeds ${dt} bytes`);let e;try{e=JSON.parse(new TextDecoder().decode(r))}catch(t){let n=t instanceof Error?t.message:String(t);throw new Error(`Invalid VFS image metadata JSON: ${n}`)}return hr(e)}function wa(r){if(r===null)return new Uint8Array(0);let e=hr(r),t=new TextEncoder().encode(JSON.stringify(e));if(t.byteLength>dt)throw new Error(`VFS image metadata exceeds ${dt} bytes`);return t}function va(r){return r.byteLength>=Mt.length&&r[0]===Mt[0]&&r[1]===Mt[1]&&r[2]===Mt[2]&&r[3]===Mt[3]?Ta(r):r}function rn(r){let e=va(r);if(e.byteLengthon)throw new Error(`VFS image lazy metadata exceeds ${on} bytes`);if(r.byteLengthan)throw new Error(`VFS image lazy archive metadata exceeds ${an} bytes`);if(r.byteLength=0?n:void 0}function za(r){return r===408||r===429||r>=500&&r<=599}function ba(r,e=Date.now()){let t=r?.get("retry-after")?.trim();if(!t)return;let n;if(/^\d+$/.test(t))n=Number(t)*1e3;else{let i=Date.parse(t);if(!Number.isFinite(i))return;n=Math.max(0,i-e)}if(!(!Number.isSafeInteger(n)||n<0))return Math.min(n,Ui)}function ka(r){if(!(typeof r!="object"||r===null||!("cause"in r)))return r.cause}function Ki(r){if(!(typeof r!="object"||r===null||!("name"in r)))return typeof r.name=="string"?r.name:void 0}function Gi(r){if(!(typeof r!="object"||r===null||!("code"in r)))return typeof r.code=="string"?r.code:void 0}function Wi(r,e){let t=new Set,n=r;for(let i=0;n!==void 0&&i<8;i+=1){if(t.has(n))return!1;if(t.add(n),e(n))return!0;n=ka(n)}return!1}function Zi(r){return Wi(r,e=>Ki(e)==="AbortError"||Gi(e)==="ABORT_ERR")}function xa(r){return Zi(r)?!1:Wi(r,e=>{let t=Ki(e),n=Gi(e);return e instanceof TypeError||t==="NetworkError"||t==="TimeoutError"||n!==void 0&&ya.has(n)})}function Ia(r,e){if(r instanceof un){if(!za(r.status))return null;if(r.retryAfterMs!==void 0)return r.retryAfterMs}else if(!xa(r))return null;return Math.min(da*2**e,Ui)}function ee(r){if(r?.aborted)throw r.reason}function Aa(r,e){return ee(e),r===0?Promise.resolve():new Promise((t,n)=>{let i=setTimeout(()=>a(!1),r),o=()=>a(!0,e.reason),s=!1;function a(c,l){s||(s=!0,clearTimeout(i),e?.removeEventListener("abort",o),c?n(l):t())}e?.addEventListener("abort",o,{once:!0}),e?.aborted&&o()})}async function or(r,e){try{await r.body?.cancel(e)}catch{}}function _a(r,e){if(r.length===1)return r[0];let t=new Uint8Array(e),n=0;for(let i of r)t.set(i,n),n+=i.byteLength;return t}function Ft(r){if(r===void 0)return;if(typeof r!="object"||r===null||Array.isArray(r))throw new Error("Lazy archive integrity must be an object");let e=r;if(Object.keys(e).length!==2||!("sha256"in e)||!("bytes"in e))throw new Error("Lazy archive integrity has unexpected fields");if(typeof e.sha256!="string"||!ha.test(e.sha256))throw new Error("Lazy archive integrity has an invalid SHA-256 digest");if(!Number.isSafeInteger(e.bytes)||Number(e.bytes)<=0||Number(e.bytes)>Ti)throw new Error(`Lazy archive integrity byte count must be between 1 and ${Ti}`);return{sha256:e.sha256,bytes:Number(e.bytes)}}function We(r,e,t){if(typeof r!="object"||r===null||Array.isArray(r))throw new Error(`${t} must be an object`);let n=r;if(Object.keys(n).length!==e.length||e.some(o=>!Object.prototype.hasOwnProperty.call(n,o)))throw new Error(`${t} has unexpected or missing fields`);return n}function fr(r,e,t,n){if(typeof r!="object"||r===null||Array.isArray(r))throw new Error(`${n} must be an object`);let i=r,o=new Set(e);if(Object.keys(i).some(s=>!o.has(s))||t.some(s=>!Object.prototype.hasOwnProperty.call(i,s)))throw new Error(`${n} has unexpected or missing fields`);return i}function xe(r,e,t,n){if(!Array.isArray(r)||r.lengthn)throw new Error(`${e} must contain ${t} to ${n} items`);return r}function Be(r,e,t){if(typeof r!="string"||r.length===0||r.includes("\0")||new TextEncoder().encode(r).byteLength>t)throw new Error(`${e} is invalid or exceeds ${t} bytes`);return r}function ne(r,e,t,n){if(!Number.isSafeInteger(r)||Number(r)n)throw new Error(`${e} must be an integer between ${t} and ${n}`);return Number(r)}function dn(r,e=1){let t=r,n=typeof t=="object"&&t!==null&&!Array.isArray(t)&&t.source!==void 0,i=typeof t=="object"&&t!==null&&!Array.isArray(t)&&t.modePolicy!==void 0,o=We(r,["decoder","mediaType","sha256","bytes","expandedBytes","sourceEntryCount","transports",...i?["modePolicy"]:[],...n?["source"]:[]],"Lazy tree content"),s=o.decoder==="zip-v1"?"application/zip":o.decoder==="homebrew-bottle-tar-gzip-v1"?"application/vnd.oci.image.layer.v1.tar+gzip":null;if(s===null||o.mediaType!==s)throw new Error("Lazy tree decoder and media type are inconsistent");let a=Ft({sha256:o.sha256,bytes:o.bytes});if(!a)throw new Error("Lazy tree integrity is required");let c=xe(o.transports,"Lazy tree transports",e,le.maxTransportsPerTree).map((y,g)=>Be(y,`Lazy tree transport ${g}`,dr));if(new Set(c).size!==c.length)throw new Error("Lazy tree transports contain duplicates");let l=ne(o.expandedBytes,"Lazy tree expanded byte count",0,aa),h=ne(o.sourceEntryCount,"Lazy tree source entry count",1,ht),f=n?La(o.source,o.decoder):void 0,d=i?o.modePolicy:void 0;if(d!==void 0&&(d!=="portable-posix-v1"||o.decoder!=="zip-v1"||n))throw new Error("Lazy tree mode policy is invalid for its decoder");if(f!==void 0&&f.entries.length!==h)throw new Error("Lazy tree source inventory count differs from its content");return{decoder:o.decoder,mediaType:s,sha256:a.sha256,bytes:a.bytes,expandedBytes:l,sourceEntryCount:h,transports:c,...d===void 0?{}:{modePolicy:d},...f===void 0?{}:{source:f}}}function Hi(r){let e={groups:r.length,archiveBytes:0,expandedBytes:0,payloadBytes:0,entries:0};for(let t of r)t.content===void 0||t.inventory===void 0||(e.archiveBytes+=t.content.bytes,e.expandedBytes+=t.content.expandedBytes,e.payloadBytes+=t.inventory.filter(n=>n.type==="file").reduce((n,i)=>n+i.size,0),e.entries+=t.inventory.length+(t.content.source?.entries.length??0));return e}function ur(r){ti(r,"Serialized lazy tree collection")}function Fi(r){ur(Hi(r))}function La(r,e){if(e!=="homebrew-bottle-tar-gzip-v1")throw new Error("Lazy tree source inventory is valid only for original bottles");let t=We(r,["schema","kind","entries"],"Lazy tree source inventory");if(t.schema!==1||t.kind!=="homebrew-bottle-tar-gzip-v1")throw new Error("Lazy tree source inventory has an unsupported identity");let n=new Map,i=xe(t.entries,"Lazy tree source entries",1,ht).map((s,a)=>{let c=s,l=typeof c=="object"&&c!==null&&!Array.isArray(c)?c.type:void 0,h=l==="directory"||l==="file"?["sourcePath","type","mode","size"]:l==="symlink"||l==="hardlink"?["sourcePath","type","mode","size","target"]:null;if(h===null)throw new Error(`Lazy tree source entry ${a} has invalid type`);let f=We(s,h,`Lazy tree source entry ${a}`),d=ue(f.sourcePath,!1,`Lazy tree source entry ${a} path`);if(n.has(d))throw new Error(`Lazy tree source inventory duplicates ${d}`);let y=ne(f.mode,`Lazy tree source entry ${d} mode`,0,4095),g=ne(f.size,`Lazy tree source entry ${d} size`,0,cn),p;if((l==="directory"||l==="symlink"||l==="hardlink")&&g!==0)throw new Error(`Lazy tree source ${d} has payload for ${String(l)}`);l==="symlink"?p=Be(f.target,`Lazy tree source symlink ${d} target`,Di):l==="hardlink"&&(p=ue(f.target,!1,`Lazy tree source hardlink ${d} target`));let w={sourcePath:d,type:l,mode:y,size:g,...p===void 0?{}:{target:p}};return n.set(d,w),w}),o=i.map(s=>s.sourcePath);if(o.some((s,a)=>a>0&&o[a-1]>=s))throw new Error("Lazy tree source inventory is not in canonical path order");return{schema:1,kind:"homebrew-bottle-tar-gzip-v1",entries:i}}function Vi(r){let e=new Map(r.map(n=>[n.sourcePath,n])),t=new Map;for(let n of r){if(n.type!=="hardlink"||t.has(n.sourcePath))continue;let i=[],o=new Set,s=n,a;for(;s.type==="hardlink"&&(a=t.get(s.sourcePath),a===void 0);){if(o.has(s.sourcePath))throw new Error(`Lazy tree source hardlink cycle includes ${s.sourcePath}`);o.add(s.sourcePath),i.push(s);let c=e.get(s.target);if(c===void 0)throw new Error(`Lazy tree source hardlink ${s.sourcePath} target is absent`);if(c.type!=="file"&&c.type!=="hardlink")throw new Error(`Lazy tree source hardlink ${s.sourcePath} target is not regular`);s=c}a===void 0&&(a=s);for(let c of i)t.set(c.sourcePath,a)}return t}function ue(r,e,t,n=!1){if(typeof r!="string"||r.length===0||new TextEncoder().encode(r).byteLength>gn||r.includes("\0")||r.includes("\\")||r.startsWith("/")!==e)throw new Error(`${t} is not a canonical ${e?"absolute":"relative"} path`);if(n&&e&&r==="/")return r;if(r.slice(e?1:0).split("/").some(o=>o===""||o==="."||o===".."))throw new Error(`${t} has an unsafe path segment`);return r}function Pa(r){let e=We(r,["uid","gid"],"Lazy tree registration owner");return{uid:ne(e.uid,"Lazy tree registration owner uid",0,Ri),gid:ne(e.gid,"Lazy tree registration owner gid",0,Ri)}}function qi(r,e,t,n,i=1){let o=dn(r,i),s=pn(t),a=We(n,["mode","capabilities","roots"],"Lazy tree activation");if(a.mode!=="boot-prefetch"&&a.mode!=="first-use")throw new Error("Lazy tree activation mode is invalid");let c=xe(a.capabilities,"Lazy tree activation capabilities",1,fa).map((E,z)=>{let S=Be(E,`Lazy tree activation capability ${z}`,le.maxActivationCapabilityBytes);if(!/^[a-z0-9][a-z0-9:._-]*$/.test(S))throw new Error(`Lazy tree activation capability ${z} is invalid`);return S}),l=xe(a.roots,"Lazy tree activation roots",1,ua).map((E,z)=>ue(E,!0,`Lazy tree activation root ${z}`,!0));if(new Set(c).size!==c.length||new Set(l).size!==l.length)throw new Error("Lazy tree activation contains duplicates");let h={mode:a.mode,capabilities:c,roots:l},f=xe(e,"Lazy tree inventory",1,ht),d=[],y=new Map,g=new Map,p=o.source===void 0?void 0:new Map(o.source.entries.map(E=>[E.sourcePath,E])),w=o.source===void 0?void 0:Vi(o.source.entries),u=0;for(let[E,z]of f.entries()){if(typeof z!="object"||z===null||Array.isArray(z))throw new Error(`Lazy tree entry ${E} must be an object`);let S=z.type,k=S==="directory"?["vfsPath","sourcePath","type","mode","size"]:S==="file"?["vfsPath","sourcePath","type","mode","size","inodeGroup"]:S==="symlink"?["vfsPath","sourcePath","type","mode","size","target"]:S==="hardlink"?["vfsPath","sourcePath","type","mode","size","target","inodeGroup"]:null;if(!k)throw new Error(`Lazy tree entry ${E} has an invalid type`);let I=We(z,[...k,...p===void 0?[]:["materialization"]],`Lazy tree entry ${E}`),A=ue(I.vfsPath,!0,`Lazy tree entry ${E} VFS path`),B=ue(I.sourcePath,!1,`Lazy tree entry ${E} source path`),N=p===void 0?void 0:I.materialization;if(p!==void 0&&N!=="archive"&&N!=="archive-homebrew-relocate"&&N!=="archive-copy"&&N!=="archive-copy-mode"&&N!=="descriptor")throw new Error(`Lazy tree entry ${A} has invalid materialization provenance`);if(s!=="/"&&A!==s&&!A.startsWith(`${s}/`))throw new Error(`Lazy tree entry ${A} escapes its mount prefix`);if(y.has(A))throw new Error(`Lazy tree duplicates VFS path ${A}`);let te=ne(I.mode,`Lazy tree entry ${A} mode`,0,4095),j=ne(I.size,`Lazy tree entry ${A} size`,0,cn),R,P;if(S==="directory"){if(j!==0)throw new Error(`Lazy tree directory ${A} has nonzero size`)}else if(S==="symlink"){if(R=Be(I.target,`Lazy tree symlink ${A} target`,Di),new TextEncoder().encode(R).byteLength!==j)throw new Error(`Lazy tree symlink ${A} size differs from its target`)}else P=Be(I.inodeGroup,`Lazy tree entry ${A} inode group`,gn),S==="hardlink"&&(R=ue(I.target,!0,`Lazy tree hardlink ${A} target`));if(S!=="hardlink"&&(u+=j,u>cn))throw new Error("Lazy tree inventory exceeds the expansion limit");let _={vfsPath:A,sourcePath:B,...N===void 0?{}:{materialization:N},type:S,mode:te,size:j,...R===void 0?{}:{target:R},...P===void 0?{}:{inodeGroup:P}};if(p===void 0){let Z=g.get(B);if(Z){if(o.decoder!=="zip-v1"||_.type!=="hardlink"||Z.inodeGroup!==_.inodeGroup)throw new Error(`Lazy tree duplicates source path ${B}`)}else{if(o.decoder==="zip-v1"&&_.type==="hardlink")throw new Error(`Lazy ZIP hardlink ${A} does not reuse a canonical source path`);g.set(B,_)}}else if(_.materialization==="descriptor"){if(_.type!=="directory"&&_.type!=="symlink")throw new Error(`Lazy tree descriptor entry ${A} is not structural`);if(p.has(B))throw new Error(`Lazy tree descriptor entry ${A} impersonates a source member`)}else{let Z=p.get(B);if(Z===void 0)throw new Error(`Lazy tree entry ${A} names absent source ${B}`);if(_.materialization==="archive-copy"||_.materialization==="archive-copy-mode"){if(_.type!=="file"||Z.type!=="file"||_.materialization==="archive-copy"&&_.mode!==Z.mode)throw new Error(`Lazy tree archive copy ${A} differs from its source`)}else if(_.materialization==="archive-homebrew-relocate"){if(_.type!=="file"&&_.type!=="hardlink"||Z.type!==_.type||_.type==="file"&&Z.mode!==_.mode)throw new Error(`Lazy tree receipt-relocated entry ${A} differs from its source`)}else if(Z.type!==_.type||_.type==="symlink"&&Z.target!==_.target||_.type!=="hardlink"&&Z.mode!==_.mode)throw new Error(`Lazy tree archive entry ${A} differs from its source`)}d.push(_),y.set(A,_)}for(let E of d){let z=E.vfsPath.split("/").filter(Boolean);for(let S=1;S({path:E.vfsPath,type:E.type,mode:E.mode,size:E.size,target:E.target,inodeGroup:E.inodeGroup})),"Lazy tree");if(p!==void 0){let E=new Set;for(let z of d){if(z.materialization!=="archive-homebrew-relocate")continue;let S=p.get(z.sourcePath),k=S.type==="file"?S:w.get(S.sourcePath);if(k?.type!=="file")throw new Error(`Lazy tree receipt-relocated entry ${z.vfsPath} is not regular`);E.add(k.sourcePath)}for(let z of d){if(z.materialization==="descriptor"||z.type!=="file"&&z.type!=="hardlink")continue;let S=p.get(z.sourcePath),k=S.type==="file"?S:w.get(S.sourcePath);if(k?.type!=="file"||!E.has(k.sourcePath)&&z.size!==k.size)throw new Error(`Lazy tree archive entry ${z.vfsPath} differs from its source`)}for(let z of d){if(z.type!=="hardlink"||z.materialization!=="archive"&&z.materialization!=="archive-homebrew-relocate")continue;let S=p.get(z.sourcePath),k=y.get(z.target),I=w.get(S.sourcePath);if(S.target!==k?.sourcePath||I?.type!=="file"||I.mode!==z.mode||k?.mode!==z.mode)throw new Error(`Lazy tree hardlink ${z.vfsPath} differs from its source`)}}if(o.sourceEntryCount!==(p===void 0?g.size:p.size))throw new Error("Lazy tree source entry count differs from its inventory");if(o.source===void 0&&o.expandedBytesz.vfsPath===E||z.vfsPath.startsWith(`${E}/`)))throw new Error(`Lazy tree activation root ${E} is not owned by its inventory`);let v=new Map;for(let E of d)E.type==="file"&&v.set(E.inodeGroup,E);if(v.size!==m.canonicalByGroup.size)throw new Error("Lazy tree regular inode inventory is inconsistent");return{content:o,entries:d,mountPrefix:s,activation:h,canonicalByGroup:v}}function hn(r){return JSON.stringify([r.sourcePath,r.type,r.inodeGroup,r.target])}function $i(r,e){let t=fr(r,["kind","content","url","mountPrefix","integrity","materialized","entries"],["url","mountPrefix","materialized","entries"],"Serialized legacy lazy archive");if(t.kind===void 0){if(!e)throw new Error("Serialized lazy archive is missing its kind discriminator")}else if(t.kind!==ln)throw new Error("Serialized legacy lazy archive has an unsupported kind");let n=Be(t.url,"Serialized legacy lazy archive URL",dr),i=pn(t.mountPrefix),o=Ft(t.integrity);if(t.content!==void 0){if(!e||t.kind!==void 0)throw new Error("Typed legacy lazy archives cannot carry generic content");let c=dn(t.content);if(c.decoder!=="zip-v1"||c.transports.length!==1||c.transports[0]!==n||!o||c.sha256!==o.sha256||c.bytes!==o.bytes)throw new Error("Untagged legacy ZIP content identity is inconsistent")}if(t.materialized!==!1)throw new Error("Serialized legacy lazy archive must describe pending content");let s=new Set,a=xe(t.entries,"Serialized legacy lazy archive entries",1,ht).map((c,l)=>{let h=fr(c,["vfsPath","ino","generation","dataSequence","size","isSymlink","deleted","materialized","archivePath","sourcePath","type","inodeGroup","target"],["vfsPath","ino","size","isSymlink","deleted"],`Serialized legacy lazy archive entry ${l}`),f=ue(h.vfsPath,!0,`Serialized legacy lazy archive entry ${l} VFS path`);if(s.has(f))throw new Error(`Serialized legacy lazy archive duplicates path ${f}`);s.add(f);let d=ne(h.ino,`Serialized legacy lazy archive entry ${f} inode`,1,Number.MAX_SAFE_INTEGER),y=h.generation===void 0?void 0:ne(h.generation,`Serialized legacy lazy archive entry ${f} generation`,0,Number.MAX_SAFE_INTEGER),g=h.dataSequence===void 0?void 0:ne(h.dataSequence,`Serialized legacy lazy archive entry ${f} data sequence`,0,Number.MAX_SAFE_INTEGER),p=ne(h.size,`Serialized legacy lazy archive entry ${f} size`,0,cn);if(h.isSymlink!==!1||h.deleted!==!1||h.materialized!==void 0&&h.materialized!==!1)throw new Error(`Serialized legacy lazy archive entry ${f} is not pending`);if(h.type!==void 0&&h.type!=="file")throw new Error(`Serialized legacy lazy archive entry ${f} has an invalid type`);let w=h.archivePath===void 0?void 0:ue(h.archivePath,!1,`Serialized legacy lazy archive entry ${f} archive path`),u=h.sourcePath===void 0?void 0:ue(h.sourcePath,!1,`Serialized legacy lazy archive entry ${f} source path`),m=h.inodeGroup===void 0?void 0:Be(h.inodeGroup,`Serialized legacy lazy archive entry ${f} inode group`,gn);if(h.target!==void 0)throw new Error(`Serialized legacy lazy archive entry ${f} has a link target`);return{vfsPath:f,ino:d,...y===void 0?{}:{generation:y},...g===void 0?{}:{dataSequence:g},size:p,isSymlink:!1,deleted:!1,materialized:!1,...w===void 0?{}:{archivePath:w},...u===void 0?{}:{sourcePath:u},type:"file",...m===void 0?{}:{inodeGroup:m}}});return{kind:ln,url:n,mountPrefix:i,...o===void 0?{}:{integrity:o},materialized:!1,entries:a}}function Oa(r,e){let t=We(r,["kind","content","inventory","activation","url","mountPrefix","integrity","materialized","entries"],"Serialized lazy tree");if(t.kind!==e)throw new Error("Serialized lazy tree has an unsupported kind");let n=qi(t.content,t.inventory,t.mountPrefix,t.activation);if(e===fn!=(n.content.source===void 0))throw new Error(e===fn?"Serialized deferred-tree-v1 cannot contain original-bottle source metadata":"Serialized deferred-tree-v2 requires original-bottle source metadata");let i=Be(t.url,"Serialized lazy tree URL",dr);if(i!==n.content.transports[0])throw new Error("Serialized lazy tree URL differs from its primary transport");let o=Ft(t.integrity);if(!o||o.sha256!==n.content.sha256||o.bytes!==n.content.bytes)throw new Error("Serialized lazy tree integrity differs from its content");if(t.materialized!==!1)throw new Error("Serialized lazy tree must describe pending content");let s=new Map(n.entries.map(f=>[f.vfsPath,f])),a=new Map(n.entries.map(f=>[hn(f),f])),c=xe(t.entries,"Serialized lazy tree entries",0,ht),l=new Set,h=c.map((f,d)=>{let y=fr(f,["vfsPath","ino","generation","dataSequence","size","isSymlink","deleted","materialized","archivePath","sourcePath","type","inodeGroup","target"],["vfsPath","ino","generation","dataSequence","size","isSymlink","deleted","materialized","archivePath","sourcePath","type","inodeGroup"],`Serialized lazy tree entry ${d}`),g=ue(y.vfsPath,!0,`Serialized lazy tree entry ${d} VFS path`);if(l.has(g))throw new Error(`Serialized lazy tree duplicates pending path ${g}`);l.add(g);let p=ue(y.sourcePath,!1,`Serialized lazy tree entry ${d} source path`),w=ue(y.archivePath,!1,`Serialized lazy tree entry ${d} archive path`),u=s.get(g),m=a.get(hn({sourcePath:p,type:typeof y.type=="string"?y.type:void 0,inodeGroup:typeof y.inodeGroup=="string"?y.inodeGroup:void 0,target:typeof y.target=="string"?y.target:void 0}))??u;if(!m||m.type!=="file"&&m.type!=="hardlink"||u?.inodeGroup!==void 0&&u.inodeGroup!==m.inodeGroup)throw new Error(`Serialized lazy tree entry ${g} is absent from its inventory`);let v=n.canonicalByGroup.get(m.inodeGroup);if(y.type!==m.type||y.inodeGroup!==m.inodeGroup||y.size!==m.size||w!==v?.sourcePath||y.target!==m.target||y.isSymlink!==!1||y.deleted!==!1||y.materialized!==!1)throw new Error(`Serialized lazy tree entry ${g} disagrees with its inventory`);let E=ne(y.ino,`Serialized lazy tree entry ${g} inode`,1,Number.MAX_SAFE_INTEGER),z=ne(y.generation,`Serialized lazy tree entry ${g} generation`,0,Number.MAX_SAFE_INTEGER),S=ne(y.dataSequence,`Serialized lazy tree entry ${g} data sequence`,0,Number.MAX_SAFE_INTEGER);return{vfsPath:g,ino:E,generation:z,dataSequence:S,size:m.size,isSymlink:!1,deleted:!1,materialized:!1,archivePath:w,sourcePath:p,type:m.type,inodeGroup:m.inodeGroup,...m.target===void 0?{}:{target:m.target}}});return{kind:e,content:n.content,inventory:n.entries,activation:n.activation,url:i,mountPrefix:n.mountPrefix,integrity:o,materialized:!1,entries:h}}async function ar(r,e,t){if(t===void 0)return;if(r.byteLength!==t.bytes)throw new Error(`Lazy ${e} byte count ${r.byteLength} does not match expected ${t.bytes}`);let n=globalThis.crypto?.subtle;if(!n)throw new Error(`Lazy ${e} integrity verification is unavailable`);let i=new Uint8Array(r.byteLength);i.set(r);let o=new Uint8Array(await n.digest("SHA-256",i)),s=Array.from(o,a=>a.toString(16).padStart(2,"0")).join("");if(s!==t.sha256)throw new Error(`Lazy ${e} SHA-256 ${s} does not match expected ${t.sha256}`)}var yn=class r{fs;imageMetadata;lazyFiles=new Map;lazyArchiveGroups=[];deferredTreeMaterializationHandles=new WeakMap;lazyArchiveInodes=new Map;lazyDownloadListeners=new Set;lazyPreparations=new Map;lazyTransport={fetcher:(e,t)=>globalThis.fetch(e,t)};constructor(e,t=null){this.fs=e,this.imageMetadata=t}static inodeKey(e,t){return`${e}:${t}`}static canAdoptLegacyLazyStub(e){return(e.mode&ut)===sr&&e.size===0&&e.dataSequence<=1}reconcileLazyIdentityState(e){for(let[t,n]of this.lazyFiles){let i=e.get(t);if(!i||i.dataSequence!==n.dataSequence||i.paths.length===0){this.lazyFiles.delete(t);continue}n.paths=new Set(i.paths),n.paths.has(n.path)||(n.path=i.paths[0])}this.lazyArchiveInodes.clear();for(let t of this.lazyArchiveGroups){let n=t.content!==void 0&&t.inventory!==void 0&&!t.materialized,i=new Map;for(let s of t.entries.values()){if(s.deleted||s.materialized||s.generation===void 0)continue;let a=r.inodeKey(s.ino,s.generation);i.has(a)||i.set(a,s)}let o=new Map;for(let[s,a]of i){let c=e.get(s);if(!(!c||c.dataSequence!==(a.dataSequence??0))){for(let l of c.paths)o.set(l,{...a,ino:c.ino,generation:c.generation,dataSequence:c.dataSequence,deleted:!1,materialized:!1});c.paths.length>0&&this.lazyArchiveInodes.set(s,t)}}t.entries=o,t.materialized=o.size===0&&!n}}lazyFileForStat(e){let t=r.inodeKey(e.ino,e.generation),n=this.lazyFiles.get(t);if(n&&n.dataSequence!==e.dataSequence){this.lazyFiles.delete(t);return}return n}lazyArchiveForStat(e){let t=r.inodeKey(e.ino,e.generation),n=this.lazyArchiveInodes.get(t);if(!n)return;let i=Array.from(n.entries.values()).filter(o=>o.ino===e.ino&&o.generation===e.generation&&!o.deleted&&!o.materialized);if(i.some(o=>o.dataSequence===e.dataSequence))return n;this.lazyArchiveInodes.delete(t);for(let o of i)o.materialized=!0}lazyBackingForStat(e){let t=r.inodeKey(e.ino,e.generation),n=this.lazyFiles.get(t);if(n)return{token:n,path:n.path};let i=this.lazyArchiveInodes.get(t);if(!i)return null;let o=Array.from(i.entries.entries()).find(([,s])=>s.ino===e.ino&&s.generation===e.generation&&!s.deleted&&!s.materialized)?.[0];return o===void 0?null:{token:i,path:o}}lazyBackingForPath(e){let t=this.lazyArchiveGroups.find(n=>!n.materialized&&n.content!==void 0&&n.inventory!==void 0&&n.activation!==void 0&&Array.from(n.entries.values()).every(i=>i.deleted||i.materialized||i.isSymlink)&&n.activation.roots.some(i=>i==="/"||e===i||e.startsWith(`${i}/`)));if(t)return{token:t,path:e,directGroup:t};try{let n=this.fs.stat(e),i=this.lazyBackingForStat(n);return i?{token:i.token,path:e}:null}catch{return null}}startLazyPreparation(e){let{path:t,token:n}=e,i={status:"pending",promise:Promise.resolve(!1)},o=e.directGroup?this.ensureArchiveMaterialized(e.directGroup).then(()=>!0):this.materializePath(t);return i.promise=o.then(s=>(i.status="fulfilled",this.lazyPreparations.get(n)===i&&this.lazyPreparations.delete(n),s),s=>{throw i.status="rejected",i.error=s,s}),i.promise.catch(()=>{}),this.lazyPreparations.set(n,i),i}guardSynchronousLazyAccess(e){let t=this.lazyBackingForPath(e);if(!t)return;let n=this.lazyPreparations.get(t.token);if(n?.status==="fulfilled"){this.lazyPreparations.delete(t.token);let o=this.lazyBackingForPath(e);if(!o)return;n=this.lazyPreparations.get(o.token)??this.startLazyPreparation(o)}else if(n?.status==="rejected"){this.lazyPreparations.delete(t.token);let o=n.error instanceof Error?n.error.message:String(n.error),s=new Error(`EIO: lazy backing for ${e} failed: ${o}`);throw s.code="EIO",s.cause=n.error,s}else n||(n=this.startLazyPreparation(t));let i=new Error(`EAGAIN: lazy backing for ${e} is being prepared`);throw i.code="EAGAIN",i}invalidateLazyData(e){let t=r.inodeKey(e.ino,e.generation);this.lazyFiles.delete(t);let n=this.lazyArchiveInodes.get(t);if(n){this.lazyArchiveInodes.delete(t);for(let i of n.entries.values())i.ino===e.ino&&i.generation===e.generation&&(i.materialized=!0)}}rewriteLazyNamespacePaths(e,t,n){let i=t.length>1?t.replace(/\/+$/,""):t,o=n.length>1?n.replace(/\/+$/,""):n,s=`${i}/`,a=`${o}/`,c=r.inodeKey(e.ino,e.generation),l=(e.mode&ut)===nn,h=f=>f===i?o:l&&f.startsWith(s)?a+f.slice(s.length):f;for(let[f,d]of this.lazyFiles)!l&&f!==c||(d.paths=new Set(Array.from(d.paths,h)),d.path=h(d.path));for(let f of this.lazyArchiveGroups){let d=new Map;for(let[y,g]of f.entries){let p=g.generation===void 0?null:r.inodeKey(g.ino,g.generation);d.set(l||p===c?h(y):y,g)}f.entries=d,f.inventory&&(f.inventory=f.inventory.map(y=>({...y,vfsPath:h(y.vfsPath),...y.type==="hardlink"&&y.target!==void 0?{target:h(y.target)}:{}}))),f.activation&&(f.activation={...f.activation,roots:f.activation.roots.map(h)})}}get sharedBuffer(){return this.fs.buffer}static create(e,t){return new r(ze.mkfs(e,t))}static fromExisting(e){return new r(ze.mount(e))}rebaseToNewFileSystem(e){if(!Number.isSafeInteger(e)||e<=0)throw new Error(`Invalid MemoryFileSystem maxByteLength: ${e}`);let t=SharedArrayBuffer,{bytes:n,identities:i}=this.fs.snapshotState();this.reconcileLazyIdentityState(i);let o=this.serializeLazyEntries(),s=this.serializeLazyArchiveEntries(),a=new t(n.byteLength);new Uint8Array(a).set(n);let c=new r(ze.mount(a,{restoreImage:!0}),this.imageMetadata);c.importLazyEntries(o),c.importLazyArchiveEntries(s);let l=Math.min(e,Math.max(n.byteLength,oa)),h=new t(l,{maxByteLength:e}),f=r.create(h,e);f.setImageMetadata(this.imageMetadata);let d=new Set(o.flatMap(g=>g.paths??[g.path])),y=new Set;for(let g of s)if(!g.materialized)for(let p of g.entries)!p.deleted&&!p.isSymlink&&y.add(p.vfsPath);return c.copyPathToFreshFileSystem("/",f,d,y,new Map),f.importLazyEntries(o.map(g=>{let p=f.fs.lstat(g.path);return{...g,ino:p.ino,generation:p.generation,dataSequence:p.dataSequence}})),f.importLazyArchiveEntries(s.map(g=>({...g,entries:g.entries.map(p=>{if(p.deleted)return{...p,ino:0,generation:void 0};let w=f.fs.lstat(p.vfsPath);return{...p,ino:w.ino,generation:w.generation,dataSequence:w.dataSequence}})}))),f}getImageMetadata(){return pa(this.imageMetadata)}setImageMetadata(e){this.imageMetadata=e===null?null:hr(e)}subscribeLazyDownloads(e){return this.lazyDownloadListeners.add(e),()=>this.lazyDownloadListeners.delete(e)}setLazyFetcher(e,t={}){this.lazyTransport={fetcher:e,...t.signal===void 0?{}:{signal:t.signal}}}emitLazyDownload(e){if(this.lazyDownloadListeners.size===0)return;let t={...e,t:Ea()};for(let n of this.lazyDownloadListeners)try{n(t)}catch{}}async fetchLazyBytes(e,t){let n=0,i=e.integrity?.bytes??e.fallbackTotalBytes,o={id:e.id,kind:e.kind,url:e.url,path:e.path,mountPrefix:e.mountPrefix};for(let s=0;se.integrity.bytes)throw new Error(`Lazy ${e.kind} exceeded expected byte count ${e.integrity.bytes}`);this.emitLazyDownload({...o,status:"progress",loadedBytes:n,totalBytes:i})}}}catch(f){try{await c.cancel(f)}catch{}throw f}}finally{c.releaseLock()}let h=_a(l,n);return ee(t.signal),await ar(h,e.kind,e.integrity),ee(t.signal),this.emitLazyDownload({...o,status:"complete",loadedBytes:n,totalBytes:i??n}),h}catch(a){if(t.signal?.aborted){let h=t.signal.reason,f=h instanceof Error?h.message:String(h);throw this.emitLazyDownload({...o,status:"error",loadedBytes:n,totalBytes:i,error:f}),h}let c=s+1({...u})),activation:f,entries:new Map},p=u=>{let m=u.split("/").filter(Boolean),v="";for(let E=0;Em.vfsPath.split("/").length-v.vfsPath.split("/").length))if(u.type==="directory"){p(u.vfsPath);try{this.fs.mkdir(u.vfsPath,u.mode),this.fs.chmod(u.vfsPath,u.mode)}catch{if((this.fs.lstat(u.vfsPath).mode&ut)!==nn)throw new Error(`Lazy tree directory collides at ${u.vfsPath}`)}}for(let u of l){if(u.type!=="symlink")continue;p(u.vfsPath),this.fs.symlink(u.target,u.vfsPath);let m=this.fs.lstat(u.vfsPath);g.entries.set(u.vfsPath,{ino:m.ino,generation:m.generation,dataSequence:m.dataSequence,size:u.size,isSymlink:!0,deleted:!1,materialized:!0,archivePath:u.sourcePath,sourcePath:u.sourcePath,type:"symlink",target:u.target})}let w=new Map;for(let u of l){if(u.type!=="file")continue;p(u.vfsPath);let m=this.fs.createLazyStub(u.vfsPath,u.mode);this.invalidateLazyData(m),w.set(u.inodeGroup,m);let v={ino:m.ino,generation:m.generation,dataSequence:m.dataSequence,size:u.size,isSymlink:!1,deleted:!1,materialized:!1,archivePath:u.sourcePath,sourcePath:u.sourcePath,type:"file",inodeGroup:u.inodeGroup};g.entries.set(u.vfsPath,v)}for(let u of l){if(u.type!=="hardlink")continue;let m=d.get(u.inodeGroup);p(u.vfsPath),this.fs.link(m.vfsPath,u.vfsPath);let v=this.fs.lstat(u.vfsPath),E=w.get(u.inodeGroup);if(v.ino!==E.ino||v.generation!==E.generation)throw new Error(`Lazy tree hardlink ${u.vfsPath} did not share its inode`);g.entries.set(u.vfsPath,{ino:v.ino,generation:v.generation,dataSequence:v.dataSequence,size:u.size,isSymlink:!1,deleted:!1,materialized:!1,archivePath:m.sourcePath,sourcePath:u.sourcePath,type:"hardlink",inodeGroup:u.inodeGroup,target:u.target})}if(y!==void 0)for(let u of l)this.lchown(u.vfsPath,y.uid,y.gid);for(let u of g.entries.values())u.isSymlink||u.generation===void 0||this.lazyArchiveInodes.set(r.inodeKey(u.ino,u.generation),g);return this.lazyArchiveGroups.push(g),g}registerLazyTreeWithMaterializationHandle(e,t,n="/",i,o){let s=this.registerLazyTreeInternal(e,t,n,i,!0,o),a=Object.freeze({[ta]:!0});return this.deferredTreeMaterializationHandles.set(a,s),a}registerLazyArchiveFromEntries(e,t,n,i,o){let s=ga(e,t,n,i);s.some(({entry:c})=>!c.isDirectory&&!c.isSymlink)&&this.assertCanRegisterPendingLazyArchiveGroup();let a={...o?{content:dn({decoder:"zip-v1",mediaType:"application/zip",sha256:o.sha256,bytes:o.bytes,expandedBytes:s.reduce((c,l)=>c+l.entry.uncompressedSize,0),sourceEntryCount:s.length,transports:[e]})}:{},url:e,mountPrefix:n,integrity:Ft(o),materialized:!1,entries:new Map};for(let{entry:c,vfsPath:l}of s){if(c.isDirectory)continue;let h=l.split("/").filter(Boolean),f="";for(let d=0;dc.deleted||c.materialized),this.lazyArchiveGroups.push(a),a}importLazyArchiveEntries(e){this.importLazyArchiveEntriesInternal(e,!1,!0)}importLazyArchiveEntriesInternal(e,t,n){let i=xe(e,"Serialized lazy archive groups",0,la).map((a,c)=>{if(typeof a!="object"||a===null||Array.isArray(a))throw new Error(`Serialized lazy archive group ${c} must be an object`);let l=a.kind;if(l===fn||l===Ni)return Oa(a,l);if(l===ln)return $i(a,!1);if(l!==void 0)throw new Error(`Serialized lazy archive group ${c} has an unsupported kind`);if(n)throw new Error(`Serialized lazy archive group ${c} is missing its kind discriminator`);return $i(a,!0)});Fi([...this.serializeLazyArchiveEntries(),...i]);let o=[],s=new Map;for(let a of i){let c=new Map,l=a.mountPrefix.replace(/\/+$/,""),h=a.content!==void 0&&a.inventory!==void 0&&a.activation!==void 0,f=h?new Map(a.inventory.map(u=>[u.vfsPath,u])):null,d=h?new Map(a.inventory.map(u=>[hn(u),u])):null,y=new Map,g=new Map;for(let u of a.entries){let m=null,v=a.materialized||u.materialized===!0||u.isSymlink;if(!u.deleted&&!v){if((u.generation===void 0||u.dataSequence===void 0)&&!t)throw new Error("Live lazy-archive metadata requires inode generation and data sequence");try{m=this.fs.lstat(u.vfsPath)}catch{if(h)throw new Error(`Serialized lazy tree stub ${u.vfsPath} is missing from the filesystem`);continue}if(m.ino!==u.ino){if(h)throw new Error(`Serialized lazy tree stub ${u.vfsPath} has a different inode`);continue}if(u.generation!==void 0&&m.generation!==u.generation){if(h)throw new Error(`Serialized lazy tree stub ${u.vfsPath} has a different generation`);continue}if(u.dataSequence===void 0){if(!r.canAdoptLegacyLazyStub(m)){if(h)throw new Error(`Serialized lazy tree stub ${u.vfsPath} is not pristine`);continue}}else if(m.dataSequence!==u.dataSequence){if(h)throw new Error(`Serialized lazy tree stub ${u.vfsPath} has a different data sequence`);continue}if(h){let z=f.get(u.vfsPath),S=d.get(hn(u))??z;if(!S||(m.mode&ut)!==sr||m.size!==0||(m.mode&4095)!==S.mode||z?.inodeGroup!==void 0&&z.inodeGroup!==S.inodeGroup)throw new Error(`Serialized lazy tree stub ${u.vfsPath} disagrees with its inventory`);let k=r.inodeKey(m.ino,m.generation),I=u.inodeGroup,A=y.get(I),B=g.get(k);if(A!==void 0&&A!==k||B!==void 0&&B!==I)throw new Error(`Serialized lazy tree inode group ${I} disagrees with the filesystem`);y.set(I,k),g.set(k,I)}}c.set(u.vfsPath,{ino:u.ino,generation:m?.generation??u.generation,dataSequence:m?.dataSequence??u.dataSequence,size:u.size,isSymlink:u.isSymlink,deleted:u.deleted,materialized:v,archivePath:u.archivePath??u.vfsPath.slice(l.length+1),sourcePath:u.sourcePath??u.archivePath??u.vfsPath.slice(l.length+1),type:u.type??(u.isSymlink?"symlink":"file"),inodeGroup:u.inodeGroup,target:u.target})}let p=a.content===void 0?void 0:dn(a.content),w={content:p,url:p?.transports[0]??a.url,mountPrefix:a.mountPrefix,integrity:p?{sha256:p.sha256,bytes:p.bytes}:Ft(a.integrity),materialized:a.materialized||!(p&&a.inventory)&&Array.from(c.values()).every(u=>u.deleted||u.materialized),inventory:a.inventory?.map(u=>({...u})),activation:a.activation?{mode:a.activation.mode,capabilities:[...a.activation.capabilities],roots:[...a.activation.roots]}:void 0,entries:c};if(o.push(w),!w.materialized){for(let[,u]of c)if(!u.deleted&&!u.materialized&&u.generation!==void 0){let m=r.inodeKey(u.ino,u.generation),v=s.get(m);if(v!==void 0&&v!==w)throw new Error(`Serialized lazy archive groups share pending inode ${m}`);if(this.lazyArchiveInodes.has(m))throw new Error(`Serialized lazy archive group collides with pending inode ${m}`);s.set(m,w)}}}this.lazyArchiveGroups.push(...o);for(let[a,c]of s)this.lazyArchiveInodes.set(a,c)}rewriteLazyArchiveUrls(e){for(let t of this.lazyArchiveGroups)t.content?(t.content={...t.content,transports:t.content.transports.map(e)},t.url=t.content.transports[0]):t.url=e(t.url)}serializeLazyArchiveEntries(){let e=[];for(let t of this.lazyArchiveGroups){let n=Array.from(t.entries,([o,s])=>({vfsPath:o,ino:s.ino,generation:s.generation,dataSequence:s.dataSequence,size:s.size,isSymlink:s.isSymlink,deleted:s.deleted,materialized:s.materialized,archivePath:s.archivePath,sourcePath:s.sourcePath,type:s.type,inodeGroup:s.inodeGroup,target:s.target})).filter(o=>!o.deleted&&!o.materialized);if(n.length===0&&!(t.content&&t.inventory&&!t.materialized))continue;let i=t.content!==void 0&&t.inventory!==void 0&&t.activation!==void 0;if(i&&t.content.transports.length===0)throw new Error("Direct-materialization tree must be materialized before serialization");e.push(i?{kind:t.content.source===void 0?fn:Ni,content:t.content,inventory:t.inventory,activation:t.activation,url:t.url,mountPrefix:t.mountPrefix,integrity:t.integrity,materialized:!1,entries:n}:{kind:ln,url:t.url,mountPrefix:t.mountPrefix,integrity:t.integrity,materialized:!1,entries:n})}return e}exportLazyArchiveEntries(){return this.reconcileLazyIdentityState(this.fs.identityState()),this.serializeLazyArchiveEntries()}pendingDeferredTreeUsage(){return this.reconcileLazyIdentityState(this.fs.identityState()),Hi(this.serializeLazyArchiveEntries())}assertCanAppendDeferredTreeUsage(e){ur(e);let t=this.pendingDeferredTreeUsage();ur({groups:t.groups+e.groups,archiveBytes:t.archiveBytes+e.archiveBytes,expandedBytes:t.expandedBytes+e.expandedBytes,payloadBytes:t.payloadBytes+e.payloadBytes,entries:t.entries+e.entries})}assertCanRegisterPendingLazyArchiveGroup(){if(this.reconcileLazyIdentityState(this.fs.identityState()),this.lazyArchiveGroups.filter(t=>!t.materialized&&(t.content!==void 0&&t.inventory!==void 0||Array.from(t.entries.values()).some(n=>!n.deleted&&!n.materialized))).length>=we.maxGroups)throw new Error(`Cannot register another lazy archive group: ${we.maxGroups} pending groups already exist`)}async preparePath(e){let t=!1,n=Math.max(3,this.lazyArchiveGroups.length+1);for(let i=0;i!o.materialized&&o.activation?.mode==="boot-prefetch"),t=0,n,i=Array.from({length:Math.min(e.length,ca)},async()=>{for(;n===void 0;){let o=t;if(t+=1,o>=e.length)return;try{await this.prepareLazyTreeGroup(e[o])}catch(s){n??=s}}});if(await Promise.all(i),n!==void 0)throw n;return e.length}async materializeRegisteredDeferredTree(e,t){let n=this.deferredTreeMaterializationHandles.get(e);if(n===void 0)throw new Error("Deferred-tree handle was not issued by this filesystem");if(n.materialized)return!1;let i=this.lazyPreparations.get(n);if(i!==void 0)return i.promise;let o=new Uint8Array(t.byteLength);o.set(t);let s={status:"pending",promise:Promise.resolve(!1)};s.promise=Promise.resolve().then(async()=>(await ar(o,"tree",n.integrity),await this.materializeArchiveBytes(n,o),!0)).then(a=>(s.status="fulfilled",a),a=>{throw s.status="rejected",s.error=a,a}),s.promise.catch(()=>{}),this.lazyPreparations.set(n,s);try{return await s.promise}finally{this.lazyPreparations.get(n)===s&&this.lazyPreparations.delete(n)}}async prepareLazyTreeGroup(e){if(e.materialized)return!1;let t={token:e,path:e.activation?.roots[0]??e.mountPrefix,directGroup:e},n=this.lazyPreparations.get(e)??this.startLazyPreparation(t);try{return await n.promise}finally{this.lazyPreparations.get(e)===n&&this.lazyPreparations.delete(e)}}async ensureMaterialized(e){return this.preparePath(e)}async materializePath(e){if(this.lazyFiles.size===0&&this.lazyArchiveInodes.size===0)return!1;let t;try{t=this.fs.stat(e)}catch{return!1}let n=r.inodeKey(t.ino,t.generation),i=this.lazyFiles.get(n);if(i){let s=this.lazyTransport,a=await this.fetchLazyBytes({id:`file:${t.ino}`,kind:"file",url:i.url,path:i.path,fallbackTotalBytes:i.size},s);for(let c=0;c<3;c++){if(this.lazyFiles.get(n)!==i)return!1;for(let l of new Set([e,...i.paths]))if(ee(s.signal),this.fs.replaceIfIdentity(l,i.ino,i.generation,i.dataSequence,a))return i.path=l,this.lazyFiles.delete(n),!0;this.reconcileLazyIdentityState(this.fs.identityState())}throw new Error(`Lazy file kept changing names while materializing: ${e}`)}let o=this.lazyArchiveInodes.get(n);return o?(await this.ensureArchiveMaterialized(o,{path:e,ino:t.ino,generation:t.generation}),!this.lazyArchiveInodes.has(n)):!1}async decodeAndValidateLazyTree(e,t){let n=e.content,i=e.inventory;if(!n||!i)throw new Error("Lazy tree is missing its decoder or complete inventory");let o=new Map,s=new Map(i.map(f=>[f.vfsPath,f]));if(n.source!==void 0)for(let f of n.source.entries)o.set(f.sourcePath,f);else for(let f of i){if(f.type==="hardlink"){let y=s.get(f.target);if(!y)throw new Error(`Lazy tree hardlink target disappeared: ${f.target}`);if(f.sourcePath===y.sourcePath)continue}if(o.get(f.sourcePath))throw new Error(`Lazy tree inventory duplicates source member ${f.sourcePath}`);o.set(f.sourcePath,{sourcePath:f.sourcePath,type:f.type,mode:f.mode,size:f.size,...f.type==="symlink"?{target:f.target}:{},...f.type==="hardlink"?{target:s.get(f.target)?.sourcePath}:{}})}let a=new Map,c=0;if(n.decoder==="zip-v1"){let{parseZipCentralDirectory:f,extractZipEntryBounded:d}=await Promise.resolve().then(()=>(Yn(),jn)),y=f(t);if(y.length!==n.sourceEntryCount||y.length!==o.size)throw new Error("Lazy ZIP tree decoded inventory counts differ from its descriptor");for(let g of y){let p=g.isDirectory?g.fileName.replace(/\/$/,""):g.fileName;if(a.has(p))throw new Error(`Lazy ZIP tree duplicates source member ${p}`);let w=o.get(p);if(!w)throw new Error(`Lazy ZIP tree has undeclared source member ${p}`);if(c+=g.uncompressedSize,c>n.expandedBytes||g.uncompressedSize!==w.size)throw new Error(`Lazy ZIP tree member ${p} exceeds its inventory`);let u=g.isDirectory?"directory":g.isSymlink?"symlink":"file",m=n.modePolicy==="portable-posix-v1"?u==="directory"?493:u==="symlink"?511:(g.mode&73)!==0?493:420:g.mode&4095;if(u!==w.type||m!==w.mode)throw new Error(`Lazy ZIP tree member ${p} differs from inventory`);if(g.isDirectory)a.set(p,{type:"directory",mode:m});else{let v=d(t,g,w.size);if(g.isSymlink){let E;try{E=new TextDecoder("utf-8",{fatal:!0}).decode(v)}catch{throw new Error(`Lazy ZIP tree symlink ${p} is not UTF-8`)}a.set(p,{type:"symlink",mode:m,target:E})}else a.set(p,{type:"file",mode:m,data:v})}}}else{let{parseTarGzip:f}=await Promise.resolve().then(()=>(Oi(),Pi)),d=f(t,{label:`Lazy tree ${n.sha256}`,limits:{maxCompressedBytes:n.bytes,maxUncompressedBytes:n.expandedBytes,maxEntries:n.sourceEntryCount}});c=new DataView(t.buffer,t.byteOffset,t.byteLength).getUint32(t.byteLength-4,!0);for(let y of d){if(a.has(y.path))throw new Error(`Lazy TAR tree duplicates source member ${y.path}`);y.type==="file"?a.set(y.path,{type:"file",mode:y.mode,data:y.data}):y.type==="directory"?a.set(y.path,{type:"directory",mode:y.mode}):a.set(y.path,{type:y.type,mode:y.mode,target:y.linkName})}}if(a.size!==n.sourceEntryCount||a.size!==o.size||c!==n.expandedBytes)throw new Error("Lazy tree decoded inventory counts differ from its descriptor");for(let[f,d]of o){let y=a.get(f);if(!y)throw new Error(`Lazy tree is missing source member ${f}`);let g=d.type;if(y.type!==g)throw new Error(`Lazy tree member ${f} is ${y.type}, expected ${g}`);if((y.mode&4095)!==d.mode)throw new Error(`Lazy tree member ${f} mode differs from inventory`);if(g==="file"&&y.data?.byteLength!==d.size)throw new Error(`Lazy tree member ${f} size differs from inventory`);if(g==="symlink"&&y.target!==d.target)throw new Error(`Lazy tree symlink ${f} target differs from inventory`);if(g==="hardlink"&&y.target!==d.target)throw new Error(`Lazy tree hardlink ${f} target differs from inventory`)}let l=new Set(i.flatMap(f=>f.materialization==="archive-homebrew-relocate"?[f.sourcePath]:[]));if(n.source!==void 0){let f=new Map(n.source.entries.map(g=>[g.sourcePath,g])),d=Vi(n.source.entries),y=n.source.entries.filter(g=>g.sourcePath==="INSTALL_RECEIPT.json"||g.sourcePath.endsWith("/INSTALL_RECEIPT.json"));if(y.length>1)throw new Error(`Lazy Homebrew bottle has ${y.length} INSTALL_RECEIPT.json source members, expected at most one`);if(y.length===0){if(l.size>0)throw new Error("Lazy Homebrew bottle marks receipt relocation without INSTALL_RECEIPT.json")}else{let g=y[0],p=g.type==="file"?g:d.get(g.sourcePath),w=p===void 0?void 0:a.get(p.sourcePath);if(p?.type!=="file"||w?.type!=="file"||w.data===void 0)throw new Error("Lazy Homebrew bottle INSTALL_RECEIPT.json is not regular");let u=si(w.data),m=g.sourcePath.lastIndexOf("/"),v=m<0?"":g.sourcePath.slice(0,m),E=new Set(u.changedFiles.map(S=>v.length===0?S:`${v}/${S}`));if(l.size!==E.size||[...l].some(S=>!E.has(S)))throw new Error("Lazy Homebrew bottle relocation markers differ from INSTALL_RECEIPT.json");let z=new Set;for(let S of E){let k=f.get(S),I=k?.type==="file"?k:k===void 0?void 0:d.get(k.sourcePath),A=I===void 0?void 0:a.get(I.sourcePath);if(I?.type!=="file"||A?.type!=="file"||A.data===void 0)throw new Error(`Lazy Homebrew bottle changed source ${S} is not regular`);z.has(I.sourcePath)||(A.data=oi(A.data,u,S),z.add(I.sourcePath))}}}else if(l.size>0)throw new Error("Lazy tree receipt relocation requires original-bottle source truth");let h=new Map;for(let f of i){if(f.type!=="file"||f.materialization==="descriptor")continue;let d=a.get(f.sourcePath);if(d?.type!=="file"||!d.data)throw new Error(`Lazy tree has no file content for ${f.sourcePath}`);h.set(f.sourcePath,d.data)}return h}async ensureArchiveMaterialized(e,t){if(e.materialized)return;let n=e.content!==void 0&&e.inventory!==void 0,i=this.lazyTransport,o=n?e.content.transports:[e.url],s=[],a=null;for(let[c,l]of o.entries())try{a=await this.fetchLazyBytes({id:`archive:${e.mountPrefix}:${e.content?.sha256??l}:${c}`,kind:n?"tree":"archive",url:l,mountPrefix:e.mountPrefix,integrity:e.integrity},i);break}catch(h){if(ee(i.signal),Zi(h))throw h;s.push(h instanceof Error?h.message:String(h))}if(ee(i.signal),a===null)throw new Error(`All ${o.length} lazy ${n?"tree":"archive"} transports failed: ${s.join("; ")}`);ee(i.signal),await this.materializeArchiveBytes(e,a,t,i.signal)}async materializeArchiveBytes(e,t,n,i){if(ee(i),e.materialized)return;let s=e.content!==void 0&&e.inventory!==void 0?await this.decodeAndValidateLazyTree(e,t):null;ee(i);let{parseZipCentralDirectory:a,extractZipEntry:c}=await Promise.resolve().then(()=>(Yn(),jn));ee(i);let l=s?[]:a(t),h=new Map;for(let g of l){if(h.has(g.fileName))throw new Error(`Lazy archive contains duplicate member: ${g.fileName}`);h.set(g.fileName,g)}let f=e.mountPrefix.replace(/\/+$/,""),d=new Map;for(let[g,p]of e.entries){if(p.deleted||p.materialized)continue;let w=p.archivePath??g.slice(f.length+1),u=s?void 0:h.get(w),m=s?.get(w);if(s){if(m===void 0||m.byteLength!==p.size)throw new Error(`Lazy tree member ${w} does not match its registered metadata`)}else if(u===void 0||u.isDirectory||u.isSymlink||u.uncompressedSize!==p.size)throw new Error(`Lazy archive member ${w} does not match its registered metadata`);if(p.generation===void 0)continue;let v=r.inodeKey(p.ino,p.generation),E=d.get(v);if(E&&E.archivePath!==w)throw new Error(`Lazy archive aliases for inode ${v} name different members`);if(!E){let z=m??c(t,u);if(z.byteLength!==p.size)throw new Error(`Lazy archive member ${w} extracted ${z.byteLength} bytes, expected ${p.size}`);d.set(v,{archivePath:w,content:z})}}let y=n?r.inodeKey(n.ino,n.generation):null;for(let g=0;g<3;g++){let p=new Map;for(let[w,u]of e.entries){if(u.deleted||u.materialized||u.generation===void 0)continue;let m=r.inodeKey(u.ino,u.generation);if(this.lazyArchiveInodes.get(m)!==e)continue;let v=d.get(m);if(!v)throw new Error(`Lazy archive has no extracted content for inode ${m}`);let E=p.get(m);E||(E={ino:u.ino,generation:u.generation,dataSequence:u.dataSequence??0,paths:new Set,content:v.content},p.set(m,E)),E.paths.add(w),n&&n.ino===u.ino&&n.generation===u.generation&&E.paths.add(n.path)}if(p.size>0&&(ee(i),!this.fs.replaceManyIfIdentities(Array.from(p.values(),u=>({paths:Array.from(u.paths),expectedIno:u.ino,expectedGeneration:u.generation,expectedDataSequence:u.dataSequence,data:u.content}))))){if(this.reconcileLazyIdentityState(this.fs.identityState()),y&&!this.lazyArchiveInodes.has(y))return;continue}ee(i);for(let[w,u]of p){this.lazyArchiveInodes.delete(w);for(let m of e.entries.values())m.ino===u.ino&&m.generation===u.generation&&(m.materialized=!0)}if(e.materialized=Array.from(e.entries.values()).every(w=>w.deleted||w.materialized),e.materialized||(this.reconcileLazyIdentityState(this.fs.identityState()),y&&!this.lazyArchiveInodes.has(y)))return}if(y&&this.lazyArchiveInodes.has(y))throw new Error(`Lazy archive member kept changing names while materializing: ${n?.path}`)}async materializeAllLazyEntries(){for(let t=0;t<3;t++){this.reconcileLazyIdentityState(this.fs.identityState());let n=this.lazyArchiveGroups.filter(s=>!s.materialized&&s.content!==void 0&&s.inventory!==void 0);if(this.lazyFiles.size===0&&this.lazyArchiveInodes.size===0&&n.length===0)return;let i=Array.from(this.lazyFiles.values(),s=>s.path);for(let s of i)await this.ensureMaterialized(s);let o=new Set(this.lazyArchiveInodes.values());for(let s of n)o.add(s);for(let s of o)await this.prepareLazyTreeGroup(s)}this.reconcileLazyIdentityState(this.fs.identityState());let e=this.lazyArchiveGroups.some(t=>!t.materialized&&t.content!==void 0&&t.inventory!==void 0);if(this.lazyFiles.size!==0||this.lazyArchiveInodes.size!==0||e)throw new Error("Cannot create a self-contained VFS image while lazy entries remain pending")}async saveImage(e){e?.materializeAll&&await this.materializeAllLazyEntries();let{bytes:t,identities:n}=this.fs.snapshotState({normalizeTimestampsMs:e?.normalizeTimestampsMs});this.reconcileLazyIdentityState(n);let i=this.serializeLazyEntries(),o=i.length>0,s=o?new TextEncoder().encode(JSON.stringify(i)):new Uint8Array(0);if(s.byteLength>on)throw new Error(`VFS image lazy metadata exceeds ${on} bytes`);let a=this.serializeLazyArchiveEntries();Fi(a);let c=a.length>0,l=c?new TextEncoder().encode(JSON.stringify(a)):new Uint8Array(0);if(l.byteLength>an)throw new Error(`VFS image lazy archive metadata exceeds ${an} bytes`);let h=e?.metadata===void 0?this.imageMetadata:e.metadata,f=wa(h),d=f.byteLength>0,y=c?4+l.byteLength:0,g=d?4+f.byteLength:0,p=se+t.byteLength+4+s.byteLength+y+g,w=new Uint8Array(p),u=new DataView(w.buffer);u.setUint32(0,cr,!0),u.setUint32(4,lr,!0),u.setUint32(8,(o?nr:0)|(c?sn:0)|(c?ir:0)|(d?rr:0),!0),u.setUint32(12,t.byteLength,!0),w.set(t,se);let m=se+t.byteLength;if(u.setUint32(m,s.byteLength,!0),s.byteLength>0&&w.set(s,m+4),c){let v=m+4+s.byteLength;u.setUint32(v,l.byteLength,!0),w.set(l,v+4)}if(d){let v=m+4+s.byteLength+y;u.setUint32(v,f.byteLength,!0),w.set(f,v+4)}return w}static readImageMetadata(e){let t=rn(e);if(!(t.flags&rr))return null;let{metadataOffset:n}=Ci(t.image,t.view,t.flags,t.sabLen);if(t.image.byteLengthdt)throw new Error(`VFS image metadata exceeds ${dt} bytes`);if(t.image.byteLength0){let w=n.subarray(g+4,g+4+p),u=xe(Mi(w,"VFS image lazy metadata"),"VFS image lazy entries",0,ht);y.importLazyEntriesInternal(u,!0)}if(o&sn){let w=a.archiveOffset,u=i.getUint32(w,!0);if(u>0){let m=n.subarray(w+4,w+4+u),v=Mi(m,"VFS image lazy archive metadata");y.importLazyArchiveEntriesInternal(v,!0,!!(o&ir))}}return y}adaptStat(e){return{dev:0,ino:e.ino,mode:e.mode,nlink:e.linkCount,uid:e.uid,gid:e.gid,size:e.size,atimeMs:e.atime,mtimeMs:e.mtime,ctimeMs:e.ctime}}adaptStatWithLazySize(e){let t=this.adaptStat(e),n=this.lazyFileForStat(e);if(n)return t.size=n.size,t;let i=this.lazyArchiveForStat(e);if(i){for(let o of i.entries.values())if(o.ino===e.ino&&o.generation===e.generation&&!o.deleted){t.size=o.size;break}}return t}open(e,t,n){(t&Tt)===0&&!((t&Pt)!==0&&(t&Un)!==0)&&this.guardSynchronousLazyAccess(e);let i=this.fs.open(e,t,n);return(t&Tt)!==0&&this.invalidateLazyData(this.fs.fstat(i)),i}close(e){return this.fs.close(e),0}read(e,t,n,i){if(i>0){let o=this.lazyBackingForStat(this.fs.fstat(e));o&&(this.reconcileLazyIdentityState(this.fs.identityState()),o=this.lazyBackingForStat(this.fs.fstat(e)),o&&this.guardSynchronousLazyAccess(o.path))}return n!==null?this.fs.readAt(e,t.subarray(0,i),n):this.fs.read(e,t.subarray(0,i))}write(e,t,n,i){if(n!==null){let s=this.fs.writeAt(e,t.subarray(0,i),n);return s>0&&this.invalidateLazyData(this.fs.fstat(e)),s}let o=this.fs.write(e,t.subarray(0,i));return o>0&&this.invalidateLazyData(this.fs.fstat(e)),o}seek(e,t,n){return this.fs.lseek(e,t,n)}fstat(e){return this.adaptStatWithLazySize(this.fs.fstat(e))}fpathconf(e,t){let n=this.fstat(e);return On(n,t,{supportsSymlinks:!0,timestampResolutionNs:1e6})}ftruncate(e,t){this.fs.ftruncate(e,t),this.invalidateLazyData(this.fs.fstat(e))}fsync(e){}fchmod(e,t){this.fs.fchmod(e,t)}fchown(e,t,n){this.fs.fchown(e,t,n)}stat(e){return this.adaptStatWithLazySize(this.fs.stat(e))}lstat(e){return this.adaptStatWithLazySize(this.fs.lstat(e))}statfs(e){this.fs.stat(e);let t=this.fs.statfs();return{type:1397114451,bsize:t.blockSize,blocks:t.totalBlocks,bfree:t.freeBlocks,bavail:t.freeBlocks,files:t.totalInodes,ffree:t.freeInodes,fsid:0,namelen:t.maxName,frsize:t.blockSize,flags:0}}pathconf(e,t){let n=this.stat(e);return On(n,t,{supportsSymlinks:!0,timestampResolutionNs:1e6})}mkdir(e,t){this.fs.mkdir(e,t)}rmdir(e){this.fs.rmdir(e)}unlink(e){let t=this.fs.unlink(e),n=r.inodeKey(t.ino,t.generation);if(t.linkCount>1&&(this.lazyFiles.has(n)||this.lazyArchiveInodes.has(n))){this.reconcileLazyIdentityState(this.fs.identityState());return}let i=this.lazyFiles.get(n);i&&(i.paths.delete(e),t.linkCount<=1?this.lazyFiles.delete(n):i.path===e&&(i.path=i.paths.values().next().value));let o=this.lazyArchiveInodes.get(n);if(o){let s=o.entries.get(e);if(t.linkCount<=1){for(let a of o.entries.values())a.ino===t.ino&&a.generation===t.generation&&(a.deleted=!0);this.lazyArchiveInodes.delete(n)}else s&&o.entries.delete(e)}}rename(e,t){let{source:n,replaced:i}=this.fs.rename(e,t);if(i&&i.ino===n.ino&&i.generation===n.generation)return;let o=!1;if(i){let s=r.inodeKey(i.ino,i.generation);i.linkCount>1&&(this.lazyFiles.has(s)||this.lazyArchiveInodes.has(s))&&(this.reconcileLazyIdentityState(this.fs.identityState()),o=!0);let a=this.lazyFiles.get(s);!o&&a&&(a.paths.delete(t),i.linkCount<=1?this.lazyFiles.delete(s):a.path===t&&(a.path=a.paths.values().next().value));let c=this.lazyArchiveInodes.get(s);if(!o&&c){let l=c.entries.get(t);i.linkCount<=1?(l&&(l.deleted=!0),this.lazyArchiveInodes.delete(s)):l&&c.entries.delete(t)}}o||this.rewriteLazyNamespacePaths(n,e,t)}link(e,t){let n=this.fs.link(e,t),i=r.inodeKey(n.ino,n.generation),o=this.lazyFiles.get(i);o&&o.paths.add(t);let s=this.lazyArchiveInodes.get(i);if(s){let a=Array.from(s.entries.values()).find(c=>c.ino===n.ino&&c.generation===n.generation);a&&s.entries.set(t,{...a})}}symlink(e,t){this.fs.symlink(e,t)}readlink(e){return this.fs.readlink(e)}chmod(e,t){this.fs.chmod(e,t)}chown(e,t,n){this.fs.chown(e,t,n)}lchown(e,t,n){this.fs.lchown(e,t,n)}createFileWithOwner(e,t,n,i,o){let s=this.open(e,577,t);o.length>0&&this.write(s,o,null,o.length),this.close(s),this.chown(e,n,i),this.chmod(e,t)}mkdirWithOwner(e,t,n,i){this.mkdir(e,t),this.chown(e,n,i),this.chmod(e,t)}symlinkWithOwner(e,t,n,i){this.symlink(e,t),this.lchown(t,n,i)}copyPathToFreshFileSystem(e,t,n,i,o){let s=this.lstat(e),a=s.mode&ut,c=s.mode&4095;if(a===nn){e==="/"?(t.chown(e,s.uid,s.gid),t.chmod(e,c)):t.mkdirWithOwner(e,c,s.uid,s.gid);let d=this.opendir(e);try{for(;;){let y=this.readdir(d);if(!y)break;y.name==="."||y.name===".."||this.copyPathToFreshFileSystem(e==="/"?`/${y.name}`:`${e}/${y.name}`,t,n,i,o)}}finally{this.closedir(d)}r.applyTimes(t,e,s);return}let l=s.nlink>1?`${s.dev}:${s.ino}`:null,h=l?o.get(l):void 0;if(h){t.link(h,e);return}if(a===na){t.symlinkWithOwner(this.readlink(e),e,s.uid,s.gid),l&&o.set(l,e);return}if(a!==sr)throw new Error(`Unsupported file type while rebasing VFS: ${e}`);if(n.has(e)||i.has(e)){t.createFileWithOwner(e,c,s.uid,s.gid,new Uint8Array(0)),r.applyTimes(t,e,s),l&&o.set(l,e);return}this.copyRegularFileToFreshFileSystem(e,t,s,c),l&&o.set(l,e)}copyRegularFileToFreshFileSystem(e,t,n,i){let o=this.open(e,ra,0),s=null;try{s=t.open(e,ia,i);let a=new Uint8Array(Math.min(sa,Math.max(1,n.size))),c=n.size;for(;c>0;){let l=Math.min(a.byteLength,c),h=this.read(o,a,null,l);if(h<=0)throw new Error(`Unexpected EOF while rebasing VFS file: ${e}`);let f=0;for(;f!e||e==="."||e===".."))throw new Error(`Binary resolver path must be a normalized portable relative path: ${JSON.stringify(r)}`);return r}var qe=new Set(["wasm32","wasm64"]);function Ce(r){if(Ka(r),!r.startsWith("programs/"))return r;let e=r.slice(9),t=e.split("/",1)[0];return qe.has(t)?r:`programs/wasm32/${e}`}function Ga(r,e=$(zn(),"wasm")){let t=Ce(r),n=[$(e,t)];return r==="kernel.wasm"?n.push($(e,"kandelo-kernel.wasm")):r==="userspace.wasm"?n.push($(e,"wasm_posix_userspace.wasm")):r==="rootfs.vfs"&&n.push($(e,"rootfs.vfs")),n}var vn=class extends Error{constructor(e){super(e),this.name="BinaryNotFoundError"}};function ss(){let r=[],e=!1;try{let n=Ve();e=!0;for(let[i,o]of[["local-binaries",$(n,"local-binaries")],["binaries",$(n,"binaries")]])r.push({label:i,root:o,identity:i==="local-binaries"?"local-generation":"program-cache",allowRegularFileClosure:!1,candidatesFor(s){return[$(o,Ce(s))]}})}catch{}let t=$(zn(),"wasm");return r.push({label:"installed package",root:t,identity:"installed-package",allowRegularFileClosure:!e,candidatesFor(n){return Ga(n,t)}}),r}function yt(r,e){return new Error(`Invalid package manifest ${r}: ${e}`)}function de(r){try{return En(r),!0}catch(e){if(e instanceof Error&&"code"in e&&e.code==="ENOENT")return!1;throw e}}function Yi(r,e,t){if(r.length===0||r.startsWith("/")||r.includes("\\")||r.includes("\0")||r.split("/").some(n=>!n||n==="."||n===".."))throw yt(e,`${t} must be a normalized portable relative path`);return r}function mn(r,e,t,n=!0){if(r.length===0||r==="."||r===".."||r.includes("/")||r.includes("\\")||r.includes("\0")||!n&&r.includes("@"))throw yt(e,`${t} must be a safe single path component`);return r}var Xi="kandelo-program-packages-v2",ve="program-packages.json",Ji=null,wn=null,gr=0;function os(){if(Object.prototype.hasOwnProperty.call(process.env,"WASM_POSIX_DEPS_REGISTRY")){let r=null;return(process.env.WASM_POSIX_DEPS_REGISTRY??"").split(":").filter(Boolean).map(e=>e.startsWith("~/")&&process.env.HOME!==void 0?$(process.env.HOME,e.slice(2)):Sn(e)?pe(e):(r??=Ve(),pe(r,e)))}try{return[$(Ve(),"packages","registry")]}catch{return null}}function Wa(){let r;try{r=Ve()}catch{return null}if(!$t($(r,"tools","xtask","Cargo.toml"))||!$t($(r,"scripts","dev-shell.sh")))return null;try{let e=me(vr()),t=me(r);return[$(t,"host"),$(t,"scripts")].some(i=>$t(i)&&kr(me(i),e))?t:null}catch{return null}}function Er(r,e,t){let n=[typeof t.stderr=="string"?t.stderr.trim():"",typeof t.stdout=="string"?t.stdout.trim():"",t.error?.message??""].filter(Boolean).join(` +var xo=Object.defineProperty;var qn=(r,e,t)=>()=>{if(t)throw t[0];try{return r&&(e=r(r=0)),e}catch(n){throw t=[n],n}};var fi=(r,e)=>{for(var t in e)xo(r,t,{get:e[t],enumerable:!0})};import{createRequire as Ba}from"module";function Rs(r,e){return xs(r,{i:2},e&&e.out,e&&e.dictionary)}var $a,wt,Ua,Wa,ne,gt,Ga,Es,ws,Za,Ss,wt,Os,Ha,Is,Va,tu,Lr,Te,K,qt,jt,K,K,K,K,vs,K,Xa,Ya,Tr,we,br,As,Tn,qa,he,xs,ja,Ja,Et,ks,Qa,ec,Pr=qn(()=>{$a=Ba("/");try{wt=$a("worker_threads"),Ua=wt.Worker,Wa=wt.isMarkedAsUntransferable}catch{}ne=Uint8Array,gt=Uint16Array,Ga=Int32Array,Es=new ne([0,0,0,0,0,0,0,0,1,1,1,1,2,2,2,2,3,3,3,3,4,4,4,4,5,5,5,5,0,0,0,0]),ws=new ne([0,0,0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,8,8,9,9,10,10,11,11,12,12,13,13,0,0]),Za=new ne([16,17,18,0,8,7,9,6,10,5,11,4,12,3,13,2,14,1,15]),Ss=function(r,e){for(var t=new gt(31),n=0;n<31;++n)t[n]=e+=1<>1|(K&21845)<<1,Te=(Te&52428)>>2|(Te&13107)<<2,Te=(Te&61680)>>4|(Te&3855)<<4,Lr[K]=((Te&65280)>>8|(Te&255)<<8)>>1;qt=(function(r,e,t){for(var n=r.length,i=0,s=new gt(e);i>c]=l}else for(a=new gt(n),i=0;i>15-r[i]);return a}),jt=new ne(288);for(K=0;K<144;++K)jt[K]=8;for(K=144;K<256;++K)jt[K]=9;for(K=256;K<280;++K)jt[K]=7;for(K=280;K<288;++K)jt[K]=8;vs=new ne(32);for(K=0;K<32;++K)vs[K]=5;Xa=qt(jt,9,1),Ya=qt(vs,5,1),Tr=function(r){for(var e=r[0],t=1;te&&(e=r[t]);return e},we=function(r,e,t){var n=e/8|0;return(r[n]|r[n+1]<<8)>>(e&7)&t},br=function(r,e){var t=e/8|0;return(r[t]|r[t+1]<<8|r[t+2]<<16)>>(e&7)},As=function(r){return(r+7)/8|0},Tn=function(r,e,t){return(e==null||e<0)&&(e=0),(t==null||t>r.length)&&(t=r.length),new ne(r.subarray(e,t))},qa=["unexpected EOF","invalid block type","invalid length/literal","invalid distance","stream finished","no stream handler",,"no callback","invalid UTF-8 data","extra field too long","date not in range 1980-2099","filename too long","stream finishing","invalid zip data"],he=function(r,e,t){var n=new Error(e||qa[r]);if(n.code=r,Error.captureStackTrace&&Error.captureStackTrace(n,he),!t)throw n;return n},xs=function(r,e,t,n){var i=r.length,s=n?n.length:0;if(!i||e.f&&!e.l)return t||new ne(0);var o=!t,a=o||e.i!=2,c=e.i;o&&(t=new ne(i*3));var l=function(Ne){var Fe=t.length;if(Ne>Fe){var an=new ne(Math.max(Fe*2,Ne));an.set(t),t=an}},p=e.f||0,u=e.p||0,m=e.b||0,_=e.l,d=e.d,h=e.m,g=e.n,f=i*8;do{if(!_){p=we(r,u,1);var y=we(r,u+1,3);if(u+=3,y)if(y==1)_=Xa,d=Ya,h=9,g=5;else if(y==2){var S=we(r,u,31)+257,A=we(r,u+10,15)+4,v=S+we(r,u+5,31)+1;u+=14;for(var x=new ne(v),L=new ne(19),N=0;N>4;if(E<16)x[N++]=E;else{var k=0,V=0;for(E==16?(V=3+we(r,u,3),u+=2,k=x[N-1]):E==17?(V=3+we(r,u,7),u+=3):E==18&&(V=11+we(r,u,127),u+=7);V--;)x[N++]=k}}var Ze=x.subarray(0,S),oe=x.subarray(S);h=Tr(Ze),g=Tr(oe),_=qt(Ze,h,1),d=qt(oe,g,1)}else he(1);else{var E=As(u)+4,w=r[E-4]|r[E-3]<<8,O=E+w;if(O>i){c&&he(0);break}a&&l(m+w),t.set(r.subarray(E,O),m),e.b=m+=w,e.p=u=O*8,e.f=p;continue}if(u>f){c&&he(0);break}}a&&l(m+131072);for(var At=(1<>4;if(u+=k&15,u>f){c&&he(0);break}if(k||he(2),Ae<256)t[m++]=Ae;else if(Ae==256){He=u,_=null;break}else{var xt=Ae-254;if(Ae>264){var N=Ae-257,Pe=Es[N];xt=we(r,u,(1<>4;rt||he(3),u+=rt&15;var oe=Va[_e];if(_e>3){var Pe=ws[_e];oe+=br(r,u)&(1<f){c&&he(0);break}a&&l(m+131072);var ze=m+xt;if(m>3&1)+(e>>4&1);n>0;n-=!r[t++]);return t+(e&2)},Et=(function(){function r(e,t){typeof e=="function"&&(t=e,e={}),this.ondata=t;var n=e&&e.dictionary&&e.dictionary.subarray(-32768);this.s={i:0,b:n?n.length:0},this.o=new ne(32768),this.p=new ne(0),n&&this.o.set(n)}return r.prototype.e=function(e){if(this.ondata||he(5),this.d&&he(4),!this.p.length)this.p=e;else if(e.length){var t=new ne(this.p.length+e.length);t.set(this.p),t.set(e,this.p.length),this.p=t}},r.prototype.c=function(e){this.s.i=+(this.d=e||!1);var t=this.s.b,n=xs(this.p,this.s,this.o);this.ondata(Tn(n,t,this.s.b),this.d),this.o=Tn(n,this.s.b-32768),this.s.b=this.o.length,this.p=Tn(this.p,this.s.p/8|0),this.s.p&=7},r.prototype.push=function(e,t){this.e(e),this.c(t)},r})();ks=(function(){function r(e,t){this.v=1,this.r=0,Et.call(this,e,t)}return r.prototype.push=function(e,t){if(Et.prototype.e.call(this,e),this.r+=e.length,this.v){var n=this.p.subarray(this.v-1),i=n.length>3?Ja(n):4;if(i>n.length){if(!t)return}else this.v>1&&this.onmember&&this.onmember(this.r-n.length);this.p=n.subarray(i),this.v=0}Et.prototype.c.call(this,0),this.s.f&&!this.s.l?(this.v=As(this.s.p)+9,this.s={i:0},this.o=new ne(0),this.push(new ne(0),t)):t&&Et.prototype.c.call(this,t)},r})(),Qa=typeof TextDecoder<"u"&&new TextDecoder,ec=0;try{Qa.decode(ja,{stream:!0}),ec=1}catch{}});var Fr={};fi(Fr,{extractZipEntry:()=>cc,extractZipEntryBounded:()=>lc,fetchZipCentralDirectory:()=>fc,parseZipCentralDirectory:()=>Jt});function Ns(r){let e=new DataView(r.buffer,r.byteOffset,r.byteLength),t=Math.max(0,r.length-Ls);for(let n=r.length-rc;n>=t;n--)if(e.getUint32(n,!0)===tc)return n;throw new Error("Zip EOCD record not found")}function Jt(r){let e=new DataView(r.buffer,r.byteOffset,r.byteLength),t=Ns(r),n=e.getUint16(t+10,!0),i=e.getUint32(t+16,!0),s=[],o=i;for(let a=0;a>8,w;E===Ts?w=h>>16&65535:y.startsWith("bin/")||y.startsWith("sbin/")||y.includes("/bin/")||y.includes("/sbin/")?w=493:w=420;let O=y.endsWith("/"),S=E===Ts&&(w&sc)===ic;s.push({fileName:y,fileNameBytes:f,compressedSize:p,uncompressedSize:u,compressionMethod:l,localHeaderOffset:g,mode:w,isDirectory:O,isSymlink:S,externalAttrs:h,creatorOS:E}),o+=zr+m+_+d}return s}function Fs(r,e){if(r.byteLength!==e.byteLength)return!1;for(let t=0;t{if(a.byteLength>t-s)throw new Error(`ZIP member ${e.fileName} expands beyond ${t} bytes`);i.set(a,s),s+=a.byteLength}).push(n,!0),s!==t)throw new Error(`ZIP member ${e.fileName} expanded ${s} bytes, expected ${t}`);return i}function uc(r,e){let t=new DataView(r.buffer,r.byteOffset,r.byteLength),n=e.localHeaderOffset;if(n<0||n>r.byteLength-Nr||t.getUint32(n,!0)!==bs)throw new Error(`Invalid local file header signature at offset ${n}`);let i=t.getUint16(n+8,!0),s=t.getUint16(n+26,!0),o=t.getUint16(n+28,!0),a=n+Nr,c=a+s+o,l=c+e.compressedSize;if(i!==e.compressionMethod||cr.byteLength||!Fs(r.subarray(a,a+s),e.fileNameBytes))throw new Error(`ZIP member ${e.fileName} has inconsistent local metadata`);return r.subarray(c,l)}async function fc(r){let e=await fetch(r,{method:"HEAD"});if(!e.ok)throw new Error(`HEAD request failed: ${e.status} ${e.statusText}`);let t=parseInt(e.headers.get("content-length")||"0",10),n=e.headers.get("accept-ranges");if(!t||n!=="bytes"){let f=await fetch(r);if(!f.ok)throw new Error(`Fetch failed: ${f.status} ${f.statusText}`);let y=new Uint8Array(await f.arrayBuffer());return{entries:Jt(y),totalSize:y.length}}let i=Math.min(t,Ls),s=t-i,o=await fetch(r,{headers:{Range:`bytes=${s}-${t-1}`}});if(o.status!==206){let f=await fetch(r);if(!f.ok)throw new Error(`Fetch failed: ${f.status} ${f.statusText}`);let y=new Uint8Array(await f.arrayBuffer());return{entries:Jt(y),totalSize:y.length}}let a=new Uint8Array(await o.arrayBuffer()),c=new DataView(a.buffer,a.byteOffset,a.byteLength),l=Ns(a),p=c.getUint32(l+12,!0),u=c.getUint32(l+16,!0);if(u>=s){let f=t,y=new Uint8Array(f);return y.set(a,s),{entries:Jt(y),totalSize:f}}let m=u+p-1,_=await fetch(r,{headers:{Range:`bytes=${u}-${m}`}});if(_.status!==206)throw new Error(`Range request for CD failed: ${_.status}`);let d=new Uint8Array(await _.arrayBuffer()),h=t,g=new Uint8Array(h);return g.set(d,u),g.set(a,s),{entries:Jt(g),totalSize:h}}var tc,nc,bs,Ls,rc,zr,Nr,Ps,zs,Ts,ic,sc,oc,ac,Cr=qn(()=>{"use strict";Pr();tc=101010256,nc=33639248,bs=67324752,Ls=65557,rc=22,zr=46,Nr=30,Ps=0,zs=8,Ts=3,ic=40960,sc=61440,oc=new TextDecoder("utf-8",{fatal:!0,ignoreBOM:!0}),ac=new TextEncoder});var Us={};fi(Us,{DEFAULT_TAR_GZIP_LIMITS:()=>$s,TarParseError:()=>T,parseTarGzip:()=>mc});function mc(r,e={}){let t=e.label??"TAR gzip archive",n=yc(e.limits,t);if(r.byteLength===0||r.byteLength>n.maxCompressedBytes)throw new T(`${t}: compressed byte count ${r.byteLength} is outside 1..${n.maxCompressedBytes}`);let i=gc(r,t);if(i===0||i>n.maxUncompressedBytes)throw new T(`${t}: declared uncompressed byte count ${i} is outside 1..${n.maxUncompressedBytes}`);let s=Ec(r,t,i);if(s.byteLength!==i)throw new T(`${t}: gzip expanded to ${s.byteLength} bytes, expected ${i}`);let o=new DataView(r.buffer,r.byteOffset,r.byteLength).getUint32(r.byteLength-8,!0);if(wc(s)!==o)throw new T(`${t}: gzip CRC32 mismatch`);return _c(s,t,n)}function _c(r,e,t){if(r.byteLength%be!==0)throw new T(`${e}: TAR byte count is not block-aligned`);let n=[],i=0,s=0,o=0,a=null,c={},l=!1;for(;i+be<=r.byteLength;){let p=r.subarray(i,i+be);if(i+=be,Dr(p)){if(i+be>r.byteLength)throw new T(`${e}: TAR end marker is truncated`);let O=r.subarray(i,i+be);if(!Dr(O))throw new T(`${e}: TAR has only one zero end block`);if(i+=be,!Dr(r.subarray(i)))throw new T(`${e}: TAR has nonzero data after its end marker`);l=!0;break}vc(p,e);let u=Qt(p,156,1,e)||"0",m=Br(p,124,12,`${e}: TAR entry size`),_=Br(p,100,8,`${e}: TAR entry mode`)&dc,d=Ac(p,e,t.maxPathBytes),h=Qt(p,157,100,e);if(u==="x"||u==="g"){if(o+=1,o>t.maxEntries+1)throw new T(`${e}: TAR extension header count exceeds ${t.maxEntries+1}`);let O=Ks(r,i,m,e);i=Ds(i,m,r.byteLength,e);let S=Oc(O,e,t);u==="x"?a=S:c={...c,...S};continue}if(s+=1,s>t.maxEntries)throw new T(`${e}: TAR entry count exceeds ${t.maxEntries}`);let g={...c,...a??{}};a=null;let f=g.size===void 0?m:Ic(g.size,`${e}: PAX entry size`),y=Ks(r,i,f,e);i=Ds(i,f,r.byteLength,e);let E=Mr(g.path??d,e,t.maxPathBytes),w=g.linkpath??h;switch(u){case"0":case"\0":n.push({path:E,type:"file",mode:_,data:y});break;case"5":Kr(f,e,"directory",E),n.push({path:E,type:"directory",mode:_});break;case"2":Kr(f,e,"symlink",E),Ms(w,e,E,t.maxLinkBytes,!1),n.push({path:E,type:"symlink",mode:_,linkName:w});break;case"1":Kr(f,e,"hardlink",E),Ms(w,e,E,t.maxLinkBytes,!0),n.push({path:E,type:"hardlink",mode:_,linkName:Mr(w,`${e}: hardlink target`,t.maxPathBytes)});break;case"3":case"4":case"6":throw new T(`${e}: unsupported TAR device/FIFO entry ${E}`);default:throw new T(`${e}: unsupported TAR entry type ${JSON.stringify(u)} for ${E}`)}}if(!l)throw new T(`${e}: TAR is missing its two-block end marker`);if(a!==null)throw new T(`${e}: local PAX header has no following entry`);return n}function yc(r,e){let t={...$s,...r};for(let[n,i]of Object.entries(t))if(!Number.isSafeInteger(i)||i<=0)throw new T(`${e}: ${n} must be a positive safe integer`);return t}function gc(r,e){if(r.byteLength<18||r[0]!==31||r[1]!==139||r[2]!==8)throw new T(`${e}: invalid gzip header`);return new DataView(r.buffer,r.byteOffset,r.byteLength).getUint32(r.byteLength-4,!0)}function Ec(r,e,t){let n=new Uint8Array(t),i=0,s=!1,o=new ks(a=>{if(a.byteLength>t-i)throw new T(`${e}: gzip expansion exceeds its declared ${t} bytes`);n.set(a,i),i+=a.byteLength});o.onmember=()=>{throw s=!0,new T(`${e}: concatenated gzip members are unsupported`)};try{o.push(r,!0)}catch(a){throw a instanceof T?a:new T(`${e}: cannot gunzip archive: ${Rc(a)}`)}if(s)throw new T(`${e}: concatenated gzip members are unsupported`);return n.subarray(0,i)}function wc(r){let e=4294967295;for(let t of r)e=pc[(e^t)&255]^e>>>8;return(e^4294967295)>>>0}function Sc(){let r=new Uint32Array(256);for(let e=0;e>>1^((t&1)===0?0:3988292384);r[e]=t>>>0}return r}function Ks(r,e,t,n){if(t>r.byteLength-e)throw new T(`${n}: TAR entry is truncated`);return r.subarray(e,e+t)}function Ds(r,e,t,n){let s=Math.ceil(e/be)*be;if(!Number.isSafeInteger(s)||s>t-r)throw new T(`${n}: TAR entry padding is truncated`);return r+s}function Oc(r,e,t){let n={},i=0;for(;i9)throw new T(`${e}: invalid PAX record length`);if(o=o*10+h,!Number.isSafeInteger(o))throw new T(`${e}: invalid PAX record length`)}let a=i+o;if(o<=s-i+2||a>r.byteLength||r[a-1]!==10)throw new T(`${e}: truncated PAX record`);let c=s+1;for(;c=a-1)throw new T(`${e}: invalid PAX record`);let l=r.subarray(s+1,c);if(l.byteLength>256)throw new T(`${e}: PAX record key is too long`);let p=$r(l,`${e}: PAX record key`),u=r.subarray(c+1,a-1),m=p==="path"?t.maxPathBytes:p==="linkpath"?t.maxLinkBytes:p==="size"?32:0;if(m===0){i=a;continue}if(u.byteLength>m)throw new T(`${e}: PAX ${p} value is too long`);let _=$r(u,`${e}: PAX record value`);n[p]=_,i=a}return n}function Ic(r,e){if(!/^(0|[1-9][0-9]*)$/.test(r))throw new T(`${e} is invalid`);let t=Number(r);if(!Number.isSafeInteger(t)||t<0)throw new T(`${e} is invalid`);return t}function vc(r,e){let t=Br(r,148,8,`${e}: TAR checksum`),n=0;for(let i=0;i=148&&i<156?32:r[i];if(t!==n)throw new T(`${e}: TAR checksum mismatch`)}function Ac(r,e,t){let n=Qt(r,0,100,e),i=Qt(r,345,155,e);return Mr(i?`${i}/${n}`:n,e,t)}function Mr(r,e,t){let n=r;for(;n.startsWith("./");)n=n.slice(2);return n=n.replace(/\/+$/g,""),xc(n,`${e}: TAR path`,t),n}function Qt(r,e,t,n){let i=e,s=e+t;for(;in||r.includes("\0"))throw new T(`${e}: link target for ${t} is invalid`);if(i&&r.includes("\\"))throw new T(`${e}: hardlink target for ${t} is invalid`)}function xc(r,e,t){if(r.length===0||r.startsWith("/")||r.includes("\0")||r.includes("\\")||Bs.encode(r).byteLength>t)throw new T(`${e} ${JSON.stringify(r)} must be a bounded relative POSIX path`);for(let n of r.split("/"))if(n.length===0||n==="."||n==="..")throw new T(`${e} ${JSON.stringify(r)} contains an unsafe path segment`)}function Dr(r){for(let e of r)if(e!==0)return!1;return!0}function $r(r,e){try{return hc.decode(r)}catch{throw new T(`${e} contains non-UTF-8 text`)}}function Rc(r){return r instanceof Error?r.message:String(r)}var be,dc,Cs,hc,Bs,pc,$s,T,Ws=qn(()=>{"use strict";Pr();be=512,dc=4095,Cs=1024*1024,hc=new TextDecoder("utf-8",{fatal:!0,ignoreBOM:!0}),Bs=new TextEncoder,pc=Sc(),$s=Object.freeze({maxCompressedBytes:256*Cs,maxUncompressedBytes:512*Cs,maxEntries:1e5,maxPathBytes:4096,maxLinkBytes:65536}),T=class extends Error{constructor(e){super(e),this.name="TarParseError"}}});import{existsSync as nn,lstatSync as Vn,readdirSync as ol,readFileSync as Qe,realpathSync as Oe,statSync as et}from"node:fs";import{createHash as _o}from"node:crypto";import{spawnSync as ii}from"node:child_process";import{basename as al,dirname as sn,isAbsolute as Xn,join as M,relative as cl,resolve as Se,sep as ll}from"node:path";import{fileURLToPath as ul}from"node:url";var Rt="kandelo.wpk_fork.linked_frames";var di=[75,76,67,70],kt=24,hi=8,jn=3,pi=[{bytes:4,chunkHeaderSize:32,nodeHeaderSize:24},{bytes:8,chunkHeaderSize:56,nodeHeaderSize:32}],Q="kandelo.wpk_fork.module_state",mi=1,_i=[75,70,77,68],cn=24,yi=8;var Jn=7;var gi=1,Ei=1,wi=1;var Si=[{bytes:4,chunkHeaderSize:40},{bytes:8,chunkHeaderSize:56}];var ln="__wpk_fork_global_";var un="__wpk_fork_table_",Qn=1,er=2,tr=3,nr=4,rr=5,st=6,Tt=7,bt=8,Lt=9,xe="kandelo.wpk_fork.capabilities",Oi=1;var Ii=7,fn=4,ye="kandelo.wpk_fork.exception_codec",vi=1,dn=8,ir=16;var hn="env",pn="__wpk_fork_unwind",ot="kandelo.wpk_fork.unwind_transport",Pt="__wpk_fork_static_root_catalog",Ce="kandelo.wpk_fork.static_root_catalog";var sr=1,or=0,Ai=1,mn=12,xi=[75,70,83,82],G="kandelo.wpk_fork.imported_globals";var Ri=[75,70,73,71],ki=1,_n=16,zt=24,Ti=1,bi=2,Li=3,Z="kandelo.wpk_fork.imported_tables",Pi=[75,70,73,84],zi=1,yn=16,Nt=24,Ni=1,Fi=1,ar="env",cr="__wpk_fork_module_activation";var at=[{module:"env",name:"__wpk_fork_frame_commit",params:["ptr"],results:[]},{module:"env",name:"__wpk_fork_frame_next",params:["ptr"],results:["ptr"]},{module:"env",name:"__wpk_fork_frame_peek",params:["ptr"],results:["ptr"]},{module:"env",name:"__wpk_fork_frame_reserve",params:["ptr"],results:["ptr"]},{module:"env",name:"__wpk_fork_module_state_record_commit",params:["ptr"],results:[]},{module:"env",name:"__wpk_fork_module_state_record_find",params:["i32","i32","i32","i32"],results:["ptr"]},{module:"env",name:"__wpk_fork_module_state_record_reserve",params:["i32","i32","i32","ptr"],results:["ptr"]},{module:"env",name:"__wpk_fork_module_state_table_dirty_count",params:["i32"],results:["i32"]},{module:"env",name:"__wpk_fork_module_state_table_dirty_mark",params:["i32","i64","i64"],results:[]},{module:"env",name:"__wpk_fork_module_state_table_dirty_page",params:["i32","i32"],results:["i64"]},{module:"env",name:"__wpk_fork_module_state_table_mutation_abort",params:[],results:[]},{module:"env",name:"__wpk_fork_module_state_table_mutation_begin",params:[],results:["i64"]},{module:"env",name:"__wpk_fork_module_state_table_mutation_commit",params:["i32","i64","i64"],results:[]},{module:"env",name:"__wpk_fork_module_state_table_reconcile",params:[],results:["i64"]},{module:"env",name:"__wpk_fork_module_state_table_state_owned",params:["i32"],results:["i32"]},{module:"env",name:"__wpk_fork_ref_decode_funcref",params:["i32"],results:["funcref"]},{module:"env",name:"__wpk_fork_ref_encode_funcref",params:["funcref"],results:["i32"]},{module:"env",name:"__wpk_fork_ref_exn_broker_encode",params:["i32"],results:["i32"]},{module:"env",name:"__wpk_fork_ref_exn_broker_throw_recipe",params:["i32"],results:[]},{module:"env",name:"__wpk_fork_ref_exn_cache_index",params:["i32"],results:["i32"]},{module:"env",name:"__wpk_fork_ref_exn_claim",params:["i32"],results:["i32"]},{module:"env",name:"__wpk_fork_ref_exn_define",params:["i32","i32","i32","i32","ptr","i32","ptr","i32"],results:[]},{module:"env",name:"__wpk_fork_ref_exn_ingress_throw",params:["i32"],results:[]},{module:"env",name:"__wpk_fork_ref_exn_load",params:["i32","i32","i32","i32","ptr","i32","ptr","i32"],results:["i32"]},{module:"env",name:"__wpk_fork_ref_exn_lookup",params:["i32"],results:["i32"]},{module:"env",name:"__wpk_fork_ref_exn_route",params:["i32","i32"],results:["i32"]},{module:"env",name:"__wpk_fork_ref_gc_broker_encode",params:["i32"],results:["i32"]},{module:"env",name:"__wpk_fork_ref_gc_capture_layout",params:["i32","i32","i32"],results:["i32"]},{module:"env",name:"__wpk_fork_ref_gc_claim",params:["i32"],results:["i32"]},{module:"env",name:"__wpk_fork_ref_gc_define",params:["i32","i32","i32","i32","i32","ptr","i32","i32"],results:[]},{module:"env",name:"__wpk_fork_ref_gc_i31",params:["i32"],results:["i32"]},{module:"env",name:"__wpk_fork_ref_gc_load",params:["i32","i32","i32","i32","i32","ptr","i32"],results:["i32"]},{module:"env",name:"__wpk_fork_ref_gc_lookup",params:["i32"],results:["i32"]},{module:"env",name:"__wpk_fork_ref_gc_payload_len",params:["i32","i32","i32"],results:["i32"]},{module:"env",name:"__wpk_fork_ref_gc_provenance_begin",params:["i32","i32","i32","i32","i64","i64","i32"],results:["i32"]},{module:"env",name:"__wpk_fork_ref_gc_provenance_end",params:["i32"],results:[]},{module:"env",name:"__wpk_fork_ref_gc_provenance_ref",params:["i32","i32","i32"],results:[]},{module:"env",name:"__wpk_fork_ref_gc_route",params:["i32","i32"],results:["i32"]},{module:"env",name:"__wpk_fork_ref_scratch_release",params:["ptr","ptr"],results:[]},{module:"env",name:"__wpk_fork_ref_scratch_reserve",params:["ptr"],results:["ptr"]},{module:"env",name:"__wpk_fork_ref_vector_append",params:["i32","i32"],results:[]},{module:"env",name:"__wpk_fork_ref_vector_begin",params:["i32"],results:["i32"]},{module:"env",name:"__wpk_fork_ref_vector_finish",params:["i32"],results:["i32"]},{module:"env",name:"__wpk_fork_ref_vector_get",params:["i32","i32"],results:["i32"]},{module:"env",name:"__wpk_fork_resume_peek",params:["i32"],results:["i32"]}],lr=[{module:"env",name:"__wpk_fork_ref_gc_transit",table64:!1,element:"anyref",minimum:1,maximum:null},{module:"env",name:"__wpk_fork_resume_table",table64:!1,element:"funcref",minimum:1,maximum:null}],Ft=[{name:"__wpk_fork_exception_materialize",params:["i32"],results:[]},{name:"__wpk_fork_ref_decode_exnref",params:["i32"],results:["exnref"]},{name:"__wpk_fork_ref_encode_exnref",params:["exnref"],results:["i32"]},{name:"__wpk_fork_ref_exn_abort",params:[],results:[]},{name:"__wpk_fork_ref_exn_clear",params:[],results:[]},{name:"__wpk_fork_ref_exn_encode_ingress",params:["i32"],results:["i32"]},{name:"__wpk_fork_ref_exn_throw_recipe",params:["i32"],results:[]},{name:"__wpk_fork_ref_exn_throw_slot",params:["i32"],results:[]},{name:"__wpk_fork_ref_gc_allocate",params:["i32"],results:[]},{name:"__wpk_fork_ref_gc_encode_slot",params:["i32"],results:["i32"]},{name:"__wpk_fork_ref_gc_fill",params:["i32"],results:[]},{name:"__wpk_fork_ref_gc_probe",params:["i32"],results:["i64"]},{name:"__wpk_fork_ref_gc_publish_externref",params:["i32","externref"],results:[]},{name:"__wpk_fork_static_root_harvest",params:[],results:[]},{name:"wpk_fork_abort_begin",params:["ptr"],results:[]},{name:"wpk_fork_abort_end",params:[],results:[]},{name:"wpk_fork_module_bootstrap",params:[],results:[]},{name:"wpk_fork_module_state_finish_restore",params:["i32"],results:[]},{name:"wpk_fork_module_state_restore",params:["i32"],results:[]},{name:"wpk_fork_module_state_save",params:["i32"],results:[]},{name:"wpk_fork_module_table_state_restore",params:["i32"],results:[]},{name:"wpk_fork_module_table_state_save",params:["i32"],results:[]},{name:"wpk_fork_module_thread_bootstrap",params:[],results:[]},{name:"wpk_fork_rewind_begin",params:["ptr"],results:[]},{name:"wpk_fork_rewind_end",params:[],results:[]},{name:"wpk_fork_state",params:[],results:["i32"]},{name:"wpk_fork_unwind_begin",params:["ptr"],results:[]},{name:"wpk_fork_unwind_end",params:[],results:[]}];var Ci=["__abi_version","kernel_alloc_scratch","kernel_create_process","kernel_create_process_with_stdio","kernel_dequeue_signal","kernel_exec_prepare","kernel_exec_setup_for_thread","kernel_fork_process","kernel_get_parent_pid","kernel_get_process_exit_signal","kernel_get_process_state","kernel_handle_channel","kernel_has_sa_nocldstop","kernel_host_adapter_manifest_len","kernel_host_adapter_manifest_ptr","kernel_ipc_shmat_for_process","kernel_ipc_shmat_for_task","kernel_ipc_shmdt_for_process","kernel_ipc_shmdt_for_task","kernel_mark_process_signaled","kernel_pipe_has_readers","kernel_posix_timer_fire","kernel_prepare_write_operation","kernel_reap_exited_child","kernel_remove_process","kernel_set_current_tid","kernel_spawn_process","kernel_thread_exit","kernel_validate_task","kernel_wait_child_poll"];var W={LINK_MAX:0,MAX_CANON:1,MAX_INPUT:2,NAME_MAX:3,PATH_MAX:4,PIPE_BUF:5,CHOWN_RESTRICTED:6,NO_TRUNC:7,VDISABLE:8,SYNC_IO:9,ASYNC_IO:10,PRIO_IO:11,SOCK_MAXBUF:12,FILESIZEBITS:13,REC_INCR_XFER_SIZE:14,REC_MAX_XFER_SIZE:15,REC_MIN_XFER_SIZE:16,REC_XFER_ALIGN:17,ALLOC_SIZE_MIN:18,SYMLINK_MAX:19,POSIX2_SYMLINKS:20,FALLOC:21,TEXTDOMAIN_MAX:22,TIMESTAMP_RESOLUTION:23};var ko=Uint8Array.from(xi);function R(r,e){let t=0,n=0,i=e;for(;;){let s=r[i++];if(t|=(s&127)<=r.length)throw new Error(`${n} is truncated`);if((r[e++]&128)===0)return e}throw new Error(`${n} has an overlong LEB128 encoding`)}function ge(r,e,t){if(e>=r.length)throw new Error(`${t} is truncated`);let n=r[e++];switch(n){case 127:case 126:case 125:case 124:case 123:case 117:case 116:case 115:case 114:case 113:case 112:case 111:case 110:case 109:case 108:case 107:case 106:case 105:case 104:return{code:n,shared:!1,next:e};case 98:case 99:case 100:{let i=r[e]===101;i&&e++;let s=Zi(r,e,5,`${t} heap type`),[o]=Gi(r,e);return{code:n,heapType:Number(o),shared:i,next:s}}default:throw new Error(`${t} has unknown value type 0x${n.toString(16)}`)}}function To(r,e,t){return r[e]===120||r[e]===119?{code:r[e],shared:!1,next:e+1}:ge(r,e,t)}function bo(r,e){let t=r[e];if(t===64||t===127||t===126||t===125||t===124||t===123||t===112||t===111)return e+1;let[,n]=fr(r,e);return e+n}function Lo(r,e,t){let[n,i]=R(r,e);e+=i;let s=[],o=[];for(let u=0;u=r.length)throw new Error(`${t} mutability is truncated`);let i=r[e++];if(i!==0&&i!==1)throw new Error(`${t} has invalid mutability ${i}`);return e}function Po(r,e,t,n){if(e===101){if(t>=r.length)throw new Error(`${n} shared type is truncated`);e=r[t++]}for(let i of[76,77]){if(e!==i)continue;let[,s]=R(r,t);if(t+=s,t>=r.length)throw new Error(`${n} descriptor is truncated`);e=r[t++]}if(e===96)return Lo(r,t,n);if(e===95){let[i,s]=R(r,t);t+=s;for(let o=0;o=r.length)throw new Error(`${t} is truncated`);let n=r[e++];if(n===79||n===80){let[i,s]=R(r,e);e+=s;for(let o=0;o=r.length)throw new Error(`${t} body is truncated`);n=r[e++]}return Po(r,n,e,t)}function zo(r,e){let[t,n]=R(r,e);e+=n;let i=[];for(let s=0;s=21&&n<=34?Kt(e,t):n===84||n>=92&&n<=99||n>=112&&n<=123||n>=124&&n<=131||n>=156&&n<=159?t+1:t:r===254?n===0||n===1||n===2?Kt(e,t):n===3?t:n>=16&&n<=79?Kt(e,t):null:null}function Fo(r,e,t){let[n,i]=R(r,e);e+=i+n;let[s,o]=R(r,e);e+=o+s;let a=r[e++];if(a===0){t.funcImports++;let[,c]=R(r,e);e+=c}else if(a===1)e=ge(r,e,"table import type").next,e=De(r,e).next;else if(a===2)e=De(r,e).next;else if(a===3)t.globalImports++,e=ge(r,e,"global import type").next,e++;else if(a===4){e++;let[,c]=R(r,e);e+=c}return e}function gn(r){return r.length>=8&&r[0]===0&&r[1]===97&&r[2]===115&&r[3]===109}function Ke(r,e){let[t,n]=R(r,e);return e+=n,[new TextDecoder().decode(r.subarray(e,e+t)),e+t]}function Co(r,e){if(e.length===0)return!0;let t=new TextEncoder().encode(e);e:for(let n=0;n<=r.length-t.length;n++){for(let i=0;ir);function Mi(r,e){switch(r.code){case 127:return Qn;case 126:return er;case 125:return tr;case 124:return nr;case 123:return rr;case 112:case 115:return st;case 111:case 114:return Tt;case 105:case 116:return bt;case 104:case 106:case 107:case 108:case 109:case 110:case 113:case 117:return Lt;case 98:case 99:case 100:{let t=r.heapType;return t===void 0?null:t===-16||t===-13?st:t===-17||t===-14?Tt:t===-23||t===-12?bt:t>=0&&e[t]!==void 0?st:Lt}default:return null}}function ur(r,e,t){if(!t)throw new Error(`function ${e} refers to an unknown type`);let n=r.get(e)??[];n.push(t),r.set(e,n)}function Ct(r,e,t){let n=r.get(e)??[];n.push(t),r.set(e,n)}function De(r,e){let[t,n]=R(r,e);e+=n;let[i,s]=R(r,e);e+=s;let o=null;if((t&1)!==0){let[a,c]=R(r,e);e+=c,o=a}return{flags:t,minimum:i,maximum:o,next:e}}function Do(r){let e=new Uint8Array(r);if(!gn(e))throw new Error("not a wasm binary");let t=[],n=[],i=[],s={functionImports:new Map,functionImportEntries:[],globalImports:new Map,tableImports:new Map,tables:[],tagImports:new Map,functionExports:new Map,globalExports:new Map,tableExports:new Map,exports:new Map,memoryPointerWidths:[],forkCapabilities:[],linkedFrameDescriptors:[],exceptionCodecDescriptors:[],importedGlobalsDescriptors:[],importedTablesDescriptors:[],moduleStateDescriptors:[],staticRootDescriptors:[],unwindTransportDescriptors:[],nativeStartCount:0,importsKernelFork:!1},o=0,a=0,c=8;for(;ce.length)throw new Error("wasm section exceeds file size");let d=m,h=!1;if(l===0){let[g,f]=Ke(e,d);g===Rt?s.linkedFrameDescriptors.push(e.slice(f,_)):g===xe?s.forkCapabilities.push(e.slice(f,_)):g===ye?s.exceptionCodecDescriptors.push(e.slice(f,_)):g===G?s.importedGlobalsDescriptors.push(e.slice(f,_)):g===Z?s.importedTablesDescriptors.push(e.slice(f,_)):g===Q?s.moduleStateDescriptors.push(e.slice(f,_)):g===Ce?s.staticRootDescriptors.push(e.slice(f,_)):g===ot&&s.unwindTransportDescriptors.push(e.slice(f,_))}else if(l===1){h=!0;let g=zo(e,d);t.push(...g.types),d=g.next}else if(l===2){h=!0;let[g,f]=R(e,d);d+=f;for(let y=0;y=e.length)throw new Error(`global import ${E}.${O} is truncated`);let x=e[d++];if((x&-4)!==0)throw new Error(`global import ${E}.${O} has invalid flags ${x}`);Ct(s.globalImports,`${E}.${O}`,{module:E,name:O,importOrdinal:y,index:o++,valueType:v.code,recipeTypeCode:Mi(v,t),mutable:(x&1)!==0,shared:(x&2)!==0})}else if(A===4){let v=e[d++];if(v!==0)throw new Error(`unsupported wasm tag attribute ${v}`);let[x,L]=R(e,d);d+=L,ur(s.tagImports,`${E}.${O}`,t[x])}else throw new Error(`unsupported wasm import kind ${A}`)}}else if(l===3){h=!0;let[g,f]=R(e,d);d+=f;for(let y=0;yr[c]===a))throw new Error("linked-frame descriptor has invalid magic");let e=new DataView(r.buffer,r.byteOffset,r.byteLength),t=e.getUint16(4,!0);if(t!==1)throw new Error(`linked-frame descriptor version ${t} is unsupported`);let n=e.getUint16(6,!0);if(n!==kt)throw new Error(`linked-frame descriptor declares size ${n}, expected ${kt}`);let i=e.getUint8(8),s=pi.find(({bytes:a})=>a===i);if(!s)throw new Error(`linked-frame descriptor pointer width ${i} is unsupported`);if(e.getUint8(9)!==hi)throw new Error(`linked-frame descriptor alignment ${e.getUint8(9)} is unsupported`);let o=e.getUint16(10,!0);if(o!==jn)throw new Error(`linked-frame descriptor flags 0x${o.toString(16)} do not equal required flags 0x${jn.toString(16)}`);if(e.getUint32(12,!0)!==s.chunkHeaderSize||e.getUint32(16,!0)!==s.nodeHeaderSize)throw new Error(`linked-frame descriptor header sizes do not match its ${i}-byte pointer width`);return s.bytes}function Bo(r){if(r.length===0)return[`missing required ${xe} capability`];if(r.length!==1)return[`has ${r.length} ${xe} sections, expected exactly one`];let e=r[0];if(e.byteLength!==2)return[`${xe} has ${e.byteLength} bytes, expected 2`];if(e[0]!==Oi)return[`${xe} version ${e[0]} is unsupported`];let t=e[1];return(t&~Ii)!==0?[`${xe} has unknown flags 0x${t.toString(16)}`]:(t&fn)!==fn?[`${xe} flags 0x${t.toString(16)} omit required activation-state safety flags 0x${fn.toString(16)}`]:[]}function $o(r){let e=[],t=`${hn}.${pn}`,n=r.tagImports.get(t);if(n?n.length!==1?e.push(`duplicate private fork-unwind tag import ${t}`):(n[0].params.length!==0||n[0].results.length!==0)&&e.push(`private fork-unwind tag ${t} must have an empty payload`):e.push(`missing required private fork-unwind tag import ${t}`),r.unwindTransportDescriptors.length===0)e.push(`missing required ${ot} descriptor`);else if(r.unwindTransportDescriptors.length!==1)e.push(`has ${r.unwindTransportDescriptors.length} ${ot} descriptors, expected exactly one`);else{let i=r.unwindTransportDescriptors[0];(i.length!==2||i[0]!==sr||i[1]!==or)&&e.push(`${ot} must be [${sr}, ${or}]`)}return e}function Uo(r,e){if(r.length===0)return[`missing required ${Q} descriptor`];if(r.length!==1)return[`has ${r.length} ${Q} descriptors, expected exactly one`];let t=r[0];if(t.byteLength!==cn)return[`${Q} has ${t.byteLength} bytes, expected ${cn}`];if(!_i.every((h,g)=>t[g]===h))return[`${Q} has invalid magic`];let n=new DataView(t.buffer,t.byteOffset,t.byteLength),i=n.getUint16(4,!0),s=n.getUint16(6,!0),o=n.getUint8(8),a=Si.find(({bytes:h})=>h===o),c=n.getUint8(9),l=n.getUint16(10,!0),p=n.getUint16(12,!0),u=n.getUint16(14,!0),m=n.getUint32(16,!0),_=n.getUint32(20,!0),d=[];return i!==mi&&d.push(`${Q} version ${i} is unsupported`),s!==cn&&d.push(`${Q} declares size ${s}`),a?e!==null&&o!==e&&d.push(`${Q} pointer width ${o} does not match linked frames ${e}`):d.push(`${Q} pointer width ${o} is unsupported`),c!==yi&&d.push(`${Q} alignment ${c} is unsupported`),l!==Jn&&d.push(`${Q} flags 0x${l.toString(16)} do not equal required flags 0x${Jn.toString(16)}`),p!==gi&&d.push(`${Q} arena version ${p} is unsupported`),u!==Ei&&d.push(`${Q} record version ${u} is unsupported`),m!==wi&&d.push(`${Q} root word ${m} is unsupported`),_!==0&&d.push(`${Q} reserved field is nonzero`),d}function Wo(r){if(r.length===0)return[`missing required ${ye} descriptor`];if(r.length!==1)return[`has ${r.length} ${ye} descriptors, expected exactly one`];let e=r[0];if(e.byteLength2147483647||o.has(p))&&n.push(`${ye} layout id ${p} is invalid or duplicated`),o.add(p)}return n}var Go=new Set([Qn,er,tr,nr,rr,st,Tt,bt,Lt]);function Bi(r){return!(r.module===ar&&(r.name===cr||r.name==="__channel_base"||r.name==="__wpk_fork_module_state_table_generation_addr"))}function Zo(r){let e=r.importedGlobalsDescriptors;if(e.length===0)return[`missing required ${G} descriptor`];if(e.length!==1)return[`has ${e.length} ${G} descriptors, expected exactly one`];let t=e[0];if(t.byteLength<_n)return[`${G} descriptor is truncated`];let n=new DataView(t.buffer,t.byteOffset,t.byteLength),i=[];Ri.every((h,g)=>t[g]===h)||i.push(`${G} has invalid magic`),n.getUint16(4,!0)!==ki&&i.push(`${G} version ${n.getUint16(4,!0)} is unsupported`),n.getUint16(6,!0)!==_n&&i.push(`${G} declares an invalid header size`);let s=n.getUint32(8,!0);n.getUint32(12,!0)!==0&&i.push(`${G} reserved field is nonzero`);let o=new Set,a=new Set,c=new TextDecoder("utf-8",{fatal:!0}),l=[],p=-1,u=_n;for(let h=0;ht.byteLength)return i.push(`${G} record ${h} header is truncated`),i;let g=n.getUint32(u,!0),f=n.getUint32(u+4,!0),y=n.getUint8(u+8),E=n.getUint8(u+9),w=n.getUint32(u+12,!0),O=n.getUint32(u+16,!0),S=n.getUint32(u+20,!0),A=zt+w+O;if(!Number.isSafeInteger(A)||g!==A||gt.byteLength)return i.push(`${G} record ${h} has invalid bounds`),i;(f===0||o.has(f))&&i.push(`${G} record ${h} has invalid or duplicated owner ${f}`),o.add(f),Go.has(y)||i.push(`${G} record ${h} has unknown value type ${y}`),(E&~Li)!==0&&i.push(`${G} record ${h} has unknown flags 0x${E.toString(16)}`),n.getUint16(u+10,!0)!==0&&i.push(`${G} record ${h} reserved fields are nonzero`),(a.has(S)||S<=p)&&i.push(`${G} record ${h} has duplicated or unordered import ordinal`),a.add(S),p=S;let v=u+zt;try{let x=c.decode(t.subarray(v,v+w)),L=c.decode(t.subarray(v+w,v+w+O));l.push({ownerId:f,typeCode:y,flags:E,importOrdinal:S,module:x,name:L})}catch{i.push(`${G} record ${h} contains invalid UTF-8`)}u+=g}u!==t.byteLength&&i.push(`${G} has trailing bytes`);let m=[...r.globalImports.values()].flat(),_=new Map(m.map(h=>[h.index,h])),d=new Set;for(let h of l){let g=`${ln}${h.ownerId}`,f=r.exports.get(g);if(!f||f.length!==1||f[0].kind!==3){i.push(`${G} owner ${h.ownerId} lacks exactly one global catalog export ${g}`);continue}let y=_.get(f[0].index);if(!y||!Bi(y)){i.push(`${G} owner ${h.ownerId} does not identify a reconstructible imported global`);continue}if(y.module!==h.module||y.name!==h.name||y.importOrdinal!==h.importOrdinal||y.recipeTypeCode!==h.typeCode||y.mutable!==((h.flags&Ti)!==0)||y.shared!==((h.flags&bi)!==0)){i.push(`${G} owner ${h.ownerId} does not match its imported global declaration`);continue}if(d.has(y.index)){i.push(`${G} repeats imported global index ${y.index}`);continue}d.add(y.index)}for(let h of m)Bi(h)&&!d.has(h.index)&&i.push(`${G} omits imported global ${h.module}.${h.name} at index ${h.index}`);for(let[h,g]of r.exports){if(!h.startsWith(ln))continue;let f=h.slice(ln.length),y=Number(f);(!/^[1-9][0-9]*$/.test(f)||!Number.isSafeInteger(y)||y>4294967295||g.length!==1||g[0].kind!==3)&&i.push(`malformed reserved fork global catalog export ${h}`)}return i}var Ho=new Set([st,Tt,bt,Lt]);function $i(r){return!lr.some(({module:e,name:t})=>r.module===e&&r.name===t)}function Vo(r){let e=r.importedTablesDescriptors;if(e.length===0)return[`missing required ${Z} descriptor`];if(e.length!==1)return[`has ${e.length} ${Z} descriptors, expected exactly one`];let t=e[0];if(t.byteLengtht[g]===h)||i.push(`${Z} has invalid magic`),n.getUint16(4,!0)!==zi&&i.push(`${Z} version ${n.getUint16(4,!0)} is unsupported`),n.getUint16(6,!0)!==yn&&i.push(`${Z} declares an invalid header size`);let s=n.getUint32(8,!0);n.getUint32(12,!0)!==0&&i.push(`${Z} reserved field is nonzero`);let o=new Set,a=new Set,c=new TextDecoder("utf-8",{fatal:!0}),l=[],p=-1,u=yn;for(let h=0;ht.byteLength)return i.push(`${Z} record ${h} header is truncated`),i;let g=n.getUint32(u,!0),f=n.getUint32(u+4,!0),y=n.getUint8(u+8),E=n.getUint8(u+9),w=n.getUint32(u+12,!0),O=n.getUint32(u+16,!0),S=n.getUint32(u+20,!0),A=Nt+w+O;if(!Number.isSafeInteger(A)||g!==A||gt.byteLength)return i.push(`${Z} record ${h} has invalid bounds`),i;(f===0||o.has(f))&&i.push(`${Z} record ${h} has invalid or duplicated owner ${f}`),o.add(f),Ho.has(y)||i.push(`${Z} record ${h} has unknown element type ${y}`),(E&~Fi)!==0&&i.push(`${Z} record ${h} has unknown flags 0x${E.toString(16)}`),n.getUint16(u+10,!0)!==0&&i.push(`${Z} record ${h} reserved fields are nonzero`),(a.has(S)||S<=p)&&i.push(`${Z} record ${h} has duplicated or unordered import ordinal`),a.add(S),p=S;let v=u+Nt;try{let x=c.decode(t.subarray(v,v+w)),L=c.decode(t.subarray(v+w,v+w+O));l.push({ownerId:f,typeCode:y,flags:E,importOrdinal:S,module:x,name:L})}catch{i.push(`${Z} record ${h} contains invalid UTF-8`)}u+=g}u!==t.byteLength&&i.push(`${Z} has trailing bytes`);let m=[...r.tableImports.values()].flat(),_=new Map(m.map(h=>[h.index,h])),d=new Set;for(let h of l){let g=`${un}${h.ownerId}`,f=r.exports.get(g);if(!f||f.length!==1||f[0].kind!==1){i.push(`${Z} owner ${h.ownerId} lacks exactly one table catalog export ${g}`);continue}let y=_.get(f[0].index);if(!y||!$i(y)){i.push(`${Z} owner ${h.ownerId} does not identify a reconstructible imported table`);continue}if(y.module!==h.module||y.name!==h.name||y.importOrdinal!==h.importOrdinal||y.recipeTypeCode!==h.typeCode||y.table64!==((h.flags&Ni)!==0)){i.push(`${Z} owner ${h.ownerId} does not match its imported table declaration`);continue}if(d.has(y.index)){i.push(`${Z} repeats imported table index ${y.index}`);continue}d.add(y.index)}for(let h of m)$i(h)&&!d.has(h.index)&&i.push(`${Z} omits imported table ${h.module}.${h.name} at index ${h.index}`);for(let[h,g]of r.exports){if(!h.startsWith(un))continue;let f=h.slice(un.length),y=Number(f);(!/^[1-9][0-9]*$/.test(f)||!Number.isSafeInteger(y)||y>4294967295||g.length!==1||g[0].kind!==1)&&i.push(`malformed reserved fork table catalog export ${h}`)}return i}function dr(r,e){switch(r){case"ptr":return e===8?126:127;case"i32":return 127;case"i64":return 126;case"anyref":return 110;case"exnref":return 105;case"externref":return 111;case"funcref":return 112}}function Ui(r,e,t,n){return r.params.length===e.length&&r.results.length===t.length&&r.params.every((i,s)=>i===dr(e[s],n))&&r.results.every((i,s)=>i===dr(t[s],n))}function Wi(r,e,t){let n=i=>i==="ptr"?t===8?"i64":"i32":i;return`(${r.map(n).join(", ")}) -> (${e.map(n).join(", ")})`}function Xo(r){let e=`${ar}.${cr}`,t=r.globalImports.get(e);return t?t.length!==1?[`duplicate exception-codec activation import ${e}`]:t[0].valueType!==127||t[0].mutable?[`exception-codec activation import ${e} must be immutable i32`]:[]:[`missing required immutable exception-codec activation import ${e}`]}function Yo(r){let e=[];for(let t of lr){let n=`${t.module}.${t.name}`,i=r.tableImports.get(n);if(!i){e.push(`missing required ABI 43 fork-runtime table import ${n}`);continue}if(i.length!==1){e.push(`duplicate ABI 43 fork-runtime table import ${n}`);continue}let s=i[0],o=dr(t.element,4);(s.elementType!==o||s.table64!==t.table64||s.minimum!==t.minimum||s.maximum!==t.maximum)&&e.push(`ABI 43 fork-runtime table import ${n} has the wrong type or limits`)}return e}function qo(r){if(r.staticRootDescriptors.length===0)return[`missing required ${Ce} descriptor`];if(r.staticRootDescriptors.length!==1)return[`has ${r.staticRootDescriptors.length} ${Ce} descriptors, expected exactly one`];let e=r.staticRootDescriptors[0];if(e.byteLength!==mn)return[`${Ce} has ${e.byteLength} bytes, expected ${mn}`];let t=[];ko.some((l,p)=>e[p]!==l)&&t.push(`${Ce} has invalid magic`);let n=new DataView(e.buffer,e.byteOffset,e.byteLength);n.getUint16(4,!0)!==Ai&&t.push(`${Ce} version ${n.getUint16(4,!0)} is unsupported`),n.getUint16(6,!0)!==mn&&t.push(`${Ce} declares an invalid header size`);let i=n.getUint32(8,!0),s=r.tableExports.get(Pt);if(!s||s.length!==1)return t.push(`missing exactly one table export ${Pt}`),t;let o=[...r.tableImports.values()].reduce((l,p)=>l+p.length,0),a=s[0],c=r.tables[a];return a!r.functionExports.has(c)).map(({name:c})=>c);t.length>0&&e.push(`incomplete wasm-fork-instrument exports; missing ${t.join(", ")}`);let n=null;if(r.linkedFrameDescriptors.length===0)e.push(`missing required ${Rt} descriptor`);else if(r.linkedFrameDescriptors.length!==1)e.push(`has ${r.linkedFrameDescriptors.length} ${Rt} descriptors, expected exactly one`);else try{n=Mo(r.linkedFrameDescriptors[0])}catch(c){e.push(c instanceof Error?c.message:String(c))}e.push(...Uo(r.moduleStateDescriptors,n));let i=at.filter(({module:c,name:l})=>r.functionImports.has(`${c}.${l}`)),s=`${hn}.${pn}`,o=r.importsKernelFork||i.length>0;if((o||r.tagImports.has(s)||r.unwindTransportDescriptors.length>0)&&e.push(...$o(r)),o){let c=at.filter(({module:l,name:p})=>!r.functionImports.has(`${l}.${p}`)).map(({module:l,name:p})=>`${l}.${p}`);c.length>0&&e.push(`incomplete ABI 43 fork-runtime imports; missing ${c.join(", ")}`);for(let l of at){let p=`${l.module}.${l.name}`,u=r.functionImports.get(p);u&&u.length!==1&&e.push(`duplicate ABI 43 fork-runtime import ${p}`)}}if(n!==null){if(r.memoryPointerWidths.length!==1)e.push(`ABI 43 fork instrumentation requires exactly one module memory, found ${r.memoryPointerWidths.length}`);else if(r.memoryPointerWidths[0]!==n){let c=n===8?"an":"a";e.push(`ABI 43 linked-frame descriptor declares ${c} ${n}-byte pointer but the module memory uses ${r.memoryPointerWidths[0]}-byte addresses`)}for(let c of Ft){let l=r.functionExports.get(c.name);l?.length===1&&!Ui(l[0],c.params,c.results,n)&&e.push(`ABI 43 wasm-fork-instrument export ${c.name} has the wrong signature; expected ${Wi(c.params,c.results,n)}`)}if(o)for(let c of at){let l=`${c.module}.${c.name}`,p=r.functionImports.get(l);p?.length===1&&!Ui(p[0],c.params,c.results,n)&&e.push(`ABI 43 fork-runtime import ${l} has the wrong signature; expected ${Wi(c.params,c.results,n)}`)}}return e}function Jo(r){let e=new Uint8Array(r);if(!gn(e))return[];let t=[],n=8;for(;nt.startsWith("reloc."))}function Vi(r,e={}){let t=[],n=null;ea(r)&&t.push("contains asyncify_"),e.expectedAbi!==void 0&&e.expectedAbi!==null&&(n=ra(r),n!==null&&n!==e.expectedAbi&&t.push(`ABI ${n}, expected ${e.expectedAbi}`));let i=new Set(Qo(r));if(e.requiredExports){let E=e.requiredExports.filter(w=>!i.has(w));E.length>0&&t.push(`missing required exports: ${E.join(", ")}`)}let s=Ko.filter(E=>i.has(E)),o=Jo(r),a=Hi(r),c=at.filter(({module:E,name:w})=>o.includes(`${E}.${w}`)),l=a.filter(E=>E===Rt).length,p=a.filter(E=>E===xe).length,u=a.filter(E=>E===Q).length,m=a.filter(E=>E===ye).length,_=a.filter(E=>E===G).length,d=a.filter(E=>E===Z).length,h=a.filter(E=>E===ot).length,g=o.includes(`${hn}.${pn}`),f=s.length>0||c.length>0||l>0||p>0||u>0||m>0||_>0||d>0||h>0||g;if(e.expectedAbi!==void 0&&e.expectedAbi!==null&&f&&n===null&&t.push(`ABI ${e.expectedAbi} fork artifact is missing __abi_version; the activation-state capability epoch cannot be verified`),e.forbidForkInstrumentation&&f&&t.push("contains ABI 43 wasm-fork-instrument metadata, imports, or exports"),(e.requireForkInstrumentation??!ta(r))&&(f||o.includes("kernel.kernel_fork")))try{t.push(...jo(Do(r)))}catch(E){t.push(`cannot validate ABI 43 fork-artifact contract: ${E instanceof Error?E.message:String(E)}`)}return t}function na(r,e){let t=new Uint8Array(r);if(t.length<8)return null;let n=0,i=null,s=null,o=8;for(;o=c)return null;let h=a;for(let y=0;y=d)return null;let[h,g]=R(t,_);_+=g;for(let f=0;fd)return null}return _}function m(_,d=0){if(d>4)return null;let h=p(_);if(!h)return null;let g=u(h.start,h.end);if(g===null)return null;let f=g,y=h.end;for(;f=32&&E<=38||E===208){let[,w]=R(t,f);f+=w}else if(E>=40&&E<=62)f=Kt(t,f);else if(E===63||E===64)f++;else if(E===66){let[,w]=Gi(t,f);f+=w}else if(E===67)f+=4;else if(E===68)f+=8;else if(E===252||E===253||E===254){let w=No(E,t,f);if(w===null)return null;f=w}}return null}return m(i)}function ra(r){return na(r,"__abi_version")}var ia=ArrayBuffer,X=Uint8Array,En=Uint16Array,sa=Int16Array;var wn=Int32Array,hr=function(r,e,t){if(X.prototype.slice)return X.prototype.slice.call(r,e,t);(e==null||e<0)&&(e=0),(t==null||t>r.length)&&(t=r.length);var n=new X(t-e);return n.set(r.subarray(e,t)),n},Mt=function(r,e,t,n){if(X.prototype.fill)return X.prototype.fill.call(r,e,t,n);for((t==null||t<0)&&(t=0),(n==null||n>r.length)&&(n=r.length);tr.length)&&(n=r.length);t2046MB)","invalid block type","FSE accuracy too high","match distance too far back","unexpected EOF"],Y=function(r,e,t){var n=new Error(e||aa[r]);if(n.code=r,Error.captureStackTrace&&Error.captureStackTrace(n,Y),!t)throw n;return n},Xi=function(r,e,t){for(var n=0,i=0;n>>0},la=function(r,e){var t=r[0]|r[1]<<8|r[2]<<16;if(t==3126568&&r[3]==253){var n=r[4],i=n>>5&1,s=n>>2&1,o=n&3,a=n>>6;n&8&&Y(0);var c=6-i,l=o==3?4:o,p=Xi(r,c,l);c+=l;var u=a?1<>3);_=d+(d>>3)*(r[5]&7)}_>2145386496&&Y(1);var h=new X((e==1?m||_:e?0:_)+12);return h[0]=1,h[4]=4,h[8]=8,{b:c+u,y:0,l:0,d:p,w:e&&e!=1?e:h.subarray(12),e:_,o:new wn(h.buffer,0,3),u:m,c:s,m:Math.min(131072,_)}}else if((t>>4|r[3]<<20)==25481893)return ca(r,4)+8;Y(0)},Xe=function(r){for(var e=0;1<t&&Y(3);for(var s=1<0;){var y=Xe(o+1),E=n>>3,w=(1<>(n&7)&w,S=(1<S&&(O-=A)),m[++a]=--O,O==-1?(o+=O,g[--p]=a):o-=O,!O)do{var x=n>>3;c=(r[x]|r[x+1]<<8)>>(n&7)&3,n+=2,a+=c}while(c==3)}(a>255||o)&&Y(0);for(var L=0,N=(s>>1)+(s>>3)+3,j=s-1,J=0;J<=a;++J){var F=m[J];if(F<1){_[J]=-F;continue}for(l=0;l=p)}}for(L&&Y(0),l=0;l>3,{b:i,s:g,n:f,t:d}]},ua=function(r,e){var t=0,n=-1,i=new X(292),s=r[e],o=i.subarray(0,256),a=i.subarray(256,268),c=new En(i.buffer,268);if(s<128){var l=Bt(r,e+1,6),p=l[0],u=l[1];e+=s;var m=p<<3,_=r[e];_||Y(0);for(var d=0,h=0,g=u.b,f=g,y=(++e<<3)-8+Xe(_);y-=g,!(y>3;if(d+=(r[E]|r[E+1]<<8)>>(y&7)&(1<>3,h+=(r[E]|r[E+1]<<8)>>(y&7)&(1<255&&Y(0)}else{for(n=s-127;t>4,o[t+1]=w&15}++e}var O=0;for(t=0;t11&&Y(0),O+=S&&1<0;--t){var J=c[t];Mt(j,t,J,c[t-1]=J+a[t]*(1<a&&u>3,_=(r[m]|r[m+1]<<8|r[m+2]<<16)>>(p&7);c=(c<>2,o=s<<1,a=s+o;Dt(r.subarray(n,n+=r[0]|r[1]<<8),e.subarray(0,s),t),Dt(r.subarray(n,n+=r[2]|r[3]<<8),e.subarray(s,o),t),Dt(r.subarray(n,n+=r[4]|r[5]<<8),e.subarray(o,a),t),Dt(r.subarray(n),e.subarray(a),t)},ya=function(r,e,t){var n,i=e.b,s=r[i],o=s>>1&3;e.l=s&1;var a=s>>3|r[i+1]<<5|r[i+2]<<13,c=(i+=3)+a;if(o==1)return i>=r.length?void 0:(e.b=i+1,t?(Mt(t,r[i],e.y,e.y+=a),t):Mt(new X(a),r[i]));if(!(c>r.length)){if(o==0)return e.b=c,t?(t.set(r.subarray(i,c),e.y),e.y+=a,t):hr(r,i,c);if(o==2){var l=r[i],p=l&3,u=l>>2&3,m=l>>4,_=0,d=0;p<2?u&1?m|=r[++i]<<4|(u&2&&r[++i]<<12):m=l>>3:(d=u,u<2?(m|=(r[++i]&63)<<4,_=r[i]>>6|r[++i]<<2):u==2?(m|=r[++i]<<4|(r[++i]&3)<<12,_=r[i]>>2|r[++i]<<6):(m|=r[++i]<<4|(r[++i]&63)<<12,_=r[i]>>6|r[++i]<<2|r[++i]<<10)),++i;var h=t?t.subarray(e.y,e.y+e.m):new X(e.m),g=h.length-m;if(p==0)h.set(r.subarray(i,i+=m),g);else if(p==1)Mt(h,r[i++],g);else{var f=e.h;if(p==2){var y=ua(r,i);_+=i-(i=y[0]),e.h=f=y[1]}else f||Y(0);(d?_a:Dt)(r.subarray(i,i+=_),h.subarray(g),f)}var E=r[i++];if(E){E==255?E=(r[i++]|r[i++]<<8)+32512:E>127&&(E=E-128<<8|r[i++]);var w=r[i++];w&3&&Y(0);for(var O=[da,ha,fa],S=2;S>-1;--S){var A=w>>(S<<1)+2&3;if(A==1){var v=new X([0,0,r[i++]]);O[S]={s:v.subarray(2,3),n:v.subarray(0,1),t:new En(v.buffer,0,1),b:0}}else A==2?(n=Bt(r,i,9-(S&1)),i=n[0],O[S]=n[1]):A==3&&(e.t||Y(0),O[S]=e.t[S])}var x=e.t=O,L=x[0],N=x[1],j=x[2],J=r[c-1];J||Y(0);var F=(c<<3)-8+Xe(J)-j.b,b=F>>3,k=0,V=(r[b]|r[b+1]<<8)>>(F&7)&(1<>3;var Ze=(r[b]|r[b+1]<<8)>>(F&7)&(1<>3;var oe=(r[b]|r[b+1]<<8)>>(F&7)&(1<>3;var rt=1<>>(F&7)&rt-1);b=(F-=mr[He])>>3;var ze=ma[He]+((r[b]|r[b+1]<<8|r[b+2]<<16)>>(F&7)&(1<>3;var Ve=pa[At]+((r[b]|r[b+1]<<8|r[b+2]<<16)>>(F&7)&(1<>3,V=j.t[V]+((r[b]|r[b+1]<<8)>>(F&7)&(1<>3,oe=L.t[oe]+((r[b]|r[b+1]<<8)>>(F&7)&(1<>3,Ze=N.t[Ze]+((r[b]|r[b+1]<<8)>>(F&7)&(1<3)e.o[2]=e.o[1],e.o[1]=e.o[0],e.o[0]=_e-=3;else{var it=_e-(Ve!=0);it?(_e=it==3?e.o[0]-1:e.o[it],it>1&&(e.o[2]=e.o[1]),e.o[1]=e.o[0],e.o[0]=_e):_e=e.o[0]}for(var S=0;Sze&&(Fe=ze);for(var S=0;S=i){let v=(_+1)*4096;try{e.grow(v)}catch{throw new I(te)}if(i=Math.floor(e.byteLength/4096),_>=i)throw new I(te)}new Uint8Array(e).fill(0);let d=new r(e);d.w32(wr,gr),d.w32(Sr,Er),d.w32(In,4096),d.w32(lt,i),d.w32(Me,o),d.w32(Ye,p),d.w32(An,u),d.w32(Qi,m),d.w32(xn,_),d.w32(ka,a),d.w32(Ta,c),d.w32(ba,l),d.w32(Wt,s),d.w32(es,256);let h=u*4096;for(let v=0;v<_;v++){let x=(h>>2)+(v>>5);d.i32[x]|=1<<(v&31)}let g=i-_;Atomics.store(d.i32,ut>>2,g),d.blockAllocHint=_;let f=p*4096;d.i32[f>>2]|=3,Atomics.store(d.i32,vn>>2,o-2),d.inodeAllocHint=2;let y=d.inodeOffset(1);d.w32(y+C,$|493),d.w32(y+B,2),d.w64(y+le,1);let E=d.blockAlloc();if(E<0)throw new I(te);d.w32(y+ee,E);let w=E*4096,O=$e(P+1),S=$e(P+2);d.w32(w,1),d.view.setUint16(w+4,O,!0),d.view.setUint16(w+6,1,!0),d.u8[w+P]=46;let A=w+O;return d.w32(A,1),d.view.setUint16(A+4,S,!0),d.view.setUint16(A+6,2,!0),d.u8[A+P]=46,d.u8[A+P+1]=46,d.w64(y+z,O+S),Atomics.store(d.i32,Or>>2,1),d}static inspectImageCapacity(e){if(e.byteLengththis.snapshotBytesUnlocked(e))}snapshotState(e){return this.withNamespaceLock(()=>({bytes:this.snapshotBytesUnlocked(e),identities:this.collectIdentityStateUnlocked()}))}identityState(){return this.withNamespaceLock(()=>this.collectIdentityStateUnlocked())}snapshotBytesUnlocked(e){let t=e?.normalizeTimestampsMs;if(t!==void 0&&(!Number.isSafeInteger(t)||t<0))throw new I(H,"Snapshot timestamp must be a non-negative safe integer in milliseconds");let n=t===void 0?void 0:BigInt(t);for(let a=0;a>2)!==0)throw new I(vr,"Cannot save a VFS image with open descriptors")}let i=this.r32(Me);for(let a=0;a=1&&this.inodeIsAllocated(a)?n:0n;o.setBigUint64(c+Gt,l,!0),o.setBigUint64(c+ae,l,!0),o.setBigUint64(c+q,l,!0)}}return s}collectIdentityStateUnlocked(){let e=new Map,t=[{ino:1,path:"/"}],n=new Set;for(;t.length>0;){let i=t.pop();if(n.has(i.ino))throw new I(D);n.add(i.ino);let s=this.inodeOffset(i.ino);if((this.r32(s+C)&U)!==$)throw new I(D);let o=this.r64(s+z),a=0;for(;a>2)>>>0,paths:[]},e.set(S,A)),A.paths.push(E),(this.r32(w+C)&U)===$&&t.push({ino:h,path:E})}}_+=g}a+=m}}return e}statfs(){let e=this.r32(In),t=this.r32(lt),n=this.r32(Wt),i=typeof this.buffer.maxByteLength=="number"?this.buffer.maxByteLength:this.buffer.byteLength,s=Math.floor(i/e),o=Math.max(t,Math.min(n,s)),a=Atomics.load(this.i32,ut>>2),c=Math.max(0,o-t);return{blockSize:e,totalBlocks:o,freeBlocks:a+c,totalInodes:this.r32(Me),freeInodes:Atomics.load(this.i32,vn>>2),maxName:255}}r32(e){return this.view.getUint32(e,!0)}w32(e,t){this.view.setUint32(e,t,!0)}r64(e){return Number(this.view.getBigUint64(e,!0))}w64(e,t){this.view.setBigUint64(e,BigInt(t),!0)}waitForAtomicChange(e,t){if(this.atomicsWaitAllowed!==!1)try{Atomics.wait(this.i32,e,t),this.atomicsWaitAllowed=!0;return}catch(n){if(!(n instanceof TypeError))throw n;this.atomicsWaitAllowed=!1}for(;Atomics.load(this.i32,e)===t;);}resetAllocationHints(){this.blockAllocHint=this.findNextFreeBlockHint(),this.inodeAllocHint=this.findNextFreeInodeHint()}findNextFreeBlockHint(){let e=this.r32(lt),t=this.r32(xn),n=this.r32(An)*4096;for(let i=t;i>2)+(i>>5),o=i&31;if((Atomics.load(this.i32,s)&1<>2)+(n>>5),s=n&31;if((Atomics.load(this.i32,i)&1<>2;for(;;){if(Atomics.compareExchange(this.i32,e,0,1)===0)return;this.waitForAtomicChange(e,1)}}sbUnlock(){let e=Rn>>2;Atomics.store(this.i32,e,0),Atomics.notify(this.i32,e,1/0)}namespaceLock(){let e=kn>>2;for(;;){if(Atomics.compareExchange(this.i32,e,0,1)===0)return;this.waitForAtomicChange(e,1)}}namespaceUnlock(){let e=kn>>2;Atomics.store(this.i32,e,0),Atomics.notify(this.i32,e,1/0)}withNamespaceLock(e){this.namespaceLock();try{return e()}finally{this.namespaceUnlock()}}resetRestoredRuntimeState(){Atomics.store(this.i32,Rn>>2,0),Atomics.store(this.i32,kn>>2,0),this.u8.fill(0,256,4096);let e=this.r32(Me),t=this.r32(Ye)*4096;for(let n=0;n>5)*4)&1<<(n&31))===0||this.r32(i+B)!==0)continue;let o=this.r32(i+C),a=this.r64(i+z);(o&U)===Ut&&a<=40?(this.u8.fill(0,i+ee,i+ee+40),this.w64(i+z,0)):this.inodeTruncate(n,0),this.inodeFree(n)}}blockAlloc(){let e=this.r32(lt),t=this.r32(An)*4096,n=this.r32(xn),i=this.blockAllocHint>=n&&this.blockAllocHint>2)+(a>>5),l=a&31,p=Atomics.load(this.i32,c);if(p&1<>2,1),this.blockAllocHint=a+1>2)+(e>>5),i=e&31;for(;;){let s=Atomics.load(this.i32,n),o=s&~(1<>2,1),e>=this.r32(xn)&&e>2)>0)return 0;let e=this.r32(lt),t=this.r32(Wt),n=this.r32(es),i=e+n;if(i>t&&(i=t,n=i-e,n===0))return te;let s=i*4096;if(this.buffer.byteLength>2,n),Atomics.add(this.i32,Or>>2,1),this.blockAllocHint=e,0}finally{this.sbUnlock()}}inodeOffset(e){let n=this.r32(Qi)+Math.floor(e/32),i=e%32*128;return n*4096+i}inodeAlloc(){let e=this.r32(Me),t=this.r32(Ye)*4096,n=this.inodeAllocHint>=2&&this.inodeAllocHint>2)+(o>>5),c=o&31,l=Atomics.load(this.i32,a);if(l&1<>2,1),this.inodeAllocHint=o+1>2,1)+1}inodeFree(e){let n=(this.r32(Ye)*4096>>2)+(e>>5),i=e&31;for(;;){let s=Atomics.load(this.i32,n);if((s&1<>2,1),e>=2&&e0&&this.w32(n+Be,i-1),i<=1&&this.r32(n+B)===0&&(this.inodeTruncate(e,0),t=!0)}finally{this.inodeWriteUnlock(e)}t&&this.inodeFree(e)}inodeDropLinkRefLocked(e){let t=this.inodeOffset(e),n=this.r32(t+B);return n>1?(this.w32(t+B,n-1),this.w64(t+q,Date.now()),!1):this.inodeOrphanLocked(e)}inodeOrphanLocked(e){let t=this.inodeOffset(e);if(this.w32(t+B,0),this.w64(t+q,Date.now()),this.r32(t+Be)>0)return!1;let n=this.r32(t+C),i=this.r64(t+z);return(n&U)===Ut&&i<=40?(this.u8.fill(0,t+ee,t+ee+40),this.w64(t+z,0)):this.inodeTruncate(e,0),!0}inodeReadLock(e){let t=this.inodeOffset(e)+ft>>2;for(;;){let n=Atomics.load(this.i32,t);if(n&as){this.waitForAtomicChange(t,n);continue}if(Atomics.compareExchange(this.i32,t,n,n+1)===n)return}}inodeReadUnlock(e){let t=this.inodeOffset(e)+ft>>2;(Atomics.sub(this.i32,t,1)&La)===1&&Atomics.notify(this.i32,t,1)}inodeWriteLock(e){let t=this.inodeOffset(e)+ft>>2;for(;;){let n=Atomics.load(this.i32,t);if(n!==0){this.waitForAtomicChange(t,n);continue}if(Atomics.compareExchange(this.i32,t,0,as)===0)return}}inodeWriteUnlock(e){let t=this.inodeOffset(e)+ft>>2;Atomics.store(this.i32,t,0),Atomics.notify(this.i32,t,1/0)}inodeBlockMap(e,t,n){let i=this.inodeOffset(e);if(t<10){let s=this.r32(i+ee+t*4);if(s!==0)return s;if(!n)return 0;let o=this.blockAllocWithGrow();return o<0||this.w32(i+ee+t*4,o),o}if(t-=10,t<1024){let s=this.r32(i+Zt),o=!1;if(s===0){if(!n)return 0;if(s=this.blockAllocWithGrow(),s<0)return s;this.w32(i+Zt,s),o=!0}let a=s*4096+t*4,c=this.r32(a);if(c!==0)return c;if(!n)return 0;let l=this.blockAllocWithGrow();return l<0?(o&&(this.w32(i+Zt,0),this.blockFree(s)),l):(this.w32(a,l),l)}if(t-=1024,t<1024*1024){let s=Math.floor(t/1024),o=t%1024,a=this.r32(i+dt),c=!1;if(a===0){if(!n)return 0;if(a=this.blockAllocWithGrow(),a<0)return a;this.w32(i+dt,a),c=!0}let l=a*4096+s*4,p=this.r32(l),u=!1;if(p===0){if(!n)return 0;if(p=this.blockAllocWithGrow(),p<0)return c&&(this.w32(i+dt,0),this.blockFree(a)),p;this.w32(l,p),u=!0}let m=p*4096+o*4,_=this.r32(m);if(_!==0)return _;if(!n)return 0;let d=this.blockAllocWithGrow();return d<0?(u&&(this.w32(l,0),this.blockFree(p)),c&&(this.w32(i+dt,0),this.blockFree(a)),d):(this.w32(m,d),d)}return H}inodeReadData(e,t,n,i){let s=this.inodeOffset(e),o=this.r64(s+z);if(t>=o)return 0;t+i>o&&(i=o-t);let a=0,c=0;for(;i>0;){let l=Math.floor(t/4096),p=t%4096,u=4096-p;u>i&&(u=i);let m=this.inodeBlockMap(e,l,!1);if(m<=0)n.fill(0,c,c+u);else{let _=m*4096+p;n.set(this.u8.subarray(_,_+u),c)}c+=u,t+=u,i-=u,a+=u}return a}inodeWriteData(e,t,n,i){let s=this.inodeOffset(e),o=this.r64(s+z);t>o&&this.zeroOldEofTail(e,o);let a=0,c=0;for(;i>0;){let l=Math.floor(t/4096),p=t%4096,u=4096-p;u>i&&(u=i);let m=this.inodeBlockMap(e,l,!0);if(m<0){if(a===0)return m;break}let _=m*4096+p;this.u8.set(n.subarray(c,c+u),_),c+=u,t+=u,i-=u,a+=u}if(a>0&&t>this.r64(s+z)&&this.w64(s+z,t),a>0){let l=Date.now();this.w64(s+ae,l),this.w64(s+q,l),Atomics.add(this.i32,s+ue>>2,1)}return a}zeroInodeRange(e,t,n){for(;t0){let c=a*4096+s;this.u8.fill(0,c,c+o)}t+=o}}zeroOldEofTail(e,t){let n=t%4096;if(n===0)return;let i=Math.floor(t/4096),s=this.inodeBlockMap(e,i,!1);if(s<=0)return;let o=s*4096+n;this.u8.fill(0,o,s*4096+4096)}freeBlocksFrom(e,t){let n=this.inodeOffset(e);for(let o=t;o<10;o++){let a=this.r32(n+ee+o*4);a&&(this.blockFree(a),this.w32(n+ee+o*4,0))}let i=this.r32(n+Zt);if(i){let o=t>10?t-10:0;for(let a=o;a<1024;a++){let c=i*4096+a*4,l=this.r32(c);l&&(this.blockFree(l),this.w32(c,0))}o===0&&(this.blockFree(i),this.w32(n+Zt,0))}let s=this.r32(n+dt);if(s){let o=t>1034?t-10-1024:0,a=Math.floor(o/1024);for(let c=a;c<1024;c++){let l=s*4096+c*4,p=this.r32(l);if(!p)continue;let u=c===a?o%1024:0;for(let m=u;m<1024;m++){let _=p*4096+m*4,d=this.r32(_);d&&(this.blockFree(d),this.w32(_,0))}u===0&&(this.blockFree(p),this.w32(l,0))}a===0&&(this.blockFree(s),this.w32(n+dt,0))}}inodeTruncate(e,t,n=!1){let i=this.inodeOffset(e),s=this.r64(i+z),o=t!==s;if(t>=s){if(t>s&&this.zeroOldEofTail(e,s),this.w64(i+z,t),o||n){let c=Date.now();this.w64(i+ae,c),this.w64(i+q,c),Atomics.add(this.i32,i+ue>>2,1)}return}t%4096!==0&&this.zeroInodeRange(e,t,Math.ceil(t/4096)*4096);let a=Math.ceil(t/4096);if(this.freeBlocksFrom(e,a),this.w64(i+z,t),o||n){let c=Date.now();this.w64(i+ae,c),this.w64(i+q,c),Atomics.add(this.i32,i+ue>>2,1)}}validateFileSize(e){if(!Number.isSafeInteger(e)||e<0)throw new I(H);if(e>ht)throw new I(Ht)}validateSeekPosition(e){if(!Number.isSafeInteger(e))throw new I(fs);if(e<0)throw new I(H);if(e>ht)throw new I(Ht)}touchDirectoryMutation(e){let t=this.inodeOffset(e),n=Date.now();this.w64(t+ae,n),this.w64(t+q,n);let i=Atomics.add(this.i32,t+rs>>2,1)+1>>>0,s=this.dirIndexes.get(e);s&&(s.mutationSequence=i,s.size=this.r64(t+z))}dirNameKey(e){return pt(e)}dirEntryNameMatches(e,t){if(this.view.getUint16(e+6,!0)!==t.length)return!1;for(let i=0;i=P&&n%4===0&&e+n<=t&&i<=n-P}inodeIsAllocated(e){let t=this.r32(Me);if(e<=0||e>=t)return!1;let n=this.r32(Ye)*4096;return(Atomics.load(this.i32,(n>>2)+(e>>5))&1<<(e&31))!==0}rebuildDirIndex(e,t,n,i){let s=new Map,o=[],a=0;for(;a4096-p&&(_=4096-p);let d=p;for(;d=P&&o.push({abs:h,recLen:f});d+=f}a+=_}let c={generation:t,mutationSequence:n,size:i,entries:s,free:o};return this.dirIndexes.set(e,c),c}getDirIndex(e){let t=this.inodeOffset(e),n=this.r64(t+z),i=this.r64(t+le),s=Atomics.load(this.i32,t+rs>>2)>>>0,o=this.dirIndexes.get(e);return o&&o.generation===i&&o.mutationSequence===s&&o.size===n?o:(o&&this.dirIndexes.delete(e),n=0;o--){let a=e.free[o];if(!(a.recLen4096-c&&(u=4096-c);let m=c;for(;mn)return-1;a=c,o+=l}return o===n?a:-1}dirAppendEntry(e,t,n,i=-1){let s=this.inodeOffset(e),o=this.r64(s+z),a=$e(P+t.length),c=o,l=Math.floor(c/4096),p=c%4096,u=0;if(p!==0&&p+a>4096){let d=4096-p,h=0;if(d>=P){if(h=this.inodeBlockMap(e,l,!1),h<=0)return D}else if(i<0&&(i=this.findLastDirEntryInBlock(e,l,p)),i<0)return D;if(u=this.inodeBlockMap(e,l+1,!0),u<0)return u;if(d>=P){let g=h*4096+p;this.w32(g,0),this.view.setUint16(g+4,d,!0),this.view.setUint16(g+6,0,!0)}else{let f=this.view.getUint16(i+4,!0)+d;this.view.setUint16(i+4,f,!0),this.updateDirIndexRecLen(e,i,f)}c=(l+1)*4096,l++,p=0}let m;if(p===0){if(m=u||this.inodeBlockMap(e,l,!0),m<0)return m}else if(m=this.inodeBlockMap(e,l,!1),m<=0)return D;let _=m*4096+p;return this.w32(_,n),this.view.setUint16(_+4,a,!0),this.view.setUint16(_+6,t.length,!0),this.u8.set(t,_+P),this.w64(s+z,c+a),this.touchDirectoryMutation(e),this.updateDirIndexAdd(e,t,n,_,a),0}dirAddEntry(e,t,n){let i=this.getDirIndex(e);if(typeof i=="number")return i;if(i)return this.useDirIndexFreeSlot(i,e,t,n)?0:this.dirAppendEntry(e,t,n);let s=this.inodeOffset(e),o=this.r64(s+z),a=$e(P+t.length),c=-1,l=0;for(;l4096-u&&(d=4096-u);let h=u;for(;hu+d||E>y-P)return D;if(f===0&&y>=a)return this.w32(g,n),this.view.setUint16(g+6,t.length,!0),this.u8.set(t,g+P),this.touchDirectoryMutation(e),this.updateDirIndexAdd(e,t,n,g,y),0;let w=$e(P+E),O=y-w;if(f!==0&&O>=a){this.view.setUint16(g+4,w,!0);let S=g+w;return this.w32(S,n),this.view.setUint16(S+4,O,!0),this.view.setUint16(S+6,t.length,!0),this.u8.set(t,S+P),this.touchDirectoryMutation(e),this.updateDirIndexAdd(e,t,n,S,O),0}c=g,h+=y}l+=d}return this.dirAppendEntry(e,t,n,c)}dirRemoveEntry(e,t){let n=this.getDirIndex(e);if(typeof n=="number")return n;if(n){let a=this.dirNameKey(t),c=n.entries.get(a);if(!c)return Ee;if(this.r32(c.abs)===c.ino&&this.view.getUint16(c.abs+4,!0)===c.recLen&&this.view.getUint16(c.abs+6,!0)===c.nameLen&&this.dirEntryNameMatches(c.abs,t))return this.w32(c.abs,0),n.entries.delete(a),n.free.push({abs:c.abs,recLen:c.recLen}),this.touchDirectoryMutation(e),0;n.entries.delete(a)}let i=this.inodeOffset(e),s=this.r64(i+z),o=0;for(;o4096-c&&(u=4096-c);let m=c;for(;m4096-l&&(m=4096-l);let _=l;for(;_4096-o&&(l=4096-o);let p=o;for(;po+l||d>_-P)throw new I(D);if(m!==0){if(d===1&&this.u8[u+P]===46){p+=_;continue}if(d===2&&this.u8[u+P]===46&&this.u8[u+P+1]===46){p+=_;continue}return!1}p+=_}i+=l}return!0}dirIsAncestor(e,t){let n=t;for(let i=0;i<8*1024;i++){if(n===e)return!0;if(n===1)return!1;let s=this.dirLookup(n,cs);if(s<0||s===n)throw new I(D);n=s}throw new I(D)}pathResolve(e,t){if(!e.startsWith("/"))return Ee;let n=1,i=e.split("/").filter(o=>o.length>0),s=0;for(let o=0;o255)return Ar;let c=fe.encode(a),l;this.inodeReadLock(n);try{let m=this.inodeOffset(n);if((this.r32(m+C)&U)!==$)return Re;l=this.dirLookup(n,c)}finally{this.inodeReadUnlock(n)}if(l<0)return l;let p=this.inodeOffset(l);if((this.r32(p+C)&U)===Ut&&(!(o===i.length-1)||t)){if(++s>8)return us;let _=this.r64(p+z),d;if(_<=40)d=pt(this.u8.subarray(p+ee,p+ee+_));else{let h=new Uint8Array(_);this.inodeReadData(l,0,h,_),d=Xt.decode(h)}if(d.startsWith("/")){n=1;let h=d.split("/").filter(f=>f.length>0),g=i.slice(o+1);i.length=0,i.push(...h,...g),o=-1}else{let h=d.split("/").filter(f=>f.length>0),g=i.slice(o+1);i.length=o,i.push(...h,...g),o--}continue}n=l}return n}pathResolveParent(e){if(!e.startsWith("/"))throw new I(H,"Path must be absolute");let t=e.split("/").filter(c=>c.length>0);if(t.length===0)throw new I(H,"Cannot operate on /");let n=t.pop();if(n.length>255)throw new I(Ar);let i="/"+t.join("/"),s=this.pathResolve(i,!0);if(s<0)throw new I(s);let o=this.inodeOffset(s);if((this.r32(o+C)&U)!==$)throw new I(Re);return{parentIno:s,name:n}}fdAlloc(e,t,n){for(let i=0;i>2;if(Atomics.compareExchange(this.i32,o,0,1)===0)return this.w32(s+is,e),this.w64(s+qe,0),this.w32(s+ss,t),this.w32(s+os,n?1:0),this.inodeAddOpenRef(e)?i:(Atomics.store(this.i32,o,0),Ee)}return ls}fdGet(e){if(e<0||e>=Sn)return null;let t=256+e*24;return Atomics.load(this.i32,t>>2)?{base:t,ino:this.r32(t+is),offset:this.r64(t+qe),flags:this.r32(t+ss),isDir:this.r32(t+os)!==0}:null}fdFree(e){if(e>=0&&e>2,0)}}buildStat(e){let t=this.inodeOffset(e);return{ino:e,generation:this.r64(t+le),dataSequence:this.r32(t+ue),mode:this.r32(t+C),linkCount:this.r32(t+B),size:this.r64(t+z),mtime:this.r64(t+ae),ctime:this.r64(t+q),atime:this.r64(t+Gt),uid:this.r32(t+ts),gid:this.r32(t+ns)}}namespaceEntryIdentity(e){let t=this.inodeOffset(e);return{ino:e,generation:this.r64(t+le),linkCount:this.r32(t+B),mode:this.r32(t+C)}}open(e,t,n=420){return this.withNamespaceLock(()=>this.openUnlocked(e,t,n))}createLazyStub(e,t){return this.withNamespaceLock(()=>{let n=this.openUnlocked(e,Ji|Vt,t);try{let i=this.fdGet(n);if(!i)throw new I(re);this.inodeWriteLock(i.ino);try{return this.inodeTruncate(i.ino,0,!0),this.buildStat(i.ino)}finally{this.inodeWriteUnlock(i.ino)}}finally{this.closeUnlocked(n)}})}replaceIfIdentity(e,t,n,i,s){return this.withNamespaceLock(()=>{let o=this.pathResolve(e,!0);if(o<0||o!==t)return!1;let a=this.inodeOffset(o);if(this.r64(a+le)!==n||this.r32(a+ue)!==i||(this.r32(a+C)&U)!==$t)return!1;this.validateFileSize(s.byteLength),this.inodeWriteLock(o);try{if(this.r64(a+le)!==n||this.r32(a+ue)!==i||this.r64(a+z)!==0)return!1;let c=this.r64(a+ae),l=this.r64(a+q);this.inodeTruncate(o,0,!0);let p=s.byteLength>0?this.inodeWriteData(o,0,s,s.byteLength):0;if(p!==s.byteLength)throw this.inodeTruncate(o,0,!0),Atomics.store(this.i32,a+ue>>2,i),this.w64(a+ae,c),this.w64(a+q,l),new I(p<0?p:te);return!0}finally{this.inodeWriteUnlock(o)}})}replaceManyIfIdentities(e){return e.length===0?!0:this.withNamespaceLock(()=>{let t=[],n=new Set;for(let s of e){this.validateFileSize(s.data.byteLength);let o=-1;for(let a of s.paths){let c=this.pathResolve(a,!0);if(c!==s.expectedIno)continue;let l=this.inodeOffset(c);if(this.r64(l+le)===s.expectedGeneration&&this.r32(l+ue)===s.expectedDataSequence&&(this.r32(l+C)&U)===$t&&this.r64(l+z)===0){o=c;break}}if(o<0)return!1;if(n.has(o))throw new I(H,"duplicate conditional replacement inode");n.add(o),t.push({...s,ino:o})}let i=[...n].sort((s,o)=>s-o);for(let s of i)this.inodeWriteLock(s);try{for(let a of t){let c=this.inodeOffset(a.ino);if(this.r64(c+le)!==a.expectedGeneration||this.r32(c+ue)!==a.expectedDataSequence||(this.r32(c+C)&U)!==$t||this.r64(c+z)!==0)return!1}let s=t.map(a=>{let c=this.inodeOffset(a.ino);return{ino:a.ino,dataSequence:this.r32(c+ue),mtime:this.r64(c+ae),ctime:this.r64(c+q)}}),o=0;try{for(let a of t){o++,this.inodeTruncate(a.ino,0,!0);let c=a.data.byteLength>0?this.inodeWriteData(a.ino,0,a.data,a.data.byteLength):0;if(c!==a.data.byteLength)throw new I(c<0?c:te)}}catch(a){for(let c=o-1;c>=0;c--){let l=s[c],p=this.inodeOffset(l.ino);this.inodeTruncate(l.ino,0,!0),Atomics.store(this.i32,p+ue>>2,l.dataSequence),this.w64(p+ae,l.mtime),this.w64(p+q,l.ctime)}throw a}return!0}finally{for(let s=i.length-1;s>=0;s--)this.inodeWriteUnlock(i[s])}})}openUnlocked(e,t,n=420){let i=t&On,s=(t&Vt)!==0,o=(t&Rr)!==0;if(s&&o){let u=this.pathResolve(e,!1);if(u>=0)throw new I(mt);if(u!==Ee)throw new I(u)}let a=this.pathResolve(e,!0);if(a<0&&a===Ee&&s){let{parentIno:u,name:m}=this.pathResolveParent(e);this.inodeWriteLock(u);try{let _=fe.encode(m),d=this.dirLookup(u,_);if(d>=0){if(o)throw new I(mt);a=d}else{let h=this.inodeAlloc();if(h<0)throw new I(te);let g=this.inodeOffset(h);this.w32(g+C,$t|n&4095),this.w32(g+B,1),this.w64(g+z,0);let f=Date.now();this.w64(g+Gt,f),this.w64(g+ae,f),this.w64(g+q,f);let y=this.dirAddEntry(u,_,h);if(y<0)throw this.inodeFree(h),new I(y);a=h}}finally{this.inodeWriteUnlock(u)}}if(a<0)throw new I(a);let c=this.inodeOffset(a),l=this.r32(c+C);if((l&U)===$&&i!==ct)throw new I(je);if(t&Ia&&(l&U)!==$)throw new I(Re);if(t&Yt){if((l&U)===$)throw new I(je);this.inodeWriteLock(a),this.inodeTruncate(a,0,!0),this.inodeWriteUnlock(a)}let p=this.fdAlloc(a,t,!1);if(p<0)throw new I(p);return p}close(e){this.withNamespaceLock(()=>this.closeUnlocked(e))}closeUnlocked(e){let t=this.fdGet(e);if(!t)throw new I(re);this.fdFree(e),this.inodeDropOpenRef(t.ino)}read(e,t){let n=this.fdGet(e);if(!n)throw new I(re);let i=this.inodeOffset(n.ino);if((this.r32(i+C)&U)===$)throw new I(je);this.inodeReadLock(n.ino);try{let o=this.inodeReadData(n.ino,n.offset,t,t.length),a=256+e*24;return this.w64(a+qe,n.offset+o),o}finally{this.inodeReadUnlock(n.ino)}}readAt(e,t,n){let i=this.fdGet(e);if(!i)throw new I(re);let s=this.inodeOffset(i.ino);if((this.r32(s+C)&U)===$)throw new I(je);this.validateSeekPosition(n),this.inodeReadLock(i.ino);try{return this.inodeReadData(i.ino,n,t,t.length)}finally{this.inodeReadUnlock(i.ino)}}write(e,t){let n=this.fdGet(e);if(!n)throw new I(re);if((n.flags&On)===ct)throw new I(re);this.inodeWriteLock(n.ino);try{let s=n.offset;if(n.flags&Oa){let c=this.inodeOffset(n.ino);s=this.r64(c+z)}if(!Number.isSafeInteger(s)||s<0)throw new I(H);if(s>ht||t.length>ht-s)throw new I(Ht);let o=this.inodeWriteData(n.ino,s,t,t.length);if(o<0)return o;let a=256+e*24;return this.w64(a+qe,s+o),o}finally{this.inodeWriteUnlock(n.ino)}}writeAt(e,t,n){let i=this.fdGet(e);if(!i)throw new I(re);if((i.flags&On)===ct)throw new I(re);this.validateSeekPosition(n),this.inodeWriteLock(i.ino);try{if(n>ht||t.length>ht-n)throw new I(Ht);return this.inodeWriteData(i.ino,n,t,t.length)}finally{this.inodeWriteUnlock(i.ino)}}lseek(e,t,n){let i=this.fdGet(e);if(!i)throw new I(re);let s;if(n===va)s=t;else if(n===Aa)s=i.offset+t;else if(n===xa){let a=this.inodeOffset(i.ino);s=this.r64(a+z)+t}else throw new I(H);this.validateSeekPosition(s);let o=256+e*24;return this.w64(o+qe,s),s}ftruncate(e,t){let n=this.fdGet(e);if(!n)throw new I(re);if((n.flags&On)===ct)throw new I(re);this.validateFileSize(t),this.inodeWriteLock(n.ino);try{this.inodeTruncate(n.ino,t,!0)}finally{this.inodeWriteUnlock(n.ino)}}fstat(e){let t=this.fdGet(e);if(!t)throw new I(re);this.inodeReadLock(t.ino);try{return this.buildStat(t.ino)}finally{this.inodeReadUnlock(t.ino)}}stat(e){return this.withNamespaceLock(()=>this.statUnlocked(e))}statUnlocked(e){let t=this.pathResolve(e,!0);if(t<0)throw new I(t);this.inodeReadLock(t);try{return this.buildStat(t)}finally{this.inodeReadUnlock(t)}}lstat(e){return this.withNamespaceLock(()=>this.lstatUnlocked(e))}lstatUnlocked(e){let t=this.pathResolve(e,!1);if(t<0)throw new I(t);this.inodeReadLock(t);try{return this.buildStat(t)}finally{this.inodeReadUnlock(t)}}unlink(e){return this.withNamespaceLock(()=>this.unlinkUnlocked(e))}unlinkUnlocked(e){let{parentIno:t,name:n}=this.pathResolveParent(e),i=fe.encode(n),s=e.length>1&&e.endsWith("/");this.inodeWriteLock(t);try{let o=this.dirLookup(t,i);if(o<0)throw new I(o);let a=this.inodeOffset(o),c=this.r32(a+C);if(s&&(c&U)!==$)throw new I(Re);if((c&U)===$)throw new I(je);let l=this.namespaceEntryIdentity(o),p=this.dirRemoveEntry(t,i);if(p<0)throw new I(p);let u=!1;this.inodeWriteLock(o);try{u=this.inodeDropLinkRefLocked(o)}finally{this.inodeWriteUnlock(o)}return u&&this.inodeFree(o),l}finally{this.inodeWriteUnlock(t)}}rename(e,t){return this.withNamespaceLock(()=>this.renameUnlocked(e,t))}renameUnlocked(e,t){let{parentIno:n,name:i}=this.pathResolveParent(e),{parentIno:s,name:o}=this.pathResolveParent(t);if(Ir(i)||Ir(o))throw new I(H);let a=fe.encode(i),c=fe.encode(o),l=e.length>1&&e.endsWith("/"),p=t.length>1&&t.endsWith("/"),u=Math.min(n,s),m=Math.max(n,s);this.inodeWriteLock(u),u!==m&&this.inodeWriteLock(m);try{let _=this.dirLookup(n,a);if(_<0)throw new I(_);let d=this.inodeOffset(_),g=this.r32(d+C)&U,f=this.namespaceEntryIdentity(_);if((l||p)&&g!==$)throw new I(Re);if(g===$&&this.dirIsAncestor(_,s))throw new I(H);let y=this.dirLookup(s,c),E=!1,w;if(y>=0){if(y===_)return{source:f,replaced:f};w=this.namespaceEntryIdentity(y);let S=this.inodeOffset(y),v=this.r32(S+C)&U;if(g===$&&v!==$)throw new I(Re);if(g!==$&&v===$)throw new I(je);let x=!1,L=y===n||y===s;L||this.inodeWriteLock(y);try{if(v===$&&!this.dirIsEmpty(y))throw new I(xr);let N=this.dirReplaceEntryIno(s,c,_);if(N<0)throw new I(N);x=v===$?this.inodeOrphanLocked(y):this.inodeDropLinkRefLocked(y)}finally{L||this.inodeWriteUnlock(y)}x&&this.inodeFree(y),E=v===$}else{let S=this.dirAddEntry(s,c,_);if(S<0)throw new I(S)}let O=this.dirRemoveEntry(n,a);if(O<0)throw new I(O);if(g===$){if(n!==s){let S=this.inodeOffset(n);this.w32(S+B,this.r32(S+B)-1);let A=this.inodeOffset(s);this.w32(A+B,this.r32(A+B)+1),this.inodeWriteLock(_);try{let v=this.dirReplaceEntryIno(_,cs,s);if(v<0)throw new I(v);this.w64(d+q,Date.now())}finally{this.inodeWriteUnlock(_)}}if(E){let S=this.inodeOffset(s);this.w32(S+B,this.r32(S+B)-1)}}else if(E){let S=this.inodeOffset(s);this.w32(S+B,this.r32(S+B)-1)}return{source:f,replaced:w}}finally{u!==m&&this.inodeWriteUnlock(m),this.inodeWriteUnlock(u)}}mkdir(e,t=493){this.withNamespaceLock(()=>this.mkdirUnlocked(e,t))}mkdirUnlocked(e,t=493){let{parentIno:n,name:i}=this.pathResolveParent(e),s=fe.encode(i);this.inodeWriteLock(n);try{if(this.dirLookup(n,s)>=0)throw new I(mt);let a=this.inodeAlloc();if(a<0)throw new I(te);let c=this.inodeOffset(a);this.w32(c+C,$|t),this.w32(c+B,2),this.w64(c+z,0);let l=Date.now();this.w64(c+Gt,l),this.w64(c+ae,l),this.w64(c+q,l);let p=this.blockAllocWithGrow();if(p<0)throw this.inodeFree(a),new I(te);this.w32(c+ee,p);let u=p*4096,m=$e(P+1),_=$e(P+2);this.w32(u,a),this.view.setUint16(u+4,m,!0),this.view.setUint16(u+6,1,!0),this.u8[u+P]=46;let d=u+m;this.w32(d,n),this.view.setUint16(d+4,_,!0),this.view.setUint16(d+6,2,!0),this.u8[d+P]=46,this.u8[d+P+1]=46,this.w64(c+z,m+_);let h=this.dirAddEntry(n,s,a);if(h<0)throw this.blockFree(p),this.inodeFree(a),new I(h);let g=this.inodeOffset(n);this.w32(g+B,this.r32(g+B)+1)}finally{this.inodeWriteUnlock(n)}}rmdir(e){this.withNamespaceLock(()=>this.rmdirUnlocked(e))}rmdirUnlocked(e){let{parentIno:t,name:n}=this.pathResolveParent(e);if(Ir(n))throw new I(H);let i=fe.encode(n);this.inodeWriteLock(t);try{let s=this.dirLookup(t,i);if(s<0)throw new I(s);let o=this.inodeOffset(s);if((this.r32(o+C)&U)!==$)throw new I(Re);let c=!1;this.inodeWriteLock(s);try{if(!this.dirIsEmpty(s))throw new I(xr);let p=this.dirRemoveEntry(t,i);if(p<0)throw new I(p);c=this.inodeOrphanLocked(s)}finally{this.inodeWriteUnlock(s)}c&&this.inodeFree(s);let l=this.inodeOffset(t);this.w32(l+B,this.r32(l+B)-1)}finally{this.inodeWriteUnlock(t)}}symlink(e,t){this.withNamespaceLock(()=>this.symlinkUnlocked(e,t))}symlinkUnlocked(e,t){let{parentIno:n,name:i}=this.pathResolveParent(t),s=fe.encode(i),o=fe.encode(e);this.inodeWriteLock(n);try{if(this.dirLookup(n,s)>=0)throw new I(mt);let c=this.inodeAlloc();if(c<0)throw new I(te);let l=this.inodeOffset(c);if(this.w32(l+C,Ut|511),this.w32(l+B,1),o.length<=40)this.u8.set(o,l+ee),this.w64(l+z,o.length);else{this.w64(l+z,0);let u=this.inodeWriteData(c,0,o,o.length);if(u!==o.length)throw u>0&&this.inodeTruncate(c,0),this.inodeFree(c),new I(u<0?u:te)}let p=this.dirAddEntry(n,s,c);if(p<0)throw o.length<=40?(this.u8.fill(0,l+ee,l+ee+40),this.w64(l+z,0)):this.inodeTruncate(c,0),this.inodeFree(c),new I(p)}finally{this.inodeWriteUnlock(n)}}chmod(e,t){this.withNamespaceLock(()=>this.chmodUnlocked(e,t))}chmodUnlocked(e,t){let n=this.pathResolve(e,!0);if(n<0)throw new I(n);this.inodeWriteLock(n);try{let i=this.inodeOffset(n),s=this.r32(i+C);this.w32(i+C,s&U|t&4095),this.w64(i+q,Date.now())}finally{this.inodeWriteUnlock(n)}}fchmod(e,t){let n=this.fdGet(e);if(!n)throw new I(re);this.inodeWriteLock(n.ino);try{let i=this.inodeOffset(n.ino),s=this.r32(i+C);this.w32(i+C,s&U|t&4095),this.w64(i+q,Date.now())}finally{this.inodeWriteUnlock(n.ino)}}chown(e,t,n){this.withNamespaceLock(()=>this.chownUnlocked(e,t,n))}chownUnlocked(e,t,n){let i=this.pathResolve(e,!0);if(i<0)throw new I(i);this.inodeWriteLock(i);try{this.chownInodeUnlocked(i,t,n)}finally{this.inodeWriteUnlock(i)}}fchown(e,t,n){let i=this.fdGet(e);if(!i)throw new I(re);this.inodeWriteLock(i.ino);try{this.chownInodeUnlocked(i.ino,t,n)}finally{this.inodeWriteUnlock(i.ino)}}lchown(e,t,n){this.withNamespaceLock(()=>this.lchownUnlocked(e,t,n))}lchownUnlocked(e,t,n){let i=this.pathResolve(e,!1);if(i<0)throw new I(i);this.inodeWriteLock(i);try{this.chownInodeUnlocked(i,t,n)}finally{this.inodeWriteUnlock(i)}}chownInodeUnlocked(e,t,n){let i=this.inodeOffset(e);t!==ji&&this.w32(i+ts,t),n!==ji&&this.w32(i+ns,n);let s=this.r32(i+C);(s&U)===$t&&(s&Sa)!==0&&this.w32(i+C,s&~(Ea|wa)),this.w64(i+q,Date.now())}utimens(e,t,n,i,s){this.withNamespaceLock(()=>this.utimensUnlocked(e,t,n,i,s))}utimensUnlocked(e,t,n,i,s){let o=this.pathResolve(e,!0);if(o<0)throw new I(o);this.inodeWriteLock(o);try{let a=this.inodeOffset(o),c=1073741823,l=1073741822,p=Date.now();if(n!==l){let u=n===c?p:t*1e3+Math.floor(n/1e6);this.w64(a+Gt,u)}if(s!==l){let u=s===c?p:i*1e3+Math.floor(s/1e6);this.w64(a+ae,u)}this.w64(a+q,p)}finally{this.inodeWriteUnlock(o)}}link(e,t){return this.withNamespaceLock(()=>this.linkUnlocked(e,t))}linkUnlocked(e,t){let n=this.pathResolve(e,!1);if(n<0)throw new I(n);let i=this.inodeOffset(n);if((this.r32(i+C)&U)===$)throw new I(Ra);let{parentIno:o,name:a}=this.pathResolveParent(t),c=fe.encode(a);this.inodeWriteLock(o);try{if(this.dirLookup(o,c)>=0)throw new I(mt);let p=this.dirAddEntry(o,c,n);if(p<0)throw new I(p);this.inodeWriteLock(n);try{let u=this.r32(i+B);this.w32(i+B,u+1),this.w64(i+q,Date.now())}finally{this.inodeWriteUnlock(n)}return{...this.namespaceEntryIdentity(n),linkCount:this.r32(i+B)}}finally{this.inodeWriteUnlock(o)}}readlink(e){return this.withNamespaceLock(()=>this.readlinkUnlocked(e))}readlinkUnlocked(e){let t=this.pathResolve(e,!1);if(t<0)throw new I(t);let n=this.inodeOffset(t);if((this.r32(n+C)&U)!==Ut)throw new I(H);let s=this.r64(n+z);if(s<=40)return pt(this.u8.subarray(n+ee,n+ee+s));this.inodeReadLock(t);try{let o=new Uint8Array(s);return this.inodeReadData(t,0,o,s),Xt.decode(o)}finally{this.inodeReadUnlock(t)}}opendir(e){return this.withNamespaceLock(()=>this.opendirUnlocked(e))}opendirUnlocked(e){let t=this.pathResolve(e,!0);if(t<0)throw new I(t);let n=this.inodeOffset(t);if((this.r32(n+C)&U)!==$)throw new I(Re);let s=this.fdAlloc(t,ct,!0);if(s<0)throw new I(s);return s}readdirEntry(e){return this.withNamespaceLock(()=>this.readdirEntryUnlocked(e))}readdirEntryUnlocked(e){let t=this.fdGet(e);if(!t||!t.isDir)throw new I(re);let n=this.inodeOffset(t.ino),i=this.r64(n+z);for(;t.offset=this.r32(Me))throw new I(D);let h=this.r32(Ye)*4096;if((this.r32(h+(p>>5)*4)&1<<(p&31))===0)throw new I(D);let f=pt(this.u8.subarray(l+P,l+P+m)),y=this.buildStat(p);return this.w64(d+qe,_),t.offset=_,{name:f,stat:y}}return null}closedir(e){this.close(e)}readdir(e){let t=this.opendir(e),n=[];try{let i;for(;(i=this.readdirEntry(t))!==null;)i.name!=="."&&i.name!==".."&&n.push(i.name)}finally{this.closedir(t)}return n}writeFile(e,t){let n=typeof t=="string"?fe.encode(t):t,i=this.open(e,Ji|Vt|Yt);try{this.write(i,n)}finally{this.close(i)}}readFile(e){let t=this.open(e,ct);try{let n=this.fstat(t),i=new Uint8Array(n.size);return this.read(t,i),i}finally{this.close(t)}}readFileText(e){return Xt.decode(this.readFile(e))}};function ds(r,e){let t=new Map,n=new Map;for(let o of r){if(t.has(o.path))throw new Error(`${e} duplicates path ${o.path}`);if(t.set(o.path,o),o.type==="file"){if(!o.inodeGroup)throw new Error(`${e} file ${o.path} has no inode group`);if(n.has(o.inodeGroup))throw new Error(`${e} inode group ${o.inodeGroup} has multiple files`);n.set(o.inodeGroup,o)}}let i=new Set,s=new Map;for(let o of r){if(o.type!=="hardlink"||s.has(o.path))continue;let a=[],c=o,l;for(;c.type==="hardlink";){let u=s.get(c.path);if(u){l=u;break}if(i.has(c.path))throw new Error(`${e} hardlink cycle reaches ${c.path}`);if(i.add(c.path),a.push(c),!c.target)throw new Error(`${e} hardlink ${c.path} has no target`);let m=t.get(c.target);if(!m)throw new Error(`${e} hardlink ${c.path} target ${c.target} is missing`);if(m.type!=="file"&&m.type!=="hardlink"||!c.inodeGroup||m.inodeGroup!==c.inodeGroup||m.size!==c.size||m.mode!==c.mode)throw new Error(`${e} hardlink ${c.path} has an invalid target`);c=m}l??=c.type==="file"?c:void 0;let p=n.get(o.inodeGroup??"");if(!l||l!==p)throw new Error(`${e} hardlink ${o.path} does not resolve to its inode`);for(let u=a.length-1;u>=0;u-=1){let m=a[u];if(n.get(m.inodeGroup??"")!==l)throw new Error(`${e} hardlink ${m.path} does not resolve to its inode`);i.delete(m.path),s.set(m.path,l)}}return{canonicalByGroup:n,canonicalTargetByPath:s}}var de={maxArchiveBytes:268435456,maxExpandedBytes:268435456,maxPayloadBytes:268435456,maxEntries:1e5,maxPathBytes:4096,maxSymlinkTargetBytes:65536,maxStringBytes:8192,maxTransportsPerTree:8,maxActivationCapabilities:32,maxActivationRoots:64,maxActivationCapabilityBytes:255},Ie={maxArchiveBytes:512*1024*1024,maxExpandedBytes:512*1024*1024,maxPayloadBytes:512*1024*1024,maxEntries:1e5,maxGroups:512};function hs(r,e="Deferred tree collection"){for(let[t,n]of Object.entries(r))if(!Number.isSafeInteger(n)||n<0)throw new Error(`${e} ${t} usage is invalid`);if(r.groups>Ie.maxGroups)throw new Error(`${e} exceeds the ${Ie.maxGroups}-group cap`);if(r.archiveBytes>Ie.maxArchiveBytes)throw new Error(`${e} exceeds the archive-byte cap`);if(r.expandedBytes>Ie.maxExpandedBytes)throw new Error(`${e} exceeds the expansion cap`);if(r.payloadBytes>Ie.maxPayloadBytes)throw new Error(`${e} exceeds the payload-byte cap`);if(r.entries>Ie.maxEntries)throw new Error(`${e} exceeds the entry-count cap`)}var _t="/home/linuxbrew/.linuxbrew",_s=[["@@HOMEBREW_PREFIX@@",_t],["@@HOMEBREW_CELLAR@@",`${_t}/Cellar`],["@@HOMEBREW_REPOSITORY@@",_t],["@@HOMEBREW_LIBRARY@@",`${_t}/Library`],["@@HOMEBREW_PERL@@",`${_t}/opt/perl/bin/perl`]],kr="@@HOMEBREW_JAVA@@",Na=/^openjdk(?:@\d+(?:\.\d+)*)?/,yt=new TextEncoder,Fa=[..._s.map(([r])=>r),kr].map(r=>({placeholder:r,bytes:yt.encode(r)}));function ys(r){let e;try{e=JSON.parse(new TextDecoder("utf-8",{fatal:!0}).decode(r))}catch(a){throw new Error("INSTALL_RECEIPT.json is not valid UTF-8 JSON: "+Ma(a))}if(typeof e!="object"||e===null||Array.isArray(e))throw new Error("INSTALL_RECEIPT.json must contain an object");let t=e,n=t.changed_files;if(n!=null&&!Array.isArray(n))throw new Error("INSTALL_RECEIPT.json changed_files must be an array or null when present");let i=Array.isArray(n)?n:[];if(i.length>1e5)throw new Error(`INSTALL_RECEIPT.json declares ${i.length} changed files, limit 100000`);let s=[],o=new Set;for(let[a,c]of i.entries()){if(typeof c!="string")throw new Error(`INSTALL_RECEIPT.json changed_files[${a}] is not a string`);if(Ka(c,"Homebrew changed file"),o.has(c))throw new Error(`INSTALL_RECEIPT.json repeats changed file ${c}`);o.add(c),s.push(c)}return{changedFiles:s,runtimeDependencies:t.runtime_dependencies}}function gs(r,e,t){let n=r;for(let[o,a]of _s)n=ms(n,yt.encode(o),yt.encode(a));let i=yt.encode(kr);if(ps(n,i)){let o=Ca(e.runtimeDependencies);if(o===void 0)throw new Error(`Homebrew changed file ${t} uses ${kr} without exactly one OpenJDK runtime dependency`);n=ms(n,i,yt.encode(o))}let s=Fa.find(({bytes:o})=>ps(n,o));if(s!==void 0)throw new Error(`Homebrew changed file ${t} retains ${s.placeholder}`);return n}function Ca(r){if(!Array.isArray(r))return;let e=[];for(let n of r){if(typeof n!="object"||n===null||Array.isArray(n))continue;let i=n,s=typeof i.full_name=="string"?i.full_name.split("/").at(-1):typeof i.name=="string"?i.name.split("/").at(-1):void 0,o=s===void 0?null:Na.exec(s);s!==void 0&&o?.[0]===s&&e.push(s)}let t=[...new Set(e)];return t.length===1?`${_t}/opt/${t[0]}/libexec`:void 0}function Ka(r,e){if(r.length===0||r.startsWith("/")||r.includes("\\")||r.includes("\0")||Da(r)||yt.encode(r).byteLength>4096||r.split("/").some(t=>t===""||t==="."||t===".."))throw new Error(`${e} has an unsafe path segment: ${r}`)}function Da(r){for(let e=0;e57343)){if(t<=56319&&e+1=56320&&r.charCodeAt(e+1)<=57343){e+=1;continue}return!0}}return!1}function ps(r,e){if(e.byteLength===0||e.byteLength>r.byteLength)return!1;e:for(let t=0;t<=r.byteLength-e.byteLength;t+=1){for(let n=0;nUn||r.includes("\0")||r.includes("\\"))throw new Error(`Lazy archive mount prefix must be an absolute POSIX path: ${JSON.stringify(r)}`);let e=r.replace(/\/+$/,"");if(e==="")return"/";if(e.slice(1).split("/").some(n=>n===""||n==="."||n===".."))throw new Error(`Lazy archive mount prefix is not canonical: ${JSON.stringify(r)}`);return e}function Uc(r,e,t,n){let i=Wn(t),s=new Map,o=e.map(a=>{let c=a.fileName,l=`Lazy archive ${JSON.stringify(r)} member ${JSON.stringify(c)}`;if(c.length===0)throw new Error(`${l} has an empty path`);if(c.includes("\0"))throw new Error(`${l} contains a NUL byte`);if(c.includes("\\"))throw new Error(`${l} contains a backslash`);if(c.startsWith("/")||/^[A-Za-z]:\//.test(c))throw new Error(`${l} must be relative, not absolute`);if(a.isDirectory&&a.isSymlink)throw new Error(`${l} has conflicting directory and symlink types`);if(a.isDirectory!==c.endsWith("/"))throw new Error(`${l} has inconsistent directory metadata`);let p=a.isDirectory?c.slice(0,-1):c,u=p.split("/");if(p.length===0||u.some(m=>m===""||m==="."||m===".."))throw new Error(`${l} is not a canonical relative POSIX path`);if(s.has(p))throw new Error(`${l} collides with another member at ${JSON.stringify(p)}`);if(a.isSymlink&&!n?.has(c))throw new Error(`Lazy archive symlink target was not provided: ${c}`);return s.set(p,a),{entry:a,archivePath:p,vfsPath:i==="/"?`/${p}`:`${i}/${p}`}});for(let{archivePath:a}of o){let c=a.split("/");for(let l=1;lOt)throw new Error(`VFS image metadata exceeds ${Ot} bytes`);let e;try{e=JSON.parse(new TextDecoder().decode(r))}catch(t){let n=t instanceof Error?t.message:String(t);throw new Error(`Invalid VFS image metadata JSON: ${n}`)}return Qr(e)}function Zc(r){if(r===null)return new Uint8Array(0);let e=Qr(r),t=new TextEncoder().encode(JSON.stringify(e));if(t.byteLength>Ot)throw new Error(`VFS image metadata exceeds ${Ot} bytes`);return t}function Hc(r){return r.byteLength>=en.length&&r[0]===en[0]&&r[1]===en[1]&&r[2]===en[2]&&r[3]===en[3]?sl(r):r}function Ln(r){let e=Hc(r);if(e.byteLengthzn)throw new Error(`VFS image lazy metadata exceeds ${zn} bytes`);if(r.byteLengthNn)throw new Error(`VFS image lazy archive metadata exceeds ${Nn} bytes`);if(r.byteLength=0?n:void 0}function Yc(r){return r===408||r===429||r>=500&&r<=599}function qc(r,e=Date.now()){let t=r?.get("retry-after")?.trim();if(!t)return;let n;if(/^\d+$/.test(t))n=Number(t)*1e3;else{let i=Date.parse(t);if(!Number.isFinite(i))return;n=Math.max(0,i-e)}if(!(!Number.isSafeInteger(n)||n<0))return Math.min(n,Qs)}function jc(r){if(!(typeof r!="object"||r===null||!("cause"in r)))return r.cause}function eo(r){if(!(typeof r!="object"||r===null||!("name"in r)))return typeof r.name=="string"?r.name:void 0}function to(r){if(!(typeof r!="object"||r===null||!("code"in r)))return typeof r.code=="string"?r.code:void 0}function no(r,e){let t=new Set,n=r;for(let i=0;n!==void 0&&i<8;i+=1){if(t.has(n))return!1;if(t.add(n),e(n))return!0;n=jc(n)}return!1}function ro(r){return no(r,e=>eo(e)==="AbortError"||to(e)==="ABORT_ERR")}function Jc(r){return ro(r)?!1:no(r,e=>{let t=eo(e),n=to(e);return e instanceof TypeError||t==="NetworkError"||t==="TimeoutError"||n!==void 0&&$c.has(n)})}function Qc(r,e){if(r instanceof Dn){if(!Yc(r.status))return null;if(r.retryAfterMs!==void 0)return r.retryAfterMs}else if(!Jc(r))return null;return Math.min(Mc*2**e,Qs)}function ie(r){if(r?.aborted)throw r.reason}function el(r,e){return ie(e),r===0?Promise.resolve():new Promise((t,n)=>{let i=setTimeout(()=>a(!1),r),s=()=>a(!0,e.reason),o=!1;function a(c,l){o||(o=!0,clearTimeout(i),e?.removeEventListener("abort",s),c?n(l):t())}e?.addEventListener("abort",s,{once:!0}),e?.aborted&&s()})}async function Hr(r,e){try{await r.body?.cancel(e)}catch{}}function tl(r,e){if(r.length===1)return r[0];let t=new Uint8Array(e),n=0;for(let i of r)t.set(i,n),n+=i.byteLength;return t}function tn(r){if(r===void 0)return;if(typeof r!="object"||r===null||Array.isArray(r))throw new Error("Lazy archive integrity must be an object");let e=r;if(Object.keys(e).length!==2||!("sha256"in e)||!("bytes"in e))throw new Error("Lazy archive integrity has unexpected fields");if(typeof e.sha256!="string"||!Bc.test(e.sha256))throw new Error("Lazy archive integrity has an invalid SHA-256 digest");if(!Number.isSafeInteger(e.bytes)||Number(e.bytes)<=0||Number(e.bytes)>Gs)throw new Error(`Lazy archive integrity byte count must be between 1 and ${Gs}`);return{sha256:e.sha256,bytes:Number(e.bytes)}}function Je(r,e,t){if(typeof r!="object"||r===null||Array.isArray(r))throw new Error(`${t} must be an object`);let n=r;if(Object.keys(n).length!==e.length||e.some(s=>!Object.prototype.hasOwnProperty.call(n,s)))throw new Error(`${t} has unexpected or missing fields`);return n}function qr(r,e,t,n){if(typeof r!="object"||r===null||Array.isArray(r))throw new Error(`${n} must be an object`);let i=r,s=new Set(e);if(Object.keys(i).some(o=>!s.has(o))||t.some(o=>!Object.prototype.hasOwnProperty.call(i,o)))throw new Error(`${n} has unexpected or missing fields`);return i}function Le(r,e,t,n){if(!Array.isArray(r)||r.lengthn)throw new Error(`${e} must contain ${t} to ${n} items`);return r}function Ue(r,e,t){if(typeof r!="string"||r.length===0||r.includes("\0")||new TextEncoder().encode(r).byteLength>t)throw new Error(`${e} is invalid or exceeds ${t} bytes`);return r}function se(r,e,t,n){if(!Number.isSafeInteger(r)||Number(r)n)throw new Error(`${e} must be an integer between ${t} and ${n}`);return Number(r)}function Mn(r,e=1){let t=r,n=typeof t=="object"&&t!==null&&!Array.isArray(t)&&t.source!==void 0,i=typeof t=="object"&&t!==null&&!Array.isArray(t)&&t.modePolicy!==void 0,s=Je(r,["decoder","mediaType","sha256","bytes","expandedBytes","sourceEntryCount","transports",...i?["modePolicy"]:[],...n?["source"]:[]],"Lazy tree content"),o=s.decoder==="zip-v1"?"application/zip":s.decoder==="homebrew-bottle-tar-gzip-v1"?"application/vnd.oci.image.layer.v1.tar+gzip":null;if(o===null||s.mediaType!==o)throw new Error("Lazy tree decoder and media type are inconsistent");let a=tn({sha256:s.sha256,bytes:s.bytes});if(!a)throw new Error("Lazy tree integrity is required");let c=Le(s.transports,"Lazy tree transports",e,de.maxTransportsPerTree).map((_,d)=>Ue(_,`Lazy tree transport ${d}`,Jr));if(new Set(c).size!==c.length)throw new Error("Lazy tree transports contain duplicates");let l=se(s.expandedBytes,"Lazy tree expanded byte count",0,Nc),p=se(s.sourceEntryCount,"Lazy tree source entry count",1,It),u=n?nl(s.source,s.decoder):void 0,m=i?s.modePolicy:void 0;if(m!==void 0&&(m!=="portable-posix-v1"||s.decoder!=="zip-v1"||n))throw new Error("Lazy tree mode policy is invalid for its decoder");if(u!==void 0&&u.entries.length!==p)throw new Error("Lazy tree source inventory count differs from its content");return{decoder:s.decoder,mediaType:o,sha256:a.sha256,bytes:a.bytes,expandedBytes:l,sourceEntryCount:p,transports:c,...m===void 0?{}:{modePolicy:m},...u===void 0?{}:{source:u}}}function io(r){let e={groups:r.length,archiveBytes:0,expandedBytes:0,payloadBytes:0,entries:0};for(let t of r)t.content===void 0||t.inventory===void 0||(e.archiveBytes+=t.content.bytes,e.expandedBytes+=t.content.expandedBytes,e.payloadBytes+=t.inventory.filter(n=>n.type==="file").reduce((n,i)=>n+i.size,0),e.entries+=t.inventory.length+(t.content.source?.entries.length??0));return e}function jr(r){hs(r,"Serialized lazy tree collection")}function qs(r){jr(io(r))}function nl(r,e){if(e!=="homebrew-bottle-tar-gzip-v1")throw new Error("Lazy tree source inventory is valid only for original bottles");let t=Je(r,["schema","kind","entries"],"Lazy tree source inventory");if(t.schema!==1||t.kind!=="homebrew-bottle-tar-gzip-v1")throw new Error("Lazy tree source inventory has an unsupported identity");let n=new Map,i=Le(t.entries,"Lazy tree source entries",1,It).map((o,a)=>{let c=o,l=typeof c=="object"&&c!==null&&!Array.isArray(c)?c.type:void 0,p=l==="directory"||l==="file"?["sourcePath","type","mode","size"]:l==="symlink"||l==="hardlink"?["sourcePath","type","mode","size","target"]:null;if(p===null)throw new Error(`Lazy tree source entry ${a} has invalid type`);let u=Je(o,p,`Lazy tree source entry ${a}`),m=pe(u.sourcePath,!1,`Lazy tree source entry ${a} path`);if(n.has(m))throw new Error(`Lazy tree source inventory duplicates ${m}`);let _=se(u.mode,`Lazy tree source entry ${m} mode`,0,4095),d=se(u.size,`Lazy tree source entry ${m} size`,0,Fn),h;if((l==="directory"||l==="symlink"||l==="hardlink")&&d!==0)throw new Error(`Lazy tree source ${m} has payload for ${String(l)}`);l==="symlink"?h=Ue(u.target,`Lazy tree source symlink ${m} target`,Js):l==="hardlink"&&(h=pe(u.target,!1,`Lazy tree source hardlink ${m} target`));let g={sourcePath:m,type:l,mode:_,size:d,...h===void 0?{}:{target:h}};return n.set(m,g),g}),s=i.map(o=>o.sourcePath);if(s.some((o,a)=>a>0&&s[a-1]>=o))throw new Error("Lazy tree source inventory is not in canonical path order");return{schema:1,kind:"homebrew-bottle-tar-gzip-v1",entries:i}}function so(r){let e=new Map(r.map(n=>[n.sourcePath,n])),t=new Map;for(let n of r){if(n.type!=="hardlink"||t.has(n.sourcePath))continue;let i=[],s=new Set,o=n,a;for(;o.type==="hardlink"&&(a=t.get(o.sourcePath),a===void 0);){if(s.has(o.sourcePath))throw new Error(`Lazy tree source hardlink cycle includes ${o.sourcePath}`);s.add(o.sourcePath),i.push(o);let c=e.get(o.target);if(c===void 0)throw new Error(`Lazy tree source hardlink ${o.sourcePath} target is absent`);if(c.type!=="file"&&c.type!=="hardlink")throw new Error(`Lazy tree source hardlink ${o.sourcePath} target is not regular`);o=c}a===void 0&&(a=o);for(let c of i)t.set(c.sourcePath,a)}return t}function pe(r,e,t,n=!1){if(typeof r!="string"||r.length===0||new TextEncoder().encode(r).byteLength>Un||r.includes("\0")||r.includes("\\")||r.startsWith("/")!==e)throw new Error(`${t} is not a canonical ${e?"absolute":"relative"} path`);if(n&&e&&r==="/")return r;if(r.slice(e?1:0).split("/").some(s=>s===""||s==="."||s===".."))throw new Error(`${t} has an unsafe path segment`);return r}function rl(r){let e=Je(r,["uid","gid"],"Lazy tree registration owner");return{uid:se(e.uid,"Lazy tree registration owner uid",0,Zs),gid:se(e.gid,"Lazy tree registration owner gid",0,Zs)}}function oo(r,e,t,n,i=1){let s=Mn(r,i),o=Wn(t),a=Je(n,["mode","capabilities","roots"],"Lazy tree activation");if(a.mode!=="boot-prefetch"&&a.mode!=="first-use")throw new Error("Lazy tree activation mode is invalid");let c=Le(a.capabilities,"Lazy tree activation capabilities",1,Kc).map((w,O)=>{let S=Ue(w,`Lazy tree activation capability ${O}`,de.maxActivationCapabilityBytes);if(!/^[a-z0-9][a-z0-9:._-]*$/.test(S))throw new Error(`Lazy tree activation capability ${O} is invalid`);return S}),l=Le(a.roots,"Lazy tree activation roots",1,Dc).map((w,O)=>pe(w,!0,`Lazy tree activation root ${O}`,!0));if(new Set(c).size!==c.length||new Set(l).size!==l.length)throw new Error("Lazy tree activation contains duplicates");let p={mode:a.mode,capabilities:c,roots:l},u=Le(e,"Lazy tree inventory",1,It),m=[],_=new Map,d=new Map,h=s.source===void 0?void 0:new Map(s.source.entries.map(w=>[w.sourcePath,w])),g=s.source===void 0?void 0:so(s.source.entries),f=0;for(let[w,O]of u.entries()){if(typeof O!="object"||O===null||Array.isArray(O))throw new Error(`Lazy tree entry ${w} must be an object`);let S=O.type,A=S==="directory"?["vfsPath","sourcePath","type","mode","size"]:S==="file"?["vfsPath","sourcePath","type","mode","size","inodeGroup"]:S==="symlink"?["vfsPath","sourcePath","type","mode","size","target"]:S==="hardlink"?["vfsPath","sourcePath","type","mode","size","target","inodeGroup"]:null;if(!A)throw new Error(`Lazy tree entry ${w} has an invalid type`);let v=Je(O,[...A,...h===void 0?[]:["materialization"]],`Lazy tree entry ${w}`),x=pe(v.vfsPath,!0,`Lazy tree entry ${w} VFS path`),L=pe(v.sourcePath,!1,`Lazy tree entry ${w} source path`),N=h===void 0?void 0:v.materialization;if(h!==void 0&&N!=="archive"&&N!=="archive-homebrew-relocate"&&N!=="archive-copy"&&N!=="archive-copy-mode"&&N!=="descriptor")throw new Error(`Lazy tree entry ${x} has invalid materialization provenance`);if(o!=="/"&&x!==o&&!x.startsWith(`${o}/`))throw new Error(`Lazy tree entry ${x} escapes its mount prefix`);if(_.has(x))throw new Error(`Lazy tree duplicates VFS path ${x}`);let j=se(v.mode,`Lazy tree entry ${x} mode`,0,4095),J=se(v.size,`Lazy tree entry ${x} size`,0,Fn),F,b;if(S==="directory"){if(J!==0)throw new Error(`Lazy tree directory ${x} has nonzero size`)}else if(S==="symlink"){if(F=Ue(v.target,`Lazy tree symlink ${x} target`,Js),new TextEncoder().encode(F).byteLength!==J)throw new Error(`Lazy tree symlink ${x} size differs from its target`)}else b=Ue(v.inodeGroup,`Lazy tree entry ${x} inode group`,Un),S==="hardlink"&&(F=pe(v.target,!0,`Lazy tree hardlink ${x} target`));if(S!=="hardlink"&&(f+=J,f>Fn))throw new Error("Lazy tree inventory exceeds the expansion limit");let k={vfsPath:x,sourcePath:L,...N===void 0?{}:{materialization:N},type:S,mode:j,size:J,...F===void 0?{}:{target:F},...b===void 0?{}:{inodeGroup:b}};if(h===void 0){let V=d.get(L);if(V){if(s.decoder!=="zip-v1"||k.type!=="hardlink"||V.inodeGroup!==k.inodeGroup)throw new Error(`Lazy tree duplicates source path ${L}`)}else{if(s.decoder==="zip-v1"&&k.type==="hardlink")throw new Error(`Lazy ZIP hardlink ${x} does not reuse a canonical source path`);d.set(L,k)}}else if(k.materialization==="descriptor"){if(k.type!=="directory"&&k.type!=="symlink")throw new Error(`Lazy tree descriptor entry ${x} is not structural`);if(h.has(L))throw new Error(`Lazy tree descriptor entry ${x} impersonates a source member`)}else{let V=h.get(L);if(V===void 0)throw new Error(`Lazy tree entry ${x} names absent source ${L}`);if(k.materialization==="archive-copy"||k.materialization==="archive-copy-mode"){if(k.type!=="file"||V.type!=="file"||k.materialization==="archive-copy"&&k.mode!==V.mode)throw new Error(`Lazy tree archive copy ${x} differs from its source`)}else if(k.materialization==="archive-homebrew-relocate"){if(k.type!=="file"&&k.type!=="hardlink"||V.type!==k.type||k.type==="file"&&V.mode!==k.mode)throw new Error(`Lazy tree receipt-relocated entry ${x} differs from its source`)}else if(V.type!==k.type||k.type==="symlink"&&V.target!==k.target||k.type!=="hardlink"&&V.mode!==k.mode)throw new Error(`Lazy tree archive entry ${x} differs from its source`)}m.push(k),_.set(x,k)}for(let w of m){let O=w.vfsPath.split("/").filter(Boolean);for(let S=1;S({path:w.vfsPath,type:w.type,mode:w.mode,size:w.size,target:w.target,inodeGroup:w.inodeGroup})),"Lazy tree");if(h!==void 0){let w=new Set;for(let O of m){if(O.materialization!=="archive-homebrew-relocate")continue;let S=h.get(O.sourcePath),A=S.type==="file"?S:g.get(S.sourcePath);if(A?.type!=="file")throw new Error(`Lazy tree receipt-relocated entry ${O.vfsPath} is not regular`);w.add(A.sourcePath)}for(let O of m){if(O.materialization==="descriptor"||O.type!=="file"&&O.type!=="hardlink")continue;let S=h.get(O.sourcePath),A=S.type==="file"?S:g.get(S.sourcePath);if(A?.type!=="file"||!w.has(A.sourcePath)&&O.size!==A.size)throw new Error(`Lazy tree archive entry ${O.vfsPath} differs from its source`)}for(let O of m){if(O.type!=="hardlink"||O.materialization!=="archive"&&O.materialization!=="archive-homebrew-relocate")continue;let S=h.get(O.sourcePath),A=_.get(O.target),v=g.get(S.sourcePath);if(S.target!==A?.sourcePath||v?.type!=="file"||v.mode!==O.mode||A?.mode!==O.mode)throw new Error(`Lazy tree hardlink ${O.vfsPath} differs from its source`)}}if(s.sourceEntryCount!==(h===void 0?d.size:h.size))throw new Error("Lazy tree source entry count differs from its inventory");if(s.source===void 0&&s.expandedBytesO.vfsPath===w||O.vfsPath.startsWith(`${w}/`)))throw new Error(`Lazy tree activation root ${w} is not owned by its inventory`);let E=new Map;for(let w of m)w.type==="file"&&E.set(w.inodeGroup,w);if(E.size!==y.canonicalByGroup.size)throw new Error("Lazy tree regular inode inventory is inconsistent");return{content:s,entries:m,mountPrefix:o,activation:p,canonicalByGroup:E}}function Bn(r){return JSON.stringify([r.sourcePath,r.type,r.inodeGroup,r.target])}function js(r,e){let t=qr(r,["kind","content","url","mountPrefix","integrity","materialized","entries"],["url","mountPrefix","materialized","entries"],"Serialized legacy lazy archive");if(t.kind===void 0){if(!e)throw new Error("Serialized lazy archive is missing its kind discriminator")}else if(t.kind!==Cn)throw new Error("Serialized legacy lazy archive has an unsupported kind");let n=Ue(t.url,"Serialized legacy lazy archive URL",Jr),i=Wn(t.mountPrefix),s=tn(t.integrity);if(t.content!==void 0){if(!e||t.kind!==void 0)throw new Error("Typed legacy lazy archives cannot carry generic content");let c=Mn(t.content);if(c.decoder!=="zip-v1"||c.transports.length!==1||c.transports[0]!==n||!s||c.sha256!==s.sha256||c.bytes!==s.bytes)throw new Error("Untagged legacy ZIP content identity is inconsistent")}if(t.materialized!==!1)throw new Error("Serialized legacy lazy archive must describe pending content");let o=new Set,a=Le(t.entries,"Serialized legacy lazy archive entries",1,It).map((c,l)=>{let p=qr(c,["vfsPath","ino","generation","dataSequence","size","isSymlink","deleted","materialized","archivePath","sourcePath","type","inodeGroup","target"],["vfsPath","ino","size","isSymlink","deleted"],`Serialized legacy lazy archive entry ${l}`),u=pe(p.vfsPath,!0,`Serialized legacy lazy archive entry ${l} VFS path`);if(o.has(u))throw new Error(`Serialized legacy lazy archive duplicates path ${u}`);o.add(u);let m=se(p.ino,`Serialized legacy lazy archive entry ${u} inode`,1,Number.MAX_SAFE_INTEGER),_=p.generation===void 0?void 0:se(p.generation,`Serialized legacy lazy archive entry ${u} generation`,0,Number.MAX_SAFE_INTEGER),d=p.dataSequence===void 0?void 0:se(p.dataSequence,`Serialized legacy lazy archive entry ${u} data sequence`,0,Number.MAX_SAFE_INTEGER),h=se(p.size,`Serialized legacy lazy archive entry ${u} size`,0,Fn);if(p.isSymlink!==!1||p.deleted!==!1||p.materialized!==void 0&&p.materialized!==!1)throw new Error(`Serialized legacy lazy archive entry ${u} is not pending`);if(p.type!==void 0&&p.type!=="file")throw new Error(`Serialized legacy lazy archive entry ${u} has an invalid type`);let g=p.archivePath===void 0?void 0:pe(p.archivePath,!1,`Serialized legacy lazy archive entry ${u} archive path`),f=p.sourcePath===void 0?void 0:pe(p.sourcePath,!1,`Serialized legacy lazy archive entry ${u} source path`),y=p.inodeGroup===void 0?void 0:Ue(p.inodeGroup,`Serialized legacy lazy archive entry ${u} inode group`,Un);if(p.target!==void 0)throw new Error(`Serialized legacy lazy archive entry ${u} has a link target`);return{vfsPath:u,ino:m,..._===void 0?{}:{generation:_},...d===void 0?{}:{dataSequence:d},size:h,isSymlink:!1,deleted:!1,materialized:!1,...g===void 0?{}:{archivePath:g},...f===void 0?{}:{sourcePath:f},type:"file",...y===void 0?{}:{inodeGroup:y}}});return{kind:Cn,url:n,mountPrefix:i,...s===void 0?{}:{integrity:s},materialized:!1,entries:a}}function il(r,e){let t=Je(r,["kind","content","inventory","activation","url","mountPrefix","integrity","materialized","entries"],"Serialized lazy tree");if(t.kind!==e)throw new Error("Serialized lazy tree has an unsupported kind");let n=oo(t.content,t.inventory,t.mountPrefix,t.activation);if(e===Kn!=(n.content.source===void 0))throw new Error(e===Kn?"Serialized deferred-tree-v1 cannot contain original-bottle source metadata":"Serialized deferred-tree-v2 requires original-bottle source metadata");let i=Ue(t.url,"Serialized lazy tree URL",Jr);if(i!==n.content.transports[0])throw new Error("Serialized lazy tree URL differs from its primary transport");let s=tn(t.integrity);if(!s||s.sha256!==n.content.sha256||s.bytes!==n.content.bytes)throw new Error("Serialized lazy tree integrity differs from its content");if(t.materialized!==!1)throw new Error("Serialized lazy tree must describe pending content");let o=new Map(n.entries.map(u=>[u.vfsPath,u])),a=new Map(n.entries.map(u=>[Bn(u),u])),c=Le(t.entries,"Serialized lazy tree entries",0,It),l=new Set,p=c.map((u,m)=>{let _=qr(u,["vfsPath","ino","generation","dataSequence","size","isSymlink","deleted","materialized","archivePath","sourcePath","type","inodeGroup","target"],["vfsPath","ino","generation","dataSequence","size","isSymlink","deleted","materialized","archivePath","sourcePath","type","inodeGroup"],`Serialized lazy tree entry ${m}`),d=pe(_.vfsPath,!0,`Serialized lazy tree entry ${m} VFS path`);if(l.has(d))throw new Error(`Serialized lazy tree duplicates pending path ${d}`);l.add(d);let h=pe(_.sourcePath,!1,`Serialized lazy tree entry ${m} source path`),g=pe(_.archivePath,!1,`Serialized lazy tree entry ${m} archive path`),f=o.get(d),y=a.get(Bn({sourcePath:h,type:typeof _.type=="string"?_.type:void 0,inodeGroup:typeof _.inodeGroup=="string"?_.inodeGroup:void 0,target:typeof _.target=="string"?_.target:void 0}))??f;if(!y||y.type!=="file"&&y.type!=="hardlink"||f?.inodeGroup!==void 0&&f.inodeGroup!==y.inodeGroup)throw new Error(`Serialized lazy tree entry ${d} is absent from its inventory`);let E=n.canonicalByGroup.get(y.inodeGroup);if(_.type!==y.type||_.inodeGroup!==y.inodeGroup||_.size!==y.size||g!==E?.sourcePath||_.target!==y.target||_.isSymlink!==!1||_.deleted!==!1||_.materialized!==!1)throw new Error(`Serialized lazy tree entry ${d} disagrees with its inventory`);let w=se(_.ino,`Serialized lazy tree entry ${d} inode`,1,Number.MAX_SAFE_INTEGER),O=se(_.generation,`Serialized lazy tree entry ${d} generation`,0,Number.MAX_SAFE_INTEGER),S=se(_.dataSequence,`Serialized lazy tree entry ${d} data sequence`,0,Number.MAX_SAFE_INTEGER);return{vfsPath:d,ino:w,generation:O,dataSequence:S,size:y.size,isSymlink:!1,deleted:!1,materialized:!1,archivePath:g,sourcePath:h,type:y.type,inodeGroup:y.inodeGroup,...y.target===void 0?{}:{target:y.target}}});return{kind:e,content:n.content,inventory:n.entries,activation:n.activation,url:i,mountPrefix:n.mountPrefix,integrity:s,materialized:!1,entries:p}}async function Vr(r,e,t){if(t===void 0)return;if(r.byteLength!==t.bytes)throw new Error(`Lazy ${e} byte count ${r.byteLength} does not match expected ${t.bytes}`);let n=globalThis.crypto?.subtle;if(!n)throw new Error(`Lazy ${e} integrity verification is unavailable`);let i=new Uint8Array(r.byteLength);i.set(r);let s=new Uint8Array(await n.digest("SHA-256",i)),o=Array.from(s,a=>a.toString(16).padStart(2,"0")).join("");if(o!==t.sha256)throw new Error(`Lazy ${e} SHA-256 ${o} does not match expected ${t.sha256}`)}var $n=class r{fs;imageMetadata;lazyFiles=new Map;lazyArchiveGroups=[];deferredTreeMaterializationHandles=new WeakMap;lazyArchiveInodes=new Map;lazyDownloadListeners=new Set;lazyPreparations=new Map;lazyTransport={fetcher:(e,t)=>globalThis.fetch(e,t)};constructor(e,t=null){this.fs=e,this.imageMetadata=t}static inodeKey(e,t){return`${e}:${t}`}static canAdoptLegacyLazyStub(e){return(e.mode&St)===Zr&&e.size===0&&e.dataSequence<=1}reconcileLazyIdentityState(e){for(let[t,n]of this.lazyFiles){let i=e.get(t);if(!i||i.dataSequence!==n.dataSequence||i.paths.length===0){this.lazyFiles.delete(t);continue}n.paths=new Set(i.paths),n.paths.has(n.path)||(n.path=i.paths[0])}this.lazyArchiveInodes.clear();for(let t of this.lazyArchiveGroups){let n=t.content!==void 0&&t.inventory!==void 0&&!t.materialized,i=new Map;for(let o of t.entries.values()){if(o.deleted||o.materialized||o.generation===void 0)continue;let a=r.inodeKey(o.ino,o.generation);i.has(a)||i.set(a,o)}let s=new Map;for(let[o,a]of i){let c=e.get(o);if(!(!c||c.dataSequence!==(a.dataSequence??0))){for(let l of c.paths)s.set(l,{...a,ino:c.ino,generation:c.generation,dataSequence:c.dataSequence,deleted:!1,materialized:!1});c.paths.length>0&&this.lazyArchiveInodes.set(o,t)}}t.entries=s,t.materialized=s.size===0&&!n}}lazyFileForStat(e){let t=r.inodeKey(e.ino,e.generation),n=this.lazyFiles.get(t);if(n&&n.dataSequence!==e.dataSequence){this.lazyFiles.delete(t);return}return n}lazyArchiveForStat(e){let t=r.inodeKey(e.ino,e.generation),n=this.lazyArchiveInodes.get(t);if(!n)return;let i=Array.from(n.entries.values()).filter(s=>s.ino===e.ino&&s.generation===e.generation&&!s.deleted&&!s.materialized);if(i.some(s=>s.dataSequence===e.dataSequence))return n;this.lazyArchiveInodes.delete(t);for(let s of i)s.materialized=!0}lazyBackingForStat(e){let t=r.inodeKey(e.ino,e.generation),n=this.lazyFiles.get(t);if(n)return{token:n,path:n.path};let i=this.lazyArchiveInodes.get(t);if(!i)return null;let s=Array.from(i.entries.entries()).find(([,o])=>o.ino===e.ino&&o.generation===e.generation&&!o.deleted&&!o.materialized)?.[0];return s===void 0?null:{token:i,path:s}}lazyBackingForPath(e){let t=this.lazyArchiveGroups.find(n=>!n.materialized&&n.content!==void 0&&n.inventory!==void 0&&n.activation!==void 0&&Array.from(n.entries.values()).every(i=>i.deleted||i.materialized||i.isSymlink)&&n.activation.roots.some(i=>i==="/"||e===i||e.startsWith(`${i}/`)));if(t)return{token:t,path:e,directGroup:t};try{let n=this.fs.stat(e),i=this.lazyBackingForStat(n);return i?{token:i.token,path:e}:null}catch{return null}}startLazyPreparation(e){let{path:t,token:n}=e,i={status:"pending",promise:Promise.resolve(!1)},s=e.directGroup?this.ensureArchiveMaterialized(e.directGroup).then(()=>!0):this.materializePath(t);return i.promise=s.then(o=>(i.status="fulfilled",this.lazyPreparations.get(n)===i&&this.lazyPreparations.delete(n),o),o=>{throw i.status="rejected",i.error=o,o}),i.promise.catch(()=>{}),this.lazyPreparations.set(n,i),i}guardSynchronousLazyAccess(e){let t=this.lazyBackingForPath(e);if(!t)return;let n=this.lazyPreparations.get(t.token);if(n?.status==="fulfilled"){this.lazyPreparations.delete(t.token);let s=this.lazyBackingForPath(e);if(!s)return;n=this.lazyPreparations.get(s.token)??this.startLazyPreparation(s)}else if(n?.status==="rejected"){this.lazyPreparations.delete(t.token);let s=n.error instanceof Error?n.error.message:String(n.error),o=new Error(`EIO: lazy backing for ${e} failed: ${s}`);throw o.code="EIO",o.cause=n.error,o}else n||(n=this.startLazyPreparation(t));let i=new Error(`EAGAIN: lazy backing for ${e} is being prepared`);throw i.code="EAGAIN",i}invalidateLazyData(e){let t=r.inodeKey(e.ino,e.generation);this.lazyFiles.delete(t);let n=this.lazyArchiveInodes.get(t);if(n){this.lazyArchiveInodes.delete(t);for(let i of n.entries.values())i.ino===e.ino&&i.generation===e.generation&&(i.materialized=!0)}}rewriteLazyNamespacePaths(e,t,n){let i=t.length>1?t.replace(/\/+$/,""):t,s=n.length>1?n.replace(/\/+$/,""):n,o=`${i}/`,a=`${s}/`,c=r.inodeKey(e.ino,e.generation),l=(e.mode&St)===bn,p=u=>u===i?s:l&&u.startsWith(o)?a+u.slice(o.length):u;for(let[u,m]of this.lazyFiles)!l&&u!==c||(m.paths=new Set(Array.from(m.paths,p)),m.path=p(m.path));for(let u of this.lazyArchiveGroups){let m=new Map;for(let[_,d]of u.entries){let h=d.generation===void 0?null:r.inodeKey(d.ino,d.generation);m.set(l||h===c?p(_):_,d)}u.entries=m,u.inventory&&(u.inventory=u.inventory.map(_=>({..._,vfsPath:p(_.vfsPath),..._.type==="hardlink"&&_.target!==void 0?{target:p(_.target)}:{}}))),u.activation&&(u.activation={...u.activation,roots:u.activation.roots.map(p)})}}get sharedBuffer(){return this.fs.buffer}static create(e,t){return new r(ke.mkfs(e,t))}static fromExisting(e){return new r(ke.mount(e))}rebaseToNewFileSystem(e){if(!Number.isSafeInteger(e)||e<=0)throw new Error(`Invalid MemoryFileSystem maxByteLength: ${e}`);let t=SharedArrayBuffer,{bytes:n,identities:i}=this.fs.snapshotState();this.reconcileLazyIdentityState(i);let s=this.serializeLazyEntries(),o=this.serializeLazyArchiveEntries(),a=new t(n.byteLength);new Uint8Array(a).set(n);let c=new r(ke.mount(a,{restoreImage:!0}),this.imageMetadata);c.importLazyEntries(s),c.importLazyArchiveEntries(o);let l=Math.min(e,Math.max(n.byteLength,zc)),p=new t(l,{maxByteLength:e}),u=r.create(p,e);u.setImageMetadata(this.imageMetadata);let m=new Set(s.flatMap(d=>d.paths??[d.path])),_=new Set;for(let d of o)if(!d.materialized)for(let h of d.entries)!h.deleted&&!h.isSymlink&&_.add(h.vfsPath);return c.copyPathToFreshFileSystem("/",u,m,_,new Map),u.importLazyEntries(s.map(d=>{let h=u.fs.lstat(d.path);return{...d,ino:h.ino,generation:h.generation,dataSequence:h.dataSequence}})),u.importLazyArchiveEntries(o.map(d=>({...d,entries:d.entries.map(h=>{if(h.deleted)return{...h,ino:0,generation:void 0};let g=u.fs.lstat(h.vfsPath);return{...h,ino:g.ino,generation:g.generation,dataSequence:g.dataSequence}})}))),u}getImageMetadata(){return Wc(this.imageMetadata)}setImageMetadata(e){this.imageMetadata=e===null?null:Qr(e)}subscribeLazyDownloads(e){return this.lazyDownloadListeners.add(e),()=>this.lazyDownloadListeners.delete(e)}setLazyFetcher(e,t={}){this.lazyTransport={fetcher:e,...t.signal===void 0?{}:{signal:t.signal}}}emitLazyDownload(e){if(this.lazyDownloadListeners.size===0)return;let t={...e,t:Vc()};for(let n of this.lazyDownloadListeners)try{n(t)}catch{}}async fetchLazyBytes(e,t){let n=0,i=e.integrity?.bytes??e.fallbackTotalBytes,s={id:e.id,kind:e.kind,url:e.url,path:e.path,mountPrefix:e.mountPrefix};for(let o=0;oe.integrity.bytes)throw new Error(`Lazy ${e.kind} exceeded expected byte count ${e.integrity.bytes}`);this.emitLazyDownload({...s,status:"progress",loadedBytes:n,totalBytes:i})}}}catch(u){try{await c.cancel(u)}catch{}throw u}}finally{c.releaseLock()}let p=tl(l,n);return ie(t.signal),await Vr(p,e.kind,e.integrity),ie(t.signal),this.emitLazyDownload({...s,status:"complete",loadedBytes:n,totalBytes:i??n}),p}catch(a){if(t.signal?.aborted){let p=t.signal.reason,u=p instanceof Error?p.message:String(p);throw this.emitLazyDownload({...s,status:"error",loadedBytes:n,totalBytes:i,error:u}),p}let c=o+1({...f})),activation:u,entries:new Map},h=f=>{let y=f.split("/").filter(Boolean),E="";for(let w=0;wy.vfsPath.split("/").length-E.vfsPath.split("/").length))if(f.type==="directory"){h(f.vfsPath);try{this.fs.mkdir(f.vfsPath,f.mode),this.fs.chmod(f.vfsPath,f.mode)}catch{if((this.fs.lstat(f.vfsPath).mode&St)!==bn)throw new Error(`Lazy tree directory collides at ${f.vfsPath}`)}}for(let f of l){if(f.type!=="symlink")continue;h(f.vfsPath),this.fs.symlink(f.target,f.vfsPath);let y=this.fs.lstat(f.vfsPath);d.entries.set(f.vfsPath,{ino:y.ino,generation:y.generation,dataSequence:y.dataSequence,size:f.size,isSymlink:!0,deleted:!1,materialized:!0,archivePath:f.sourcePath,sourcePath:f.sourcePath,type:"symlink",target:f.target})}let g=new Map;for(let f of l){if(f.type!=="file")continue;h(f.vfsPath);let y=this.fs.createLazyStub(f.vfsPath,f.mode);this.invalidateLazyData(y),g.set(f.inodeGroup,y);let E={ino:y.ino,generation:y.generation,dataSequence:y.dataSequence,size:f.size,isSymlink:!1,deleted:!1,materialized:!1,archivePath:f.sourcePath,sourcePath:f.sourcePath,type:"file",inodeGroup:f.inodeGroup};d.entries.set(f.vfsPath,E)}for(let f of l){if(f.type!=="hardlink")continue;let y=m.get(f.inodeGroup);h(f.vfsPath),this.fs.link(y.vfsPath,f.vfsPath);let E=this.fs.lstat(f.vfsPath),w=g.get(f.inodeGroup);if(E.ino!==w.ino||E.generation!==w.generation)throw new Error(`Lazy tree hardlink ${f.vfsPath} did not share its inode`);d.entries.set(f.vfsPath,{ino:E.ino,generation:E.generation,dataSequence:E.dataSequence,size:f.size,isSymlink:!1,deleted:!1,materialized:!1,archivePath:y.sourcePath,sourcePath:f.sourcePath,type:"hardlink",inodeGroup:f.inodeGroup,target:f.target})}if(_!==void 0)for(let f of l)this.lchown(f.vfsPath,_.uid,_.gid);for(let f of d.entries.values())f.isSymlink||f.generation===void 0||this.lazyArchiveInodes.set(r.inodeKey(f.ino,f.generation),d);return this.lazyArchiveGroups.push(d),d}registerLazyTreeWithMaterializationHandle(e,t,n="/",i,s){let o=this.registerLazyTreeInternal(e,t,n,i,!0,s),a=Object.freeze({[kc]:!0});return this.deferredTreeMaterializationHandles.set(a,o),a}registerLazyArchiveFromEntries(e,t,n,i,s){let o=Uc(e,t,n,i);o.some(({entry:c})=>!c.isDirectory&&!c.isSymlink)&&this.assertCanRegisterPendingLazyArchiveGroup();let a={...s?{content:Mn({decoder:"zip-v1",mediaType:"application/zip",sha256:s.sha256,bytes:s.bytes,expandedBytes:o.reduce((c,l)=>c+l.entry.uncompressedSize,0),sourceEntryCount:o.length,transports:[e]})}:{},url:e,mountPrefix:n,integrity:tn(s),materialized:!1,entries:new Map};for(let{entry:c,vfsPath:l}of o){if(c.isDirectory)continue;let p=l.split("/").filter(Boolean),u="";for(let m=0;mc.deleted||c.materialized),this.lazyArchiveGroups.push(a),a}importLazyArchiveEntries(e){this.importLazyArchiveEntriesInternal(e,!1,!0)}importLazyArchiveEntriesInternal(e,t,n){let i=Le(e,"Serialized lazy archive groups",0,Cc).map((a,c)=>{if(typeof a!="object"||a===null||Array.isArray(a))throw new Error(`Serialized lazy archive group ${c} must be an object`);let l=a.kind;if(l===Kn||l===Vs)return il(a,l);if(l===Cn)return js(a,!1);if(l!==void 0)throw new Error(`Serialized lazy archive group ${c} has an unsupported kind`);if(n)throw new Error(`Serialized lazy archive group ${c} is missing its kind discriminator`);return js(a,!0)});qs([...this.serializeLazyArchiveEntries(),...i]);let s=[],o=new Map;for(let a of i){let c=new Map,l=a.mountPrefix.replace(/\/+$/,""),p=a.content!==void 0&&a.inventory!==void 0&&a.activation!==void 0,u=p?new Map(a.inventory.map(f=>[f.vfsPath,f])):null,m=p?new Map(a.inventory.map(f=>[Bn(f),f])):null,_=new Map,d=new Map;for(let f of a.entries){let y=null,E=a.materialized||f.materialized===!0||f.isSymlink;if(!f.deleted&&!E){if((f.generation===void 0||f.dataSequence===void 0)&&!t)throw new Error("Live lazy-archive metadata requires inode generation and data sequence");try{y=this.fs.lstat(f.vfsPath)}catch{if(p)throw new Error(`Serialized lazy tree stub ${f.vfsPath} is missing from the filesystem`);continue}if(y.ino!==f.ino){if(p)throw new Error(`Serialized lazy tree stub ${f.vfsPath} has a different inode`);continue}if(f.generation!==void 0&&y.generation!==f.generation){if(p)throw new Error(`Serialized lazy tree stub ${f.vfsPath} has a different generation`);continue}if(f.dataSequence===void 0){if(!r.canAdoptLegacyLazyStub(y)){if(p)throw new Error(`Serialized lazy tree stub ${f.vfsPath} is not pristine`);continue}}else if(y.dataSequence!==f.dataSequence){if(p)throw new Error(`Serialized lazy tree stub ${f.vfsPath} has a different data sequence`);continue}if(p){let O=u.get(f.vfsPath),S=m.get(Bn(f))??O;if(!S||(y.mode&St)!==Zr||y.size!==0||(y.mode&4095)!==S.mode||O?.inodeGroup!==void 0&&O.inodeGroup!==S.inodeGroup)throw new Error(`Serialized lazy tree stub ${f.vfsPath} disagrees with its inventory`);let A=r.inodeKey(y.ino,y.generation),v=f.inodeGroup,x=_.get(v),L=d.get(A);if(x!==void 0&&x!==A||L!==void 0&&L!==v)throw new Error(`Serialized lazy tree inode group ${v} disagrees with the filesystem`);_.set(v,A),d.set(A,v)}}c.set(f.vfsPath,{ino:f.ino,generation:y?.generation??f.generation,dataSequence:y?.dataSequence??f.dataSequence,size:f.size,isSymlink:f.isSymlink,deleted:f.deleted,materialized:E,archivePath:f.archivePath??f.vfsPath.slice(l.length+1),sourcePath:f.sourcePath??f.archivePath??f.vfsPath.slice(l.length+1),type:f.type??(f.isSymlink?"symlink":"file"),inodeGroup:f.inodeGroup,target:f.target})}let h=a.content===void 0?void 0:Mn(a.content),g={content:h,url:h?.transports[0]??a.url,mountPrefix:a.mountPrefix,integrity:h?{sha256:h.sha256,bytes:h.bytes}:tn(a.integrity),materialized:a.materialized||!(h&&a.inventory)&&Array.from(c.values()).every(f=>f.deleted||f.materialized),inventory:a.inventory?.map(f=>({...f})),activation:a.activation?{mode:a.activation.mode,capabilities:[...a.activation.capabilities],roots:[...a.activation.roots]}:void 0,entries:c};if(s.push(g),!g.materialized){for(let[,f]of c)if(!f.deleted&&!f.materialized&&f.generation!==void 0){let y=r.inodeKey(f.ino,f.generation),E=o.get(y);if(E!==void 0&&E!==g)throw new Error(`Serialized lazy archive groups share pending inode ${y}`);if(this.lazyArchiveInodes.has(y))throw new Error(`Serialized lazy archive group collides with pending inode ${y}`);o.set(y,g)}}}this.lazyArchiveGroups.push(...s);for(let[a,c]of o)this.lazyArchiveInodes.set(a,c)}rewriteLazyArchiveUrls(e){for(let t of this.lazyArchiveGroups)t.content?(t.content={...t.content,transports:t.content.transports.map(e)},t.url=t.content.transports[0]):t.url=e(t.url)}serializeLazyArchiveEntries(){let e=[];for(let t of this.lazyArchiveGroups){let n=Array.from(t.entries,([s,o])=>({vfsPath:s,ino:o.ino,generation:o.generation,dataSequence:o.dataSequence,size:o.size,isSymlink:o.isSymlink,deleted:o.deleted,materialized:o.materialized,archivePath:o.archivePath,sourcePath:o.sourcePath,type:o.type,inodeGroup:o.inodeGroup,target:o.target})).filter(s=>!s.deleted&&!s.materialized);if(n.length===0&&!(t.content&&t.inventory&&!t.materialized))continue;let i=t.content!==void 0&&t.inventory!==void 0&&t.activation!==void 0;if(i&&t.content.transports.length===0)throw new Error("Direct-materialization tree must be materialized before serialization");e.push(i?{kind:t.content.source===void 0?Kn:Vs,content:t.content,inventory:t.inventory,activation:t.activation,url:t.url,mountPrefix:t.mountPrefix,integrity:t.integrity,materialized:!1,entries:n}:{kind:Cn,url:t.url,mountPrefix:t.mountPrefix,integrity:t.integrity,materialized:!1,entries:n})}return e}exportLazyArchiveEntries(){return this.reconcileLazyIdentityState(this.fs.identityState()),this.serializeLazyArchiveEntries()}pendingDeferredTreeUsage(){return this.reconcileLazyIdentityState(this.fs.identityState()),io(this.serializeLazyArchiveEntries())}assertCanAppendDeferredTreeUsage(e){jr(e);let t=this.pendingDeferredTreeUsage();jr({groups:t.groups+e.groups,archiveBytes:t.archiveBytes+e.archiveBytes,expandedBytes:t.expandedBytes+e.expandedBytes,payloadBytes:t.payloadBytes+e.payloadBytes,entries:t.entries+e.entries})}assertCanRegisterPendingLazyArchiveGroup(){if(this.reconcileLazyIdentityState(this.fs.identityState()),this.lazyArchiveGroups.filter(t=>!t.materialized&&(t.content!==void 0&&t.inventory!==void 0||Array.from(t.entries.values()).some(n=>!n.deleted&&!n.materialized))).length>=Ie.maxGroups)throw new Error(`Cannot register another lazy archive group: ${Ie.maxGroups} pending groups already exist`)}async preparePath(e){let t=!1,n=Math.max(3,this.lazyArchiveGroups.length+1);for(let i=0;i!s.materialized&&s.activation?.mode==="boot-prefetch"),t=0,n,i=Array.from({length:Math.min(e.length,Fc)},async()=>{for(;n===void 0;){let s=t;if(t+=1,s>=e.length)return;try{await this.prepareLazyTreeGroup(e[s])}catch(o){n??=o}}});if(await Promise.all(i),n!==void 0)throw n;return e.length}async materializeRegisteredDeferredTree(e,t){let n=this.deferredTreeMaterializationHandles.get(e);if(n===void 0)throw new Error("Deferred-tree handle was not issued by this filesystem");if(n.materialized)return!1;let i=this.lazyPreparations.get(n);if(i!==void 0)return i.promise;let s=new Uint8Array(t.byteLength);s.set(t);let o={status:"pending",promise:Promise.resolve(!1)};o.promise=Promise.resolve().then(async()=>(await Vr(s,"tree",n.integrity),await this.materializeArchiveBytes(n,s),!0)).then(a=>(o.status="fulfilled",a),a=>{throw o.status="rejected",o.error=a,a}),o.promise.catch(()=>{}),this.lazyPreparations.set(n,o);try{return await o.promise}finally{this.lazyPreparations.get(n)===o&&this.lazyPreparations.delete(n)}}async prepareLazyTreeGroup(e){if(e.materialized)return!1;let t={token:e,path:e.activation?.roots[0]??e.mountPrefix,directGroup:e},n=this.lazyPreparations.get(e)??this.startLazyPreparation(t);try{return await n.promise}finally{this.lazyPreparations.get(e)===n&&this.lazyPreparations.delete(e)}}async ensureMaterialized(e){return this.preparePath(e)}async materializePath(e){if(this.lazyFiles.size===0&&this.lazyArchiveInodes.size===0)return!1;let t;try{t=this.fs.stat(e)}catch{return!1}let n=r.inodeKey(t.ino,t.generation),i=this.lazyFiles.get(n);if(i){let o=this.lazyTransport,a=await this.fetchLazyBytes({id:`file:${t.ino}`,kind:"file",url:i.url,path:i.path,fallbackTotalBytes:i.size},o);for(let c=0;c<3;c++){if(this.lazyFiles.get(n)!==i)return!1;for(let l of new Set([e,...i.paths]))if(ie(o.signal),this.fs.replaceIfIdentity(l,i.ino,i.generation,i.dataSequence,a))return i.path=l,this.lazyFiles.delete(n),!0;this.reconcileLazyIdentityState(this.fs.identityState())}throw new Error(`Lazy file kept changing names while materializing: ${e}`)}let s=this.lazyArchiveInodes.get(n);return s?(await this.ensureArchiveMaterialized(s,{path:e,ino:t.ino,generation:t.generation}),!this.lazyArchiveInodes.has(n)):!1}async decodeAndValidateLazyTree(e,t){let n=e.content,i=e.inventory;if(!n||!i)throw new Error("Lazy tree is missing its decoder or complete inventory");let s=new Map,o=new Map(i.map(u=>[u.vfsPath,u]));if(n.source!==void 0)for(let u of n.source.entries)s.set(u.sourcePath,u);else for(let u of i){if(u.type==="hardlink"){let _=o.get(u.target);if(!_)throw new Error(`Lazy tree hardlink target disappeared: ${u.target}`);if(u.sourcePath===_.sourcePath)continue}if(s.get(u.sourcePath))throw new Error(`Lazy tree inventory duplicates source member ${u.sourcePath}`);s.set(u.sourcePath,{sourcePath:u.sourcePath,type:u.type,mode:u.mode,size:u.size,...u.type==="symlink"?{target:u.target}:{},...u.type==="hardlink"?{target:o.get(u.target)?.sourcePath}:{}})}let a=new Map,c=0;if(n.decoder==="zip-v1"){let{parseZipCentralDirectory:u,extractZipEntryBounded:m}=await Promise.resolve().then(()=>(Cr(),Fr)),_=u(t);if(_.length!==n.sourceEntryCount||_.length!==s.size)throw new Error("Lazy ZIP tree decoded inventory counts differ from its descriptor");for(let d of _){let h=d.isDirectory?d.fileName.replace(/\/$/,""):d.fileName;if(a.has(h))throw new Error(`Lazy ZIP tree duplicates source member ${h}`);let g=s.get(h);if(!g)throw new Error(`Lazy ZIP tree has undeclared source member ${h}`);if(c+=d.uncompressedSize,c>n.expandedBytes||d.uncompressedSize!==g.size)throw new Error(`Lazy ZIP tree member ${h} exceeds its inventory`);let f=d.isDirectory?"directory":d.isSymlink?"symlink":"file",y=n.modePolicy==="portable-posix-v1"?f==="directory"?493:f==="symlink"?511:(d.mode&73)!==0?493:420:d.mode&4095;if(f!==g.type||y!==g.mode)throw new Error(`Lazy ZIP tree member ${h} differs from inventory`);if(d.isDirectory)a.set(h,{type:"directory",mode:y});else{let E=m(t,d,g.size);if(d.isSymlink){let w;try{w=new TextDecoder("utf-8",{fatal:!0}).decode(E)}catch{throw new Error(`Lazy ZIP tree symlink ${h} is not UTF-8`)}a.set(h,{type:"symlink",mode:y,target:w})}else a.set(h,{type:"file",mode:y,data:E})}}}else{let{parseTarGzip:u}=await Promise.resolve().then(()=>(Ws(),Us)),m=u(t,{label:`Lazy tree ${n.sha256}`,limits:{maxCompressedBytes:n.bytes,maxUncompressedBytes:n.expandedBytes,maxEntries:n.sourceEntryCount}});c=new DataView(t.buffer,t.byteOffset,t.byteLength).getUint32(t.byteLength-4,!0);for(let _ of m){if(a.has(_.path))throw new Error(`Lazy TAR tree duplicates source member ${_.path}`);_.type==="file"?a.set(_.path,{type:"file",mode:_.mode,data:_.data}):_.type==="directory"?a.set(_.path,{type:"directory",mode:_.mode}):a.set(_.path,{type:_.type,mode:_.mode,target:_.linkName})}}if(a.size!==n.sourceEntryCount||a.size!==s.size||c!==n.expandedBytes)throw new Error("Lazy tree decoded inventory counts differ from its descriptor");for(let[u,m]of s){let _=a.get(u);if(!_)throw new Error(`Lazy tree is missing source member ${u}`);let d=m.type;if(_.type!==d)throw new Error(`Lazy tree member ${u} is ${_.type}, expected ${d}`);if((_.mode&4095)!==m.mode)throw new Error(`Lazy tree member ${u} mode differs from inventory`);if(d==="file"&&_.data?.byteLength!==m.size)throw new Error(`Lazy tree member ${u} size differs from inventory`);if(d==="symlink"&&_.target!==m.target)throw new Error(`Lazy tree symlink ${u} target differs from inventory`);if(d==="hardlink"&&_.target!==m.target)throw new Error(`Lazy tree hardlink ${u} target differs from inventory`)}let l=new Set(i.flatMap(u=>u.materialization==="archive-homebrew-relocate"?[u.sourcePath]:[]));if(n.source!==void 0){let u=new Map(n.source.entries.map(d=>[d.sourcePath,d])),m=so(n.source.entries),_=n.source.entries.filter(d=>d.sourcePath==="INSTALL_RECEIPT.json"||d.sourcePath.endsWith("/INSTALL_RECEIPT.json"));if(_.length>1)throw new Error(`Lazy Homebrew bottle has ${_.length} INSTALL_RECEIPT.json source members, expected at most one`);if(_.length===0){if(l.size>0)throw new Error("Lazy Homebrew bottle marks receipt relocation without INSTALL_RECEIPT.json")}else{let d=_[0],h=d.type==="file"?d:m.get(d.sourcePath),g=h===void 0?void 0:a.get(h.sourcePath);if(h?.type!=="file"||g?.type!=="file"||g.data===void 0)throw new Error("Lazy Homebrew bottle INSTALL_RECEIPT.json is not regular");let f=ys(g.data),y=d.sourcePath.lastIndexOf("/"),E=y<0?"":d.sourcePath.slice(0,y),w=new Set(f.changedFiles.map(S=>E.length===0?S:`${E}/${S}`));if(l.size!==w.size||[...l].some(S=>!w.has(S)))throw new Error("Lazy Homebrew bottle relocation markers differ from INSTALL_RECEIPT.json");let O=new Set;for(let S of w){let A=u.get(S),v=A?.type==="file"?A:A===void 0?void 0:m.get(A.sourcePath),x=v===void 0?void 0:a.get(v.sourcePath);if(v?.type!=="file"||x?.type!=="file"||x.data===void 0)throw new Error(`Lazy Homebrew bottle changed source ${S} is not regular`);O.has(v.sourcePath)||(x.data=gs(x.data,f,S),O.add(v.sourcePath))}}}else if(l.size>0)throw new Error("Lazy tree receipt relocation requires original-bottle source truth");let p=new Map;for(let u of i){if(u.type!=="file"||u.materialization==="descriptor")continue;let m=a.get(u.sourcePath);if(m?.type!=="file"||!m.data)throw new Error(`Lazy tree has no file content for ${u.sourcePath}`);p.set(u.sourcePath,m.data)}return p}async ensureArchiveMaterialized(e,t){if(e.materialized)return;let n=e.content!==void 0&&e.inventory!==void 0,i=this.lazyTransport,s=n?e.content.transports:[e.url],o=[],a=null;for(let[c,l]of s.entries())try{a=await this.fetchLazyBytes({id:`archive:${e.mountPrefix}:${e.content?.sha256??l}:${c}`,kind:n?"tree":"archive",url:l,mountPrefix:e.mountPrefix,integrity:e.integrity},i);break}catch(p){if(ie(i.signal),ro(p))throw p;o.push(p instanceof Error?p.message:String(p))}if(ie(i.signal),a===null)throw new Error(`All ${s.length} lazy ${n?"tree":"archive"} transports failed: ${o.join("; ")}`);ie(i.signal),await this.materializeArchiveBytes(e,a,t,i.signal)}async materializeArchiveBytes(e,t,n,i){if(ie(i),e.materialized)return;let o=e.content!==void 0&&e.inventory!==void 0?await this.decodeAndValidateLazyTree(e,t):null;ie(i);let{parseZipCentralDirectory:a,extractZipEntry:c}=await Promise.resolve().then(()=>(Cr(),Fr));ie(i);let l=o?[]:a(t),p=new Map;for(let d of l){if(p.has(d.fileName))throw new Error(`Lazy archive contains duplicate member: ${d.fileName}`);p.set(d.fileName,d)}let u=e.mountPrefix.replace(/\/+$/,""),m=new Map;for(let[d,h]of e.entries){if(h.deleted||h.materialized)continue;let g=h.archivePath??d.slice(u.length+1),f=o?void 0:p.get(g),y=o?.get(g);if(o){if(y===void 0||y.byteLength!==h.size)throw new Error(`Lazy tree member ${g} does not match its registered metadata`)}else if(f===void 0||f.isDirectory||f.isSymlink||f.uncompressedSize!==h.size)throw new Error(`Lazy archive member ${g} does not match its registered metadata`);if(h.generation===void 0)continue;let E=r.inodeKey(h.ino,h.generation),w=m.get(E);if(w&&w.archivePath!==g)throw new Error(`Lazy archive aliases for inode ${E} name different members`);if(!w){let O=y??c(t,f);if(O.byteLength!==h.size)throw new Error(`Lazy archive member ${g} extracted ${O.byteLength} bytes, expected ${h.size}`);m.set(E,{archivePath:g,content:O})}}let _=n?r.inodeKey(n.ino,n.generation):null;for(let d=0;d<3;d++){let h=new Map;for(let[g,f]of e.entries){if(f.deleted||f.materialized||f.generation===void 0)continue;let y=r.inodeKey(f.ino,f.generation);if(this.lazyArchiveInodes.get(y)!==e)continue;let E=m.get(y);if(!E)throw new Error(`Lazy archive has no extracted content for inode ${y}`);let w=h.get(y);w||(w={ino:f.ino,generation:f.generation,dataSequence:f.dataSequence??0,paths:new Set,content:E.content},h.set(y,w)),w.paths.add(g),n&&n.ino===f.ino&&n.generation===f.generation&&w.paths.add(n.path)}if(h.size>0&&(ie(i),!this.fs.replaceManyIfIdentities(Array.from(h.values(),f=>({paths:Array.from(f.paths),expectedIno:f.ino,expectedGeneration:f.generation,expectedDataSequence:f.dataSequence,data:f.content}))))){if(this.reconcileLazyIdentityState(this.fs.identityState()),_&&!this.lazyArchiveInodes.has(_))return;continue}ie(i);for(let[g,f]of h){this.lazyArchiveInodes.delete(g);for(let y of e.entries.values())y.ino===f.ino&&y.generation===f.generation&&(y.materialized=!0)}if(e.materialized=Array.from(e.entries.values()).every(g=>g.deleted||g.materialized),e.materialized||(this.reconcileLazyIdentityState(this.fs.identityState()),_&&!this.lazyArchiveInodes.has(_)))return}if(_&&this.lazyArchiveInodes.has(_))throw new Error(`Lazy archive member kept changing names while materializing: ${n?.path}`)}async materializeAllLazyEntries(){for(let t=0;t<3;t++){this.reconcileLazyIdentityState(this.fs.identityState());let n=this.lazyArchiveGroups.filter(o=>!o.materialized&&o.content!==void 0&&o.inventory!==void 0);if(this.lazyFiles.size===0&&this.lazyArchiveInodes.size===0&&n.length===0)return;let i=Array.from(this.lazyFiles.values(),o=>o.path);for(let o of i)await this.ensureMaterialized(o);let s=new Set(this.lazyArchiveInodes.values());for(let o of n)s.add(o);for(let o of s)await this.prepareLazyTreeGroup(o)}this.reconcileLazyIdentityState(this.fs.identityState());let e=this.lazyArchiveGroups.some(t=>!t.materialized&&t.content!==void 0&&t.inventory!==void 0);if(this.lazyFiles.size!==0||this.lazyArchiveInodes.size!==0||e)throw new Error("Cannot create a self-contained VFS image while lazy entries remain pending")}async saveImage(e){e?.materializeAll&&await this.materializeAllLazyEntries();let{bytes:t,identities:n}=this.fs.snapshotState({normalizeTimestampsMs:e?.normalizeTimestampsMs});this.reconcileLazyIdentityState(n);let i=this.serializeLazyEntries(),s=i.length>0,o=s?new TextEncoder().encode(JSON.stringify(i)):new Uint8Array(0);if(o.byteLength>zn)throw new Error(`VFS image lazy metadata exceeds ${zn} bytes`);let a=this.serializeLazyArchiveEntries();qs(a);let c=a.length>0,l=c?new TextEncoder().encode(JSON.stringify(a)):new Uint8Array(0);if(l.byteLength>Nn)throw new Error(`VFS image lazy archive metadata exceeds ${Nn} bytes`);let p=e?.metadata===void 0?this.imageMetadata:e.metadata,u=Zc(p),m=u.byteLength>0,_=c?4+l.byteLength:0,d=m?4+u.byteLength:0,h=ce+t.byteLength+4+o.byteLength+_+d,g=new Uint8Array(h),f=new DataView(g.buffer);f.setUint32(0,Xr,!0),f.setUint32(4,Yr,!0),f.setUint32(8,(s?Ur:0)|(c?Pn:0)|(c?Gr:0)|(m?Wr:0),!0),f.setUint32(12,t.byteLength,!0),g.set(t,ce);let y=ce+t.byteLength;if(f.setUint32(y,o.byteLength,!0),o.byteLength>0&&g.set(o,y+4),c){let E=y+4+o.byteLength;f.setUint32(E,l.byteLength,!0),g.set(l,E+4)}if(m){let E=y+4+o.byteLength+_;f.setUint32(E,u.byteLength,!0),g.set(u,E+4)}return g}static readImageMetadata(e){let t=Ln(e);if(!(t.flags&Wr))return null;let{metadataOffset:n}=Xs(t.image,t.view,t.flags,t.sabLen);if(t.image.byteLengthOt)throw new Error(`VFS image metadata exceeds ${Ot} bytes`);if(t.image.byteLength0){let g=n.subarray(d+4,d+4+h),f=Le(Ys(g,"VFS image lazy metadata"),"VFS image lazy entries",0,It);_.importLazyEntriesInternal(f,!0)}if(s&Pn){let g=a.archiveOffset,f=i.getUint32(g,!0);if(f>0){let y=n.subarray(g+4,g+4+f),E=Ys(y,"VFS image lazy archive metadata");_.importLazyArchiveEntriesInternal(E,!0,!!(s&Gr))}}return _}adaptStat(e){return{dev:0,ino:e.ino,mode:e.mode,nlink:e.linkCount,uid:e.uid,gid:e.gid,size:e.size,atimeMs:e.atime,mtimeMs:e.mtime,ctimeMs:e.ctime}}adaptStatWithLazySize(e){let t=this.adaptStat(e),n=this.lazyFileForStat(e);if(n)return t.size=n.size,t;let i=this.lazyArchiveForStat(e);if(i){for(let s of i.entries.values())if(s.ino===e.ino&&s.generation===e.generation&&!s.deleted){t.size=s.size;break}}return t}open(e,t,n){(t&Yt)===0&&!((t&Vt)!==0&&(t&Rr)!==0)&&this.guardSynchronousLazyAccess(e);let i=this.fs.open(e,t,n);return(t&Yt)!==0&&this.invalidateLazyData(this.fs.fstat(i)),i}close(e){return this.fs.close(e),0}read(e,t,n,i){if(i>0){let s=this.lazyBackingForStat(this.fs.fstat(e));s&&(this.reconcileLazyIdentityState(this.fs.identityState()),s=this.lazyBackingForStat(this.fs.fstat(e)),s&&this.guardSynchronousLazyAccess(s.path))}return n!==null?this.fs.readAt(e,t.subarray(0,i),n):this.fs.read(e,t.subarray(0,i))}write(e,t,n,i){if(n!==null){let o=this.fs.writeAt(e,t.subarray(0,i),n);return o>0&&this.invalidateLazyData(this.fs.fstat(e)),o}let s=this.fs.write(e,t.subarray(0,i));return s>0&&this.invalidateLazyData(this.fs.fstat(e)),s}seek(e,t,n){return this.fs.lseek(e,t,n)}fstat(e){return this.adaptStatWithLazySize(this.fs.fstat(e))}fpathconf(e,t){let n=this.fstat(e);return yr(n,t,{supportsSymlinks:!0,timestampResolutionNs:1e6})}ftruncate(e,t){this.fs.ftruncate(e,t),this.invalidateLazyData(this.fs.fstat(e))}fsync(e){}fchmod(e,t){this.fs.fchmod(e,t)}fchown(e,t,n){this.fs.fchown(e,t,n)}stat(e){return this.adaptStatWithLazySize(this.fs.stat(e))}lstat(e){return this.adaptStatWithLazySize(this.fs.lstat(e))}statfs(e){this.fs.stat(e);let t=this.fs.statfs();return{type:1397114451,bsize:t.blockSize,blocks:t.totalBlocks,bfree:t.freeBlocks,bavail:t.freeBlocks,files:t.totalInodes,ffree:t.freeInodes,fsid:0,namelen:t.maxName,frsize:t.blockSize,flags:0}}pathconf(e,t){let n=this.stat(e);return yr(n,t,{supportsSymlinks:!0,timestampResolutionNs:1e6})}mkdir(e,t){this.fs.mkdir(e,t)}rmdir(e){this.fs.rmdir(e)}unlink(e){let t=this.fs.unlink(e),n=r.inodeKey(t.ino,t.generation);if(t.linkCount>1&&(this.lazyFiles.has(n)||this.lazyArchiveInodes.has(n))){this.reconcileLazyIdentityState(this.fs.identityState());return}let i=this.lazyFiles.get(n);i&&(i.paths.delete(e),t.linkCount<=1?this.lazyFiles.delete(n):i.path===e&&(i.path=i.paths.values().next().value));let s=this.lazyArchiveInodes.get(n);if(s){let o=s.entries.get(e);if(t.linkCount<=1){for(let a of s.entries.values())a.ino===t.ino&&a.generation===t.generation&&(a.deleted=!0);this.lazyArchiveInodes.delete(n)}else o&&s.entries.delete(e)}}rename(e,t){let{source:n,replaced:i}=this.fs.rename(e,t);if(i&&i.ino===n.ino&&i.generation===n.generation)return;let s=!1;if(i){let o=r.inodeKey(i.ino,i.generation);i.linkCount>1&&(this.lazyFiles.has(o)||this.lazyArchiveInodes.has(o))&&(this.reconcileLazyIdentityState(this.fs.identityState()),s=!0);let a=this.lazyFiles.get(o);!s&&a&&(a.paths.delete(t),i.linkCount<=1?this.lazyFiles.delete(o):a.path===t&&(a.path=a.paths.values().next().value));let c=this.lazyArchiveInodes.get(o);if(!s&&c){let l=c.entries.get(t);i.linkCount<=1?(l&&(l.deleted=!0),this.lazyArchiveInodes.delete(o)):l&&c.entries.delete(t)}}s||this.rewriteLazyNamespacePaths(n,e,t)}link(e,t){let n=this.fs.link(e,t),i=r.inodeKey(n.ino,n.generation),s=this.lazyFiles.get(i);s&&s.paths.add(t);let o=this.lazyArchiveInodes.get(i);if(o){let a=Array.from(o.entries.values()).find(c=>c.ino===n.ino&&c.generation===n.generation);a&&o.entries.set(t,{...a})}}symlink(e,t){this.fs.symlink(e,t)}readlink(e){return this.fs.readlink(e)}chmod(e,t){this.fs.chmod(e,t)}chown(e,t,n){this.fs.chown(e,t,n)}lchown(e,t,n){this.fs.lchown(e,t,n)}createFileWithOwner(e,t,n,i,s){let o=this.open(e,577,t);s.length>0&&this.write(o,s,null,s.length),this.close(o),this.chown(e,n,i),this.chmod(e,t)}mkdirWithOwner(e,t,n,i){this.mkdir(e,t),this.chown(e,n,i),this.chmod(e,t)}symlinkWithOwner(e,t,n,i){this.symlink(e,t),this.lchown(t,n,i)}copyPathToFreshFileSystem(e,t,n,i,s){let o=this.lstat(e),a=o.mode&St,c=o.mode&4095;if(a===bn){e==="/"?(t.chown(e,o.uid,o.gid),t.chmod(e,c)):t.mkdirWithOwner(e,c,o.uid,o.gid);let m=this.opendir(e);try{for(;;){let _=this.readdir(m);if(!_)break;_.name==="."||_.name===".."||this.copyPathToFreshFileSystem(e==="/"?`/${_.name}`:`${e}/${_.name}`,t,n,i,s)}}finally{this.closedir(m)}r.applyTimes(t,e,o);return}let l=o.nlink>1?`${o.dev}:${o.ino}`:null,p=l?s.get(l):void 0;if(p){t.link(p,e);return}if(a===Tc){t.symlinkWithOwner(this.readlink(e),e,o.uid,o.gid),l&&s.set(l,e);return}if(a!==Zr)throw new Error(`Unsupported file type while rebasing VFS: ${e}`);if(n.has(e)||i.has(e)){t.createFileWithOwner(e,c,o.uid,o.gid,new Uint8Array(0)),r.applyTimes(t,e,o),l&&s.set(l,e);return}this.copyRegularFileToFreshFileSystem(e,t,o,c),l&&s.set(l,e)}copyRegularFileToFreshFileSystem(e,t,n,i){let s=this.open(e,bc,0),o=null;try{o=t.open(e,Lc,i);let a=new Uint8Array(Math.min(Pc,Math.max(1,n.size))),c=n.size;for(;c>0;){let l=Math.min(a.byteLength,c),p=this.read(s,a,null,l);if(p<=0)throw new Error(`Unexpected EOF while rebasing VFS file: ${e}`);let u=0;for(;u!e||e==="."||e===".."))throw new Error(`Binary resolver path must be a normalized portable relative path: ${JSON.stringify(r)}`);return r}var nt=new Set(["wasm32","wasm64"]);function Ge(r){if(ml(r),!r.startsWith("programs/"))return r;let e=r.slice(9),t=e.split("/",1)[0];return nt.has(t)?r:`programs/wasm32/${e}`}function _l(r,e=M(Yn(),"wasm")){let t=Ge(r),n=[M(e,t)];return r==="kernel.wasm"?n.push(M(e,"kandelo-kernel.wasm")):r==="userspace.wasm"?n.push(M(e,"wasm_posix_userspace.wasm")):r==="rootfs.vfs"&&n.push(M(e,"rootfs.vfs")),n}var Hn=class extends Error{constructor(e){super(e),this.name="BinaryNotFoundError"}};function go(){let r=[],e=!1;try{let n=tt();e=!0;for(let[i,s]of[["local-binaries",M(n,"local-binaries")],["binaries",M(n,"binaries")]])r.push({label:i,root:s,identity:i==="local-binaries"?"local-generation":"program-cache",allowRegularFileClosure:!1,candidatesFor(o){return[M(s,Ge(o))]}})}catch{}let t=M(Yn(),"wasm");return r.push({label:"installed package",root:t,identity:"installed-package",allowRegularFileClosure:!e,candidatesFor(n){return _l(n,t)}}),r}function vt(r,e){return new Error(`Invalid package manifest ${r}: ${e}`)}function me(r){try{return Vn(r),!0}catch(e){if(e instanceof Error&&"code"in e&&e.code==="ENOENT")return!1;throw e}}function co(r,e,t){if(r.length===0||r.startsWith("/")||r.includes("\\")||r.includes("\0")||r.split("/").some(n=>!n||n==="."||n===".."))throw vt(e,`${t} must be a normalized portable relative path`);return r}function Gn(r,e,t,n=!0){if(r.length===0||r==="."||r===".."||r.includes("/")||r.includes("\\")||r.includes("\0")||!n&&r.includes("@"))throw vt(e,`${t} must be a safe single path component`);return r}var lo="kandelo-program-packages-v2",ve="program-packages.json",uo=null,Zn=null,ti=0;function Eo(){if(Object.prototype.hasOwnProperty.call(process.env,"WASM_POSIX_DEPS_REGISTRY")){let r=null;return(process.env.WASM_POSIX_DEPS_REGISTRY??"").split(":").filter(Boolean).map(e=>e.startsWith("~/")&&process.env.HOME!==void 0?M(process.env.HOME,e.slice(2)):Xn(e)?Se(e):(r??=tt(),Se(r,e)))}try{return[M(tt(),"packages","registry")]}catch{return null}}function yl(){let r;try{r=tt()}catch{return null}if(!nn(M(r,"tools","xtask","Cargo.toml"))||!nn(M(r,"scripts","dev-shell.sh")))return null;try{let e=Oe(si()),t=Oe(r);return[M(t,"host"),M(t,"scripts")].some(i=>nn(i)&&ui(Oe(i),e))?t:null}catch{return null}}function oi(r,e,t){let n=[typeof t.stderr=="string"?t.stderr.trim():"",typeof t.stdout=="string"?t.stdout.trim():"",t.error?.message??""].filter(Boolean).join(` `);return`${r} ${e.join(" ")} failed${t.status===null?"":` with status ${t.status}`}${n?`: -${n}`:""}`}function Za(r){let e=process.env.KANDELO_DEV_SHELL_TOOL_PATH!==void 0,t=e?"rustc":"bash",n=e?["-vV"]:[$(r,"scripts","dev-shell.sh"),"rustc","-vV"],i=wr(t,n,{cwd:r,encoding:"utf8"});if(i.status!==0)throw new Error(Er(t,n,i));let o=i.stdout.split(/\r?\n/).find(s=>s.startsWith("host: "))?.slice(6).trim();if(!o)throw new Error(`Could not determine the Rust host target for ${r}`);return o}function pr(r){try{if(En(r).isFile())return me(r)}catch{}throw new Error(`Prepared xtask is not a regular file: ${r}`)}function Ha(r){let e=process.env.WASM_POSIX_XTASK_BIN;if(e!==void 0){let l=Sn(e)?pe(e):pe(r,e);return pr(l)}if(wn?.sourceRepoRoot===r)return pr(wn.xtaskPath);let t=Za(r),n=$(r,"target",t,"release",process.platform==="win32"?"xtask.exe":"xtask"),i=["build","--release","-p","xtask","--target",t,"--quiet"],o=process.env.KANDELO_DEV_SHELL_TOOL_PATH!==void 0,s=o?"cargo":"bash",a=o?i:[$(r,"scripts","dev-shell.sh"),"cargo",...i],c=wr(s,a,{cwd:r,encoding:"utf8"});if(c.status!==0)throw new Error(Er(s,a,c));return wn={sourceRepoRoot:r,xtaskPath:pr(n)},wn.xtaskPath}function Va(){let r=Wa();if(r===null)return;let e=os();if(e===null)return;if(Ji){Ji(r,e);return}let t=Ha(r),n=["build-deps","program-index-context-check","--source-repo-root",r],i=wr(t,n,{cwd:r,encoding:"utf8",env:{...process.env,WASM_POSIX_DEPS_REGISTRY:e.join(":")}});if(i.status!==0)throw new Error(`Program package source projection is not current: -${Er(t,n,i)}`)}function qa(r,e){if(gr>0||!r.some(t=>t.startsWith("programs/")))return e();gr+=1;try{return Va(),e()}finally{gr-=1}}function Ne(r,e){let t=Object.keys(r).sort(),n=[...e].sort();return t.length===n.length&&t.every((i,o)=>i===n[o])}function mr(r){let e;try{e=JSON.parse(Ze(r,"utf8"))}catch(s){throw new Error(`Invalid program package index ${r}: ${s instanceof Error?s.message:String(s)}`)}if(typeof e!="object"||e===null||!Ne(e,["format","identities","packages"])||e.format!==Xi||typeof e.identities!="object"||e.identities===null||Array.isArray(e.identities)||typeof e.packages!="object"||e.packages===null||Array.isArray(e.packages))throw new Error(`Invalid program package index ${r}: expected ${Xi}`);let t=new Map,n=e.identities;for(let[s,a]of Object.entries(n)){if(mn(s,r,"identity package name",!1),typeof a!="object"||a===null||!Ne(a,["manifestSha256","cacheKeys"])||typeof a.manifestSha256!="string"||!/^[a-f0-9]{64}$/.test(a.manifestSha256)||typeof a.cacheKeys!="object"||a.cacheKeys===null||Array.isArray(a.cacheKeys))throw new Error(`Invalid program package index ${r}: malformed identity ${JSON.stringify(s)}`);let c=a.cacheKeys;if(!Ne(c,["wasm32","wasm64"])||Object.values(c).some(l=>typeof l!="string"||!/^[a-f0-9]{64}$/.test(l)))throw new Error(`Invalid program package index ${r}: identity ${JSON.stringify(s)} has invalid contextual cache keys`);t.set(s,{manifestSha256:a.manifestSha256,cacheKeys:c})}let i=new Map,o=e.packages;for(let[s,a]of Object.entries(o)){if(mn(s,r,"package name",!1),typeof a!="object"||a===null||!Ne(a,["manifestSha256","arches","cacheKeys","dependencyClosures","members"])||!Array.isArray(a.arches)||typeof a.cacheKeys!="object"||a.cacheKeys===null||Array.isArray(a.cacheKeys)||typeof a.dependencyClosures!="object"||a.dependencyClosures===null||Array.isArray(a.dependencyClosures)||!Array.isArray(a.members)||typeof a.manifestSha256!="string"||!/^[a-f0-9]{64}$/.test(a.manifestSha256))throw new Error(`Invalid program package index ${r}: malformed package ${JSON.stringify(s)}`);let c=a.arches;if(c.length===0||new Set(c).size!==c.length||c.some(p=>typeof p!="string"||!qe.has(p)))throw new Error(`Invalid program package index ${r}: package ${JSON.stringify(s)} has invalid arches`);let l=a.cacheKeys;if(!Ne(l,c)||Object.values(l).some(p=>typeof p!="string"||!/^[a-f0-9]{64}$/.test(p)))throw new Error(`Invalid program package index ${r}: package ${JSON.stringify(s)} has invalid cache keys`);let h=a.dependencyClosures;if(!Ne(h,c))throw new Error(`Invalid program package index ${r}: package ${JSON.stringify(s)} has invalid dependency closure arches`);let f={};for(let p of c){let w=h[p];if(!Array.isArray(w))throw new Error(`Invalid program package index ${r}: package ${JSON.stringify(s)} has a malformed dependency closure for ${p}`);let u=new Set;f[p]=w.map((m,v)=>{if(typeof m!="object"||m===null||!Ne(m,["packageName","manifestSha256","cacheKey"])||typeof m.packageName!="string"||typeof m.manifestSha256!="string"||!/^[a-f0-9]{64}$/.test(m.manifestSha256)||typeof m.cacheKey!="string"||!/^[a-f0-9]{64}$/.test(m.cacheKey))throw new Error(`Invalid program package index ${r}: package ${JSON.stringify(s)} dependency ${v+1} for ${p} is malformed`);let E=m;if(mn(E.packageName,r,`${s} dependency packageName`,!1),E.packageName===s||u.has(E.packageName))throw new Error(`Invalid program package index ${r}: package ${JSON.stringify(s)} dependency closure for ${p} must contain unique dependencies other than itself`);u.add(E.packageName);let z=t.get(E.packageName);if(!z||z.manifestSha256!==E.manifestSha256||z.cacheKeys[p]!==E.cacheKey)throw new Error(`Invalid program package index ${r}: package ${JSON.stringify(s)} dependency ${JSON.stringify(E.packageName)} for ${p} does not match the index's authoritative contextual identity`);return E})}let d=a.members.map((p,w)=>{if(typeof p!="object"||p===null||p.kind!=="output"&&p.kind!=="runtime-file"||typeof p.sourceArtifact!="string"||typeof p.mirrorPath!="string")throw new Error(`Invalid program package index ${r}: package ${JSON.stringify(s)} member ${w+1} is malformed`);let u=p,m=u.kind==="output"?["kind","sourceArtifact","mirrorPath","outputName","forkInstrumentation"]:["kind","sourceArtifact","mirrorPath","guestPath","mode"];if(!Ne(u,m))throw new Error(`Invalid program package index ${r}: package ${JSON.stringify(s)} member ${w+1} has unknown or missing fields`);if(Yi(u.sourceArtifact,r,`${s} sourceArtifact`),Yi(u.mirrorPath,r,`${s} mirrorPath`),u.kind==="output"){if(typeof u.outputName!="string"||u.forkInstrumentation!=="auto"&&u.forkInstrumentation!=="disabled")throw new Error(`Invalid program package index ${r}: package ${JSON.stringify(s)} output member lacks outputName or forkInstrumentation`);mn(u.outputName,r,`${s} outputName`)}else if(typeof u.guestPath!="string"||!u.guestPath.startsWith("/")||!Number.isInteger(u.mode)||u.mode<0||u.mode>511)throw new Error(`Invalid program package index ${r}: package ${JSON.stringify(s)} runtime member lacks valid guestPath or mode`);return u});if(d.length===0||new Set(d.map(p=>p.sourceArtifact)).size!==d.length||new Set(d.map(p=>p.mirrorPath)).size!==d.length||d.length===1&&d[0].mirrorPath.includes("/")||d.length>1&&d.some(p=>!p.mirrorPath.startsWith(`${s}/`)))throw new Error(`Invalid program package index ${r}: package ${JSON.stringify(s)} members are empty, collide, or violate scalar/package-directory layout`);let y=a.manifestSha256,g=t.get(s);if(!g||g.manifestSha256!==y||c.some(p=>g.cacheKeys[p]!==l[p]))throw new Error(`Invalid program package index ${r}: package ${JSON.stringify(s)} does not match its contextual package identity`);i.set(s,{manifestSha256:y,arches:c,cacheKeys:l,dependencyClosures:f,members:d})}return{identities:t,packages:i,indexPath:r}}function as(r){return JSON.stringify({manifestSha256:r.manifestSha256,arches:r.arches,cacheKeys:Object.fromEntries(r.arches.map(e=>[e,r.cacheKeys[e]])),dependencyClosures:Object.fromEntries(r.arches.map(e=>[e,[...r.dependencyClosures[e]].sort((t,n)=>t.packageNamen.packageName?1:0)])),members:r.members.map(e=>e.kind==="output"?{kind:e.kind,sourceArtifact:e.sourceArtifact,mirrorPath:e.mirrorPath,outputName:e.outputName,forkInstrumentation:e.forkInstrumentation}:{kind:e.kind,sourceArtifact:e.sourceArtifact,mirrorPath:e.mirrorPath,guestPath:e.guestPath,mode:e.mode})})}function Sr(){let r=$(zn(),"wasm",ve);return de(r)?mr(r):null}function ja(r){let e=Sr();if(!e)return null;let t=r.split("/");if(t[0]!=="programs"||!qe.has(t[1]))return null;let n=t[1];if(t.length>=4){let o=t[2];return e.packages.get(o)?.arches.includes(n)?o:null}if(t.length!==3)return null;let i=t[2];for(let[o,s]of e.packages)if(s.arches.includes(n)&&s.members.some(a=>a.kind==="output"&&a.mirrorPath.split("/").at(-1)===i))return o;return null}function Qi(r){let e=ja(r);if(e)throw new Error(`Installed package resolver path ${JSON.stringify(r)} is owned by ${JSON.stringify(e)}, but that package is not selected by the configured program registry`)}function cs(){let r=os(),e=new Map,t=new Map,n=new Map,i=new Map,o=[];if(r===null){let l=$(zn(),"wasm",ve);if(!de(l))return{identities:e,unidentifiedPackages:t,packages:n,unprojectedPackages:i,physicalProgramClaims:o};let h=mr(l);for(let[f,d]of h.identities)e.set(f,{...d,packageName:f,policyPath:`${h.indexPath}#identities.${f}`});for(let[f,d]of h.packages)o.push({packageName:f,projection:d,selected:!0}),n.set(f,{...d,packageName:f,policyPath:`${h.indexPath}#${f}`});return{identities:e,unidentifiedPackages:t,packages:n,unprojectedPackages:i,physicalProgramClaims:o}}let s=new Set,a=null,c=null;for(let l of r){if(!de(l))continue;if(!He(l).isDirectory())throw new Error(`Program registry root is not a directory: ${l}`);let h=$(l,ve);if(!de(h))throw new Error(`Program registry ${l} is missing ${ve}; generate it with xtask build-deps program-index`);let f=mr(h);a??=f.identities,c??=f.packages;let d=Ra(l,{withFileTypes:!0}).filter(y=>y.isDirectory()||y.isSymbolicLink()).sort((y,g)=>y.name.localeCompare(g.name));for(let y of d){let g=y.name,p=$(l,g,"package.toml");if(!de(p))continue;let w=!1;try{w=He(p).isFile()}catch{w=!1}if(!w)continue;let u=f.packages.get(g),m=!s.has(g);if(u&&o.push({packageName:g,projection:u,selected:m}),!m)continue;s.add(g);let v=a.get(g);v?e.set(g,{...v,packageName:g,manifestPath:p,policyPath:p}):t.set(g,p);let E=c.get(g);if(!E){i.set(g,p);continue}n.set(g,{...E,packageName:g,manifestPath:p,policyPath:p})}}return{identities:e,unidentifiedPackages:t,packages:n,unprojectedPackages:i,physicalProgramClaims:o}}function es(r){if(!r.manifestPath)return;let e;try{e=Ze(r.manifestPath)}catch(n){throw new Error(`Program package identity cannot verify ${r.manifestPath}: ${n instanceof Error?n.message:String(n)}`)}if(rs("sha256").update(e).digest("hex")!==r.manifestSha256)throw new Error(`Program package identity is stale for ${r.manifestPath}; regenerate ${ve}`)}function Ya(r){if(!r.manifestPath)return;let e;try{e=Ze(r.manifestPath)}catch(n){throw new Error(`Program package projection cannot verify ${r.manifestPath}: ${n instanceof Error?n.message:String(n)}`)}if(rs("sha256").update(e).digest("hex")!==r.manifestSha256)throw new Error(`Program package projection is stale for ${r.manifestPath}; regenerate ${ve}`)}function Dt(r){let e=zr(),t=e.packages.get(r);if(t)return Ya(t),t;let n=e.unprojectedPackages.get(r);if(n)throw new Error(`Package ${JSON.stringify(r)} is selected at ${n} but is absent from ${ve}; regenerate the registry projection`);return null}function Xa(r,e){let t=r.dependencyClosures[e];if(!t)throw yt(r.policyPath,`package ${JSON.stringify(r.packageName)} lacks a dependency identity closure for ${e}`);let n=cs(),i=n.identities.get(r.packageName);if(!i){let s=n.unidentifiedPackages.get(r.packageName);throw new Error(`Program package ${JSON.stringify(r.packageName)} has no authoritative contextual identity for ${e}${s?` at ${s}`:""}; regenerate ${ve} with the exact ordered registry roots`)}es(i);let o=i.cacheKeys[e];if(i.manifestSha256!==r.manifestSha256||o!==r.cacheKeys[e])throw new Error(`Program package ${JSON.stringify(r.packageName)} was projected with manifest ${r.manifestSha256} and cache key ${r.cacheKeys[e]} for ${e}, but the authoritative first-hit registry context at ${i.policyPath} requires manifest ${i.manifestSha256} and cache key ${o??""}. Regenerate this program projection with the exact ordered registry roots; the highest-priority index must carry the complete combined-context projection rather than relying on a lower suffix-context build identity.`);for(let s of t){let a=n.identities.get(s.packageName);if(!a){let l=n.unidentifiedPackages.get(s.packageName);throw l?new Error(`Program package ${JSON.stringify(r.packageName)} was generated against dependency ${JSON.stringify(s.packageName)}, but the first-hit package at ${l} has no contextual identity in ${ve}`):new Error(`Program package ${JSON.stringify(r.packageName)} was generated against dependency ${JSON.stringify(s.packageName)}, but that dependency is absent from the configured first-hit registry roots`)}es(a);let c=a.cacheKeys[e];if(a.manifestSha256!==s.manifestSha256||c!==s.cacheKey)throw new Error(`Program package ${JSON.stringify(r.packageName)} has a contextual cache identity mismatch for ${e}: its projection expects dependency ${JSON.stringify(s.packageName)} manifest ${s.manifestSha256} and cache key ${s.cacheKey}, but first-hit selection at ${a.policyPath} provides manifest ${a.manifestSha256} and cache key ${c??""}. Regenerate the program projection with the exact ordered registry roots; the complete highest-priority projection must bind every selected program to the same combined dependency context.`)}}function zr(){let r=cs(),{physicalProgramClaims:e,...t}=r,n={...t,legacyFlatOutputs:new Map,forkInstrumentationDisabledOutputs:new Map},i=[];for(let o of r.packages.values()){let s=o.members.length>1;for(let a of o.arches)for(let c of o.members){let l=i.find(y=>y.arch===a&&(y.path===c.mirrorPath||y.path.startsWith(`${c.mirrorPath}/`)||c.mirrorPath.startsWith(`${y.path}/`)));if(l)throw new Error(`Program resolver paths programs/${a}/${l.path} and programs/${a}/${c.mirrorPath} conflict between selected packages ${JSON.stringify(l.packageName)} and ${JSON.stringify(o.packageName)}`);if(i.push({arch:a,path:c.mirrorPath,packageName:o.packageName}),c.kind!=="output")continue;let h=c.mirrorPath.split("/").at(-1),f=`${a}/${h}`,d=n.legacyFlatOutputs.get(f);d||(d={scalarOwners:new Set,packagePaths:new Map,shadowedOwners:new Set},n.legacyFlatOutputs.set(f,d)),s?d.packagePaths.set(`programs/${a}/${c.mirrorPath}`,o.packageName):d.scalarOwners.add(o.packageName),c.forkInstrumentation==="disabled"&&n.forkInstrumentationDisabledOutputs.set(`${a}/${c.mirrorPath}`,o.packageName)}}for(let{packageName:o,projection:s,selected:a}of e)if(!(a&&r.packages.has(o)))for(let c of s.arches)for(let l of s.members){if(l.kind!=="output")continue;let h=l.mirrorPath.split("/").at(-1),f=`${c}/${h}`,d=n.legacyFlatOutputs.get(f);d||(d={scalarOwners:new Set,packagePaths:new Map,shadowedOwners:new Set},n.legacyFlatOutputs.set(f,d)),d.shadowedOwners.add(o)}return n}function Ja(r){let e=r.split("/");if(e.length!==3||e[0]!=="programs"||!qe.has(e[1]))return null;let t=zr().legacyFlatOutputs.get(`${e[1]}/${e[2]}`);if(!t)return null;for(let n of t.scalarOwners){let i=Dt(n);if(i)return i}for(let n of t.packagePaths.values())Dt(n);if(t.packagePaths.size>0)throw new Error(`Legacy flat resolver path ${JSON.stringify(r)} belongs to a multi-member package; use ${[...t.packagePaths.keys()].sort().map(n=>JSON.stringify(n)).join(" or ")}`);for(let n of t.shadowedOwners){let i=Dt(n);if(i)return i;throw new Error(`Legacy flat resolver path ${JSON.stringify(r)} is claimed by a lower-root program package ${JSON.stringify(n)}, but its first-hit selected package does not project that program; stale scalar mirror fallback is forbidden`)}return null}function ts(r,e,t){if(!r.arches.includes(e))throw yt(r.policyPath,`package ${JSON.stringify(r.packageName)} does not declare resolver artifacts for ${e}`);let n=r.cacheKeys[e];if(!n)throw yt(r.policyPath,`package ${JSON.stringify(r.packageName)} lacks a cache identity for ${e}`);Xa(r,e);let i=as(r),o=r.members.map(s=>({packageName:r.packageName,relPath:`programs/${e}/${s.mirrorPath}`,sourceArtifact:s.sourceArtifact,cacheKey:n,forkInstrumentation:s.kind==="output"?s.forkInstrumentation??null:null,projectionIdentity:i}));if(!o.some(s=>s.relPath===t))throw yt(r.policyPath,`resolver path ${JSON.stringify(t)} is not a declared member of package ${JSON.stringify(r.packageName)}`);return{manifestPath:r.policyPath,packageName:r.packageName,members:o}}function Qa(r){let e=Ce(r),t=e.split("/");if(t[0]==="programs"&&!Da()&&Sr()===null)throw new Error(`Installed host package is missing wasm/${ve}; program artifacts cannot be resolved without packaged policy`);if(t.length===3){let s=Ja(e);return s?ts(s,t[1],e):(Qi(e),null)}if(t.length<4||t[0]!=="programs"||!qe.has(t[1]))return null;let n=t[1],i=t[2],o=Dt(i);return o?ts(o,n,e):(Qi(e),null)}function ec(r){let e=Ce(r);for(let t of["programs/wasm32/","programs/wasm64/"])if(e.startsWith(t))return e.slice(t.length);return null}function tc(r){let e=Ce(r);for(let t of qe){let n=`programs/${t}/`;if(e.startsWith(n)){let i=zr().forkInstrumentationDisabledOutputs.get(`${t}/${e.slice(n.length)}`);return i?Dt(i)!==null:!1}}return!1}function nc(r){let e=Ce(r);if(e==="kernel.wasm")return Lr;let t=ec(e);if(t&&t.endsWith(".wasm"))return Fa}function rc(r,e,t){if(!r.endsWith(".wasm"))return!1;try{let n=Ze(r),i=n.buffer.slice(n.byteOffset,n.byteOffset+n.byteLength),o=t===void 0?tc(e):t==="disabled";return Nr(i,{expectedAbi:42,requiredExports:nc(e),requireForkInstrumentation:o?!1:void 0,forbidForkInstrumentation:o}).length>0}catch{return!0}}function ic(r){if(!r.endsWith(".vfs")&&!r.endsWith(".vfs.zst"))return!1;try{let t=yn.readImageMetadata(Ze(r))?.kernelAbi;return t!==void 0&&t!==42}catch{return!0}}function br(r,e,t){return rc(r,e,t)||ic(r)}function ls(r,e,t){let n=r.filter(de);return n.length===0?null:n.find(i=>{try{return He(i).isFile()&&!br(i,e,t)}catch{return!1}})??null}function fs(r,e,t){try{if(!En(r).isSymbolicLink())return r;let i=me(r);if(!He(i).isFile()||br(i,e,t))throw new Error("canonical target is not an accepted regular file");if(Ce(e).startsWith("programs/")&&sc(i))throw new Error("resolver-owned program generation has no matching selected package projection");return i}catch(n){throw new Error(`Binary changed or became invalid while pinning ${e}: ${n instanceof Error?n.message:String(n)}`)}}function sc(r){let e=[is()];try{e.push($(Ve(),"local-binaries",".kandelo-local-generations"))}catch{}return e.some(t=>{try{return de(t)&&kr(me(t),r)}catch{return!1}})}function kr(r,e){let t=Na(r,e);return t===""||t!==".."&&!t.startsWith(`..${Ca}`)&&!Sn(t)}function oc(r,e){let t=e.split("/"),n=r;for(let i=0;ia.packageName!==o))return"declared package members do not share a valid program namespace";if(!He(e).isDirectory())return"shared package generation root is not a directory";let s=t[0].cacheKey;if(!/^[a-f0-9]{64}$/.test(s)||t.some(a=>a.cacheKey!==s))return"declared package members do not share one valid cache identity";if(r.identity==="local-generation"){let a=$(r.root,".kandelo-local-generations",i,o,s);if(!de(a))return"local mirror targets are not one direct immutable local generation";let c=me(a);return Ut(e)===c?null:"local mirror targets are not one direct immutable local generation"}if(r.identity==="program-cache"){let a=is();if(!de(a))return"fetched mirror targets are not one canonical program-cache generation";let c=me(a),l=Ba(e),h=l.startsWith(`${o}-`)&&new RegExp(`-rev[0-9]+-${i}-${s}$`).test(l);return Ut(e)===c&&h?null:"fetched mirror targets are not one canonical program-cache generation"}return"installed-package symlink closures are not an immutable installed identity"}function cc(r,e,t){if(e.length!==t.length)return{failure:"internal member/path count mismatch"};try{let n=e.map(l=>{let h=En(l);return h.isSymbolicLink()?"symlink":h.isFile()?"file":"other"});if(n.includes("other"))return{failure:"a selected mirror member is neither a regular file nor a symlink"};let i=n.every(l=>l==="symlink"),o=n.every(l=>l==="file");if(!i&&!o)return{failure:"regular files and symlinks cannot share one package identity"};if(o){if(!r.allowRegularFileClosure)return{failure:"a mutable source-checkout wasm tree is not an installed package identity"};let l=t[0].packageName,h=t[0].projectionIdentity;if(t.some(p=>p.packageName!==l||p.projectionIdentity!==h))return{failure:"declared members do not share one selected package projection"};let d=Sr()?.packages.get(l);if(!d||as(d)!==h)return{failure:"installed bytes do not match the selected package projection"};let y=me(r.root),g=[];for(let p of e){let w=me(p);if(!kr(y,w)||!He(w).isFile())return{failure:"an installed-package member escapes its immutable wasm tree"};g.push(w)}return{paths:g}}let s=null,a=[];for(let l=0;llc(r))}function lc(r){let e=Ce(r),t=Qa(e);if(t){let s=fc(t.members.map(a=>a.relPath),t.members);if(s)return s[t.members.findIndex(a=>a.relPath===e)];throw new vn(`Package artifacts not found for ${t.packageName}: ${e}`)}let n=[],i=[];for(let s of ss())for(let a of s.candidatesFor(r))n.push(a),i.push(a);let o=ls(i,r);if(o)return fs(o,r);throw i.some(de)?new Error(`Binary exists but was rejected by artifact policy: ${r} -`+n.map(s=>` checked: ${s}`).join(` -`)):new vn(`Binary not found: ${r} -`+n.map(s=>` checked: ${s}`).join(` +${n}`:""}`}function gl(r){let e=process.env.KANDELO_DEV_SHELL_TOOL_PATH!==void 0,t=e?"rustc":"bash",n=e?["-vV"]:[M(r,"scripts","dev-shell.sh"),"rustc","-vV"],i=ii(t,n,{cwd:r,encoding:"utf8"});if(i.status!==0)throw new Error(oi(t,n,i));let s=i.stdout.split(/\r?\n/).find(o=>o.startsWith("host: "))?.slice(6).trim();if(!s)throw new Error(`Could not determine the Rust host target for ${r}`);return s}function ni(r){try{if(Vn(r).isFile())return Oe(r)}catch{}throw new Error(`Prepared xtask is not a regular file: ${r}`)}function El(r){let e=process.env.WASM_POSIX_XTASK_BIN;if(e!==void 0){let l=Xn(e)?Se(e):Se(r,e);return ni(l)}if(Zn?.sourceRepoRoot===r)return ni(Zn.xtaskPath);let t=gl(r),n=M(r,"target",t,"release",process.platform==="win32"?"xtask.exe":"xtask"),i=["build","--release","-p","xtask","--target",t,"--quiet"],s=process.env.KANDELO_DEV_SHELL_TOOL_PATH!==void 0,o=s?"cargo":"bash",a=s?i:[M(r,"scripts","dev-shell.sh"),"cargo",...i],c=ii(o,a,{cwd:r,encoding:"utf8"});if(c.status!==0)throw new Error(oi(o,a,c));return Zn={sourceRepoRoot:r,xtaskPath:ni(n)},Zn.xtaskPath}function wl(){let r=yl();if(r===null)return;let e=Eo();if(e===null)return;if(uo){uo(r,e);return}let t=El(r),n=["build-deps","program-index-context-check","--source-repo-root",r],i=ii(t,n,{cwd:r,encoding:"utf8",env:{...process.env,WASM_POSIX_DEPS_REGISTRY:e.join(":")}});if(i.status!==0)throw new Error(`Program package source projection is not current: +${oi(t,n,i)}`)}function Sl(r,e){if(ti>0||!r.some(t=>t.startsWith("programs/")))return e();ti+=1;try{return wl(),e()}finally{ti-=1}}function We(r,e){let t=Object.keys(r).sort(),n=[...e].sort();return t.length===n.length&&t.every((i,s)=>i===n[s])}function ri(r){let e;try{e=JSON.parse(Qe(r,"utf8"))}catch(o){throw new Error(`Invalid program package index ${r}: ${o instanceof Error?o.message:String(o)}`)}if(typeof e!="object"||e===null||!We(e,["format","identities","packages"])||e.format!==lo||typeof e.identities!="object"||e.identities===null||Array.isArray(e.identities)||typeof e.packages!="object"||e.packages===null||Array.isArray(e.packages))throw new Error(`Invalid program package index ${r}: expected ${lo}`);let t=new Map,n=e.identities;for(let[o,a]of Object.entries(n)){if(Gn(o,r,"identity package name",!1),typeof a!="object"||a===null||!We(a,["manifestSha256","cacheKeys"])||typeof a.manifestSha256!="string"||!/^[a-f0-9]{64}$/.test(a.manifestSha256)||typeof a.cacheKeys!="object"||a.cacheKeys===null||Array.isArray(a.cacheKeys))throw new Error(`Invalid program package index ${r}: malformed identity ${JSON.stringify(o)}`);let c=a.cacheKeys;if(!We(c,["wasm32","wasm64"])||Object.values(c).some(l=>typeof l!="string"||!/^[a-f0-9]{64}$/.test(l)))throw new Error(`Invalid program package index ${r}: identity ${JSON.stringify(o)} has invalid contextual cache keys`);t.set(o,{manifestSha256:a.manifestSha256,cacheKeys:c})}let i=new Map,s=e.packages;for(let[o,a]of Object.entries(s)){if(Gn(o,r,"package name",!1),typeof a!="object"||a===null||!We(a,["manifestSha256","arches","cacheKeys","dependencyClosures","members"])||!Array.isArray(a.arches)||typeof a.cacheKeys!="object"||a.cacheKeys===null||Array.isArray(a.cacheKeys)||typeof a.dependencyClosures!="object"||a.dependencyClosures===null||Array.isArray(a.dependencyClosures)||!Array.isArray(a.members)||typeof a.manifestSha256!="string"||!/^[a-f0-9]{64}$/.test(a.manifestSha256))throw new Error(`Invalid program package index ${r}: malformed package ${JSON.stringify(o)}`);let c=a.arches;if(c.length===0||new Set(c).size!==c.length||c.some(h=>typeof h!="string"||!nt.has(h)))throw new Error(`Invalid program package index ${r}: package ${JSON.stringify(o)} has invalid arches`);let l=a.cacheKeys;if(!We(l,c)||Object.values(l).some(h=>typeof h!="string"||!/^[a-f0-9]{64}$/.test(h)))throw new Error(`Invalid program package index ${r}: package ${JSON.stringify(o)} has invalid cache keys`);let p=a.dependencyClosures;if(!We(p,c))throw new Error(`Invalid program package index ${r}: package ${JSON.stringify(o)} has invalid dependency closure arches`);let u={};for(let h of c){let g=p[h];if(!Array.isArray(g))throw new Error(`Invalid program package index ${r}: package ${JSON.stringify(o)} has a malformed dependency closure for ${h}`);let f=new Set;u[h]=g.map((y,E)=>{if(typeof y!="object"||y===null||!We(y,["packageName","manifestSha256","cacheKey"])||typeof y.packageName!="string"||typeof y.manifestSha256!="string"||!/^[a-f0-9]{64}$/.test(y.manifestSha256)||typeof y.cacheKey!="string"||!/^[a-f0-9]{64}$/.test(y.cacheKey))throw new Error(`Invalid program package index ${r}: package ${JSON.stringify(o)} dependency ${E+1} for ${h} is malformed`);let w=y;if(Gn(w.packageName,r,`${o} dependency packageName`,!1),w.packageName===o||f.has(w.packageName))throw new Error(`Invalid program package index ${r}: package ${JSON.stringify(o)} dependency closure for ${h} must contain unique dependencies other than itself`);f.add(w.packageName);let O=t.get(w.packageName);if(!O||O.manifestSha256!==w.manifestSha256||O.cacheKeys[h]!==w.cacheKey)throw new Error(`Invalid program package index ${r}: package ${JSON.stringify(o)} dependency ${JSON.stringify(w.packageName)} for ${h} does not match the index's authoritative contextual identity`);return w})}let m=a.members.map((h,g)=>{if(typeof h!="object"||h===null||h.kind!=="output"&&h.kind!=="runtime-file"||typeof h.sourceArtifact!="string"||typeof h.mirrorPath!="string")throw new Error(`Invalid program package index ${r}: package ${JSON.stringify(o)} member ${g+1} is malformed`);let f=h,y=f.kind==="output"?["kind","sourceArtifact","mirrorPath","outputName","forkInstrumentation"]:["kind","sourceArtifact","mirrorPath","guestPath","mode"];if(!We(f,y))throw new Error(`Invalid program package index ${r}: package ${JSON.stringify(o)} member ${g+1} has unknown or missing fields`);if(co(f.sourceArtifact,r,`${o} sourceArtifact`),co(f.mirrorPath,r,`${o} mirrorPath`),f.kind==="output"){if(typeof f.outputName!="string"||f.forkInstrumentation!=="auto"&&f.forkInstrumentation!=="disabled")throw new Error(`Invalid program package index ${r}: package ${JSON.stringify(o)} output member lacks outputName or forkInstrumentation`);Gn(f.outputName,r,`${o} outputName`)}else if(typeof f.guestPath!="string"||!f.guestPath.startsWith("/")||!Number.isInteger(f.mode)||f.mode<0||f.mode>511)throw new Error(`Invalid program package index ${r}: package ${JSON.stringify(o)} runtime member lacks valid guestPath or mode`);return f});if(m.length===0||new Set(m.map(h=>h.sourceArtifact)).size!==m.length||new Set(m.map(h=>h.mirrorPath)).size!==m.length||m.length===1&&m[0].mirrorPath.includes("/")||m.length>1&&m.some(h=>!h.mirrorPath.startsWith(`${o}/`)))throw new Error(`Invalid program package index ${r}: package ${JSON.stringify(o)} members are empty, collide, or violate scalar/package-directory layout`);let _=a.manifestSha256,d=t.get(o);if(!d||d.manifestSha256!==_||c.some(h=>d.cacheKeys[h]!==l[h]))throw new Error(`Invalid program package index ${r}: package ${JSON.stringify(o)} does not match its contextual package identity`);i.set(o,{manifestSha256:_,arches:c,cacheKeys:l,dependencyClosures:u,members:m})}return{identities:t,packages:i,indexPath:r}}function wo(r){return JSON.stringify({manifestSha256:r.manifestSha256,arches:r.arches,cacheKeys:Object.fromEntries(r.arches.map(e=>[e,r.cacheKeys[e]])),dependencyClosures:Object.fromEntries(r.arches.map(e=>[e,[...r.dependencyClosures[e]].sort((t,n)=>t.packageNamen.packageName?1:0)])),members:r.members.map(e=>e.kind==="output"?{kind:e.kind,sourceArtifact:e.sourceArtifact,mirrorPath:e.mirrorPath,outputName:e.outputName,forkInstrumentation:e.forkInstrumentation}:{kind:e.kind,sourceArtifact:e.sourceArtifact,mirrorPath:e.mirrorPath,guestPath:e.guestPath,mode:e.mode})})}function ai(){let r=M(Yn(),"wasm",ve);return me(r)?ri(r):null}function Ol(r){let e=ai();if(!e)return null;let t=r.split("/");if(t[0]!=="programs"||!nt.has(t[1]))return null;let n=t[1];if(t.length>=4){let s=t[2];return e.packages.get(s)?.arches.includes(n)?s:null}if(t.length!==3)return null;let i=t[2];for(let[s,o]of e.packages)if(o.arches.includes(n)&&o.members.some(a=>a.kind==="output"&&a.mirrorPath.split("/").at(-1)===i))return s;return null}function fo(r){let e=Ol(r);if(e)throw new Error(`Installed package resolver path ${JSON.stringify(r)} is owned by ${JSON.stringify(e)}, but that package is not selected by the configured program registry`)}function So(){let r=Eo(),e=new Map,t=new Map,n=new Map,i=new Map,s=[];if(r===null){let l=M(Yn(),"wasm",ve);if(!me(l))return{identities:e,unidentifiedPackages:t,packages:n,unprojectedPackages:i,physicalProgramClaims:s};let p=ri(l);for(let[u,m]of p.identities)e.set(u,{...m,packageName:u,policyPath:`${p.indexPath}#identities.${u}`});for(let[u,m]of p.packages)s.push({packageName:u,projection:m,selected:!0}),n.set(u,{...m,packageName:u,policyPath:`${p.indexPath}#${u}`});return{identities:e,unidentifiedPackages:t,packages:n,unprojectedPackages:i,physicalProgramClaims:s}}let o=new Set,a=null,c=null;for(let l of r){if(!me(l))continue;if(!et(l).isDirectory())throw new Error(`Program registry root is not a directory: ${l}`);let p=M(l,ve);if(!me(p))throw new Error(`Program registry ${l} is missing ${ve}; generate it with xtask build-deps program-index`);let u=ri(p);a??=u.identities,c??=u.packages;let m=ol(l,{withFileTypes:!0}).filter(_=>_.isDirectory()||_.isSymbolicLink()).sort((_,d)=>_.name.localeCompare(d.name));for(let _ of m){let d=_.name,h=M(l,d,"package.toml");if(!me(h))continue;let g=!1;try{g=et(h).isFile()}catch{g=!1}if(!g)continue;let f=u.packages.get(d),y=!o.has(d);if(f&&s.push({packageName:d,projection:f,selected:y}),!y)continue;o.add(d);let E=a.get(d);E?e.set(d,{...E,packageName:d,manifestPath:h,policyPath:h}):t.set(d,h);let w=c.get(d);if(!w){i.set(d,h);continue}n.set(d,{...w,packageName:d,manifestPath:h,policyPath:h})}}return{identities:e,unidentifiedPackages:t,packages:n,unprojectedPackages:i,physicalProgramClaims:s}}function ho(r){if(!r.manifestPath)return;let e;try{e=Qe(r.manifestPath)}catch(n){throw new Error(`Program package identity cannot verify ${r.manifestPath}: ${n instanceof Error?n.message:String(n)}`)}if(_o("sha256").update(e).digest("hex")!==r.manifestSha256)throw new Error(`Program package identity is stale for ${r.manifestPath}; regenerate ${ve}`)}function Il(r){if(!r.manifestPath)return;let e;try{e=Qe(r.manifestPath)}catch(n){throw new Error(`Program package projection cannot verify ${r.manifestPath}: ${n instanceof Error?n.message:String(n)}`)}if(_o("sha256").update(e).digest("hex")!==r.manifestSha256)throw new Error(`Program package projection is stale for ${r.manifestPath}; regenerate ${ve}`)}function rn(r){let e=ci(),t=e.packages.get(r);if(t)return Il(t),t;let n=e.unprojectedPackages.get(r);if(n)throw new Error(`Package ${JSON.stringify(r)} is selected at ${n} but is absent from ${ve}; regenerate the registry projection`);return null}function vl(r,e){let t=r.dependencyClosures[e];if(!t)throw vt(r.policyPath,`package ${JSON.stringify(r.packageName)} lacks a dependency identity closure for ${e}`);let n=So(),i=n.identities.get(r.packageName);if(!i){let o=n.unidentifiedPackages.get(r.packageName);throw new Error(`Program package ${JSON.stringify(r.packageName)} has no authoritative contextual identity for ${e}${o?` at ${o}`:""}; regenerate ${ve} with the exact ordered registry roots`)}ho(i);let s=i.cacheKeys[e];if(i.manifestSha256!==r.manifestSha256||s!==r.cacheKeys[e])throw new Error(`Program package ${JSON.stringify(r.packageName)} was projected with manifest ${r.manifestSha256} and cache key ${r.cacheKeys[e]} for ${e}, but the authoritative first-hit registry context at ${i.policyPath} requires manifest ${i.manifestSha256} and cache key ${s??""}. Regenerate this program projection with the exact ordered registry roots; the highest-priority index must carry the complete combined-context projection rather than relying on a lower suffix-context build identity.`);for(let o of t){let a=n.identities.get(o.packageName);if(!a){let l=n.unidentifiedPackages.get(o.packageName);throw l?new Error(`Program package ${JSON.stringify(r.packageName)} was generated against dependency ${JSON.stringify(o.packageName)}, but the first-hit package at ${l} has no contextual identity in ${ve}`):new Error(`Program package ${JSON.stringify(r.packageName)} was generated against dependency ${JSON.stringify(o.packageName)}, but that dependency is absent from the configured first-hit registry roots`)}ho(a);let c=a.cacheKeys[e];if(a.manifestSha256!==o.manifestSha256||c!==o.cacheKey)throw new Error(`Program package ${JSON.stringify(r.packageName)} has a contextual cache identity mismatch for ${e}: its projection expects dependency ${JSON.stringify(o.packageName)} manifest ${o.manifestSha256} and cache key ${o.cacheKey}, but first-hit selection at ${a.policyPath} provides manifest ${a.manifestSha256} and cache key ${c??""}. Regenerate the program projection with the exact ordered registry roots; the complete highest-priority projection must bind every selected program to the same combined dependency context.`)}}function ci(){let r=So(),{physicalProgramClaims:e,...t}=r,n={...t,legacyFlatOutputs:new Map,forkInstrumentationDisabledOutputs:new Map},i=[];for(let s of r.packages.values()){let o=s.members.length>1;for(let a of s.arches)for(let c of s.members){let l=i.find(_=>_.arch===a&&(_.path===c.mirrorPath||_.path.startsWith(`${c.mirrorPath}/`)||c.mirrorPath.startsWith(`${_.path}/`)));if(l)throw new Error(`Program resolver paths programs/${a}/${l.path} and programs/${a}/${c.mirrorPath} conflict between selected packages ${JSON.stringify(l.packageName)} and ${JSON.stringify(s.packageName)}`);if(i.push({arch:a,path:c.mirrorPath,packageName:s.packageName}),c.kind!=="output")continue;let p=c.mirrorPath.split("/").at(-1),u=`${a}/${p}`,m=n.legacyFlatOutputs.get(u);m||(m={scalarOwners:new Set,packagePaths:new Map,shadowedOwners:new Set},n.legacyFlatOutputs.set(u,m)),o?m.packagePaths.set(`programs/${a}/${c.mirrorPath}`,s.packageName):m.scalarOwners.add(s.packageName),c.forkInstrumentation==="disabled"&&n.forkInstrumentationDisabledOutputs.set(`${a}/${c.mirrorPath}`,s.packageName)}}for(let{packageName:s,projection:o,selected:a}of e)if(!(a&&r.packages.has(s)))for(let c of o.arches)for(let l of o.members){if(l.kind!=="output")continue;let p=l.mirrorPath.split("/").at(-1),u=`${c}/${p}`,m=n.legacyFlatOutputs.get(u);m||(m={scalarOwners:new Set,packagePaths:new Map,shadowedOwners:new Set},n.legacyFlatOutputs.set(u,m)),m.shadowedOwners.add(s)}return n}function Al(r){let e=r.split("/");if(e.length!==3||e[0]!=="programs"||!nt.has(e[1]))return null;let t=ci().legacyFlatOutputs.get(`${e[1]}/${e[2]}`);if(!t)return null;for(let n of t.scalarOwners){let i=rn(n);if(i)return i}for(let n of t.packagePaths.values())rn(n);if(t.packagePaths.size>0)throw new Error(`Legacy flat resolver path ${JSON.stringify(r)} belongs to a multi-member package; use ${[...t.packagePaths.keys()].sort().map(n=>JSON.stringify(n)).join(" or ")}`);for(let n of t.shadowedOwners){let i=rn(n);if(i)return i;throw new Error(`Legacy flat resolver path ${JSON.stringify(r)} is claimed by a lower-root program package ${JSON.stringify(n)}, but its first-hit selected package does not project that program; stale scalar mirror fallback is forbidden`)}return null}function po(r,e,t){if(!r.arches.includes(e))throw vt(r.policyPath,`package ${JSON.stringify(r.packageName)} does not declare resolver artifacts for ${e}`);let n=r.cacheKeys[e];if(!n)throw vt(r.policyPath,`package ${JSON.stringify(r.packageName)} lacks a cache identity for ${e}`);vl(r,e);let i=wo(r),s=r.members.map(o=>({packageName:r.packageName,relPath:`programs/${e}/${o.mirrorPath}`,sourceArtifact:o.sourceArtifact,cacheKey:n,forkInstrumentation:o.kind==="output"?o.forkInstrumentation??null:null,projectionIdentity:i}));if(!s.some(o=>o.relPath===t))throw vt(r.policyPath,`resolver path ${JSON.stringify(t)} is not a declared member of package ${JSON.stringify(r.packageName)}`);return{manifestPath:r.policyPath,packageName:r.packageName,members:s}}function xl(r){let e=Ge(r),t=e.split("/");if(t[0]==="programs"&&!hl()&&ai()===null)throw new Error(`Installed host package is missing wasm/${ve}; program artifacts cannot be resolved without packaged policy`);if(t.length===3){let o=Al(e);return o?po(o,t[1],e):(fo(e),null)}if(t.length<4||t[0]!=="programs"||!nt.has(t[1]))return null;let n=t[1],i=t[2],s=rn(i);return s?po(s,n,e):(fo(e),null)}function Rl(r){let e=Ge(r);for(let t of["programs/wasm32/","programs/wasm64/"])if(e.startsWith(t))return e.slice(t.length);return null}function kl(r){let e=Ge(r);for(let t of nt){let n=`programs/${t}/`;if(e.startsWith(n)){let i=ci().forkInstrumentationDisabledOutputs.get(`${t}/${e.slice(n.length)}`);return i?rn(i)!==null:!1}}return!1}function Tl(r){let e=Ge(r);if(e==="kernel.wasm")return Ci;let t=Rl(e);if(t&&t.endsWith(".wasm"))return fl}function bl(r,e,t){if(!r.endsWith(".wasm"))return!1;try{let n=Qe(r),i=n.buffer.slice(n.byteOffset,n.byteOffset+n.byteLength),s=t===void 0?kl(e):t==="disabled";return Vi(i,{expectedAbi:43,requiredExports:Tl(e),requireForkInstrumentation:s?!1:void 0,forbidForkInstrumentation:s}).length>0}catch{return!0}}function Ll(r){if(!r.endsWith(".vfs")&&!r.endsWith(".vfs.zst"))return!1;try{let t=$n.readImageMetadata(Qe(r))?.kernelAbi;return t!==void 0&&t!==43}catch{return!0}}function li(r,e,t){return bl(r,e,t)||Ll(r)}function Oo(r,e,t){let n=r.filter(me);return n.length===0?null:n.find(i=>{try{return et(i).isFile()&&!li(i,e,t)}catch{return!1}})??null}function Io(r,e,t){try{if(!Vn(r).isSymbolicLink())return r;let i=Oe(r);if(!et(i).isFile()||li(i,e,t))throw new Error("canonical target is not an accepted regular file");if(Ge(e).startsWith("programs/")&&Pl(i))throw new Error("resolver-owned program generation has no matching selected package projection");return i}catch(n){throw new Error(`Binary changed or became invalid while pinning ${e}: ${n instanceof Error?n.message:String(n)}`)}}function Pl(r){let e=[yo()];try{e.push(M(tt(),"local-binaries",".kandelo-local-generations"))}catch{}return e.some(t=>{try{return me(t)&&ui(Oe(t),r)}catch{return!1}})}function ui(r,e){let t=cl(r,e);return t===""||t!==".."&&!t.startsWith(`..${ll}`)&&!Xn(t)}function zl(r,e){let t=e.split("/"),n=r;for(let i=0;ia.packageName!==s))return"declared package members do not share a valid program namespace";if(!et(e).isDirectory())return"shared package generation root is not a directory";let o=t[0].cacheKey;if(!/^[a-f0-9]{64}$/.test(o)||t.some(a=>a.cacheKey!==o))return"declared package members do not share one valid cache identity";if(r.identity==="local-generation"){let a=M(r.root,".kandelo-local-generations",i,s,o);if(!me(a))return"local mirror targets are not one direct immutable local generation";let c=Oe(a);return sn(e)===c?null:"local mirror targets are not one direct immutable local generation"}if(r.identity==="program-cache"){let a=yo();if(!me(a))return"fetched mirror targets are not one canonical program-cache generation";let c=Oe(a),l=al(e),p=l.startsWith(`${s}-`)&&new RegExp(`-rev[0-9]+-${i}-${o}$`).test(l);return sn(e)===c&&p?null:"fetched mirror targets are not one canonical program-cache generation"}return"installed-package symlink closures are not an immutable installed identity"}function Fl(r,e,t){if(e.length!==t.length)return{failure:"internal member/path count mismatch"};try{let n=e.map(l=>{let p=Vn(l);return p.isSymbolicLink()?"symlink":p.isFile()?"file":"other"});if(n.includes("other"))return{failure:"a selected mirror member is neither a regular file nor a symlink"};let i=n.every(l=>l==="symlink"),s=n.every(l=>l==="file");if(!i&&!s)return{failure:"regular files and symlinks cannot share one package identity"};if(s){if(!r.allowRegularFileClosure)return{failure:"a mutable source-checkout wasm tree is not an installed package identity"};let l=t[0].packageName,p=t[0].projectionIdentity;if(t.some(h=>h.packageName!==l||h.projectionIdentity!==p))return{failure:"declared members do not share one selected package projection"};let m=ai()?.packages.get(l);if(!m||wo(m)!==p)return{failure:"installed bytes do not match the selected package projection"};let _=Oe(r.root),d=[];for(let h of e){let g=Oe(h);if(!ui(_,g)||!et(g).isFile())return{failure:"an installed-package member escapes its immutable wasm tree"};d.push(g)}return{paths:d}}let o=null,a=[];for(let l=0;lCl(r))}function Cl(r){let e=Ge(r),t=xl(e);if(t){let o=Kl(t.members.map(a=>a.relPath),t.members);if(o)return o[t.members.findIndex(a=>a.relPath===e)];throw new Hn(`Package artifacts not found for ${t.packageName}: ${e}`)}let n=[],i=[];for(let o of go())for(let a of o.candidatesFor(r))n.push(a),i.push(a);let s=Oo(i,r);if(s)return Io(s,r);throw i.some(me)?new Error(`Binary exists but was rejected by artifact policy: ${r} +`+n.map(o=>` checked: ${o}`).join(` +`)):new Hn(`Binary not found: ${r} +`+n.map(o=>` checked: ${o}`).join(` `)+` - Run scripts/fetch-binaries.sh, place a file at local-binaries/${e}, or install a package that includes wasm/${r}.`)}function fc(r,e){if(r.length===0)return[];let t=!1,n=[];for(let i of ss()){let o=[],s=[];if(e){let[a,c,l]=e[0].relPath.split("/");a==="programs"&&c&&l&&(t||=de($(i.root,a,c,l)))}for(let[a,c]of r.entries()){let l=i.candidatesFor(c),h=l.filter(de);t||=h.length>0;let f=ls(l,c,e?.[a]?.forkInstrumentation);f?o.push(f):h.length>0?s.push(`${c} (rejected by artifact policy)`):s.push(`${c} (missing)`)}if(s.length===0&&e){let a=cc(i,o,e);if("failure"in a)s.push(`shared package identity rejected: ${a.failure}`);else{let c=a.paths.flatMap((l,h)=>br(l,r[h],e[h].forkInstrumentation)?[r[h]]:[]);if(c.length>0)s.push(`pinned package generation rejected by artifact policy: ${c.join(", ")}`);else return a.paths}}if(s.length===0)return o.map((a,c)=>fs(a,r[c],e?.[c]?.forkInstrumentation));n.push(` ${i.label} (${i.root}): ${s.join(", ")}`)}if(!t)return null;throw new Error(`Package artifact closure is incomplete: no single provenance tier contains every accepted artifact, and tiers will not be mixed. + Run scripts/fetch-binaries.sh, place a file at local-binaries/${e}, or install a package that includes wasm/${r}.`)}function Kl(r,e){if(r.length===0)return[];let t=!1,n=[];for(let i of go()){let s=[],o=[];if(e){let[a,c,l]=e[0].relPath.split("/");a==="programs"&&c&&l&&(t||=me(M(i.root,a,c,l)))}for(let[a,c]of r.entries()){let l=i.candidatesFor(c),p=l.filter(me);t||=p.length>0;let u=Oo(l,c,e?.[a]?.forkInstrumentation);u?s.push(u):p.length>0?o.push(`${c} (rejected by artifact policy)`):o.push(`${c} (missing)`)}if(o.length===0&&e){let a=Fl(i,s,e);if("failure"in a)o.push(`shared package identity rejected: ${a.failure}`);else{let c=a.paths.flatMap((l,p)=>li(l,r[p],e[p].forkInstrumentation)?[r[p]]:[]);if(c.length>0)o.push(`pinned package generation rejected by artifact policy: ${c.join(", ")}`);else return a.paths}}if(o.length===0)return s.map((a,c)=>Io(a,r[c],e?.[c]?.forkInstrumentation));n.push(` ${i.label} (${i.root}): ${o.join(", ")}`)}if(!t)return null;throw new Error(`Package artifact closure is incomplete: no single provenance tier contains every accepted artifact, and tiers will not be mixed. `+n.join(` -`))}var[ds,...uc]=process.argv.slice(2);(!ds||uc.length>0)&&(console.error("usage: scripts/resolve-binary.sh "),process.exit(2));try{process.stdout.write(`${us(ds)} +`))}var[Ao,...Dl]=process.argv.slice(2);(!Ao||Dl.length>0)&&(console.error("usage: scripts/resolve-binary.sh "),process.exit(2));try{process.stdout.write(`${vo(Ao)} `)}catch(r){console.error(r instanceof Error?r.message:String(r)),process.exit(1)} diff --git a/scripts/test-wasm-artifact-guards.sh b/scripts/test-wasm-artifact-guards.sh index 66c4a9a975..78ba3dab74 100755 --- a/scripts/test-wasm-artifact-guards.sh +++ b/scripts/test-wasm-artifact-guards.sh @@ -23,6 +23,7 @@ wat2wasm --debug-names "$work/abi.wat" -o "$work/abi.wasm" real_objdump="$(command -v wasm-objdump)" mkdir "$work/bin" +missing_structural_tool="$work/bin/missing-wasm-fork-instrument" cat >"$work/bin/wasm-objdump" <<'SH' #!/usr/bin/env bash if [ "${1:-}" = "-d" ] && [ "${2:-}" = "${FAIL_WASM_OBJDUMP_PATH:-}" ]; then @@ -43,8 +44,9 @@ assert_extracts_abi() { exit 1 } actual="$( - PATH="$work/bin:$PATH" REAL_WASM_OBJDUMP="$real_objdump" FAIL_WASM_OBJDUMP_PATH="$path" \ - wasm_extract_abi_version "$path" + WASM_POSIX_FORK_INSTRUMENT="$missing_structural_tool" \ + PATH="$work/bin:$PATH" REAL_WASM_OBJDUMP="$real_objdump" \ + FAIL_WASM_OBJDUMP_PATH="$path" wasm_extract_abi_version "$path" )" [ "$actual" = 18 ] || { echo "ERROR: Binaryen ABI extraction returned $actual for $description" >&2 @@ -60,7 +62,9 @@ assert_rejects_abi() { echo "ERROR: primary ABI extraction accepted $description" >&2 exit 1 fi - if PATH="$work/bin:$PATH" REAL_WASM_OBJDUMP="$real_objdump" FAIL_WASM_OBJDUMP_PATH="$path" \ + if WASM_POSIX_FORK_INSTRUMENT="$missing_structural_tool" \ + PATH="$work/bin:$PATH" REAL_WASM_OBJDUMP="$real_objdump" \ + FAIL_WASM_OBJDUMP_PATH="$path" \ wasm_extract_abi_version "$path" >/dev/null 2>&1; then echo "ERROR: Binaryen ABI extraction accepted $description" >&2 exit 1 @@ -83,19 +87,97 @@ assert_classifies_unsafe_abi() { fi extract_status=0 - PATH="$work/bin:$PATH" REAL_WASM_OBJDUMP="$real_objdump" FAIL_WASM_OBJDUMP_PATH="$path" \ + WASM_POSIX_FORK_INSTRUMENT="$missing_structural_tool" \ + PATH="$work/bin:$PATH" REAL_WASM_OBJDUMP="$real_objdump" \ + FAIL_WASM_OBJDUMP_PATH="$path" \ wasm_extract_abi_version "$path" >/dev/null 2>&1 || extract_status=$? [ "$extract_status" -gt 1 ] || { echo "ERROR: fallback ABI extraction classified $description as absent (status $extract_status)" >&2 exit 1 } - if ! PATH="$work/bin:$PATH" REAL_WASM_OBJDUMP="$real_objdump" FAIL_WASM_OBJDUMP_PATH="$path" \ - wasm_has_stale_abi "$path" 18; then + if ! WASM_POSIX_FORK_INSTRUMENT="$missing_structural_tool" \ + PATH="$work/bin:$PATH" REAL_WASM_OBJDUMP="$real_objdump" \ + FAIL_WASM_OBJDUMP_PATH="$path" wasm_has_stale_abi "$path" 18; then echo "ERROR: stale-ABI predicate accepted $description after the primary decoder failed" >&2 exit 1 fi } +mkdir "$work/no-objdump-bin" +cat >"$work/no-objdump-bin/wasm-objdump" <<'SH' +#!/usr/bin/env bash +exit 99 +SH +chmod +x "$work/no-objdump-bin/wasm-objdump" + +cat >"$work/bin/structural-identity-tool" <<'SH' +#!/usr/bin/env bash +if [ "${1:-}" != "--artifact-identity" ] || [ "$#" -ne 2 ]; then + exit 64 +fi +if [ -n "${MOCK_IDENTITY_RECORD:-}" ]; then + printf '%s\n' "$MOCK_IDENTITY_RECORD" + exit 0 +fi +state="${MOCK_ABI_STATE:-present}" +case "$state" in + present) version="${MOCK_ABI_VERSION:-18}" ;; + missing|invalid) version=- ;; + *) exit 65 ;; +esac +printf '0\t1\t0\t%s\t%s\t%s\t0\n' \ + "$state" "$version" "${MOCK_IMPORTS_FORK:-1}" +SH +chmod +x "$work/bin/structural-identity-tool" + +# The structural identity decoder owns these predicates when installed. A +# deliberately unusable WABT binary proves neither helper silently falls back +# to full-module text decoding for a large ABI 43 artifact. +structural_path="$work/bin/structural-identity-tool" +actual="$( + WASM_POSIX_FORK_INSTRUMENT="$structural_path" \ + PATH="$work/no-objdump-bin:$PATH" wasm_extract_abi_version "$work/abi.wasm" +)" +[ "$actual" = 18 ] || { + echo "ERROR: structural ABI extraction returned $actual" >&2 + exit 1 +} +if ! WASM_POSIX_FORK_INSTRUMENT="$structural_path" \ + PATH="$work/no-objdump-bin:$PATH" wasm_imports_kernel_fork "$work/abi.wasm"; then + echo "ERROR: structural identity lost the kernel_fork import" >&2 + exit 1 +fi +if WASM_POSIX_FORK_INSTRUMENT="$structural_path" MOCK_IMPORTS_FORK=0 \ + PATH="$work/no-objdump-bin:$PATH" wasm_imports_kernel_fork "$work/abi.wasm"; then + echo "ERROR: structural identity invented a kernel_fork import" >&2 + exit 1 +fi + +structural_status=0 +WASM_POSIX_FORK_INSTRUMENT="$structural_path" MOCK_ABI_STATE=missing \ + PATH="$work/no-objdump-bin:$PATH" \ + wasm_extract_abi_version "$work/abi.wasm" >/dev/null 2>&1 || structural_status=$? +[ "$structural_status" -eq 1 ] || { + echo "ERROR: structural identity returned $structural_status for a missing ABI export" >&2 + exit 1 +} +structural_status=0 +WASM_POSIX_FORK_INSTRUMENT="$structural_path" MOCK_ABI_STATE=invalid \ + PATH="$work/no-objdump-bin:$PATH" \ + wasm_extract_abi_version "$work/abi.wasm" >/dev/null 2>&1 || structural_status=$? +[ "$structural_status" -gt 1 ] || { + echo "ERROR: structural identity returned $structural_status for an invalid ABI export" >&2 + exit 1 +} +structural_status=0 +WASM_POSIX_FORK_INSTRUMENT="$structural_path" MOCK_IDENTITY_RECORD=malformed \ + PATH="$work/no-objdump-bin:$PATH" \ + wasm_extract_abi_version "$work/abi.wasm" >/dev/null 2>&1 || structural_status=$? +[ "$structural_status" -eq 2 ] || { + echo "ERROR: malformed structural identity returned $structural_status instead of 2" >&2 + exit 1 +} + assert_extracts_abi "$work/abi.wasm" "an implicit return" cat >"$work/folded-command-wrapper-abi.wat" <<'WAT' @@ -324,6 +406,11 @@ limit = 16 * 1024 * 1024 environment = os.environ.copy() environment["PATH"] = f"{inflated_bin}:{environment['PATH']}" environment["REAL_WASM_OBJDUMP"] = real_objdump +# Exercise the bounded source-only decoder under the file-size limit instead +# of letting the installed structural decoder make this fallback test vacuous. +environment["WASM_POSIX_FORK_INSTRUMENT"] = os.path.join( + os.path.dirname(inflated_bin), "missing-wasm-fork-instrument" +) def set_file_limit() -> None: @@ -388,6 +475,7 @@ cat >"$work/complete-fork.wat" <<'WAT' (module (@custom "kandelo.wpk_fork.linked_frames" "KLCF\01\00\18\00\04\08\03\00\20\00\00\00\18\00\00\00\10\00\00\00") + (@custom "kandelo.wpk_fork.capabilities" "\01\04") (import "kernel" "kernel_fork" (func $kernel_fork)) (import "env" "__wpk_fork_frame_reserve" (func $frame_reserve (param i32) (result i32))) @@ -418,10 +506,137 @@ if wasm_has_missing_fork_instrumentation "$work/complete-fork.wasm"; then fi wasm_require_fork_instrumentation_if_needed "$work/complete-fork.wasm" +awk ' + { print } + /\(import "kernel" "kernel_fork"/ { + print " (import \"env\" \"__wasm_dlopen\"" + print " (func (param i32 i32 i32 i32 i32) (result i32)))" + } +' "$work/complete-fork.wat" >"$work/legacy-loader-fork.wat" +wat2wasm --enable-annotations \ + "$work/legacy-loader-fork.wat" -o "$work/legacy-loader-fork.wasm" +if wasm_has_complete_fork_instrumentation "$work/legacy-loader-fork.wasm"; then + echo "ERROR: complete-fork predicate accepted the reentrant legacy loader import" >&2 + exit 1 +fi +if ! wasm_has_missing_fork_instrumentation "$work/legacy-loader-fork.wasm"; then + echo "ERROR: missing-fork predicate accepted the reentrant legacy loader import" >&2 + exit 1 +fi +if wasm_require_fork_instrumentation_if_needed \ + "$work/legacy-loader-fork.wasm" 2>"$work/legacy-loader-fork.error"; then + echo "ERROR: fork guard accepted the reentrant legacy loader import" >&2 + exit 1 +fi +grep -F ' loader: retains reentrant env.__wasm_dlopen' \ + "$work/legacy-loader-fork.error" >/dev/null || { + echo "ERROR: fork guard did not identify the legacy loader failure" >&2 + cat "$work/legacy-loader-fork.error" >&2 + exit 1 +} + +awk ' + /\(func \(export "_start"/ { + sub(/\(func /, "(func $native_start ") + } + { + line[NR] = $0 + } + END { + if (sub(/\)\)$/, ")", line[NR]) != 1) exit 2 + for (row = 1; row <= NR; row++) print line[row] + print " (start $native_start))" + } +' "$work/complete-fork.wat" >"$work/native-start-fork.wat" +wat2wasm --enable-annotations \ + "$work/native-start-fork.wat" -o "$work/native-start-fork.wasm" +if wasm_has_complete_fork_instrumentation "$work/native-start-fork.wasm"; then + echo "ERROR: complete-fork predicate accepted a retained native start section" >&2 + exit 1 +fi +if ! wasm_has_missing_fork_instrumentation "$work/native-start-fork.wasm"; then + echo "ERROR: missing-fork predicate accepted a retained native start section" >&2 + exit 1 +fi +if wasm_require_fork_instrumentation_if_needed \ + "$work/native-start-fork.wasm" 2>"$work/native-start-fork.error"; then + echo "ERROR: fork guard accepted a retained native start section" >&2 + exit 1 +fi +grep -F ' start: retains a native Wasm start section' \ + "$work/native-start-fork.error" >/dev/null || { + echo "ERROR: fork guard did not identify the native start failure" >&2 + cat "$work/native-start-fork.error" >&2 + exit 1 +} + +assert_rejects_fork_capability() { + local wat_path="$1" + local description="$2" + local wasm_path="${wat_path%.wat}.wasm" + local error_path="${wat_path%.wat}.error" + + wat2wasm --enable-annotations "$wat_path" -o "$wasm_path" + if wasm_has_complete_fork_instrumentation "$wasm_path"; then + echo "ERROR: complete-fork predicate accepted $description" >&2 + exit 1 + fi + if ! wasm_has_missing_fork_instrumentation "$wasm_path"; then + echo "ERROR: missing-fork predicate accepted $description" >&2 + exit 1 + fi + if wasm_require_fork_instrumentation_if_needed "$wasm_path" 2>"$error_path"; then + echo "ERROR: fork guard accepted $description" >&2 + exit 1 + fi + grep -F ' capability:' "$error_path" >/dev/null || { + echo "ERROR: fork guard did not identify the capability failure for $description" >&2 + cat "$error_path" >&2 + exit 1 + } +} + +sed '/kandelo\.wpk_fork\.capabilities/d' \ + "$work/complete-fork.wat" >"$work/missing-fork-capability.wat" +assert_rejects_fork_capability \ + "$work/missing-fork-capability.wat" \ + "an ABI 42-style artifact with no activation-state capability" + +sed 's/"\\01\\04"/"\\01\\00"/' \ + "$work/complete-fork.wat" >"$work/unsafe-fork-capability.wat" +assert_rejects_fork_capability \ + "$work/unsafe-fork-capability.wat" \ + "a capability that omits activation-state safety" + +sed 's/"\\01\\04"/"\\02\\04"/' \ + "$work/complete-fork.wat" >"$work/versioned-fork-capability.wat" +assert_rejects_fork_capability \ + "$work/versioned-fork-capability.wat" \ + "an unsupported capability version" + +sed 's/"\\01\\04"/"\\01\\84"/' \ + "$work/complete-fork.wat" >"$work/unknown-fork-capability.wat" +assert_rejects_fork_capability \ + "$work/unknown-fork-capability.wat" \ + "a capability with unknown flags" + +sed 's/"\\01\\04"/"\\01"/' \ + "$work/complete-fork.wat" >"$work/malformed-fork-capability.wat" +assert_rejects_fork_capability \ + "$work/malformed-fork-capability.wat" \ + "a malformed capability payload" + +sed '/kandelo\.wpk_fork\.capabilities/p' \ + "$work/complete-fork.wat" >"$work/duplicate-fork-capability.wat" +assert_rejects_fork_capability \ + "$work/duplicate-fork-capability.wat" \ + "duplicate capability sections" + cat >"$work/complete-fork-wasm64.wat" <<'WAT' (module (@custom "kandelo.wpk_fork.linked_frames" "KLCF\01\00\18\00\08\08\03\00\38\00\00\00\20\00\00\00\10\00\00\00") + (@custom "kandelo.wpk_fork.capabilities" "\01\04") (import "kernel" "kernel_fork" (func $kernel_fork)) (import "env" "__wpk_fork_frame_reserve" (func $frame_reserve (param i64) (result i64))) @@ -453,6 +668,7 @@ cat >"$work/partial-fork.wat" <<'WAT' (module (@custom "kandelo.wpk_fork.linked_frames" "KLCF\01\00\18\00\04\08\03\00\20\00\00\00\18\00\00\00\10\00\00\00") + (@custom "kandelo.wpk_fork.capabilities" "\01\04") (import "kernel" "kernel_fork" (func $kernel_fork)) (import "env" "__wpk_fork_frame_reserve" (func $frame_reserve (param i32) (result i32))) @@ -485,7 +701,8 @@ grep -Fqx ' missing: wpk_fork_state' "$partial_fork_error" || { # A section name is not sufficient evidence. Publication must reject a missing # payload, malformed layout fields, or a partially installed transaction hook. -sed '/(@custom/,+1d' "$work/complete-fork.wat" >"$work/missing-fork-descriptor.wat" +sed '/kandelo\.wpk_fork\.linked_frames/,+1d' \ + "$work/complete-fork.wat" >"$work/missing-fork-descriptor.wat" wat2wasm --enable-annotations "$work/missing-fork-descriptor.wat" \ -o "$work/missing-fork-descriptor.wasm" if wasm_require_fork_instrumentation_if_needed \ @@ -573,6 +790,7 @@ cat >"$work/inert-fork.wat" <<'WAT' (module (@custom "kandelo.wpk_fork.linked_frames" "KLCF\01\00\18\00\04\08\03\00\20\00\00\00\18\00\00\00\10\00\00\00") + (@custom "kandelo.wpk_fork.capabilities" "\01\04") (memory 1) (func (export "wpk_fork_abort_begin") (param i32)) (func (export "wpk_fork_abort_end")) @@ -608,25 +826,89 @@ mkdir "$work/counting-bin" cat >"$work/counting-bin/wasm-objdump" <<'SH' #!/usr/bin/env bash printf '%s\n' "${1:-}" >> "$WASM_OBJDUMP_COUNT_FILE" +if [ "${FAIL_WASM_OBJDUMP_DETAILS:-0}" = 1 ] && [ "${1:-}" = "-x" ]; then + exit 1 +fi exec "$REAL_WASM_OBJDUMP" "$@" SH chmod +x "$work/counting-bin/wasm-objdump" + +real_inventory_tool="$REPO_ROOT/tools/bin/wasm-fork-instrument" +[ -x "$real_inventory_tool" ] || { + echo "ERROR: shell guard test requires the built wasm-fork-instrument tool" >&2 + exit 1 +} +cat >"$work/counting-bin/wasm-fork-instrument" <<'SH' +#!/usr/bin/env bash +printf '%s\n' "$*" >> "$WASM_FORK_INVENTORY_COUNT_FILE" +exec "$REAL_WASM_FORK_INSTRUMENT" "$@" +SH +chmod +x "$work/counting-bin/wasm-fork-instrument" + count_file="$work/wasm-objdump.count" +inventory_count_file="$work/wasm-fork-instrument.count" : >"$count_file" +: >"$inventory_count_file" ( export PATH="$work/counting-bin:$PATH" export REAL_WASM_OBJDUMP="$real_objdump" export WASM_OBJDUMP_COUNT_FILE="$count_file" + export REAL_WASM_FORK_INSTRUMENT="$real_inventory_tool" + export WASM_FORK_INVENTORY_COUNT_FILE="$inventory_count_file" + export WASM_POSIX_FORK_INSTRUMENT="$work/counting-bin/wasm-fork-instrument" + export FAIL_WASM_OBJDUMP_DETAILS=1 + wasm_require_fork_instrumentation_if_needed "$work/complete-fork.wasm" +) +[ "$(grep -c '^-x$' "$count_file" || true)" = 0 ] && + [ "$(grep -c '^-s$' "$count_file" || true)" = 0 ] && + [ "$(wc -l <"$count_file" | tr -d ' ')" = 0 ] && + [ "$(grep -c -- '--contract-inventory' "$inventory_count_file")" = 1 ] && + [ "$(grep -c -- '--fork-capability-hex' "$inventory_count_file")" = 1 ] && + [ "$(grep -c -- '--linked-frame-descriptor-hex' "$inventory_count_file")" = 1 ] && + [ "$(wc -l <"$inventory_count_file" | tr -d ' ')" = 3 ] || { + echo "ERROR: fork validation did not use three binary contract passes" >&2 + cat "$count_file" >&2 + cat "$inventory_count_file" >&2 + exit 1 +} + +# The standalone guard remains usable before the Rust tool is installed. Its +# WABT compatibility path must still perform one structural pass and must not +# mistake an absent configured tool for successful validation. +: >"$count_file" +( + export PATH="$work/counting-bin:$PATH" + export REAL_WASM_OBJDUMP="$real_objdump" + export WASM_OBJDUMP_COUNT_FILE="$count_file" + export WASM_POSIX_FORK_INSTRUMENT="$work/not-installed/wasm-fork-instrument" wasm_require_fork_instrumentation_if_needed "$work/complete-fork.wasm" ) [ "$(grep -c '^-x$' "$count_file")" = 1 ] && - [ "$(grep -c '^-s$' "$count_file")" = 1 ] && - [ "$(wc -l <"$count_file" | tr -d ' ')" = 2 ] || { - echo "ERROR: fork validation did not use one structure pass and one descriptor pass" >&2 + [ "$(grep -c '^-s$' "$count_file")" = 2 ] && + [ "$(wc -l <"$count_file" | tr -d ' ')" = 3 ] || { + echo "ERROR: fork validation did not preserve the truthful WABT fallback" >&2 cat "$count_file" >&2 exit 1 } +if ( + export PATH="$work/counting-bin:$PATH" + export REAL_WASM_OBJDUMP="$real_objdump" + export WASM_OBJDUMP_COUNT_FILE="$count_file" + export WASM_POSIX_FORK_INSTRUMENT="$work/not-installed/wasm-fork-instrument" + wasm_require_fork_instrumentation_if_needed \ + "$work/native-start-fork.wasm" 2>"$work/native-start-fallback.error" +); then + echo "ERROR: WABT fork guard accepted a retained native start section" >&2 + exit 1 +fi +grep -F ' start: retains a native Wasm start section' \ + "$work/native-start-fallback.error" >/dev/null || { + echo "ERROR: WABT fork guard did not identify the native start failure" >&2 + cat "$work/native-start-fallback.error" >&2 + exit 1 +} + mkdir "$work/failing-bin" cat >"$work/failing-bin/wasm-objdump" <<'SH' #!/usr/bin/env bash @@ -635,7 +917,9 @@ SH chmod +x "$work/failing-bin/wasm-objdump" decoder_path="$work/failing-bin:$PATH" -if ! PATH="$decoder_path" wasm_has_stale_abi "$work/abi.wasm" 18; then +missing_inventory_tool="$work/not-installed/wasm-fork-instrument" +if ! WASM_POSIX_FORK_INSTRUMENT="$missing_structural_tool" \ + PATH="$decoder_path" wasm_has_stale_abi "$work/abi.wasm" 18; then echo "ERROR: stale-ABI predicate accepted an artifact after decoder failure" >&2 exit 1 fi @@ -647,15 +931,20 @@ if PATH="$decoder_path" wasm_require_exports "$work/abi.wasm" __abi_version >/de echo "ERROR: required-export guard accepted an artifact after decoder failure" >&2 exit 1 fi -if ! PATH="$decoder_path" wasm_has_missing_fork_instrumentation "$work/abi.wasm"; then +if ! WASM_POSIX_FORK_INSTRUMENT="$missing_inventory_tool" \ + PATH="$decoder_path" wasm_has_missing_fork_instrumentation "$work/abi.wasm"; then echo "ERROR: fork predicate accepted an artifact after decoder failure" >&2 exit 1 fi -if PATH="$decoder_path" wasm_require_fork_instrumentation_if_needed "$work/abi.wasm" >/dev/null 2>&1; then +if WASM_POSIX_FORK_INSTRUMENT="$missing_inventory_tool" \ + PATH="$decoder_path" \ + wasm_require_fork_instrumentation_if_needed "$work/abi.wasm" >/dev/null 2>&1; then echo "ERROR: fork guard accepted an artifact after decoder failure" >&2 exit 1 fi -if PATH="$decoder_path" wasm_require_no_fork_instrumentation "$work/abi.wasm" >/dev/null 2>&1; then +if WASM_POSIX_FORK_INSTRUMENT="$missing_inventory_tool" \ + PATH="$decoder_path" \ + wasm_require_no_fork_instrumentation "$work/abi.wasm" >/dev/null 2>&1; then echo "ERROR: disabled-fork guard accepted an artifact after decoder failure" >&2 exit 1 fi @@ -674,11 +963,14 @@ if ! wasm_has_missing_fork_instrumentation "$work/fake-fork-exports.wasm"; then echo "ERROR: fork guard accepted data-segment strings as instrumentation exports" >&2 exit 1 fi -if ! PATH=/usr/bin:/bin wasm_has_missing_fork_instrumentation "$work/fake-fork-exports.wasm"; then +if ! WASM_POSIX_FORK_INSTRUMENT="$missing_inventory_tool" \ + PATH=/usr/bin:/bin \ + wasm_has_missing_fork_instrumentation "$work/fake-fork-exports.wasm"; then echo "ERROR: decoder-free fork predicate accepted raw export-name strings" >&2 exit 1 fi -if PATH=/usr/bin:/bin wasm_require_fork_instrumentation_if_needed \ +if WASM_POSIX_FORK_INSTRUMENT="$missing_inventory_tool" \ + PATH=/usr/bin:/bin wasm_require_fork_instrumentation_if_needed \ "$work/fake-fork-exports.wasm" >/dev/null 2>&1; then echo "ERROR: decoder-free fork guard accepted raw export-name strings" >&2 exit 1 diff --git a/scripts/wasm-artifact-guards.sh b/scripts/wasm-artifact-guards.sh index 7c23233914..c7ccd20e99 100644 --- a/scripts/wasm-artifact-guards.sh +++ b/scripts/wasm-artifact-guards.sh @@ -307,12 +307,67 @@ wasm_extract_abi_version_with_binaryen() { printf '%s\n' "$abi" } +# Validate and print the stable structural-identity record. Return 127 only +# when the Rust decoder is unavailable so callers can distinguish a truthful +# source-only fallback from a decoder failure that must remain fail-closed. +_wasm_structural_artifact_identity() { + local path="${1:-}" + local identity identity_status=0 + local relocatable memory_count memory64_count abi_state abi_version + local imports_fork has_fork_exports extra + + identity="$(wasm_artifact_identity "$path")" || identity_status=$? + [ "$identity_status" -eq 0 ] || return "$identity_status" + IFS=$'\t' read -r relocatable memory_count memory64_count abi_state abi_version \ + imports_fork has_fork_exports extra <<< "$identity" + + [[ "$relocatable" =~ ^[01]$ ]] && + [[ "$memory_count" =~ ^[0-9]+$ ]] && + [[ "$memory64_count" =~ ^[0-9]+$ ]] && + [[ "$imports_fork" =~ ^[01]$ ]] && + [[ "$has_fork_exports" =~ ^[01]$ ]] && + [ -z "$extra" ] || return 2 + case "$abi_state" in + present) [[ "$abi_version" =~ ^[0-9]+$ ]] || return 2 ;; + missing|invalid) [ "$abi_version" = - ] || return 2 ;; + *) return 2 ;; + esac + + printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \ + "$relocatable" "$memory_count" "$memory64_count" "$abi_state" \ + "$abi_version" "$imports_fork" "$has_fork_exports" +} + # Print a constant ABI export and return 0. Return 1 only when a valid Wasm # module genuinely has no optional ABI export; all inspection or semantic # failures return a status greater than 1 so resolver predicates fail closed. wasm_extract_abi_version() { local path="${1:-}" wasm_is_binary "$path" || return 2 + + local identity identity_status=0 + local relocatable memory_count memory64_count abi_state abi_version + local imports_fork has_fork_exports + identity="$(_wasm_structural_artifact_identity "$path")" || identity_status=$? + if [ "$identity_status" -eq 0 ]; then + IFS=$'\t' read -r relocatable memory_count memory64_count abi_state abi_version \ + imports_fork has_fork_exports <<< "$identity" + case "$abi_state" in + present) + printf '%s\n' "$abi_version" + return 0 + ;; + missing) return 1 ;; + invalid) return 3 ;; + *) return 2 ;; + esac + elif [ "$identity_status" -ne 127 ]; then + return "$identity_status" + fi + + # WHY: source-only callers may not have the Rust decoder yet. Preserve the + # bounded WABT/Binaryen compatibility path, but never fall back after an + # installed structural decoder reports malformed or undecodable bytes. command -v wasm-objdump >/dev/null 2>&1 || return 2 # The export name and the function's optional debug name are separate Wasm # concepts. SDK binaries export the internal function @@ -637,6 +692,20 @@ wasm_has_stale_abi() { wasm_imports_kernel_fork() { local path="${1:-}" wasm_is_binary "$path" || return 1 + + local identity identity_status=0 + local relocatable memory_count memory64_count abi_state abi_version + local imports_fork has_fork_exports + identity="$(_wasm_structural_artifact_identity "$path")" || identity_status=$? + if [ "$identity_status" -eq 0 ]; then + IFS=$'\t' read -r relocatable memory_count memory64_count abi_state abi_version \ + imports_fork has_fork_exports <<< "$identity" + [ "$imports_fork" = 1 ] + return + elif [ "$identity_status" -ne 127 ]; then + return "$identity_status" + fi + if command -v wasm-objdump >/dev/null 2>&1; then _wasm_stream_awk ' /<- kernel\.kernel_fork/ { found = 1 } @@ -649,19 +718,78 @@ wasm_imports_kernel_fork() { grep -a -q 'kernel_fork' "$path" 2>/dev/null } +# Resolve the in-tree instrumenter without building it as a side effect of an +# artifact policy check. Release/package jobs install this binary alongside the +# guard; source-only environments can still use the WABT fallback below. +_wasm_fork_contract_inventory_tool() { + local configured="${WASM_POSIX_FORK_INSTRUMENT:-}" + if [ -n "$configured" ]; then + if [ -x "$configured" ]; then + printf '%s\n' "$configured" + return 0 + fi + if [[ "$configured" != */* ]] && command -v "$configured" >/dev/null 2>&1; then + command -v "$configured" + return 0 + fi + # An explicit tool selection is an ownership boundary. Do not silently + # substitute a different binary when that exact path is unavailable. + return 1 + fi + + local repo_root repo_tool + repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." 2>/dev/null && pwd)" || return 1 + repo_tool="$repo_root/tools/bin/wasm-fork-instrument" + if [ -x "$repo_tool" ]; then + printf '%s\n' "$repo_tool" + return 0 + fi + command -v wasm-fork-instrument 2>/dev/null +} + +wasm_artifact_identity() { + local path="${1:-}" + wasm_is_binary "$path" || return 2 + + local inventory_tool + inventory_tool="$(_wasm_fork_contract_inventory_tool)" || return 127 + "$inventory_tool" --artifact-identity "$path" 2>/dev/null || return 2 +} + +_wasm_fork_contract_inventory_decoder_available() { + _wasm_fork_contract_inventory_tool >/dev/null || + command -v wasm-objdump >/dev/null 2>&1 +} + # Inspect the complete fork-instrumentation contract with one structural -# decoder pass. Large programs such as Ruby produce tens of megabytes of -# `wasm-objdump -x` output; decoding that output once also keeps a transient -# decoder failure from being misreported as one arbitrarily missing export. +# decoder pass. The wasmparser-backed tool emits only the stable TSV record, so +# large programs do not materialize tens of megabytes of `wasm-objdump -x` +# text. Keep the WABT parser as a truthful compatibility fallback when the +# instrumenter binary is not installed. # # Output fields are, in order: # relocatable, imports kernel.kernel_fork, frame reserve/commit/next imports, -# linked-frame descriptor count, abort begin/end, rewind begin/end, state, -# unwind begin/end exports, module-memory count, memory64 count, and -# signature mismatches against the module memory's pointer type. +# linked-frame descriptor and capability counts, abort begin/end, rewind begin/end, state, +# unwind begin/end exports, module-memory count, memory64 count, +# signature mismatches against the module memory's pointer type, and the +# count of reentrant legacy env.__wasm_dlopen imports, and native start +# sections retained by the final artifact. _wasm_fork_contract_inventory() { local path="${1:-}" wasm_is_binary "$path" || return 1 + + local inventory_tool inventory_status=0 + if inventory_tool="$(_wasm_fork_contract_inventory_tool)"; then + "$inventory_tool" --contract-inventory "$path" 2>/dev/null || + inventory_status=$? + if [ "$inventory_status" -eq 1 ]; then + # Preserve the tri-state contract: status 1 means "predicate did + # not match", while a decoder failure must fail artifact policy. + return 2 + fi + return "$inventory_status" + fi + command -v wasm-objdump >/dev/null 2>&1 || return 2 _wasm_stream_awk ' @@ -691,6 +819,7 @@ _wasm_fork_contract_inventory() { function_signatures[function_index($0)] = function_types[signature_index($0)] } /^ - func\[.* <- kernel\.kernel_fork$/ { imports_fork = 1 } + /^ - func\[.* <- env\.__wasm_dlopen$/ { legacy_dlopen++ } /^ - func\[.* <- env\.__wpk_fork_frame_reserve$/ { frame_reserve++ frame_reserve_signatures[frame_reserve] = function_signatures[function_index($0)] @@ -704,6 +833,8 @@ _wasm_fork_contract_inventory() { frame_next_signatures[frame_next] = function_signatures[function_index($0)] } /^ - name: "kandelo\.wpk_fork\.linked_frames"$/ { linked_descriptor++ } + /^ - name: "kandelo\.wpk_fork\.capabilities"$/ { fork_capability++ } + /^Start:$/ { native_start++ } /^ - memory\[[0-9]+\] pages:/ { memory_count++ if ($0 ~ / i64( |$)/) memory64_count++ @@ -762,21 +893,80 @@ _wasm_fork_contract_inventory() { for (i = 1; i <= unwind_end; i++) if (unwind_end_signatures[i] != nil_to_nil) signature_mismatch++ - printf "%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\n", + printf "%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\n", relocatable + 0, imports_fork + 0, frame_reserve + 0, frame_commit + 0, frame_next + 0, - linked_descriptor + 0, + linked_descriptor + 0, fork_capability + 0, abort_begin + 0, abort_end + 0, rewind_begin + 0, rewind_end + 0, state + 0, unwind_begin + 0, unwind_end + 0, - memory_count + 0, memory64_count + 0, signature_mismatch + 0 + memory_count + 0, memory64_count + 0, signature_mismatch + 0, + legacy_dlopen + 0, native_start + 0 } ' wasm-objdump -x "$path" } +_wasm_fork_capability_hex() { + local path="${1:-}" + wasm_is_binary "$path" || return 2 + + local inventory_tool + if inventory_tool="$(_wasm_fork_contract_inventory_tool)"; then + "$inventory_tool" --fork-capability-hex "$path" 2>/dev/null + return + fi + + command -v wasm-objdump >/dev/null 2>&1 || return 2 + + _wasm_stream_awk ' + /^Contents of section Custom:$/ { + sections++ + next + } + sections > 0 && /^[0-9a-fA-F]+:/ { + line = $0 + sub(/^[^:]*:[[:space:]]*/, "", line) + sub(/[[:space:]][[:space:]].*$/, "", line) + gsub(/[[:space:]]/, "", line) + if (line !~ /^[0-9a-fA-F]+$/) exit 3 + hex = hex tolower(line) + } + END { + if (sections != 1 || hex == "") exit 1 + print hex + } + ' wasm-objdump -s -j kandelo.wpk_fork.capabilities "$path" +} + +wasm_has_activation_state_safe_capability() { + local path="${1:-}" + local section_hex capability_hex flags_hex flags + section_hex="$(_wasm_fork_capability_hex "$path")" || return $? + + # One-byte name length (29), UTF-8 section name, then [version, flags]. + local name_prefix="1d6b616e64656c6f2e77706b5f666f726b2e6361706162696c6974696573" + case "$section_hex" in + "$name_prefix"*) capability_hex="${section_hex#"$name_prefix"}" ;; + *) return 3 ;; + esac + [ "${#capability_hex}" -eq 4 ] || return 3 + [ "${capability_hex:0:2}" = "01" ] || return 3 + flags_hex="${capability_hex:2:2}" + flags=$((16#$flags_hex)) + [ $((flags & ~7)) -eq 0 ] || return 3 + [ $((flags & 4)) -eq 4 ] +} + _wasm_linked_frame_descriptor_hex() { local path="${1:-}" wasm_is_binary "$path" || return 2 + + local inventory_tool + if inventory_tool="$(_wasm_fork_contract_inventory_tool)"; then + "$inventory_tool" --linked-frame-descriptor-hex "$path" 2>/dev/null + return + fi + command -v wasm-objdump >/dev/null 2>&1 || return 2 # `wasm-objdump -x` reports a custom section's name but not its payload. @@ -908,20 +1098,23 @@ wasm_require_exports() { wasm_has_complete_fork_instrumentation() { local path="${1:-}" local inventory inventory_status=0 - local relocatable imports_fork frame_reserve frame_commit frame_next linked_descriptor + local relocatable imports_fork frame_reserve frame_commit frame_next linked_descriptor fork_capability local abort_begin abort_end rewind_begin rewind_end state unwind_begin unwind_end - local memory_count memory64_count signature_mismatch extra + local memory_count memory64_count signature_mismatch legacy_dlopen native_start extra inventory="$(_wasm_fork_contract_inventory "$path")" || inventory_status=$? [ "$inventory_status" -eq 0 ] || return "$inventory_status" IFS=$'\t' read -r relocatable imports_fork frame_reserve frame_commit frame_next \ - linked_descriptor abort_begin abort_end rewind_begin rewind_end state \ - unwind_begin unwind_end memory_count memory64_count signature_mismatch extra <<< "$inventory" + linked_descriptor fork_capability abort_begin abort_end rewind_begin rewind_end state \ + unwind_begin unwind_end memory_count memory64_count signature_mismatch legacy_dlopen native_start extra <<< "$inventory" [ -z "$extra" ] || return 2 [ "$frame_reserve$frame_commit$frame_next" = 111 ] || return 1 [ "$linked_descriptor" = 1 ] || return 1 + [ "$fork_capability" = 1 ] || return 1 + wasm_has_activation_state_safe_capability "$path" || return $? [ "$abort_begin$abort_end$rewind_begin$rewind_end$state$unwind_begin$unwind_end" = 1111111 ] || return 1 - [ "$memory_count" = 1 ] && [ "$signature_mismatch" = 0 ] || return 1 + [ "$memory_count" = 1 ] && [ "$signature_mismatch" = 0 ] && + [ "$legacy_dlopen" = 0 ] && [ "$native_start" = 0 ] || return 1 local descriptor_pointer_width descriptor_pointer_width="$(wasm_linked_frame_descriptor_pointer_width "$path")" || return $? [ "$descriptor_pointer_width" = 8 ] && [ "$memory64_count" = 1 ] && return 0 @@ -963,9 +1156,9 @@ wasm_memory_arch() { wasm_has_any_wpk_fork_export() { local path="${1:-}" local inventory inventory_status=0 - local relocatable imports_fork frame_reserve frame_commit frame_next linked_descriptor + local relocatable imports_fork frame_reserve frame_commit frame_next linked_descriptor fork_capability local abort_begin abort_end rewind_begin rewind_end state unwind_begin unwind_end - local memory_count memory64_count signature_mismatch extra + local memory_count memory64_count signature_mismatch legacy_dlopen native_start extra inventory="$(_wasm_fork_contract_inventory "$path")" || inventory_status=$? case "$inventory_status" in 0) ;; @@ -973,8 +1166,8 @@ wasm_has_any_wpk_fork_export() { *) return 0 ;; # Decoder failure: classify as unsafe/present. esac IFS=$'\t' read -r relocatable imports_fork frame_reserve frame_commit frame_next \ - linked_descriptor abort_begin abort_end rewind_begin rewind_end state \ - unwind_begin unwind_end memory_count memory64_count signature_mismatch extra <<< "$inventory" + linked_descriptor fork_capability abort_begin abort_end rewind_begin rewind_end state \ + unwind_begin unwind_end memory_count memory64_count signature_mismatch legacy_dlopen native_start extra <<< "$inventory" [ -z "$extra" ] || return 0 [ "$abort_begin$abort_end$rewind_begin$rewind_end$state$unwind_begin$unwind_end" != 0000000 ] } @@ -982,9 +1175,9 @@ wasm_has_any_wpk_fork_export() { wasm_has_any_fork_instrumentation() { local path="${1:-}" local inventory inventory_status=0 - local relocatable imports_fork frame_reserve frame_commit frame_next linked_descriptor + local relocatable imports_fork frame_reserve frame_commit frame_next linked_descriptor fork_capability local abort_begin abort_end rewind_begin rewind_end state unwind_begin unwind_end - local memory_count memory64_count signature_mismatch extra + local memory_count memory64_count signature_mismatch legacy_dlopen native_start extra inventory="$(_wasm_fork_contract_inventory "$path")" || inventory_status=$? case "$inventory_status" in 0) ;; @@ -992,23 +1185,24 @@ wasm_has_any_fork_instrumentation() { *) return 0 ;; # Decoder failure: classify as unsafe/present. esac IFS=$'\t' read -r relocatable imports_fork frame_reserve frame_commit frame_next \ - linked_descriptor abort_begin abort_end rewind_begin rewind_end state \ - unwind_begin unwind_end memory_count memory64_count signature_mismatch extra <<< "$inventory" + linked_descriptor fork_capability abort_begin abort_end rewind_begin rewind_end state \ + unwind_begin unwind_end memory_count memory64_count signature_mismatch legacy_dlopen native_start extra <<< "$inventory" [ -z "$extra" ] || return 0 [ "$frame_reserve$frame_commit$frame_next" != 000 ] || [ "$linked_descriptor" != 0 ] || + [ "$fork_capability" != 0 ] || [ "$abort_begin$abort_end$rewind_begin$rewind_end$state$unwind_begin$unwind_end" != 0000000 ] } wasm_has_missing_fork_instrumentation() { local path="${1:-}" local inventory inventory_status=0 - local relocatable imports_fork frame_reserve frame_commit frame_next linked_descriptor + local relocatable imports_fork frame_reserve frame_commit frame_next linked_descriptor fork_capability local abort_begin abort_end rewind_begin rewind_end state unwind_begin unwind_end - local memory_count memory64_count signature_mismatch extra + local memory_count memory64_count signature_mismatch legacy_dlopen native_start extra wasm_is_binary "$path" || return 1 - if ! command -v wasm-objdump >/dev/null 2>&1; then + if ! _wasm_fork_contract_inventory_decoder_available; then case "$path" in *.o) return 1 ;; *) return 0 ;; @@ -1018,21 +1212,25 @@ wasm_has_missing_fork_instrumentation() { inventory="$(_wasm_fork_contract_inventory "$path")" || inventory_status=$? [ "$inventory_status" -eq 0 ] || return 0 # Decoder failure: unsafe. IFS=$'\t' read -r relocatable imports_fork frame_reserve frame_commit frame_next \ - linked_descriptor abort_begin abort_end rewind_begin rewind_end state \ - unwind_begin unwind_end memory_count memory64_count signature_mismatch extra <<< "$inventory" + linked_descriptor fork_capability abort_begin abort_end rewind_begin rewind_end state \ + unwind_begin unwind_end memory_count memory64_count signature_mismatch legacy_dlopen native_start extra <<< "$inventory" [ -z "$extra" ] || return 0 [ "$relocatable" = 1 ] && return 1 local frame_imports="$frame_reserve$frame_commit$frame_next" local exports="$abort_begin$abort_end$rewind_begin$rewind_end$state$unwind_begin$unwind_end" [ "$imports_fork" = 0 ] && [ "$frame_imports" = 000 ] && - [ "$linked_descriptor" = 0 ] && [ "$exports" = 0000000 ] && return 1 + [ "$linked_descriptor" = 0 ] && [ "$fork_capability" = 0 ] && + [ "$exports" = 0000000 ] && return 1 [ "$linked_descriptor" = 1 ] || return 0 + [ "$fork_capability" = 1 ] || return 0 + wasm_has_activation_state_safe_capability "$path" || return 0 local descriptor_pointer_width descriptor_pointer_width="$(wasm_linked_frame_descriptor_pointer_width "$path")" || return 0 [ "$exports" = 1111111 ] || return 0 - [ "$memory_count" = 1 ] && [ "$signature_mismatch" = 0 ] || return 0 + [ "$memory_count" = 1 ] && [ "$signature_mismatch" = 0 ] && + [ "$legacy_dlopen" = 0 ] && [ "$native_start" = 0 ] || return 0 if [ "$descriptor_pointer_width" = 8 ]; then [ "$memory64_count" = 1 ] || return 0 else @@ -1052,31 +1250,31 @@ wasm_require_fork_instrumentation_if_needed() { local path="${1:-}" wasm_is_binary "$path" || return 0 - if ! command -v wasm-objdump >/dev/null 2>&1; then + if ! _wasm_fork_contract_inventory_decoder_available; then case "$path" in *.o) return 0 ;; esac echo "ERROR: unable to inspect fork instrumentation: $path" >&2 - echo " wasm-objdump is required for structural export validation." >&2 + echo " wasm-fork-instrument or wasm-objdump is required for structural validation." >&2 return 1 fi local inventory inventory_status=0 - local relocatable imports_fork frame_reserve frame_commit frame_next linked_descriptor + local relocatable imports_fork frame_reserve frame_commit frame_next linked_descriptor fork_capability local abort_begin abort_end rewind_begin rewind_end state unwind_begin unwind_end - local memory_count memory64_count signature_mismatch extra + local memory_count memory64_count signature_mismatch legacy_dlopen native_start extra inventory="$(_wasm_fork_contract_inventory "$path")" || inventory_status=$? if [ "$inventory_status" -ne 0 ]; then echo "ERROR: unable to inspect fork instrumentation: $path" >&2 - echo " wasm-objdump failed with status $inventory_status." >&2 + echo " structural decoder failed with status $inventory_status." >&2 return 1 fi IFS=$'\t' read -r relocatable imports_fork frame_reserve frame_commit frame_next \ - linked_descriptor abort_begin abort_end rewind_begin rewind_end state \ - unwind_begin unwind_end memory_count memory64_count signature_mismatch extra <<< "$inventory" + linked_descriptor fork_capability abort_begin abort_end rewind_begin rewind_end state \ + unwind_begin unwind_end memory_count memory64_count signature_mismatch legacy_dlopen native_start extra <<< "$inventory" if [ -n "$extra" ]; then echo "ERROR: unable to inspect fork instrumentation: $path" >&2 - echo " wasm-objdump returned an invalid fork-contract inventory." >&2 + echo " structural decoder returned an invalid fork-contract inventory." >&2 return 1 fi [ "$relocatable" = 1 ] && return 0 @@ -1084,7 +1282,8 @@ wasm_require_fork_instrumentation_if_needed() { local frame_imports="$frame_reserve$frame_commit$frame_next" local exports="$abort_begin$abort_end$rewind_begin$rewind_end$state$unwind_begin$unwind_end" [ "$imports_fork" = 0 ] && [ "$frame_imports" = 000 ] && - [ "$linked_descriptor" = 0 ] && [ "$exports" = 0000000 ] && return 0 + [ "$linked_descriptor" = 0 ] && [ "$fork_capability" = 0 ] && + [ "$exports" = 0000000 ] && return 0 local missing=() local duplicates=() @@ -1122,9 +1321,18 @@ wasm_require_fork_instrumentation_if_needed() { descriptor_error="kandelo.wpk_fork.linked_frames descriptor is malformed or unsupported" fi + local capability_error="" + if [ "$fork_capability" = 0 ]; then + capability_error="missing kandelo.wpk_fork.capabilities" + elif [ "$fork_capability" != 1 ]; then + capability_error="found $fork_capability kandelo.wpk_fork.capabilities sections; expected exactly one" + elif ! wasm_has_activation_state_safe_capability "$path"; then + capability_error="capability is malformed or omits activation-state safety" + fi + local memory_error="" if [ "$memory_count" != 1 ]; then - memory_error="ABI 42 fork instrumentation requires exactly one module memory; found $memory_count" + memory_error="ABI 43 fork instrumentation requires exactly one module memory; found $memory_count" elif [ -n "$descriptor_pointer_width" ]; then local memory_width_mismatch=0 if [ "$descriptor_pointer_width" = 8 ] && [ "$memory64_count" != 1 ]; then @@ -1146,20 +1354,31 @@ wasm_require_fork_instrumentation_if_needed() { local signature_error="" [ "$signature_mismatch" = 0 ] || - signature_error="$signature_mismatch ABI 42 fork import/export signatures do not match module memory" + signature_error="$signature_mismatch ABI 43 fork import/export signatures do not match module memory" + local legacy_loader_error="" + [ "$legacy_dlopen" = 0 ] || + legacy_loader_error="retains reentrant env.__wasm_dlopen instead of the staged loader lowering" + local native_start_error="" + [ "$native_start" = 0 ] || + native_start_error="retains a native Wasm start section instead of deferring initialization to wpk_fork_module_bootstrap" if [ ${#missing[@]} -eq 0 ] && [ ${#duplicates[@]} -eq 0 ] && - [ -z "$descriptor_error" ] && [ -z "$memory_error" ] && - [ -z "$signature_error" ]; then + [ -z "$descriptor_error" ] && [ -z "$capability_error" ] && + [ -z "$memory_error" ] && + [ -z "$signature_error" ] && [ -z "$legacy_loader_error" ] && + [ -z "$native_start_error" ]; then return 0 fi - echo "ERROR: refusing wasm artifact with incomplete ABI 42 fork instrumentation: $path" >&2 + echo "ERROR: refusing wasm artifact with incomplete ABI 43 fork instrumentation: $path" >&2 [ ${#missing[@]} -eq 0 ] || printf ' missing: %s\n' "${missing[*]}" >&2 [ ${#duplicates[@]} -eq 0 ] || printf ' duplicate: %s\n' "${duplicates[*]}" >&2 [ -z "$descriptor_error" ] || printf ' descriptor: %s\n' "$descriptor_error" >&2 + [ -z "$capability_error" ] || printf ' capability: %s\n' "$capability_error" >&2 [ -z "$memory_error" ] || printf ' memory: %s\n' "$memory_error" >&2 [ -z "$signature_error" ] || printf ' signatures: %s\n' "$signature_error" >&2 + [ -z "$legacy_loader_error" ] || printf ' loader: %s\n' "$legacy_loader_error" >&2 + [ -z "$native_start_error" ] || printf ' start: %s\n' "$native_start_error" >&2 echo " Fork-capable binaries must be processed with scripts/run-wasm-fork-instrument.sh from the current ABI." >&2 return 1 } @@ -1168,23 +1387,24 @@ wasm_require_no_fork_instrumentation() { local path="${1:-}" wasm_is_binary "$path" || return 0 local inventory inventory_status=0 - local relocatable imports_fork frame_reserve frame_commit frame_next linked_descriptor + local relocatable imports_fork frame_reserve frame_commit frame_next linked_descriptor fork_capability local abort_begin abort_end rewind_begin rewind_end state unwind_begin unwind_end - local memory_count memory64_count signature_mismatch extra + local memory_count memory64_count signature_mismatch legacy_dlopen native_start extra inventory="$(_wasm_fork_contract_inventory "$path")" || inventory_status=$? if [ "$inventory_status" -ne 0 ]; then echo "ERROR: unable to inspect fork instrumentation policy: $path" >&2 return 1 fi IFS=$'\t' read -r relocatable imports_fork frame_reserve frame_commit frame_next \ - linked_descriptor abort_begin abort_end rewind_begin rewind_end state \ - unwind_begin unwind_end memory_count memory64_count signature_mismatch extra <<< "$inventory" + linked_descriptor fork_capability abort_begin abort_end rewind_begin rewind_end state \ + unwind_begin unwind_end memory_count memory64_count signature_mismatch legacy_dlopen native_start extra <<< "$inventory" if [ -n "$extra" ]; then echo "ERROR: unable to inspect fork instrumentation policy: $path" >&2 return 1 fi if [ "$frame_reserve$frame_commit$frame_next" != 000 ] || [ "$linked_descriptor" != 0 ] || + [ "$fork_capability" != 0 ] || [ "$abort_begin$abort_end$rewind_begin$rewind_end$state$unwind_begin$unwind_end" != 0000000 ]; then echo "ERROR: refusing wasm artifact with disabled fork instrumentation policy: $path" >&2 echo " Rebuild it without scripts/run-wasm-fork-instrument.sh." >&2 diff --git a/tests/package-system/program-resolver-literals.test.ts b/tests/package-system/program-resolver-literals.test.ts index af66db0f03..c4f15c4314 100644 --- a/tests/package-system/program-resolver-literals.test.ts +++ b/tests/package-system/program-resolver-literals.test.ts @@ -1,8 +1,4 @@ -import { - existsSync, - readFileSync, - readdirSync, -} from "node:fs"; +import { existsSync, readFileSync, readdirSync } from "node:fs"; import { basename, extname, join, resolve } from "node:path"; import { describe, expect, it } from "vitest"; @@ -172,10 +168,7 @@ function sourceFilesUnder(relPath: string): string[] { if (entry.isSymbolicLink()) return []; const child = join(relPath, entry.name); if (entry.isDirectory()) { - if ( - excludedDirectories.has(entry.name) - || child === "docs/plans" - ) { + if (excludedDirectories.has(entry.name) || child === "docs/plans") { return []; } return sourceFilesUnder(child); @@ -195,15 +188,30 @@ function auditedSourceFiles(): string[] { function staleLiteralFailures( candidates: ReadonlyMap, ): string[] { + if (candidates.size === 0) return []; + const pattern = new RegExp( + [...candidates.keys()] + .sort((left, right) => right.length - left.length) + .map((path) => path.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")) + .join("|"), + "g", + ); const failures: string[] = []; for (const relPath of auditedSourceFiles()) { const bytes = readFileSync(join(repoRoot, relPath)); if (bytes.includes(0)) continue; const lines = bytes.toString("utf8").split("\n"); - for (const [stalePath, owners] of candidates) { - lines.forEach((line, index) => { - if (!line.includes(stalePath)) return; - const replacements = [...new Set(owners.map((owner) => owner.replacement))] + lines.forEach((line, index) => { + // WHY: the projection contains hundreds of possible stale paths. One + // combined literal pattern keeps this audit linear in source size + // instead of rescanning every source line once per package output. + const stalePaths = new Set(line.match(pattern) ?? []); + for (const stalePath of stalePaths) { + const owners = candidates.get(stalePath); + if (!owners) continue; + const replacements = [ + ...new Set(owners.map((owner) => owner.replacement)), + ] .sort() .map((replacement) => JSON.stringify(replacement)) .join(" or "); @@ -212,11 +220,11 @@ function staleLiteralFailures( .map((packageName) => JSON.stringify(packageName)) .join(", "); failures.push( - `${relPath}:${index + 1}: ${JSON.stringify(stalePath)} is a stale flat ` - + `resolver path owned by package ${packages}; use ${replacements}`, + `${relPath}:${index + 1}: ${JSON.stringify(stalePath)} is a stale flat ` + + `resolver path owned by package ${packages}; use ${replacements}`, ); - }); - } + } + }); } return failures; } diff --git a/tests/scripts/ci-run-test-suite-groups.test.sh b/tests/scripts/ci-run-test-suite-groups.test.sh index 4c2eb0a98a..dfb580ab04 100755 --- a/tests/scripts/ci-run-test-suite-groups.test.sh +++ b/tests/scripts/ci-run-test-suite-groups.test.sh @@ -45,6 +45,12 @@ fi exit 2 EOF +cat > "$FIXTURE/bin/cargo" <<'EOF' +#!/usr/bin/env bash +printf '%s\n' "$*" >> "$CARGO_CAPTURE" +exit 0 +EOF + cat > "$FIXTURE/run.sh" <<'EOF' #!/usr/bin/env bash printf '%s\n' "$*" > "$RUN_CAPTURE" @@ -74,13 +80,34 @@ fi EOF chmod +x "$FIXTURE/scripts/$runner" done + +prepared_xtask="$FIXTURE/target/fixture-host/release/xtask" +mkdir -p "$(dirname "$prepared_xtask")" +cat > "$prepared_xtask" <<'EOF' +#!/usr/bin/env bash +if [ "${1:-}" = "build-deps" ] && [ "${2:-}" = "cache-root" ] && + [ "$#" -eq 2 ]; then + case "${WASM_POSIX_BINARY_CACHE_ROOT:-}" in + /*) printf '%s\n' "$WASM_POSIX_BINARY_CACHE_ROOT" ;; + *) printf '%s\n' "$PWD/${WASM_POSIX_BINARY_CACHE_ROOT:-.cache/kandelo}" ;; + esac + exit 0 +fi +exit 2 +EOF + chmod +x \ + "$FIXTURE/bin/cargo" \ "$FIXTURE/bin/npm" \ "$FIXTURE/bin/npx" \ "$FIXTURE/bin/rustc" \ "$FIXTURE/bin/uname" \ "$FIXTURE/run.sh" \ - "$FIXTURE/scripts/ci-check-browser-assets.sh" + "$FIXTURE/scripts/ci-check-browser-assets.sh" \ + "$prepared_xtask" + +CARGO_CAPTURE="$TMP_DIR/cargo-build.args" +export CARGO_CAPTURE run_group() { local suite="$1" @@ -124,6 +151,20 @@ grep -Fxq -- \ "--already-materialized --fetch-only prepare-browser" \ "$browser_capture" +if ! awk ' + $0 != "build --release -p xtask --target fixture-host --quiet" { + exit 1 + } +' "$CARGO_CAPTURE"; then + echo "ci-run-test-suite.sh used an unexpected package-checker build command:" >&2 + cat "$CARGO_CAPTURE" >&2 + exit 1 +fi +[ -s "$CARGO_CAPTURE" ] || { + echo "ci-run-test-suite.sh did not prepare the source-workspace package checker" >&2 + exit 1 +} + for workflow in \ "$REPO_ROOT/.github/workflows/staging-build.yml" \ "$REPO_ROOT/.github/workflows/prepare-merge.yml"; do @@ -155,25 +196,10 @@ for workflow in \ fi done -prepared_xtask="$FIXTURE/target/fixture-host/release/xtask" -mkdir -p "$(dirname "$prepared_xtask")" -cat > "$prepared_xtask" <<'EOF' -#!/usr/bin/env bash -if [ "${1:-}" = "build-deps" ] && [ "${2:-}" = "cache-root" ] && - [ "$#" -eq 2 ]; then - case "${WASM_POSIX_BINARY_CACHE_ROOT:-}" in - /*) printf '%s\n' "$WASM_POSIX_BINARY_CACHE_ROOT" ;; - *) printf '%s\n' "$PWD/${WASM_POSIX_BINARY_CACHE_ROOT:-.cache/kandelo}" ;; - esac - exit 0 -fi -exit 2 -EOF -chmod +x "$prepared_xtask" - mkdir -p "$FIXTURE/.ci-test-binary-cache/programs" cache_capture="$TMP_DIR/portable-cache-root" xtask_capture="$TMP_DIR/portable-xtask" +: > "$CARGO_CAPTURE" PATH="$FIXTURE/bin:$PATH" \ TEST_CAPTURE="$TMP_DIR/portable-cache-suite.args" \ CACHE_CAPTURE="$cache_capture" \ @@ -189,6 +215,10 @@ grep -Fxq "$prepared_xtask" "$xtask_capture" || { echo "ci-run-test-suite.sh did not select the transported package checker" >&2 exit 1 } +[ ! -s "$CARGO_CAPTURE" ] || { + echo "ci-run-test-suite.sh rebuilt a transported package checker" >&2 + exit 1 +} missing_xtask_capture="$TMP_DIR/missing-xtask-suite.args" chmod -x "$prepared_xtask" if PATH="$FIXTURE/bin:$PATH" \ diff --git a/tools/xtask/src/archive_stage_cli.rs b/tools/xtask/src/archive_stage_cli.rs index 96491228bf..95800ce61d 100644 --- a/tools/xtask/src/archive_stage_cli.rs +++ b/tools/xtask/src/archive_stage_cli.rs @@ -16,7 +16,7 @@ use std::fs; use std::path::{Path, PathBuf}; use crate::archive_stage::{self, StageOptions}; -use crate::build_deps::{self, default_cache_root, parse_target_arch, Registry, ResolveOpts}; +use crate::build_deps::{self, Registry, ResolveOpts, default_cache_root, parse_target_arch}; use crate::pkg_manifest::{BuildToml, DepsManifest, ManifestKind, TargetArch}; use crate::repo_root; use crate::util::hex; @@ -784,9 +784,11 @@ built_by = "test" let suffix = ".tar.zst"; let short = &name[prefix.len()..name.len() - suffix.len()]; assert_eq!(short.len(), 8, "short_sha slot must be 8 chars: {short:?}"); - assert!(short - .chars() - .all(|c| c.is_ascii_hexdigit() && !c.is_uppercase())); + assert!( + short + .chars() + .all(|c| c.is_ascii_hexdigit() && !c.is_uppercase()) + ); let index_path = dir.join("index.toml"); crate::build_index::run(vec![ diff --git a/tools/xtask/src/build_deps.rs b/tools/xtask/src/build_deps.rs index fec3ab6114..32b55f7ffb 100644 --- a/tools/xtask/src/build_deps.rs +++ b/tools/xtask/src/build_deps.rs @@ -1281,15 +1281,7 @@ pub fn compute_sha( memo: &mut BTreeMap, chain: &mut Vec, ) -> Result<[u8; 32], String> { - compute_sha_with_global_toolchain_inputs( - target, - registry, - arch, - abi_version, - memo, - chain, - None, - ) + compute_sha_with_global_toolchain_inputs(target, registry, arch, abi_version, memo, chain, None) } fn compute_sha_with_global_toolchain_inputs( @@ -1354,12 +1346,10 @@ fn compute_sha_with_global_toolchain_inputs( let build_inputs = build_input_digests(target, registry)?; let global_toolchain_inputs = match target.kind { - ManifestKind::Library | ManifestKind::Program => { - match global_toolchain_inputs_override { - Some(inputs) => inputs.to_vec(), - None => global_package_toolchain_digests()?, - } - } + ManifestKind::Library | ManifestKind::Program => match global_toolchain_inputs_override { + Some(inputs) => inputs.to_vec(), + None => global_package_toolchain_digests()?, + }, ManifestKind::Source => Vec::new(), }; let fork_instrument_tool_inputs = if package_uses_fork_instrument_tool(target) { @@ -1551,8 +1541,7 @@ const FORK_INSTRUMENT_TOOL_INPUTS: &[&str] = &[ "scripts/run-wasm-fork-instrument.sh", ]; -type RootDigestCache = - OnceLock, String>>>>; +type RootDigestCache = OnceLock, String>>>>; static GLOBAL_PACKAGE_TOOLCHAIN_DIGESTS: RootDigestCache = OnceLock::new(); static FORK_INSTRUMENT_TOOL_DIGESTS: RootDigestCache = OnceLock::new(); @@ -1627,14 +1616,17 @@ struct CargoLockPackage { checksum: Option, } +const FORK_INSTRUMENT_CARGO_METADATA_ARGS: &[&str] = + &["metadata", "--format-version=1", "--locked"]; + fn fork_instrument_cargo_dependency_digest(root: &Path) -> Result<[u8; 32], String> { - let host_target = host_target_triple()?; + // WHY: program cache paths have no build-host dimension. Filtering this + // graph through the current macOS or Linux host made one source tree + // compute different identities. Cargo's unfiltered graph is the stable + // union, so any dependency that can build the instrumenter invalidates the + // shared generation without making the key host-specific. let output = Command::new("cargo") - .arg("metadata") - .arg("--format-version=1") - .arg("--locked") - .arg("--filter-platform") - .arg(&host_target) + .args(FORK_INSTRUMENT_CARGO_METADATA_ARGS) .current_dir(root) .output() .map_err(|e| format!("run cargo metadata for fork-instrument cache key: {e}"))?; @@ -1654,25 +1646,6 @@ fn fork_instrument_cargo_dependency_digest(root: &Path) -> Result<[u8; 32], Stri fork_instrument_cargo_dependency_digest_from_metadata(root, &metadata, &lock) } -fn host_target_triple() -> Result { - let output = Command::new("rustc") - .arg("-vV") - .output() - .map_err(|e| format!("run rustc -vV: {e}"))?; - if !output.status.success() { - return Err(format!( - "rustc -vV failed: {}", - String::from_utf8_lossy(&output.stderr).trim() - )); - } - let stdout = String::from_utf8_lossy(&output.stdout); - stdout - .lines() - .find_map(|line| line.strip_prefix("host: ").map(str::to_owned)) - .filter(|host| !host.is_empty()) - .ok_or_else(|| "rustc -vV did not report host target".to_string()) -} - fn fork_instrument_cargo_dependency_digest_from_metadata( root: &Path, metadata: &serde_json::Value, @@ -1747,7 +1720,7 @@ fn fork_instrument_cargo_dependency_digest_from_metadata( entries.sort_by(|a, b| a.0.cmp(&b.0)); let mut h = Sha256::new(); - h.update(b"fork-instrument-cargo-build-deps-v1\n"); + h.update(b"fork-instrument-cargo-build-deps-v2-host-union\n"); for (stable_id, features, deps, checksum) in entries { h.update(b"package\0"); h.update(stable_id.as_bytes()); @@ -2011,11 +1984,9 @@ fn resolve_build_input_path_from_repo( if let Ok(registry_relative) = Path::new(input).strip_prefix("packages/registry") { let (package_name, package_relative) = split_registry_build_input(registry_relative, input)?; - if let Some(selected) = selected_registry_package_dir( - registry, - main_repo_root, - package_name, - ) { + if let Some(selected) = + selected_registry_package_dir(registry, main_repo_root, package_name) + { return require_selected_registry_build_input( target, input, @@ -2108,9 +2079,7 @@ fn split_registry_build_input<'a>( } }; let package_name = package_component.to_str().ok_or_else(|| { - format!( - "canonical registry build input has a non-UTF-8 package name: {authored_input:?}", - ) + format!("canonical registry build input has a non-UTF-8 package name: {authored_input:?}",) })?; Ok((package_name, components.as_path())) } @@ -4323,7 +4292,9 @@ struct WasmArtifactFacts { function_imports: BTreeMap<(String, String), Vec>, function_exports: BTreeMap>, memory_pointer_widths: Vec, + fork_capabilities: Vec>, linked_frame_descriptors: Vec>, + native_start_count: usize, is_relocatable_object: bool, } @@ -4454,6 +4425,9 @@ fn wasm_artifact_facts(bytes: &[u8]) -> Result { } } } + Payload::StartSection { .. } => { + facts.native_start_count += 1; + } Payload::CustomSection(c) => { let name = c.name(); if name == "linking" || name.starts_with("reloc.") { @@ -4461,6 +4435,8 @@ fn wasm_artifact_facts(bytes: &[u8]) -> Result { } if name == wasm_posix_shared::abi::WPK_FORK_LINKED_FRAME_FORMAT_SECTION { facts.linked_frame_descriptors.push(c.data().to_vec()); + } else if name == wasm_posix_shared::abi::WPK_FORK_CAPABILITIES_SECTION { + facts.fork_capabilities.push(c.data().to_vec()); } } _ => {} @@ -4567,6 +4543,52 @@ fn validate_linked_frame_descriptor( Ok(LinkedFrameDescriptorFacts { pointer_width }) } +fn validate_fork_capabilities(sections: &[Vec]) -> Result<(), String> { + use wasm_posix_shared::abi; + + let [capability] = sections else { + return Err(match sections.len() { + 0 => format!( + "is missing required {} capability", + abi::WPK_FORK_CAPABILITIES_SECTION + ), + count => format!( + "has {count} {} sections, expected exactly one", + abi::WPK_FORK_CAPABILITIES_SECTION + ), + }); + }; + if capability.len() != 2 { + return Err(format!( + "{} has {} bytes, expected 2", + abi::WPK_FORK_CAPABILITIES_SECTION, + capability.len() + )); + } + if capability[0] != abi::WPK_FORK_CAPABILITIES_VERSION { + return Err(format!( + "{} version {} is unsupported", + abi::WPK_FORK_CAPABILITIES_SECTION, + capability[0] + )); + } + let flags = capability[1]; + if flags & !abi::WPK_FORK_CAP_KNOWN_MASK != 0 { + return Err(format!( + "{} has unknown flags 0x{flags:02x}", + abi::WPK_FORK_CAPABILITIES_SECTION + )); + } + if flags & abi::WPK_FORK_CAP_REQUIRED_FLAGS != abi::WPK_FORK_CAP_REQUIRED_FLAGS { + return Err(format!( + "{} flags 0x{flags:02x} omit required activation-state safety flags 0x{:02x}", + abi::WPK_FORK_CAPABILITIES_SECTION, + abi::WPK_FORK_CAP_REQUIRED_FLAGS + )); + } + Ok(()) +} + fn program_artifact_signature_matches( actual: &wasmparser::FuncType, params: &[wasm_posix_shared::abi::ProgramArtifactValueType], @@ -4583,6 +4605,11 @@ fn program_artifact_signature_matches( _ => false, }, ProgramArtifactValueType::I32 => *actual == ValType::I32, + ProgramArtifactValueType::I64 => *actual == ValType::I64, + ProgramArtifactValueType::FuncRef => *actual == ValType::FUNCREF, + ProgramArtifactValueType::ExternRef => *actual == ValType::EXTERNREF, + ProgramArtifactValueType::ExnRef => *actual == ValType::EXNREF, + ProgramArtifactValueType::AnyRef => *actual == ValType::Ref(wasmparser::RefType::ANYREF), }; actual.params().len() == params.len() @@ -4610,6 +4637,11 @@ fn program_artifact_signature_text( ProgramArtifactValueType::Pointer if pointer_width == 8 => "i64", ProgramArtifactValueType::Pointer => "i32", ProgramArtifactValueType::I32 => "i32", + ProgramArtifactValueType::I64 => "i64", + ProgramArtifactValueType::FuncRef => "funcref", + ProgramArtifactValueType::ExternRef => "externref", + ProgramArtifactValueType::ExnRef => "exnref", + ProgramArtifactValueType::AnyRef => "anyref", }; let params = params.iter().map(value_name).collect::>().join(","); let results = results.iter().map(value_name).collect::>().join(","); @@ -4680,13 +4712,16 @@ fn wasm_artifact_policy_failures_for( }) .count(); let descriptor_count = facts.linked_frame_descriptors.len(); - let has_fork_artifact_surface = - present_fork_exports > 0 || present_fork_imports > 0 || descriptor_count > 0; + let capability_count = facts.fork_capabilities.len(); + let has_fork_artifact_surface = present_fork_exports > 0 + || present_fork_imports > 0 + || descriptor_count > 0 + || capability_count > 0; if fork_instrumentation == ForkInstrumentationPolicy::Disabled { if has_fork_artifact_surface { failures.push( - "has ABI 42 wasm-fork-instrument metadata, imports, or exports but this output disables fork instrumentation".to_string(), + "has ABI 43 wasm-fork-instrument metadata, imports, or exports but this output disables fork instrumentation".to_string(), ); } return failures; @@ -4697,6 +4732,29 @@ fn wasm_artifact_policy_failures_for( } let contract_failure_start = failures.len(); + if facts.native_start_count != 0 { + // WHY: staged dlopen instantiates modules inside a host import. ABI 43 + // moves the source start function behind an explicit bootstrap so + // instantiation itself cannot reenter guest Wasm. + failures.push(format!( + "retains {} native Wasm start section{} instead of deferring initialization to wpk_fork_module_bootstrap", + facts.native_start_count, + if facts.native_start_count == 1 { "" } else { "s" }, + )); + } + if facts + .function_imports + .contains_key(&("env".to_string(), "__wasm_dlopen".to_string())) + { + // ABI 43's instrumenter rewrites this monolithic callback into local + // prepare/next/commit control flow. Seeing the import beside the safety + // claim therefore proves that publication received stale or forged + // instrumentation metadata. + failures.push( + "retains reentrant env.__wasm_dlopen instead of the ABI 43 staged loader lowering" + .to_string(), + ); + } let missing_exports = fork_exports .iter() .filter(|requirement| !facts.function_exports.contains_key(requirement.name)) @@ -4704,7 +4762,7 @@ fn wasm_artifact_policy_failures_for( .collect::>(); if !missing_exports.is_empty() { failures.push(format!( - "has incomplete ABI 42 wasm-fork-instrument exports; missing {}", + "has incomplete ABI 43 wasm-fork-instrument exports; missing {}", missing_exports.join(", ") )); } @@ -4715,12 +4773,16 @@ fn wasm_artifact_policy_failures_for( .is_some_and(|signatures| signatures.len() != 1) { failures.push(format!( - "has duplicate ABI 42 wasm-fork-instrument export {}", + "has duplicate ABI 43 wasm-fork-instrument export {}", requirement.name )); } } + if let Err(error) = validate_fork_capabilities(&facts.fork_capabilities) { + failures.push(error); + } + let descriptor = match facts.linked_frame_descriptors.as_slice() { [] => { failures.push(format!( @@ -4765,7 +4827,7 @@ fn wasm_artifact_policy_failures_for( .join(", "); if !missing_imports.is_empty() { failures.push(format!( - "has incomplete ABI 42 linked-frame imports; missing {missing_imports}" + "has incomplete ABI 43 linked-frame imports; missing {missing_imports}" )); } for requirement in fork_imports { @@ -4776,7 +4838,7 @@ fn wasm_artifact_policy_failures_for( .is_some_and(|signatures| signatures.len() != 1) { failures.push(format!( - "has duplicate ABI 42 linked-frame import {}.{}", + "has duplicate ABI 43 linked-frame import {}.{}", requirement.module, requirement.name )); } @@ -4793,12 +4855,12 @@ fn wasm_artifact_policy_failures_for( "a" }; failures.push(format!( - "ABI 42 linked-frame descriptor declares {article} {}-byte pointer but the module memory uses {}-byte addresses", + "ABI 43 linked-frame descriptor declares {article} {}-byte pointer but the module memory uses {}-byte addresses", descriptor.pointer_width, pointer_width )); } pointer_widths => failures.push(format!( - "ABI 42 fork instrumentation requires exactly one module memory, found {}", + "ABI 43 fork instrumentation requires exactly one module memory, found {}", pointer_widths.len() )), } @@ -4818,7 +4880,7 @@ fn wasm_artifact_policy_failures_for( descriptor.pointer_width, ) { failures.push(format!( - "ABI 42 wasm-fork-instrument export {} has the wrong signature; expected {}", + "ABI 43 wasm-fork-instrument export {} has the wrong signature; expected {}", requirement.name, program_artifact_signature_text( requirement.params, @@ -4842,7 +4904,7 @@ fn wasm_artifact_policy_failures_for( descriptor.pointer_width, ) { failures.push(format!( - "ABI 42 linked-frame import {}.{} has the wrong signature; expected {}", + "ABI 43 linked-frame import {}.{} has the wrong signature; expected {}", requirement.module, requirement.name, program_artifact_signature_text( @@ -4858,7 +4920,7 @@ fn wasm_artifact_policy_failures_for( if facts.imports_kernel_fork && failures.len() != contract_failure_start { failures.push( - "imports kernel.kernel_fork without the complete ABI 42 wasm-fork-instrument contract" + "imports kernel.kernel_fork without the complete ABI 43 wasm-fork-instrument contract" .to_string(), ); } @@ -7417,9 +7479,10 @@ impl LocalFileTransaction { if self.published || self.yielded_to_other_writer || !self.old_moved - || self.backup_snapshot.as_ref().is_none_or(|snapshot| { - validate_local_mirror_entry(&self.backup, snapshot).is_err() - }) + || self + .backup_snapshot + .as_ref() + .is_none_or(|snapshot| validate_local_mirror_entry(&self.backup, snapshot).is_err()) { return; } @@ -8968,13 +9031,7 @@ index_url = "https://example.test/releases/download/binaries-abi-v{{abi}}/index. .unwrap(); } - fn write_build_with_input( - dir: &Path, - name: &str, - revision: u32, - input: &str, - contents: &str, - ) { + fn write_build_with_input(dir: &Path, name: &str, revision: u32, input: &str, contents: &str) { let input_path = dir.join(name).join(input); fs::write(&input_path, contents).unwrap(); fs::write( @@ -9893,23 +9950,67 @@ wasm = "second.wasm" ty } - fn wasm_fork_artifact( - descriptor_pointer_width: u8, + fn wasm_contract_value_type( + value: wasm_posix_shared::abi::ProgramArtifactValueType, + pointer_width: u8, + ) -> u8 { + use wasm_posix_shared::abi::ProgramArtifactValueType; + + match value { + ProgramArtifactValueType::Pointer if pointer_width == 4 => 0x7f, + ProgramArtifactValueType::Pointer if pointer_width == 8 => 0x7e, + ProgramArtifactValueType::Pointer => { + panic!("unsupported fixture pointer width {pointer_width}") + } + ProgramArtifactValueType::I32 => 0x7f, + ProgramArtifactValueType::I64 => 0x7e, + ProgramArtifactValueType::FuncRef => 0x70, + ProgramArtifactValueType::ExternRef => 0x6f, + ProgramArtifactValueType::ExnRef => 0x69, + ProgramArtifactValueType::AnyRef => 0x6e, + } + } + + fn wasm_contract_function_type( + params: &[wasm_posix_shared::abi::ProgramArtifactValueType], + results: &[wasm_posix_shared::abi::ProgramArtifactValueType], + pointer_width: u8, + ) -> Vec { + let params = params + .iter() + .copied() + .map(|value| wasm_contract_value_type(value, pointer_width)) + .collect::>(); + let results = results + .iter() + .copied() + .map(|value| wasm_contract_value_type(value, pointer_width)) + .collect::>(); + wasm_function_type(¶ms, &results) + } + + fn wasm_fork_artifact_with_capabilities( + _descriptor_pointer_width: u8, signature_pointer_width: u8, memory_pointer_width: u8, include_kernel_fork: bool, - frame_imports: &[&str], + fork_imports: &[&str], fork_exports: &[&str], descriptors: &[Vec], + capabilities: &[Vec], + include_legacy_dlopen: bool, + include_native_start: bool, ) -> Vec { use wasm_posix_shared::abi; + use wasm_posix_shared::abi::ProgramArtifactValueType::{I32, Pointer}; - let pointer_type = match signature_pointer_width { - 4 => 0x7f, // i32 - 8 => 0x7e, // i64 - other => panic!("unsupported fixture pointer width {other}"), - }; let mut bytes = b"\0asm\x01\0\0\0".to_vec(); + for capability in capabilities { + bytes.extend(wasm_custom_section( + abi::WPK_FORK_CAPABILITIES_SECTION, + capability, + )); + } for descriptor in descriptors { bytes.extend(wasm_custom_section( abi::WPK_FORK_LINKED_FRAME_FORMAT_SECTION, @@ -9917,30 +10018,67 @@ wasm = "second.wasm" )); } - let types = [ - wasm_function_type(&[], &[0x7f]), - wasm_function_type(&[pointer_type], &[pointer_type]), - wasm_function_type(&[pointer_type], &[]), - wasm_function_type(&[], &[]), + // Derive this raw-Wasm fixture from the same declarations enforced by + // publication policy. ABI 43 deliberately has a larger private + // surface than the original three linked-frame hooks, and a + // hand-maintained type-index switch silently went stale as reference + // ownership was added. + let mut types = vec![ + wasm_contract_function_type(&[], &[I32], signature_pointer_width), + wasm_contract_function_type(&[], &[], signature_pointer_width), ]; + let kernel_fork_type = 0u32; + let empty_function_type = 1u32; + + let mut imports = Vec::new(); + if include_kernel_fork { + imports.push(("kernel", "kernel_fork", kernel_fork_type)); + } + if include_legacy_dlopen { + let type_index = types.len() as u32; + types.push(wasm_contract_function_type( + &[Pointer, I32, Pointer, I32, I32], + &[I32], + signature_pointer_width, + )); + imports.push(("env", "__wasm_dlopen", type_index)); + } + for name in fork_imports { + let requirement = abi::WPK_FORK_REQUIRED_IMPORTS + .iter() + .find(|requirement| requirement.name == *name) + .unwrap_or_else(|| panic!("unknown ABI 43 fork import fixture {name}")); + let type_index = types.len() as u32; + types.push(wasm_contract_function_type( + requirement.params, + requirement.results, + signature_pointer_width, + )); + imports.push((requirement.module, requirement.name, type_index)); + } + + // Keep a local function for every required export even in a + // missing-export fixture. Removing an export must not renumber the + // remaining functions or turn a policy test into malformed Wasm. + let mut local_functions = Vec::new(); + for requirement in abi::WPK_FORK_REQUIRED_EXPORTS { + let type_index = types.len() as u32; + types.push(wasm_contract_function_type( + requirement.params, + requirement.results, + signature_pointer_width, + )); + local_functions.push((requirement.name, type_index, requirement.results)); + } + local_functions.push(("__abi_version", kernel_fork_type, &[I32])); + local_functions.push(("_start", empty_function_type, &[])); + let mut type_section = uleb(types.len() as u32); for ty in types { type_section.extend(ty); } bytes.extend(wasm_section(1, type_section)); - let mut imports = Vec::new(); - if include_kernel_fork { - imports.push(("kernel", "kernel_fork", 0u32)); - } - for name in frame_imports { - let type_index = match *name { - abi::WPK_FORK_FRAME_IMPORT_COMMIT => 2, - abi::WPK_FORK_FRAME_IMPORT_NEXT | abi::WPK_FORK_FRAME_IMPORT_RESERVE => 1, - other => panic!("unknown linked-frame import fixture {other}"), - }; - imports.push((abi::WPK_FORK_FRAME_IMPORT_MODULE, *name, type_index)); - } if !imports.is_empty() { let mut import_section = uleb(imports.len() as u32); for (module, name, type_index) in &imports { @@ -9952,14 +10090,9 @@ wasm = "second.wasm" bytes.extend(wasm_section(2, import_section)); } - // Seven control functions plus __abi_version and _start. Keeping every - // local function present lets negative fixtures remove one export - // without changing function indices or accidentally testing malformed - // Wasm instead of the publication contract. - let function_types = [2u32, 3, 2, 3, 0, 2, 3, 0, 3]; - let mut function_section = uleb(function_types.len() as u32); - for type_index in function_types { - function_section.extend(uleb(type_index)); + let mut function_section = uleb(local_functions.len() as u32); + for (_, type_index, _) in &local_functions { + function_section.extend(uleb(*type_index)); } bytes.extend(wasm_section(3, function_section)); @@ -9970,38 +10103,37 @@ wasm = "second.wasm" }; bytes.extend(wasm_section(5, vec![0x01, memory_flags, 0x01])); - let local_exports = [ - (abi::WPK_FORK_EXPORT_ABORT_BEGIN, 0u32), - (abi::WPK_FORK_EXPORT_ABORT_END, 1), - (abi::WPK_FORK_EXPORT_REWIND_BEGIN, 2), - (abi::WPK_FORK_EXPORT_REWIND_END, 3), - (abi::WPK_FORK_EXPORT_STATE, 4), - (abi::WPK_FORK_EXPORT_UNWIND_BEGIN, 5), - (abi::WPK_FORK_EXPORT_UNWIND_END, 6), - ("__abi_version", 7), - ("_start", 8), - ]; - let exported = local_exports + let exported = local_functions .iter() - .filter(|(name, _)| { + .enumerate() + .filter(|(_, (name, _, _))| { *name == "__abi_version" || *name == "_start" || fork_exports.contains(name) }) .collect::>(); let mut export_section = uleb(exported.len() as u32); - for (name, local_index) in exported { + for (local_index, (name, _, _)) in exported { export_section.extend(wasm_name(name)); export_section.push(0x00); // function export - export_section.extend(uleb(imports.len() as u32 + *local_index)); + export_section.extend(uleb(imports.len() as u32 + local_index as u32)); } bytes.extend(wasm_section(7, export_section)); - let mut code_section = uleb(function_types.len() as u32); - for type_index in function_types { - let body = if type_index == 0 { - vec![0x00, 0x41, descriptor_pointer_width, 0x0b] - } else { - vec![0x00, 0x0b] - }; + if include_native_start { + let start_index = imports.len() as u32 + local_functions.len() as u32 - 1; + bytes.extend(wasm_section(8, uleb(start_index))); + } + + let mut code_section = uleb(local_functions.len() as u32); + for (_, _, results) in local_functions { + let mut body = vec![0x00]; // no local declarations + for result in results { + match wasm_contract_value_type(*result, signature_pointer_width) { + 0x7f => body.extend([0x41, 0x00]), // i32.const 0 + 0x7e => body.extend([0x42, 0x00]), // i64.const 0 + heap_type => body.extend([0xd0, heap_type]), // ref.null + } + } + body.push(0x0b); code_section.extend(uleb(body.len() as u32)); code_section.extend(body); } @@ -10009,6 +10141,34 @@ wasm = "second.wasm" bytes } + fn wasm_fork_artifact( + descriptor_pointer_width: u8, + signature_pointer_width: u8, + memory_pointer_width: u8, + include_kernel_fork: bool, + frame_imports: &[&str], + fork_exports: &[&str], + descriptors: &[Vec], + ) -> Vec { + use wasm_posix_shared::abi; + + wasm_fork_artifact_with_capabilities( + descriptor_pointer_width, + signature_pointer_width, + memory_pointer_width, + include_kernel_fork, + frame_imports, + fork_exports, + descriptors, + &[vec![ + abi::WPK_FORK_CAPABILITIES_VERSION, + abi::WPK_FORK_CAP_ACTIVATION_STATE_SAFE, + ]], + false, + false, + ) + } + fn complete_wasm_fork_artifact(pointer_width: u8) -> Vec { let imports = wasm_posix_shared::abi::WPK_FORK_REQUIRED_IMPORTS .iter() @@ -10661,6 +10821,19 @@ index_url = "https://example.test/releases/binaries-abi-v{abi}/index.toml" ); } + #[test] + fn fork_instrument_dependency_metadata_is_not_build_host_filtered() { + assert_eq!( + FORK_INSTRUMENT_CARGO_METADATA_ARGS, + ["metadata", "--format-version=1", "--locked"], + "shared package cache keys must hash Cargo's cross-host dependency union" + ); + assert!( + !FORK_INSTRUMENT_CARGO_METADATA_ARGS.contains(&"--filter-platform"), + "a host-filtered dependency graph gives macOS and Linux different package identities" + ); + } + #[test] fn fork_instrument_cargo_dependency_digest_ignores_unrelated_lockfile_entries() { let root = tempdir("fork-cargo-closure"); @@ -10981,7 +11154,8 @@ index_url = "https://example.test/releases/download/binaries-abi-v{abi}/index.to .unwrap_err(); assert!( missing_selected_input.contains("first-hit registry package") - && missing_selected_input.contains("lower-priority package roots were not consulted"), + && missing_selected_input + .contains("lower-priority package roots were not consulted"), "a selected external package must not be completed with a lower package's file: {missing_selected_input}", ); @@ -16199,7 +16373,7 @@ wasm = "bad.wasm" } #[test] - fn program_artifact_policy_accepts_complete_abi42_fork_contracts() { + fn program_artifact_policy_accepts_complete_abi43_fork_contracts() { for pointer_width in [4, 8] { let bytes = complete_wasm_fork_artifact(pointer_width); let failures = wasm_artifact_policy_failures_for( @@ -16232,7 +16406,7 @@ wasm = "bad.wasm" } #[test] - fn program_artifact_policy_rejects_each_missing_abi42_fork_import() { + fn program_artifact_policy_rejects_each_missing_abi43_fork_import() { let all_imports = wasm_posix_shared::abi::WPK_FORK_REQUIRED_IMPORTS .iter() .map(|requirement| requirement.name) @@ -16266,7 +16440,7 @@ wasm = "bad.wasm" } #[test] - fn program_artifact_policy_rejects_each_missing_abi42_fork_export() { + fn program_artifact_policy_rejects_each_missing_abi43_fork_export() { let all_imports = wasm_posix_shared::abi::WPK_FORK_REQUIRED_IMPORTS .iter() .map(|requirement| requirement.name) @@ -16383,6 +16557,146 @@ wasm = "bad.wasm" } } + #[test] + fn program_artifact_policy_rejects_every_unsafe_activation_capability_shape() { + use wasm_posix_shared::abi; + + let imports = abi::WPK_FORK_REQUIRED_IMPORTS + .iter() + .map(|requirement| requirement.name) + .collect::>(); + let exports = abi::WPK_FORK_REQUIRED_EXPORTS + .iter() + .map(|requirement| requirement.name) + .collect::>(); + let safe = vec![ + abi::WPK_FORK_CAPABILITIES_VERSION, + abi::WPK_FORK_CAP_ACTIVATION_STATE_SAFE, + ]; + let cases: Vec<(&str, Vec>)> = vec![ + ("missing required", vec![]), + ("expected exactly one", vec![safe.clone(), safe]), + ( + "bytes, expected 2", + vec![vec![abi::WPK_FORK_CAPABILITIES_VERSION]], + ), + ( + "version", + vec![vec![ + abi::WPK_FORK_CAPABILITIES_VERSION + 1, + abi::WPK_FORK_CAP_ACTIVATION_STATE_SAFE, + ]], + ), + ( + "unknown flags", + vec![vec![ + abi::WPK_FORK_CAPABILITIES_VERSION, + abi::WPK_FORK_CAP_KNOWN_MASK | 0x80, + ]], + ), + ( + "omit required activation-state safety", + vec![vec![abi::WPK_FORK_CAPABILITIES_VERSION, 0]], + ), + ]; + + for (expected, capabilities) in cases { + let bytes = wasm_fork_artifact_with_capabilities( + 4, + 4, + 4, + true, + &imports, + &exports, + &[linked_frame_descriptor(4)], + &capabilities, + false, + false, + ); + let failures = wasm_artifact_policy_failures(&bytes, ForkInstrumentationPolicy::Auto); + assert!( + failures.iter().any(|failure| failure.contains(expected)), + "capability case {expected:?} was not reported: {failures:?}" + ); + } + } + + #[test] + fn program_artifact_policy_rejects_reentrant_legacy_loader_claims() { + use wasm_posix_shared::abi; + + let imports = abi::WPK_FORK_REQUIRED_IMPORTS + .iter() + .map(|requirement| requirement.name) + .collect::>(); + let exports = abi::WPK_FORK_REQUIRED_EXPORTS + .iter() + .map(|requirement| requirement.name) + .collect::>(); + let bytes = wasm_fork_artifact_with_capabilities( + 4, + 4, + 4, + true, + &imports, + &exports, + &[linked_frame_descriptor(4)], + &[vec![ + abi::WPK_FORK_CAPABILITIES_VERSION, + abi::WPK_FORK_CAP_ACTIVATION_STATE_SAFE, + ]], + true, + false, + ); + let failures = + wasm_artifact_policy_failures(&bytes, ForkInstrumentationPolicy::Auto); + assert!( + failures.iter().any(|failure| { + failure.contains("reentrant env.__wasm_dlopen") + && failure.contains("staged loader lowering") + }), + "got: {failures:?}", + ); + } + + #[test] + fn program_artifact_policy_rejects_native_start_claims() { + use wasm_posix_shared::abi; + + let imports = abi::WPK_FORK_REQUIRED_IMPORTS + .iter() + .map(|requirement| requirement.name) + .collect::>(); + let exports = abi::WPK_FORK_REQUIRED_EXPORTS + .iter() + .map(|requirement| requirement.name) + .collect::>(); + let bytes = wasm_fork_artifact_with_capabilities( + 4, + 4, + 4, + true, + &imports, + &exports, + &[linked_frame_descriptor(4)], + &[vec![ + abi::WPK_FORK_CAPABILITIES_VERSION, + abi::WPK_FORK_CAP_ACTIVATION_STATE_SAFE, + ]], + false, + true, + ); + let failures = + wasm_artifact_policy_failures(&bytes, ForkInstrumentationPolicy::Auto); + assert!( + failures.iter().any(|failure| { + failure.contains("native Wasm start section") + && failure.contains("wpk_fork_module_bootstrap") + }), + "got: {failures:?}", + ); + } + #[test] fn program_artifact_policy_rejects_pointer_width_signature_drift() { let imports = wasm_posix_shared::abi::WPK_FORK_REQUIRED_IMPORTS @@ -17639,16 +17953,20 @@ libs = ["lib/libF3b.a"] #[test] fn extract_source_repo_root_flag_rejects_missing_or_duplicate_values() { - assert!(extract_source_repo_root_flag(vec!["--source-repo-root".into()]) + assert!( + extract_source_repo_root_flag(vec!["--source-repo-root".into()]) + .unwrap_err() + .contains("requires a path") + ); + assert!( + extract_source_repo_root_flag(vec![ + "--source-repo-root=/a".into(), + "--source-repo-root".into(), + "/b".into(), + ]) .unwrap_err() - .contains("requires a path")); - assert!(extract_source_repo_root_flag(vec![ - "--source-repo-root=/a".into(), - "--source-repo-root".into(), - "/b".into(), - ]) - .unwrap_err() - .contains("more than once")); + .contains("more than once") + ); } #[test] @@ -17709,19 +18027,14 @@ libs = ["lib/libF3b.a"] fs::write(first.join("identity.txt"), "first source projection").unwrap(); fs::write(second.join("identity.txt"), "second source projection").unwrap(); let cache: RootDigestCache = OnceLock::new(); - let compute = |root: &Path| { - global_package_build_input_digests_for(root, &["identity.txt"]) - }; + let compute = |root: &Path| global_package_build_input_digests_for(root, &["identity.txt"]); - let first_digest = - root_scoped_build_input_digests(&cache, &first, compute).unwrap(); - let second_digest = - root_scoped_build_input_digests(&cache, &second, compute).unwrap(); + let first_digest = root_scoped_build_input_digests(&cache, &first, compute).unwrap(); + let second_digest = root_scoped_build_input_digests(&cache, &second, compute).unwrap(); assert_ne!(first_digest[0].digest, second_digest[0].digest); fs::write(first.join("identity.txt"), "changed after memoization").unwrap(); - let first_cached = - root_scoped_build_input_digests(&cache, &first, compute).unwrap(); + let first_cached = root_scoped_build_input_digests(&cache, &first, compute).unwrap(); assert_eq!(first_cached[0].digest, first_digest[0].digest); } diff --git a/tools/xtask/src/build_index.rs b/tools/xtask/src/build_index.rs index 0d8e59a3b5..224dd5e37e 100644 --- a/tools/xtask/src/build_index.rs +++ b/tools/xtask/src/build_index.rs @@ -505,12 +505,7 @@ build_timestamp = "2026-05-05T12:34:56Z" .as_ref() .expect("test archive manifest must have compatibility") .target_arch; - let fname = crate::package_archive_name::render( - &manifest, - target_arch, - abi, - cache_key_sha, - ); + let fname = crate::package_archive_name::render(&manifest, target_arch, abi, cache_key_sha); let path = dir.join(&fname); fs::write(&path, &bytes).unwrap(); path @@ -535,42 +530,10 @@ build_timestamp = "2026-05-05T12:34:56Z" fs::create_dir_all(&archives).unwrap(); let out = dir.join("index.toml"); - write_real_archive( - &archives, - "alpha", - "1.0.0", - 1, - 6, - "wasm32", - &"a".repeat(64), - ); - write_real_archive( - &archives, - "alpha", - "1.0.0", - 1, - 6, - "wasm64", - &"b".repeat(64), - ); - write_real_archive( - &archives, - "beta", - "2.0.0", - 1, - 6, - "wasm32", - &"c".repeat(64), - ); - write_real_archive( - &archives, - "beta", - "2.0.0", - 1, - 6, - "wasm64", - &"d".repeat(64), - ); + write_real_archive(&archives, "alpha", "1.0.0", 1, 6, "wasm32", &"a".repeat(64)); + write_real_archive(&archives, "alpha", "1.0.0", 1, 6, "wasm64", &"b".repeat(64)); + write_real_archive(&archives, "beta", "2.0.0", 1, 6, "wasm32", &"c".repeat(64)); + write_real_archive(&archives, "beta", "2.0.0", 1, 6, "wasm64", &"d".repeat(64)); super::run(vec![ "--abi".into(), @@ -671,15 +634,7 @@ build_timestamp = "2026-05-05T12:34:56Z" fs::create_dir_all(&archives).unwrap(); let out = dir.join("index.toml"); - write_real_archive( - &archives, - "solo", - "1.0.0", - 1, - 6, - "wasm32", - &"e".repeat(64), - ); + write_real_archive(&archives, "solo", "1.0.0", 1, 6, "wasm32", &"e".repeat(64)); super::run(vec![ "--abi".into(), @@ -716,33 +671,9 @@ build_timestamp = "2026-05-05T12:34:56Z" let archives = dir.join("archives"); fs::create_dir_all(&archives).unwrap(); - write_real_archive( - &archives, - "alpha", - "1.0.0", - 1, - 6, - "wasm32", - &"a".repeat(64), - ); - write_real_archive( - &archives, - "alpha", - "1.0.0", - 1, - 6, - "wasm64", - &"b".repeat(64), - ); - write_real_archive( - &archives, - "beta", - "2.3.4", - 7, - 6, - "wasm32", - &"c".repeat(64), - ); + write_real_archive(&archives, "alpha", "1.0.0", 1, 6, "wasm32", &"a".repeat(64)); + write_real_archive(&archives, "alpha", "1.0.0", 1, 6, "wasm64", &"b".repeat(64)); + write_real_archive(&archives, "beta", "2.3.4", 7, 6, "wasm32", &"c".repeat(64)); let common = |out: PathBuf| { super::run(vec![ @@ -782,24 +713,8 @@ build_timestamp = "2026-05-05T12:34:56Z" fs::create_dir_all(&archives).unwrap(); let out = dir.join("index.toml"); - write_real_archive( - &archives, - "x", - "1.0.0", - 1, - 6, - "wasm32", - &"a".repeat(64), - ); - write_real_archive( - &archives, - "x", - "1.0.1", - 1, - 6, - "wasm64", - &"b".repeat(64), - ); + write_real_archive(&archives, "x", "1.0.0", 1, 6, "wasm32", &"a".repeat(64)); + write_real_archive(&archives, "x", "1.0.1", 1, 6, "wasm64", &"b".repeat(64)); let err = super::run(vec![ "--abi".into(), @@ -823,15 +738,7 @@ build_timestamp = "2026-05-05T12:34:56Z" fs::create_dir_all(&archives).unwrap(); let out = dir.join("index.toml"); - write_real_archive( - &archives, - "x", - "1.0.0", - 1, - 5, - "wasm32", - &"a".repeat(64), - ); + write_real_archive(&archives, "x", "1.0.0", 1, 5, "wasm32", &"a".repeat(64)); let err = super::run(vec![ "--abi".into(), @@ -902,10 +809,8 @@ build_timestamp = "2026-05-05T12:34:56Z" fs::create_dir_all(&archives).unwrap(); let out = dir.join("index.toml"); - let first_key = - "0f5290453e6ea7f68e5ee1e50bd6dbf23221368e7aeb7a54c34953cef453920d"; - let second_key = - "a88651d0cd72a9100a67c90fa4b5600659258b10890c852ff10ab125cf770212"; + let first_key = "0f5290453e6ea7f68e5ee1e50bd6dbf23221368e7aeb7a54c34953cef453920d"; + let second_key = "a88651d0cd72a9100a67c90fa4b5600659258b10890c852ff10ab125cf770212"; let first = write_real_archive( &archives, "spidermonkey-node", @@ -952,7 +857,10 @@ build_timestamp = "2026-05-05T12:34:56Z" ] { assert!(err.contains(value), "missing {value:?} from: {err}"); } - assert!(!out.exists(), "a rejected inventory must not write an index"); + assert!( + !out.exists(), + "a rejected inventory must not write an index" + ); } #[test] diff --git a/tools/xtask/src/dump_abi.rs b/tools/xtask/src/dump_abi.rs index 3fcd7681d7..b6213784a8 100644 --- a/tools/xtask/src/dump_abi.rs +++ b/tools/xtask/src/dump_abi.rs @@ -249,6 +249,997 @@ fn render_ts_module() -> String { )); } out.push_str("] as const;\n"); + out.push_str(&format!( + "export const WPK_FORK_MODULE_STATE_FORMAT_SECTION = {:?} as const;\n", + shared::abi::WPK_FORK_MODULE_STATE_FORMAT_SECTION + )); + out.push_str(&format!( + "export const WPK_FORK_MODULE_STATE_FORMAT_VERSION = {} as const;\n", + shared::abi::WPK_FORK_MODULE_STATE_FORMAT_VERSION + )); + out.push_str(&format!( + "export const WPK_FORK_MODULE_STATE_FORMAT_MAGIC = {:?} as const;\n", + shared::abi::WPK_FORK_MODULE_STATE_FORMAT_MAGIC + )); + out.push_str(&format!( + "export const WPK_FORK_MODULE_STATE_DESCRIPTOR_SIZE = {} as const;\n", + shared::abi::WPK_FORK_MODULE_STATE_DESCRIPTOR_SIZE + )); + out.push_str(&format!( + "export const WPK_FORK_MODULE_STATE_RECORD_ALIGNMENT = {} as const;\n", + shared::abi::WPK_FORK_MODULE_STATE_RECORD_ALIGNMENT + )); + out.push_str(&format!( + "export const WPK_FORK_MODULE_STATE_FLAG_ROOT_PREFIX_POINTER = {} as const;\n", + shared::abi::WPK_FORK_MODULE_STATE_FLAG_ROOT_PREFIX_POINTER + )); + out.push_str(&format!( + "export const WPK_FORK_MODULE_STATE_FLAG_EXPLICIT_OWNERS = {} as const;\n", + shared::abi::WPK_FORK_MODULE_STATE_FLAG_EXPLICIT_OWNERS + )); + out.push_str(&format!( + "export const WPK_FORK_MODULE_STATE_FLAG_SPARSE_TABLES = {} as const;\n", + shared::abi::WPK_FORK_MODULE_STATE_FLAG_SPARSE_TABLES + )); + out.push_str(&format!( + "export const WPK_FORK_MODULE_STATE_REQUIRED_FLAGS = {} as const;\n", + shared::abi::WPK_FORK_MODULE_STATE_REQUIRED_FLAGS + )); + out.push_str(&format!( + "export const WPK_FORK_MODULE_STATE_KNOWN_FLAGS = {} as const;\n", + shared::abi::WPK_FORK_MODULE_STATE_KNOWN_FLAGS + )); + out.push_str(&format!( + "export const WPK_FORK_MODULE_STATE_ARENA_VERSION = {} as const;\n", + shared::abi::WPK_FORK_MODULE_STATE_ARENA_VERSION + )); + out.push_str(&format!( + "export const WPK_FORK_MODULE_STATE_RECORD_VERSION = {} as const;\n", + shared::abi::WPK_FORK_MODULE_STATE_RECORD_VERSION + )); + out.push_str(&format!( + "export const WPK_FORK_MODULE_STATE_ROOT_POINTER_WORD_OFFSET = {} as const;\n", + shared::abi::WPK_FORK_MODULE_STATE_ROOT_POINTER_WORD_OFFSET + )); + out.push_str(&format!( + "export const WPK_FORK_MODULE_STATE_CHUNK_MAGIC = {:?} as const;\n", + shared::abi::WPK_FORK_MODULE_STATE_CHUNK_MAGIC + )); + out.push_str(&format!( + "export const WPK_FORK_MODULE_STATE_CHUNK_FLAG_ROOT = {} as const;\n", + shared::abi::WPK_FORK_MODULE_STATE_CHUNK_FLAG_ROOT + )); + out.push_str(&format!( + "export const WPK_FORK_MODULE_STATE_CHUNK_FLAG_SEALED = {} as const;\n", + shared::abi::WPK_FORK_MODULE_STATE_CHUNK_FLAG_SEALED + )); + out.push_str(&format!( + "export const WPK_FORK_MODULE_STATE_CHUNK_KNOWN_FLAGS = {} as const;\n", + shared::abi::WPK_FORK_MODULE_STATE_CHUNK_KNOWN_FLAGS + )); + out.push_str(&format!( + "export const WPK_FORK_MODULE_STATE_RECORD_MAGIC = {:?} as const;\n", + shared::abi::WPK_FORK_MODULE_STATE_RECORD_MAGIC + )); + out.push_str(&format!( + "export const WPK_FORK_MODULE_STATE_RECORD_HEADER_SIZE = {} as const;\n", + shared::abi::WPK_FORK_MODULE_STATE_RECORD_HEADER_SIZE + )); + for (name, value) in [ + ( + "MODULE", + shared::abi::WPK_FORK_MODULE_STATE_RECORD_KIND_MODULE, + ), + ( + "REFERENCE_RECIPE", + shared::abi::WPK_FORK_MODULE_STATE_RECORD_KIND_REFERENCE_RECIPE, + ), + ( + "MUTABLE_GLOBAL", + shared::abi::WPK_FORK_MODULE_STATE_RECORD_KIND_MUTABLE_GLOBAL, + ), + ( + "TABLE", + shared::abi::WPK_FORK_MODULE_STATE_RECORD_KIND_TABLE, + ), + ( + "TABLE_PAGE", + shared::abi::WPK_FORK_MODULE_STATE_RECORD_KIND_TABLE_PAGE, + ), + ( + "ELEMENT_SEGMENTS", + shared::abi::WPK_FORK_MODULE_STATE_RECORD_KIND_ELEMENT_SEGMENTS, + ), + ( + "DATA_SEGMENTS", + shared::abi::WPK_FORK_MODULE_STATE_RECORD_KIND_DATA_SEGMENTS, + ), + ( + "REPLAY_EVENTS", + shared::abi::WPK_FORK_MODULE_STATE_RECORD_KIND_REPLAY_EVENTS, + ), + ( + "IMPORTED_GLOBAL_BINDINGS", + shared::abi::WPK_FORK_MODULE_STATE_RECORD_KIND_IMPORTED_GLOBAL_BINDINGS, + ), + ( + "ACTIVATION_CONTINUATIONS", + shared::abi::WPK_FORK_MODULE_STATE_RECORD_KIND_ACTIVATION_CONTINUATIONS, + ), + ( + "IMPORTED_TABLE_BINDINGS", + shared::abi::WPK_FORK_MODULE_STATE_RECORD_KIND_IMPORTED_TABLE_BINDINGS, + ), + ( + "REFERENCE_RECIPE_SEGMENT", + shared::abi::WPK_FORK_MODULE_STATE_RECORD_KIND_REFERENCE_RECIPE_SEGMENT, + ), + ( + "REPLAY_EVENT_SEGMENT", + shared::abi::WPK_FORK_MODULE_STATE_RECORD_KIND_REPLAY_EVENT_SEGMENT, + ), + ] { + out.push_str(&format!( + "export const WPK_FORK_MODULE_STATE_RECORD_KIND_{name} = {value} as const;\n" + )); + } + out.push_str("export const WPK_FORK_MODULE_STATE_RECORD_KINDS = [\n"); + for kind in shared::abi::WPK_FORK_MODULE_STATE_RECORD_KINDS { + out.push_str(&format!( + " {{ number: {}, name: {:?} }},\n", + kind.number, kind.name + )); + } + out.push_str("] as const;\n"); + out.push_str("export const WPK_FORK_MODULE_STATE_POINTER_WIDTHS = [\n"); + for pointer_width in shared::abi::WPK_FORK_MODULE_STATE_POINTER_WIDTHS { + out.push_str(&format!( + " {{ bytes: {}, chunkHeaderSize: {} }},\n", + pointer_width, + shared::abi::wpk_fork_module_state_chunk_header_size(*pointer_width) + .expect("supported pointer width must have a module-state chunk header"), + )); + } + out.push_str("] as const;\n"); + out.push_str(&format!( + "export const WPK_FORK_MODULE_STATE_REPLAY_EVENTS_MAGIC = {:?} as const;\n", + shared::abi::WPK_FORK_MODULE_STATE_REPLAY_EVENTS_MAGIC, + )); + out.push_str(&format!( + "export const WPK_FORK_REFERENCE_TRANSACTION_MAGIC = {:?} as const;\n", + shared::abi::WPK_FORK_REFERENCE_TRANSACTION_MAGIC, + )); + out.push_str(&format!( + "export const WPK_FORK_REFERENCE_SEGMENT_MAGIC = {:?} as const;\n", + shared::abi::WPK_FORK_REFERENCE_SEGMENT_MAGIC, + )); + for (name, value) in [ + ( + "TRANSACTION_OWNER", + shared::abi::WPK_FORK_REFERENCE_TRANSACTION_OWNER, + ), + ( + "TRANSACTION_VERSION", + u32::from(shared::abi::WPK_FORK_REFERENCE_TRANSACTION_VERSION), + ), + ( + "TRANSACTION_MANIFEST_SIZE", + u32::from(shared::abi::WPK_FORK_REFERENCE_TRANSACTION_MANIFEST_SIZE), + ), + ( + "TRANSACTION_FLAG_SEALED", + shared::abi::WPK_FORK_REFERENCE_TRANSACTION_FLAG_SEALED, + ), + ( + "TRANSACTION_KNOWN_FLAGS", + shared::abi::WPK_FORK_REFERENCE_TRANSACTION_KNOWN_FLAGS, + ), + ( + "SEGMENT_HEADER_SIZE", + u32::from(shared::abi::WPK_FORK_REFERENCE_SEGMENT_HEADER_SIZE), + ), + ( + "SEGMENT_KNOWN_FLAGS", + u32::from(shared::abi::WPK_FORK_REFERENCE_SEGMENT_KNOWN_FLAGS), + ), + ( + "NODE_RECORD_SIZE", + u32::from(shared::abi::WPK_FORK_REFERENCE_NODE_RECORD_SIZE), + ), + ( + "VECTOR_INDEX_SIZE", + u32::from(shared::abi::WPK_FORK_REFERENCE_VECTOR_INDEX_SIZE), + ), + ( + "SECTION_NODES", + u32::from(shared::abi::WPK_FORK_REFERENCE_SECTION_NODES), + ), + ( + "SECTION_EDGES", + u32::from(shared::abi::WPK_FORK_REFERENCE_SECTION_EDGES), + ), + ( + "SECTION_SCALARS", + u32::from(shared::abi::WPK_FORK_REFERENCE_SECTION_SCALARS), + ), + ( + "SECTION_VECTOR_INDEX", + u32::from(shared::abi::WPK_FORK_REFERENCE_SECTION_VECTOR_INDEX), + ), + ( + "SECTION_VECTOR_ENTRIES", + u32::from(shared::abi::WPK_FORK_REFERENCE_SECTION_VECTOR_ENTRIES), + ), + ] { + out.push_str(&format!( + "export const WPK_FORK_REFERENCE_{name} = {value} as const;\n" + )); + } + for (name, value) in [ + ( + "MODULE_TEMPLATE_ID_SIZE", + u32::from(shared::abi::WPK_FORK_MODULE_STATE_MODULE_TEMPLATE_ID_SIZE), + ), + ( + "MODULE_RECORD_PAYLOAD_SIZE", + u32::from(shared::abi::WPK_FORK_MODULE_STATE_MODULE_RECORD_PAYLOAD_SIZE), + ), + ( + "MODULE_RECORD_KNOWN_FLAGS", + shared::abi::WPK_FORK_MODULE_STATE_MODULE_RECORD_KNOWN_FLAGS, + ), + ( + "GLOBAL_HEADER_SIZE", + u32::from(shared::abi::WPK_FORK_MODULE_STATE_GLOBAL_HEADER_SIZE), + ), + ( + "TABLE_BASELINE_FINGERPRINT_SIZE", + u32::from(shared::abi::WPK_FORK_MODULE_STATE_TABLE_BASELINE_FINGERPRINT_SIZE), + ), + ( + "TABLE_DESCRIPTOR_PAYLOAD_SIZE", + u32::from(shared::abi::WPK_FORK_MODULE_STATE_TABLE_DESCRIPTOR_PAYLOAD_SIZE), + ), + ( + "TABLE_FLAG_SPARSE_OVERRIDES", + shared::abi::WPK_FORK_MODULE_STATE_TABLE_FLAG_SPARSE_OVERRIDES, + ), + ( + "TABLE_KNOWN_FLAGS", + shared::abi::WPK_FORK_MODULE_STATE_TABLE_KNOWN_FLAGS, + ), + ( + "TABLE_PAGE_HEADER_SIZE", + u32::from(shared::abi::WPK_FORK_MODULE_STATE_TABLE_PAGE_HEADER_SIZE), + ), + ( + "TABLE_RUN_HEADER_SIZE", + u32::from(shared::abi::WPK_FORK_MODULE_STATE_TABLE_RUN_HEADER_SIZE), + ), + ( + "ELEMENT_SEGMENT_HEADER_SIZE", + u32::from(shared::abi::WPK_FORK_MODULE_STATE_ELEMENT_SEGMENT_HEADER_SIZE), + ), + ( + "DATA_SEGMENT_HEADER_SIZE", + u32::from(shared::abi::WPK_FORK_MODULE_STATE_DATA_SEGMENT_HEADER_SIZE), + ), + ( + "REPLAY_EVENTS_OWNER", + shared::abi::WPK_FORK_MODULE_STATE_REPLAY_EVENTS_OWNER, + ), + ( + "REPLAY_EVENTS_VERSION", + u32::from(shared::abi::WPK_FORK_MODULE_STATE_REPLAY_EVENTS_VERSION), + ), + ( + "REPLAY_EVENTS_HEADER_SIZE", + u32::from(shared::abi::WPK_FORK_MODULE_STATE_REPLAY_EVENTS_HEADER_SIZE), + ), + ( + "REPLAY_EVENT_SIZE", + u32::from(shared::abi::WPK_FORK_MODULE_STATE_REPLAY_EVENT_SIZE), + ), + ( + "REPLAY_EVENTS_KNOWN_FLAGS", + u32::from(shared::abi::WPK_FORK_MODULE_STATE_REPLAY_EVENTS_KNOWN_FLAGS), + ), + ( + "REPLAY_EVENT_SEGMENT_VERSION", + u32::from(shared::abi::WPK_FORK_MODULE_STATE_REPLAY_EVENT_SEGMENT_VERSION), + ), + ( + "REPLAY_EVENT_SEGMENT_HEADER_SIZE", + u32::from(shared::abi::WPK_FORK_MODULE_STATE_REPLAY_EVENT_SEGMENT_HEADER_SIZE), + ), + ( + "REPLAY_EVENT_SEGMENT_CAPACITY", + shared::abi::WPK_FORK_MODULE_STATE_REPLAY_EVENT_SEGMENT_CAPACITY, + ), + ( + "REPLAY_EVENT_SEGMENT_KNOWN_FLAGS", + u32::from(shared::abi::WPK_FORK_MODULE_STATE_REPLAY_EVENT_SEGMENT_KNOWN_FLAGS), + ), + ( + "MIN_TABLE_PAGE_SHIFT", + u32::from(shared::abi::WPK_FORK_MODULE_STATE_MIN_TABLE_PAGE_SHIFT), + ), + ( + "MAX_TABLE_PAGE_SHIFT", + u32::from(shared::abi::WPK_FORK_MODULE_STATE_MAX_TABLE_PAGE_SHIFT), + ), + ( + "TABLE_PAGE_SHIFT", + u32::from(shared::abi::WPK_FORK_MODULE_STATE_TABLE_PAGE_SHIFT), + ), + ] { + out.push_str(&format!( + "export const WPK_FORK_MODULE_STATE_{name} = {value} as const;\n" + )); + } + out.push_str(&format!( + "export const WPK_FORK_IMPORTED_GLOBAL_BINDINGS_MAGIC = {:?} as const;\n", + shared::abi::WPK_FORK_IMPORTED_GLOBAL_BINDINGS_MAGIC, + )); + for (name, value) in [ + ( + "OWNER", + shared::abi::WPK_FORK_IMPORTED_GLOBAL_BINDINGS_OWNER, + ), + ( + "VERSION", + u32::from(shared::abi::WPK_FORK_IMPORTED_GLOBAL_BINDINGS_VERSION), + ), + ( + "HEADER_SIZE", + u32::from(shared::abi::WPK_FORK_IMPORTED_GLOBAL_BINDINGS_HEADER_SIZE), + ), + ( + "ENTRY_SIZE", + u32::from(shared::abi::WPK_FORK_IMPORTED_GLOBAL_BINDINGS_ENTRY_SIZE), + ), + ( + "KNOWN_FLAGS", + u32::from(shared::abi::WPK_FORK_IMPORTED_GLOBAL_BINDINGS_KNOWN_FLAGS), + ), + ] { + out.push_str(&format!( + "export const WPK_FORK_IMPORTED_GLOBAL_BINDINGS_{name} = {value} as const;\n" + )); + } + for (name, value) in [ + ( + "RAW_NUMBER", + shared::abi::WPK_FORK_IMPORTED_GLOBAL_BINDING_RAW_NUMBER, + ), + ( + "RAW_BIGINT", + shared::abi::WPK_FORK_IMPORTED_GLOBAL_BINDING_RAW_BIGINT, + ), + ( + "RAW_REFERENCE", + shared::abi::WPK_FORK_IMPORTED_GLOBAL_BINDING_RAW_REFERENCE, + ), + ( + "ACTIVATION_GLOBAL", + shared::abi::WPK_FORK_IMPORTED_GLOBAL_BINDING_ACTIVATION_GLOBAL, + ), + ( + "BASE_IMPORT", + shared::abi::WPK_FORK_IMPORTED_GLOBAL_BINDING_BASE_IMPORT, + ), + ] { + out.push_str(&format!( + "export const WPK_FORK_IMPORTED_GLOBAL_BINDING_{name} = {value} as const;\n" + )); + } + out.push_str(&format!( + "export const WPK_FORK_GLOBAL_CATALOG_EXPORT_PREFIX = {:?} as const;\n", + shared::abi::WPK_FORK_GLOBAL_CATALOG_EXPORT_PREFIX, + )); + out.push_str(&format!( + "export const WPK_FORK_ACTIVATION_CONTINUATIONS_MAGIC = {:?} as const;\n", + shared::abi::WPK_FORK_ACTIVATION_CONTINUATIONS_MAGIC, + )); + for (name, value) in [ + ( + "OWNER", + u32::from(shared::abi::WPK_FORK_ACTIVATION_CONTINUATIONS_OWNER), + ), + ( + "VERSION", + u32::from(shared::abi::WPK_FORK_ACTIVATION_CONTINUATIONS_VERSION), + ), + ( + "HEADER_SIZE", + u32::from(shared::abi::WPK_FORK_ACTIVATION_CONTINUATIONS_HEADER_SIZE), + ), + ( + "ENTRY_SIZE", + u32::from(shared::abi::WPK_FORK_ACTIVATION_CONTINUATION_ENTRY_SIZE), + ), + ( + "KNOWN_FLAGS", + u32::from(shared::abi::WPK_FORK_ACTIVATION_CONTINUATIONS_KNOWN_FLAGS), + ), + ( + "ENTRY_KNOWN_FLAGS", + shared::abi::WPK_FORK_ACTIVATION_CONTINUATION_ENTRY_KNOWN_FLAGS, + ), + ] { + out.push_str(&format!( + "export const WPK_FORK_ACTIVATION_CONTINUATIONS_{name} = {value} as const;\n" + )); + } + out.push_str(&format!( + "export const WPK_FORK_IMPORTED_TABLE_BINDINGS_MAGIC = {:?} as const;\n", + shared::abi::WPK_FORK_IMPORTED_TABLE_BINDINGS_MAGIC, + )); + for (name, value) in [ + ("OWNER", shared::abi::WPK_FORK_IMPORTED_TABLE_BINDINGS_OWNER), + ( + "VERSION", + u32::from(shared::abi::WPK_FORK_IMPORTED_TABLE_BINDINGS_VERSION), + ), + ( + "HEADER_SIZE", + u32::from(shared::abi::WPK_FORK_IMPORTED_TABLE_BINDINGS_HEADER_SIZE), + ), + ( + "ENTRY_SIZE", + u32::from(shared::abi::WPK_FORK_IMPORTED_TABLE_BINDINGS_ENTRY_SIZE), + ), + ( + "KNOWN_FLAGS", + u32::from(shared::abi::WPK_FORK_IMPORTED_TABLE_BINDINGS_KNOWN_FLAGS), + ), + ] { + out.push_str(&format!( + "export const WPK_FORK_IMPORTED_TABLE_BINDINGS_{name} = {value} as const;\n" + )); + } + for (name, value) in [ + ( + "ACTIVATION_TABLE", + shared::abi::WPK_FORK_IMPORTED_TABLE_BINDING_ACTIVATION_TABLE, + ), + ( + "BASE_IMPORT", + shared::abi::WPK_FORK_IMPORTED_TABLE_BINDING_BASE_IMPORT, + ), + ] { + out.push_str(&format!( + "export const WPK_FORK_IMPORTED_TABLE_BINDING_{name} = {value} as const;\n" + )); + } + out.push_str(&format!( + "export const WPK_FORK_TABLE_CATALOG_EXPORT_PREFIX = {:?} as const;\n", + shared::abi::WPK_FORK_TABLE_CATALOG_EXPORT_PREFIX, + )); + for (name, value) in [ + ("I32", shared::abi::WPK_FORK_MODULE_STATE_GLOBAL_TYPE_I32), + ("I64", shared::abi::WPK_FORK_MODULE_STATE_GLOBAL_TYPE_I64), + ("F32", shared::abi::WPK_FORK_MODULE_STATE_GLOBAL_TYPE_F32), + ("F64", shared::abi::WPK_FORK_MODULE_STATE_GLOBAL_TYPE_F64), + ("V128", shared::abi::WPK_FORK_MODULE_STATE_GLOBAL_TYPE_V128), + ( + "FUNCREF", + shared::abi::WPK_FORK_MODULE_STATE_GLOBAL_TYPE_FUNCREF, + ), + ( + "EXTERNREF", + shared::abi::WPK_FORK_MODULE_STATE_GLOBAL_TYPE_EXTERNREF, + ), + ( + "EXNREF", + shared::abi::WPK_FORK_MODULE_STATE_GLOBAL_TYPE_EXNREF, + ), + ( + "ANYREF", + shared::abi::WPK_FORK_MODULE_STATE_GLOBAL_TYPE_ANYREF, + ), + ] { + out.push_str(&format!( + "export const WPK_FORK_MODULE_STATE_GLOBAL_TYPE_{name} = {value} as const;\n" + )); + } + out.push_str(&format!( + "export const WPK_FORK_CAPABILITIES_SECTION = {:?} as const;\n", + shared::abi::WPK_FORK_CAPABILITIES_SECTION + )); + out.push_str(&format!( + "export const WPK_FORK_CAPABILITIES_VERSION = {} as const;\n", + shared::abi::WPK_FORK_CAPABILITIES_VERSION + )); + out.push_str(&format!( + "export const WPK_FORK_CAP_SIDE_ENTRY = {} as const;\n", + shared::abi::WPK_FORK_CAP_SIDE_ENTRY + )); + out.push_str(&format!( + "export const WPK_FORK_CAP_DYLINK_MAIN = {} as const;\n", + shared::abi::WPK_FORK_CAP_DYLINK_MAIN + )); + out.push_str(&format!( + "export const WPK_FORK_CAP_ACTIVATION_STATE_SAFE = {} as const;\n", + shared::abi::WPK_FORK_CAP_ACTIVATION_STATE_SAFE + )); + out.push_str(&format!( + "export const WPK_FORK_CAP_KNOWN_MASK = {} as const;\n", + shared::abi::WPK_FORK_CAP_KNOWN_MASK + )); + out.push_str(&format!( + "export const WPK_FORK_CAP_REQUIRED_FLAGS = {} as const;\n", + shared::abi::WPK_FORK_CAP_REQUIRED_FLAGS + )); + out.push_str(&format!( + "export const WPK_FORK_EXCEPTION_CODEC_SECTION = {:?} as const;\n", + shared::abi::WPK_FORK_EXCEPTION_CODEC_SECTION + )); + for (name, value) in [ + ( + "VERSION", + u32::from(shared::abi::WPK_FORK_EXCEPTION_CODEC_VERSION), + ), + ( + "HEADER_SIZE", + u32::from(shared::abi::WPK_FORK_EXCEPTION_CODEC_HEADER_SIZE), + ), + ( + "TAG_RECORD_SIZE", + u32::from(shared::abi::WPK_FORK_EXCEPTION_CODEC_TAG_RECORD_SIZE), + ), + ] { + out.push_str(&format!( + "export const WPK_FORK_EXCEPTION_CODEC_{name} = {value} as const;\n" + )); + } + out.push_str(&format!( + "export const WPK_FORK_GC_CODEC_SECTION = {:?} as const;\n", + shared::abi::WPK_FORK_GC_CODEC_SECTION + )); + out.push_str(&format!( + "export const WPK_FORK_GC_CODEC_MAGIC = {:?} as const;\n", + shared::abi::WPK_FORK_GC_CODEC_MAGIC, + )); + for (name, value) in [ + ("VERSION", u32::from(shared::abi::WPK_FORK_GC_CODEC_VERSION)), + ( + "HEADER_SIZE", + u32::from(shared::abi::WPK_FORK_GC_CODEC_HEADER_SIZE), + ), + ( + "LAYOUT_RECORD_SIZE", + u32::from(shared::abi::WPK_FORK_GC_CODEC_LAYOUT_RECORD_SIZE), + ), + ( + "FIELD_RECORD_SIZE", + u32::from(shared::abi::WPK_FORK_GC_CODEC_FIELD_RECORD_SIZE), + ), + ] { + out.push_str(&format!( + "export const WPK_FORK_GC_CODEC_{name} = {value} as const;\n" + )); + } + for (name, value) in [ + ( + "WPK_FORK_UNWIND_TAG_IMPORT_MODULE", + shared::abi::WPK_FORK_UNWIND_TAG_IMPORT_MODULE, + ), + ( + "WPK_FORK_UNWIND_TAG_IMPORT_NAME", + shared::abi::WPK_FORK_UNWIND_TAG_IMPORT_NAME, + ), + ( + "WPK_FORK_UNWIND_TRANSPORT_SECTION", + shared::abi::WPK_FORK_UNWIND_TRANSPORT_SECTION, + ), + ( + "WPK_FORK_STATIC_ROOT_CATALOG_EXPORT", + shared::abi::WPK_FORK_STATIC_ROOT_CATALOG_EXPORT, + ), + ( + "WPK_FORK_STATIC_ROOT_CATALOG_SECTION", + shared::abi::WPK_FORK_STATIC_ROOT_CATALOG_SECTION, + ), + ( + "WPK_FORK_STATIC_ROOT_HARVEST_EXPORT", + shared::abi::WPK_FORK_STATIC_ROOT_HARVEST_EXPORT, + ), + ] { + out.push_str(&format!("export const {name} = {value:?} as const;\n")); + } + for (name, value) in [ + ( + "WPK_FORK_UNWIND_TRANSPORT_VERSION", + u32::from(shared::abi::WPK_FORK_UNWIND_TRANSPORT_VERSION), + ), + ( + "WPK_FORK_UNWIND_TRANSPORT_PAYLOAD_ARITY", + u32::from(shared::abi::WPK_FORK_UNWIND_TRANSPORT_PAYLOAD_ARITY), + ), + ( + "WPK_FORK_STATIC_ROOT_CATALOG_VERSION", + u32::from(shared::abi::WPK_FORK_STATIC_ROOT_CATALOG_VERSION), + ), + ( + "WPK_FORK_STATIC_ROOT_CATALOG_HEADER_SIZE", + u32::from(shared::abi::WPK_FORK_STATIC_ROOT_CATALOG_HEADER_SIZE), + ), + ] { + out.push_str(&format!("export const {name} = {value} as const;\n")); + } + out.push_str(&format!( + "export const WPK_FORK_STATIC_ROOT_CATALOG_MAGIC = {:?} as const;\n", + shared::abi::WPK_FORK_STATIC_ROOT_CATALOG_MAGIC, + )); + for (name, value) in [ + ( + "WPK_FORK_IMPORTED_GLOBALS_SECTION", + shared::abi::WPK_FORK_IMPORTED_GLOBALS_SECTION, + ), + ( + "WPK_FORK_FRAME_IMPORT_COMMIT", + shared::abi::WPK_FORK_FRAME_IMPORT_COMMIT, + ), + ( + "WPK_FORK_FRAME_IMPORT_NEXT", + shared::abi::WPK_FORK_FRAME_IMPORT_NEXT, + ), + ( + "WPK_FORK_FRAME_IMPORT_PEEK", + shared::abi::WPK_FORK_FRAME_IMPORT_PEEK, + ), + ( + "WPK_FORK_FRAME_IMPORT_RESERVE", + shared::abi::WPK_FORK_FRAME_IMPORT_RESERVE, + ), + ( + "WPK_FORK_RESUME_IMPORT_PEEK", + shared::abi::WPK_FORK_RESUME_IMPORT_PEEK, + ), + ( + "WPK_FORK_RESUME_IMPORT_TABLE", + shared::abi::WPK_FORK_RESUME_IMPORT_TABLE, + ), + ] { + out.push_str(&format!("export const {name} = {value:?} as const;\n")); + } + out.push_str(&format!( + "export const WPK_FORK_IMPORTED_GLOBALS_MAGIC = {:?} as const;\n", + shared::abi::WPK_FORK_IMPORTED_GLOBALS_MAGIC, + )); + for (name, value) in [ + ( + "VERSION", + u32::from(shared::abi::WPK_FORK_IMPORTED_GLOBALS_VERSION), + ), + ( + "HEADER_SIZE", + u32::from(shared::abi::WPK_FORK_IMPORTED_GLOBALS_HEADER_SIZE), + ), + ( + "RECORD_HEADER_SIZE", + u32::from(shared::abi::WPK_FORK_IMPORTED_GLOBALS_RECORD_HEADER_SIZE), + ), + ( + "FLAG_MUTABLE", + u32::from(shared::abi::WPK_FORK_IMPORTED_GLOBAL_FLAG_MUTABLE), + ), + ( + "FLAG_SHARED", + u32::from(shared::abi::WPK_FORK_IMPORTED_GLOBAL_FLAG_SHARED), + ), + ( + "KNOWN_FLAGS", + u32::from(shared::abi::WPK_FORK_IMPORTED_GLOBAL_KNOWN_FLAGS), + ), + ] { + out.push_str(&format!( + "export const WPK_FORK_IMPORTED_GLOBALS_{name} = {value} as const;\n" + )); + } + out.push_str(&format!( + "export const WPK_FORK_IMPORTED_TABLES_SECTION = {:?} as const;\n", + shared::abi::WPK_FORK_IMPORTED_TABLES_SECTION, + )); + out.push_str(&format!( + "export const WPK_FORK_IMPORTED_TABLES_MAGIC = {:?} as const;\n", + shared::abi::WPK_FORK_IMPORTED_TABLES_MAGIC, + )); + for (name, value) in [ + ( + "VERSION", + u32::from(shared::abi::WPK_FORK_IMPORTED_TABLES_VERSION), + ), + ( + "HEADER_SIZE", + u32::from(shared::abi::WPK_FORK_IMPORTED_TABLES_HEADER_SIZE), + ), + ( + "RECORD_HEADER_SIZE", + u32::from(shared::abi::WPK_FORK_IMPORTED_TABLES_RECORD_HEADER_SIZE), + ), + ( + "FLAG_TABLE64", + u32::from(shared::abi::WPK_FORK_IMPORTED_TABLE_FLAG_TABLE64), + ), + ( + "KNOWN_FLAGS", + u32::from(shared::abi::WPK_FORK_IMPORTED_TABLE_KNOWN_FLAGS), + ), + ] { + out.push_str(&format!( + "export const WPK_FORK_IMPORTED_TABLES_{name} = {value} as const;\n" + )); + } + for (name, value) in [ + ( + "WPK_FORK_EXCEPTION_CODEC_IMPORT_MODULE", + shared::abi::WPK_FORK_EXCEPTION_CODEC_IMPORT_MODULE, + ), + ( + "WPK_FORK_EXCEPTION_IMPORT_ACTIVATION", + shared::abi::WPK_FORK_EXCEPTION_IMPORT_ACTIVATION, + ), + ( + "WPK_FORK_EXCEPTION_IMPORT_BROKER_ENCODE", + shared::abi::WPK_FORK_EXCEPTION_IMPORT_BROKER_ENCODE, + ), + ( + "WPK_FORK_EXCEPTION_IMPORT_BROKER_THROW_RECIPE", + shared::abi::WPK_FORK_EXCEPTION_IMPORT_BROKER_THROW_RECIPE, + ), + ( + "WPK_FORK_EXCEPTION_IMPORT_CACHE_INDEX", + shared::abi::WPK_FORK_EXCEPTION_IMPORT_CACHE_INDEX, + ), + ( + "WPK_FORK_EXCEPTION_IMPORT_CLAIM", + shared::abi::WPK_FORK_EXCEPTION_IMPORT_CLAIM, + ), + ( + "WPK_FORK_EXCEPTION_IMPORT_DEFINE", + shared::abi::WPK_FORK_EXCEPTION_IMPORT_DEFINE, + ), + ( + "WPK_FORK_EXCEPTION_IMPORT_INGRESS_THROW", + shared::abi::WPK_FORK_EXCEPTION_IMPORT_INGRESS_THROW, + ), + ( + "WPK_FORK_EXCEPTION_IMPORT_LOAD", + shared::abi::WPK_FORK_EXCEPTION_IMPORT_LOAD, + ), + ( + "WPK_FORK_EXCEPTION_IMPORT_LOOKUP", + shared::abi::WPK_FORK_EXCEPTION_IMPORT_LOOKUP, + ), + ( + "WPK_FORK_EXCEPTION_IMPORT_ROUTE", + shared::abi::WPK_FORK_EXCEPTION_IMPORT_ROUTE, + ), + ( + "WPK_FORK_EXCEPTION_EXPORT_ABORT", + shared::abi::WPK_FORK_EXCEPTION_EXPORT_ABORT, + ), + ( + "WPK_FORK_EXCEPTION_EXPORT_CLEAR", + shared::abi::WPK_FORK_EXCEPTION_EXPORT_CLEAR, + ), + ( + "WPK_FORK_EXCEPTION_EXPORT_DECODE", + shared::abi::WPK_FORK_EXCEPTION_EXPORT_DECODE, + ), + ( + "WPK_FORK_EXCEPTION_EXPORT_ENCODE", + shared::abi::WPK_FORK_EXCEPTION_EXPORT_ENCODE, + ), + ( + "WPK_FORK_EXCEPTION_EXPORT_ENCODE_INGRESS", + shared::abi::WPK_FORK_EXCEPTION_EXPORT_ENCODE_INGRESS, + ), + ( + "WPK_FORK_EXCEPTION_EXPORT_MATERIALIZE", + shared::abi::WPK_FORK_EXCEPTION_EXPORT_MATERIALIZE, + ), + ( + "WPK_FORK_EXCEPTION_EXPORT_THROW_RECIPE", + shared::abi::WPK_FORK_EXCEPTION_EXPORT_THROW_RECIPE, + ), + ( + "WPK_FORK_EXCEPTION_EXPORT_THROW_SLOT", + shared::abi::WPK_FORK_EXCEPTION_EXPORT_THROW_SLOT, + ), + ( + "WPK_FORK_MODULE_STATE_IMPORT_RECORD_COMMIT", + shared::abi::WPK_FORK_MODULE_STATE_IMPORT_RECORD_COMMIT, + ), + ( + "WPK_FORK_MODULE_STATE_IMPORT_RECORD_FIND", + shared::abi::WPK_FORK_MODULE_STATE_IMPORT_RECORD_FIND, + ), + ( + "WPK_FORK_MODULE_STATE_IMPORT_RECORD_RESERVE", + shared::abi::WPK_FORK_MODULE_STATE_IMPORT_RECORD_RESERVE, + ), + ( + "WPK_FORK_MODULE_STATE_IMPORT_TABLE_DIRTY_COUNT", + shared::abi::WPK_FORK_MODULE_STATE_IMPORT_TABLE_DIRTY_COUNT, + ), + ( + "WPK_FORK_MODULE_STATE_IMPORT_TABLE_DIRTY_MARK", + shared::abi::WPK_FORK_MODULE_STATE_IMPORT_TABLE_DIRTY_MARK, + ), + ( + "WPK_FORK_MODULE_STATE_IMPORT_TABLE_DIRTY_PAGE", + shared::abi::WPK_FORK_MODULE_STATE_IMPORT_TABLE_DIRTY_PAGE, + ), + ( + "WPK_FORK_MODULE_STATE_IMPORT_TABLE_STATE_OWNED", + shared::abi::WPK_FORK_MODULE_STATE_IMPORT_TABLE_STATE_OWNED, + ), + ( + "WPK_FORK_EXPORT_MODULE_BOOTSTRAP", + shared::abi::WPK_FORK_EXPORT_MODULE_BOOTSTRAP, + ), + ( + "WPK_FORK_EXPORT_MODULE_STATE_FINISH_RESTORE", + shared::abi::WPK_FORK_EXPORT_MODULE_STATE_FINISH_RESTORE, + ), + ( + "WPK_FORK_EXPORT_MODULE_STATE_RESTORE", + shared::abi::WPK_FORK_EXPORT_MODULE_STATE_RESTORE, + ), + ( + "WPK_FORK_EXPORT_MODULE_STATE_SAVE", + shared::abi::WPK_FORK_EXPORT_MODULE_STATE_SAVE, + ), + ( + "WPK_FORK_EXPORT_MODULE_THREAD_BOOTSTRAP", + shared::abi::WPK_FORK_EXPORT_MODULE_THREAD_BOOTSTRAP, + ), + ( + "WPK_FORK_EXPORT_RESUME_START", + shared::abi::WPK_FORK_EXPORT_RESUME_START, + ), + ( + "WPK_FORK_EXPORT_RESUME_THREAD", + shared::abi::WPK_FORK_EXPORT_RESUME_THREAD, + ), + ( + "WPK_FORK_REFERENCE_IMPORT_DECODE_ANYREF", + shared::abi::WPK_FORK_REFERENCE_IMPORT_DECODE_ANYREF, + ), + ( + "WPK_FORK_REFERENCE_IMPORT_DECODE_EXNREF", + shared::abi::WPK_FORK_REFERENCE_IMPORT_DECODE_EXNREF, + ), + ( + "WPK_FORK_REFERENCE_IMPORT_DECODE_EXTERNREF", + shared::abi::WPK_FORK_REFERENCE_IMPORT_DECODE_EXTERNREF, + ), + ( + "WPK_FORK_REFERENCE_IMPORT_DECODE_FUNCREF", + shared::abi::WPK_FORK_REFERENCE_IMPORT_DECODE_FUNCREF, + ), + ( + "WPK_FORK_REFERENCE_IMPORT_ENCODE_ANYREF", + shared::abi::WPK_FORK_REFERENCE_IMPORT_ENCODE_ANYREF, + ), + ( + "WPK_FORK_REFERENCE_IMPORT_ENCODE_EXNREF", + shared::abi::WPK_FORK_REFERENCE_IMPORT_ENCODE_EXNREF, + ), + ( + "WPK_FORK_REFERENCE_IMPORT_ENCODE_EXTERNREF", + shared::abi::WPK_FORK_REFERENCE_IMPORT_ENCODE_EXTERNREF, + ), + ( + "WPK_FORK_REFERENCE_IMPORT_ENCODE_FUNCREF", + shared::abi::WPK_FORK_REFERENCE_IMPORT_ENCODE_FUNCREF, + ), + ( + "WPK_FORK_REFERENCE_CODEC_IMPORT_MODULE", + shared::abi::WPK_FORK_REFERENCE_CODEC_IMPORT_MODULE, + ), + ( + "WPK_FORK_REFERENCE_IMPORT_GC_BROKER_ENCODE", + shared::abi::WPK_FORK_REFERENCE_IMPORT_GC_BROKER_ENCODE, + ), + ( + "WPK_FORK_REFERENCE_IMPORT_GC_CAPTURE_LAYOUT", + shared::abi::WPK_FORK_REFERENCE_IMPORT_GC_CAPTURE_LAYOUT, + ), + ( + "WPK_FORK_REFERENCE_IMPORT_GC_CLAIM", + shared::abi::WPK_FORK_REFERENCE_IMPORT_GC_CLAIM, + ), + ( + "WPK_FORK_REFERENCE_IMPORT_GC_DEFINE", + shared::abi::WPK_FORK_REFERENCE_IMPORT_GC_DEFINE, + ), + ( + "WPK_FORK_REFERENCE_IMPORT_GC_I31", + shared::abi::WPK_FORK_REFERENCE_IMPORT_GC_I31, + ), + ( + "WPK_FORK_REFERENCE_IMPORT_GC_LOAD", + shared::abi::WPK_FORK_REFERENCE_IMPORT_GC_LOAD, + ), + ( + "WPK_FORK_REFERENCE_IMPORT_GC_LOOKUP", + shared::abi::WPK_FORK_REFERENCE_IMPORT_GC_LOOKUP, + ), + ( + "WPK_FORK_REFERENCE_IMPORT_GC_PAYLOAD_LEN", + shared::abi::WPK_FORK_REFERENCE_IMPORT_GC_PAYLOAD_LEN, + ), + ( + "WPK_FORK_REFERENCE_IMPORT_GC_PROVENANCE_BEGIN", + shared::abi::WPK_FORK_REFERENCE_IMPORT_GC_PROVENANCE_BEGIN, + ), + ( + "WPK_FORK_REFERENCE_IMPORT_GC_PROVENANCE_END", + shared::abi::WPK_FORK_REFERENCE_IMPORT_GC_PROVENANCE_END, + ), + ( + "WPK_FORK_REFERENCE_IMPORT_GC_PROVENANCE_REF", + shared::abi::WPK_FORK_REFERENCE_IMPORT_GC_PROVENANCE_REF, + ), + ( + "WPK_FORK_REFERENCE_IMPORT_GC_ROUTE", + shared::abi::WPK_FORK_REFERENCE_IMPORT_GC_ROUTE, + ), + ( + "WPK_FORK_REFERENCE_IMPORT_GC_TRANSIT", + shared::abi::WPK_FORK_REFERENCE_IMPORT_GC_TRANSIT, + ), + ( + "WPK_FORK_REFERENCE_EXPORT_GC_ALLOCATE", + shared::abi::WPK_FORK_REFERENCE_EXPORT_GC_ALLOCATE, + ), + ( + "WPK_FORK_REFERENCE_EXPORT_GC_ENCODE_SLOT", + shared::abi::WPK_FORK_REFERENCE_EXPORT_GC_ENCODE_SLOT, + ), + ( + "WPK_FORK_REFERENCE_EXPORT_GC_FILL", + shared::abi::WPK_FORK_REFERENCE_EXPORT_GC_FILL, + ), + ( + "WPK_FORK_REFERENCE_EXPORT_GC_PUBLISH_EXTERNREF", + shared::abi::WPK_FORK_REFERENCE_EXPORT_GC_PUBLISH_EXTERNREF, + ), + ( + "WPK_FORK_REFERENCE_EXPORT_GC_PROBE", + shared::abi::WPK_FORK_REFERENCE_EXPORT_GC_PROBE, + ), + ( + "WPK_FORK_REFERENCE_IMPORT_SCRATCH_RELEASE", + shared::abi::WPK_FORK_REFERENCE_IMPORT_SCRATCH_RELEASE, + ), + ( + "WPK_FORK_REFERENCE_IMPORT_SCRATCH_RESERVE", + shared::abi::WPK_FORK_REFERENCE_IMPORT_SCRATCH_RESERVE, + ), + ( + "WPK_FORK_REFERENCE_IMPORT_VECTOR_APPEND", + shared::abi::WPK_FORK_REFERENCE_IMPORT_VECTOR_APPEND, + ), + ( + "WPK_FORK_REFERENCE_IMPORT_VECTOR_BEGIN", + shared::abi::WPK_FORK_REFERENCE_IMPORT_VECTOR_BEGIN, + ), + ( + "WPK_FORK_REFERENCE_IMPORT_VECTOR_FINISH", + shared::abi::WPK_FORK_REFERENCE_IMPORT_VECTOR_FINISH, + ), + ( + "WPK_FORK_REFERENCE_IMPORT_VECTOR_GET", + shared::abi::WPK_FORK_REFERENCE_IMPORT_VECTOR_GET, + ), + ] { + out.push_str(&format!("export const {name} = {value:?} as const;\n")); + } out.push_str("export const WPK_FORK_REQUIRED_IMPORTS = [\n"); for requirement in shared::abi::WPK_FORK_REQUIRED_IMPORTS { out.push_str(&format!( @@ -260,6 +1251,21 @@ fn render_ts_module() -> String { )); } out.push_str("] as const;\n"); + out.push_str("export const WPK_FORK_REQUIRED_TABLE_IMPORTS = [\n"); + for requirement in shared::abi::WPK_FORK_REQUIRED_TABLE_IMPORTS { + out.push_str(&format!( + " {{ module: {:?}, name: {:?}, table64: {}, element: {:?}, minimum: {}, maximum: {} }},\n", + requirement.module, + requirement.name, + requirement.table64, + program_artifact_type_name(requirement.element), + requirement.minimum, + requirement + .maximum + .map_or_else(|| "null".to_owned(), |maximum| maximum.to_string()), + )); + } + out.push_str("] as const;\n"); out.push_str("export const WPK_FORK_REQUIRED_EXPORTS = [\n"); for requirement in shared::abi::WPK_FORK_REQUIRED_EXPORTS { out.push_str(&format!( @@ -379,6 +1385,14 @@ fn render_ts_module() -> String { "export const CH_ERRNO = {} as const;\n", channel::ERRNO_OFFSET )); + out.push_str(&format!( + "export const CH_REQUEST_FLAGS = {} as const;\n", + channel::REQUEST_FLAGS_OFFSET + )); + out.push_str(&format!( + "export const CH_REQUEST_FLAG_DEFER_SIGNAL_DELIVERY = {} as const;\n", + channel::REQUEST_FLAG_DEFER_SIGNAL_DELIVERY + )); out.push_str(&format!( "export const CH_DATA = {} as const;\n", channel::DATA_OFFSET @@ -672,19 +1686,27 @@ fn render_ts_module() -> String { } fn render_ts_program_artifact_types(values: &[shared::abi::ProgramArtifactValueType]) -> String { - use shared::abi::ProgramArtifactValueType; - let values = values .iter() - .map(|value| match value { - ProgramArtifactValueType::Pointer => "\"ptr\"", - ProgramArtifactValueType::I32 => "\"i32\"", - }) + .map(|value| format!("{:?}", program_artifact_type_name(*value))) .collect::>() .join(", "); format!("[{values}]") } +fn program_artifact_type_name(value: shared::abi::ProgramArtifactValueType) -> &'static str { + use shared::abi::ProgramArtifactValueType; + match value { + ProgramArtifactValueType::Pointer => "ptr", + ProgramArtifactValueType::I32 => "i32", + ProgramArtifactValueType::I64 => "i64", + ProgramArtifactValueType::FuncRef => "funcref", + ProgramArtifactValueType::ExternRef => "externref", + ProgramArtifactValueType::ExnRef => "exnref", + ProgramArtifactValueType::AnyRef => "anyref", + } +} + fn ts_syscall_arg_desc(desc: &shared::host_abi::SyscallArgDesc) -> String { let mut s = format!( "{{ argIndex: {}, direction: {:?}, size: {}", @@ -856,6 +1878,7 @@ fn build_snapshot(kernel_wasm: &std::path::Path) -> Result { root.insert("abi_version".into(), json!(shared::ABI_VERSION)); root.insert("channel_header".into(), channel_header()); + root.insert("channel_request_flags".into(), channel_request_flags()); root.insert("channel_signal_area".into(), channel_signal_area()); root.insert("channel_buffers".into(), channel_buffers()); @@ -898,6 +1921,7 @@ fn channel_header() -> Value { ("args", ARGS_OFFSET, ARGS_COUNT * ARG_SIZE, "[i64; 6]"), ("ret", RETURN_OFFSET, 8, "i64"), ("errno", ERRNO_OFFSET, 4, "i32"), + ("request_flags", REQUEST_FLAGS_OFFSET, 4, "u32"), ]; let mut covered: usize = 0; @@ -929,6 +1953,19 @@ fn channel_header() -> Value { Value::Object(m.into_iter().collect()) } +fn channel_request_flags() -> Value { + let mut flag: JsonMap = BTreeMap::new(); + flag.insert( + "name".into(), + json!("defer_signal_delivery"), + ); + flag.insert( + "bit".into(), + json!(shared::channel::REQUEST_FLAG_DEFER_SIGNAL_DELIVERY), + ); + Value::Array(vec![Value::Object(flag.into_iter().collect())]) +} + fn channel_buffers() -> Value { use shared::channel::*; let mut m: JsonMap = BTreeMap::new(); @@ -1089,17 +2126,16 @@ fn channel_signal_area() -> Value { fn marshalled_structs() -> Value { use shared::dri::{ WpkDrmBindForeignTexture, WpkDrmEventVblank, WpkDrmGemClose, WpkDrmGetCap, - WpkDrmGpuBoCreate, WpkDrmModeCardRes, WpkDrmModeCreateDumb, - WpkDrmModeCrtcPageFlip, WpkDrmModeDestroyDumb, WpkDrmModeFbCmd2, - WpkDrmModeGetConnector, WpkDrmModeGetCrtc, WpkDrmModeGetEncoder, - WpkDrmModeMapDumb, WpkDrmModeModeinfo, WpkDrmPrimeHandle, WpkDrmVersion, - WpkDrmWaitVblankReply, WpkDrmWaitVblankRequest, + WpkDrmGpuBoCreate, WpkDrmModeCardRes, WpkDrmModeCreateDumb, WpkDrmModeCrtcPageFlip, + WpkDrmModeDestroyDumb, WpkDrmModeFbCmd2, WpkDrmModeGetConnector, WpkDrmModeGetCrtc, + WpkDrmModeGetEncoder, WpkDrmModeMapDumb, WpkDrmModeModeinfo, WpkDrmPrimeHandle, + WpkDrmVersion, WpkDrmWaitVblankReply, WpkDrmWaitVblankRequest, }; use shared::fbdev::{FbBitfield, FbFixScreenInfo, FbVarScreenInfo}; use shared::gl::{GlContextAttrs, GlQueryInfo, GlSubmitInfo, GlSurfaceAttrs}; use shared::{ - KernelWaitResult, WasmDirent, WasmFlock, WasmPollFd, WasmRusageWire, WasmStat, - WasmStatfs, WasmTimespec, + KernelWaitResult, WasmDirent, WasmFlock, WasmPollFd, WasmRusageWire, WasmStat, WasmStatfs, + WasmTimespec, }; let mut structs: JsonMap = BTreeMap::new(); @@ -1330,11 +2366,7 @@ fn marshalled_structs() -> Value { ); structs.insert( "WpkDrmPrimeHandle".into(), - struct_layout!(WpkDrmPrimeHandle { - handle, - flags, - fd - }), + struct_layout!(WpkDrmPrimeHandle { handle, flags, fd }), ); structs.insert( "WpkDrmGetCap".into(), @@ -1535,10 +2567,7 @@ fn wait_contract() -> Value { for (name, value) in [ ("WAIT_EVENT_EXITED", json!(shared::wait::EVENT_EXITED)), ("WAIT_EVENT_STOPPED", json!(shared::wait::EVENT_STOPPED)), - ( - "WAIT_EVENT_CONTINUED", - json!(shared::wait::EVENT_CONTINUED), - ), + ("WAIT_EVENT_CONTINUED", json!(shared::wait::EVENT_CONTINUED)), ("WAIT_WNOHANG", json!(shared::wait::WNOHANG)), ("WAIT_WUNTRACED", json!(shared::wait::WUNTRACED)), ("WAIT_WSTOPPED", json!(shared::wait::WSTOPPED)), @@ -1548,10 +2577,7 @@ fn wait_contract() -> Value { ("WAIT_CLD_EXITED", json!(shared::wait::CLD_EXITED)), ("WAIT_CLD_KILLED", json!(shared::wait::CLD_KILLED)), ("WAIT_CLD_STOPPED", json!(shared::wait::CLD_STOPPED)), - ( - "WAIT_CLD_CONTINUED", - json!(shared::wait::CLD_CONTINUED), - ), + ("WAIT_CLD_CONTINUED", json!(shared::wait::CLD_CONTINUED)), ( "PROCESS_STATE_RUNNING", json!(shared::wait::PROCESS_STATE_RUNNING), @@ -1874,7 +2900,15 @@ fn channel_status_codes() -> Value { fn custom_sections() -> Value { let mut sections = vec![ shared::abi::ABI_CUSTOM_SECTION, + shared::abi::WPK_FORK_CAPABILITIES_SECTION, + shared::abi::WPK_FORK_EXCEPTION_CODEC_SECTION, + shared::abi::WPK_FORK_GC_CODEC_SECTION, shared::abi::WPK_FORK_LINKED_FRAME_FORMAT_SECTION, + shared::abi::WPK_FORK_IMPORTED_GLOBALS_SECTION, + shared::abi::WPK_FORK_IMPORTED_TABLES_SECTION, + shared::abi::WPK_FORK_MODULE_STATE_FORMAT_SECTION, + shared::abi::WPK_FORK_STATIC_ROOT_CATALOG_SECTION, + shared::abi::WPK_FORK_UNWIND_TRANSPORT_SECTION, ]; sections.sort(); Value::Array(sections.into_iter().map(Value::from).collect()) @@ -1888,13 +2922,94 @@ fn process_expected_globals() -> Value { fn program_artifact() -> Value { use shared::abi::{ - ProgramArtifactValueType, WPK_FORK_LINKED_FRAME_DESCRIPTOR_SIZE, - WPK_FORK_LINKED_FRAME_FLAG_ABORT_UNWINDING, WPK_FORK_LINKED_FRAME_FLAG_TRANSACTIONAL_NODES, - WPK_FORK_LINKED_FRAME_FORMAT_MAGIC, WPK_FORK_LINKED_FRAME_FORMAT_SECTION, - WPK_FORK_LINKED_FRAME_FORMAT_VERSION, WPK_FORK_LINKED_FRAME_POINTER_WIDTHS, - WPK_FORK_LINKED_FRAME_RECORD_ALIGNMENT, WPK_FORK_LINKED_FRAME_REQUIRED_FLAGS, - WPK_FORK_REQUIRED_EXPORTS, WPK_FORK_REQUIRED_IMPORTS, wpk_fork_linked_chunk_header_size, - wpk_fork_linked_node_header_size, + ProgramArtifactValueType, WPK_FORK_ACTIVATION_CONTINUATION_ENTRY_KNOWN_FLAGS, + WPK_FORK_ACTIVATION_CONTINUATION_ENTRY_SIZE, WPK_FORK_ACTIVATION_CONTINUATIONS_HEADER_SIZE, + WPK_FORK_ACTIVATION_CONTINUATIONS_KNOWN_FLAGS, WPK_FORK_ACTIVATION_CONTINUATIONS_MAGIC, + WPK_FORK_ACTIVATION_CONTINUATIONS_OWNER, WPK_FORK_ACTIVATION_CONTINUATIONS_VERSION, + WPK_FORK_CAP_ACTIVATION_STATE_SAFE, WPK_FORK_CAP_DYLINK_MAIN, WPK_FORK_CAP_KNOWN_MASK, + WPK_FORK_CAP_REQUIRED_FLAGS, WPK_FORK_CAP_SIDE_ENTRY, WPK_FORK_CAPABILITIES_SECTION, + WPK_FORK_CAPABILITIES_VERSION, WPK_FORK_EXCEPTION_CODEC_HEADER_SIZE, + WPK_FORK_EXCEPTION_CODEC_IMPORT_MODULE, WPK_FORK_EXCEPTION_CODEC_SECTION, + WPK_FORK_EXCEPTION_CODEC_TAG_RECORD_SIZE, WPK_FORK_EXCEPTION_CODEC_VERSION, + WPK_FORK_EXCEPTION_IMPORT_ACTIVATION, WPK_FORK_GC_CODEC_FIELD_RECORD_SIZE, + WPK_FORK_GC_CODEC_HEADER_SIZE, WPK_FORK_GC_CODEC_LAYOUT_RECORD_SIZE, + WPK_FORK_GC_CODEC_MAGIC, WPK_FORK_GC_CODEC_SECTION, WPK_FORK_GC_CODEC_VERSION, + WPK_FORK_IMPORTED_GLOBAL_BINDING_ACTIVATION_GLOBAL, + WPK_FORK_IMPORTED_GLOBAL_BINDING_BASE_IMPORT, WPK_FORK_IMPORTED_GLOBAL_BINDING_RAW_BIGINT, + WPK_FORK_IMPORTED_GLOBAL_BINDING_RAW_NUMBER, + WPK_FORK_IMPORTED_GLOBAL_BINDING_RAW_REFERENCE, + WPK_FORK_IMPORTED_GLOBAL_BINDINGS_ENTRY_SIZE, + WPK_FORK_IMPORTED_GLOBAL_BINDINGS_HEADER_SIZE, + WPK_FORK_IMPORTED_GLOBAL_BINDINGS_KNOWN_FLAGS, WPK_FORK_IMPORTED_GLOBAL_BINDINGS_MAGIC, + WPK_FORK_IMPORTED_GLOBAL_BINDINGS_OWNER, WPK_FORK_IMPORTED_GLOBAL_BINDINGS_VERSION, + WPK_FORK_IMPORTED_GLOBAL_FLAG_MUTABLE, WPK_FORK_IMPORTED_GLOBAL_FLAG_SHARED, + WPK_FORK_IMPORTED_GLOBAL_KNOWN_FLAGS, WPK_FORK_IMPORTED_GLOBALS_HEADER_SIZE, + WPK_FORK_IMPORTED_GLOBALS_MAGIC, WPK_FORK_IMPORTED_GLOBALS_RECORD_HEADER_SIZE, + WPK_FORK_IMPORTED_GLOBALS_SECTION, WPK_FORK_IMPORTED_GLOBALS_VERSION, + WPK_FORK_IMPORTED_TABLE_BINDING_ACTIVATION_TABLE, + WPK_FORK_IMPORTED_TABLE_BINDING_BASE_IMPORT, WPK_FORK_IMPORTED_TABLE_BINDINGS_ENTRY_SIZE, + WPK_FORK_IMPORTED_TABLE_BINDINGS_HEADER_SIZE, WPK_FORK_IMPORTED_TABLE_BINDINGS_KNOWN_FLAGS, + WPK_FORK_IMPORTED_TABLE_BINDINGS_MAGIC, WPK_FORK_IMPORTED_TABLE_BINDINGS_OWNER, + WPK_FORK_IMPORTED_TABLE_BINDINGS_VERSION, WPK_FORK_IMPORTED_TABLE_FLAG_TABLE64, + WPK_FORK_IMPORTED_TABLE_KNOWN_FLAGS, WPK_FORK_IMPORTED_TABLES_HEADER_SIZE, + WPK_FORK_IMPORTED_TABLES_MAGIC, WPK_FORK_IMPORTED_TABLES_RECORD_HEADER_SIZE, + WPK_FORK_IMPORTED_TABLES_SECTION, WPK_FORK_IMPORTED_TABLES_VERSION, + WPK_FORK_LINKED_FRAME_DESCRIPTOR_SIZE, WPK_FORK_LINKED_FRAME_FLAG_ABORT_UNWINDING, + WPK_FORK_LINKED_FRAME_FLAG_TRANSACTIONAL_NODES, WPK_FORK_LINKED_FRAME_FORMAT_MAGIC, + WPK_FORK_LINKED_FRAME_FORMAT_SECTION, WPK_FORK_LINKED_FRAME_FORMAT_VERSION, + WPK_FORK_LINKED_FRAME_POINTER_WIDTHS, WPK_FORK_LINKED_FRAME_RECORD_ALIGNMENT, + WPK_FORK_LINKED_FRAME_REQUIRED_FLAGS, WPK_FORK_MODULE_STATE_ARENA_VERSION, + WPK_FORK_MODULE_STATE_CHUNK_FLAG_ROOT, WPK_FORK_MODULE_STATE_CHUNK_FLAG_SEALED, + WPK_FORK_MODULE_STATE_CHUNK_KNOWN_FLAGS, WPK_FORK_MODULE_STATE_CHUNK_MAGIC, + WPK_FORK_MODULE_STATE_DATA_SEGMENT_HEADER_SIZE, WPK_FORK_MODULE_STATE_DESCRIPTOR_SIZE, + WPK_FORK_MODULE_STATE_ELEMENT_SEGMENT_HEADER_SIZE, + WPK_FORK_MODULE_STATE_FLAG_EXPLICIT_OWNERS, WPK_FORK_MODULE_STATE_FLAG_ROOT_PREFIX_POINTER, + WPK_FORK_MODULE_STATE_FLAG_SPARSE_TABLES, WPK_FORK_MODULE_STATE_FORMAT_MAGIC, + WPK_FORK_MODULE_STATE_FORMAT_SECTION, WPK_FORK_MODULE_STATE_FORMAT_VERSION, + WPK_FORK_MODULE_STATE_GLOBAL_HEADER_SIZE, WPK_FORK_MODULE_STATE_GLOBAL_TYPE_ANYREF, + WPK_FORK_MODULE_STATE_GLOBAL_TYPE_EXNREF, WPK_FORK_MODULE_STATE_GLOBAL_TYPE_EXTERNREF, + WPK_FORK_MODULE_STATE_GLOBAL_TYPE_F32, WPK_FORK_MODULE_STATE_GLOBAL_TYPE_F64, + WPK_FORK_MODULE_STATE_GLOBAL_TYPE_FUNCREF, WPK_FORK_MODULE_STATE_GLOBAL_TYPE_I32, + WPK_FORK_MODULE_STATE_GLOBAL_TYPE_I64, WPK_FORK_MODULE_STATE_GLOBAL_TYPE_V128, + WPK_FORK_MODULE_STATE_KNOWN_FLAGS, WPK_FORK_MODULE_STATE_MAX_TABLE_PAGE_SHIFT, + WPK_FORK_MODULE_STATE_MIN_TABLE_PAGE_SHIFT, + WPK_FORK_MODULE_STATE_MODULE_RECORD_KNOWN_FLAGS, + WPK_FORK_MODULE_STATE_MODULE_RECORD_PAYLOAD_SIZE, + WPK_FORK_MODULE_STATE_MODULE_TEMPLATE_ID_SIZE, WPK_FORK_MODULE_STATE_POINTER_WIDTHS, + WPK_FORK_MODULE_STATE_RECORD_ALIGNMENT, WPK_FORK_MODULE_STATE_RECORD_HEADER_SIZE, + WPK_FORK_MODULE_STATE_RECORD_KINDS, WPK_FORK_MODULE_STATE_RECORD_MAGIC, + WPK_FORK_MODULE_STATE_RECORD_VERSION, WPK_FORK_MODULE_STATE_REPLAY_EVENT_SIZE, + WPK_FORK_MODULE_STATE_REPLAY_EVENT_SEGMENT_CAPACITY, + WPK_FORK_MODULE_STATE_REPLAY_EVENT_SEGMENT_HEADER_SIZE, + WPK_FORK_MODULE_STATE_REPLAY_EVENT_SEGMENT_KNOWN_FLAGS, + WPK_FORK_MODULE_STATE_REPLAY_EVENT_SEGMENT_VERSION, + WPK_FORK_MODULE_STATE_REPLAY_EVENTS_HEADER_SIZE, + WPK_FORK_MODULE_STATE_REPLAY_EVENTS_KNOWN_FLAGS, WPK_FORK_MODULE_STATE_REPLAY_EVENTS_MAGIC, + WPK_FORK_MODULE_STATE_REPLAY_EVENTS_OWNER, + WPK_FORK_MODULE_STATE_REPLAY_EVENTS_VERSION, WPK_FORK_MODULE_STATE_REQUIRED_FLAGS, + WPK_FORK_MODULE_STATE_ROOT_POINTER_WORD_OFFSET, + WPK_FORK_MODULE_STATE_TABLE_BASELINE_FINGERPRINT_SIZE, + WPK_FORK_MODULE_STATE_TABLE_DESCRIPTOR_PAYLOAD_SIZE, + WPK_FORK_MODULE_STATE_TABLE_FLAG_SPARSE_OVERRIDES, WPK_FORK_MODULE_STATE_TABLE_KNOWN_FLAGS, + WPK_FORK_MODULE_STATE_TABLE_PAGE_HEADER_SIZE, WPK_FORK_MODULE_STATE_TABLE_PAGE_SHIFT, + WPK_FORK_MODULE_STATE_TABLE_RUN_HEADER_SIZE, WPK_FORK_REFERENCE_NODE_RECORD_SIZE, + WPK_FORK_REFERENCE_SECTION_EDGES, WPK_FORK_REFERENCE_SECTION_NODES, + WPK_FORK_REFERENCE_SECTION_SCALARS, WPK_FORK_REFERENCE_SECTION_VECTOR_ENTRIES, + WPK_FORK_REFERENCE_SECTION_VECTOR_INDEX, WPK_FORK_REFERENCE_SEGMENT_HEADER_SIZE, + WPK_FORK_REFERENCE_SEGMENT_KNOWN_FLAGS, WPK_FORK_REFERENCE_SEGMENT_MAGIC, + WPK_FORK_REFERENCE_TRANSACTION_FLAG_SEALED, + WPK_FORK_REFERENCE_TRANSACTION_KNOWN_FLAGS, + WPK_FORK_REFERENCE_TRANSACTION_MAGIC, WPK_FORK_REFERENCE_TRANSACTION_MANIFEST_SIZE, + WPK_FORK_REFERENCE_TRANSACTION_OWNER, WPK_FORK_REFERENCE_TRANSACTION_VERSION, + WPK_FORK_REFERENCE_VECTOR_INDEX_SIZE, WPK_FORK_REQUIRED_EXPORTS, + WPK_FORK_REQUIRED_IMPORTS, WPK_FORK_REQUIRED_TABLE_IMPORTS, + WPK_FORK_STATIC_ROOT_CATALOG_EXPORT, WPK_FORK_STATIC_ROOT_CATALOG_HEADER_SIZE, + WPK_FORK_STATIC_ROOT_CATALOG_MAGIC, WPK_FORK_STATIC_ROOT_CATALOG_SECTION, + WPK_FORK_STATIC_ROOT_CATALOG_VERSION, WPK_FORK_STATIC_ROOT_HARVEST_EXPORT, + WPK_FORK_UNWIND_TAG_IMPORT_MODULE, WPK_FORK_UNWIND_TAG_IMPORT_NAME, + WPK_FORK_UNWIND_TRANSPORT_PAYLOAD_ARITY, WPK_FORK_UNWIND_TRANSPORT_SECTION, + WPK_FORK_UNWIND_TRANSPORT_VERSION, wpk_fork_linked_chunk_header_size, + wpk_fork_linked_node_header_size, wpk_fork_module_state_chunk_header_size, }; let value_types = |values: &[ProgramArtifactValueType]| { @@ -1905,13 +3020,18 @@ fn program_artifact() -> Value { Value::from(match value { ProgramArtifactValueType::Pointer => "ptr", ProgramArtifactValueType::I32 => "i32", + ProgramArtifactValueType::I64 => "i64", + ProgramArtifactValueType::FuncRef => "funcref", + ProgramArtifactValueType::ExternRef => "externref", + ProgramArtifactValueType::ExnRef => "exnref", + ProgramArtifactValueType::AnyRef => "anyref", }) }) .collect(), ) }; - let imports = WPK_FORK_REQUIRED_IMPORTS + let mut imports: Vec = WPK_FORK_REQUIRED_IMPORTS .iter() .map(|requirement| { let mut item: JsonMap = BTreeMap::new(); @@ -1923,87 +3043,844 @@ fn program_artifact() -> Value { Value::Object(item.into_iter().collect()) }) .collect(); + imports.extend(WPK_FORK_REQUIRED_TABLE_IMPORTS.iter().map(|requirement| { + let mut item: JsonMap = BTreeMap::new(); + item.insert("kind".into(), json!("table")); + item.insert("module".into(), json!(requirement.module)); + item.insert("name".into(), json!(requirement.name)); + item.insert("table64".into(), json!(requirement.table64)); + item.insert( + "element".into(), + Value::from(match requirement.element { + ProgramArtifactValueType::FuncRef => "funcref", + ProgramArtifactValueType::ExternRef => "externref", + ProgramArtifactValueType::ExnRef => "exnref", + ProgramArtifactValueType::AnyRef => "anyref", + other => panic!("table element requirement is not a reference: {other:?}"), + }), + ); + item.insert("minimum".into(), json!(requirement.minimum)); + item.insert("maximum".into(), json!(requirement.maximum)); + Value::Object(item.into_iter().collect()) + })); + + let exports = WPK_FORK_REQUIRED_EXPORTS + .iter() + .map(|requirement| { + let mut item: JsonMap = BTreeMap::new(); + item.insert("kind".into(), json!("func")); + item.insert("name".into(), json!(requirement.name)); + item.insert("params".into(), value_types(requirement.params)); + item.insert("results".into(), value_types(requirement.results)); + Value::Object(item.into_iter().collect()) + }) + .collect(); + + let pointer_widths = WPK_FORK_LINKED_FRAME_POINTER_WIDTHS + .iter() + .map(|pointer_width| { + let mut item: JsonMap = BTreeMap::new(); + item.insert("bytes".into(), json!(pointer_width)); + item.insert( + "chunk_header_size".into(), + json!( + wpk_fork_linked_chunk_header_size(*pointer_width) + .expect("supported pointer width must have a chunk header") + ), + ); + item.insert( + "node_header_size".into(), + json!( + wpk_fork_linked_node_header_size(*pointer_width) + .expect("supported pointer width must have a node header") + ), + ); + Value::Object(item.into_iter().collect()) + }) + .collect(); + + let mut descriptor: JsonMap = BTreeMap::new(); + descriptor.insert( + "alignment".into(), + json!(WPK_FORK_LINKED_FRAME_RECORD_ALIGNMENT), + ); + descriptor.insert( + "descriptor_size".into(), + json!(WPK_FORK_LINKED_FRAME_DESCRIPTOR_SIZE), + ); + descriptor.insert( + "flags".into(), + json!([ + { + "bit": WPK_FORK_LINKED_FRAME_FLAG_ABORT_UNWINDING, + "name": "abort_unwinding" + }, + { + "bit": WPK_FORK_LINKED_FRAME_FLAG_TRANSACTIONAL_NODES, + "name": "transactional_nodes" + } + ]), + ); + descriptor.insert( + "magic_bytes".into(), + json!(WPK_FORK_LINKED_FRAME_FORMAT_MAGIC), + ); + descriptor.insert("pointer_widths".into(), Value::Array(pointer_widths)); + descriptor.insert( + "required_flags".into(), + json!(WPK_FORK_LINKED_FRAME_REQUIRED_FLAGS), + ); + descriptor.insert( + "section".into(), + json!(WPK_FORK_LINKED_FRAME_FORMAT_SECTION), + ); + descriptor.insert( + "version".into(), + json!(WPK_FORK_LINKED_FRAME_FORMAT_VERSION), + ); + + let module_state_pointer_widths = WPK_FORK_MODULE_STATE_POINTER_WIDTHS + .iter() + .map(|pointer_width| { + let mut item: JsonMap = BTreeMap::new(); + item.insert("bytes".into(), json!(pointer_width)); + item.insert( + "chunk_header_size".into(), + json!( + wpk_fork_module_state_chunk_header_size(*pointer_width) + .expect("supported pointer width must have a module-state chunk header") + ), + ); + Value::Object(item.into_iter().collect()) + }) + .collect(); + let module_state_record_kinds = WPK_FORK_MODULE_STATE_RECORD_KINDS + .iter() + .map(|kind| { + let mut item: JsonMap = BTreeMap::new(); + item.insert("name".into(), json!(kind.name)); + item.insert("number".into(), json!(kind.number)); + Value::Object(item.into_iter().collect()) + }) + .collect(); + + let mut module_state_descriptor: JsonMap = BTreeMap::new(); + module_state_descriptor.insert( + "alignment".into(), + json!(WPK_FORK_MODULE_STATE_RECORD_ALIGNMENT), + ); + module_state_descriptor.insert( + "descriptor_size".into(), + json!(WPK_FORK_MODULE_STATE_DESCRIPTOR_SIZE), + ); + module_state_descriptor.insert( + "flags".into(), + json!([ + { + "bit": WPK_FORK_MODULE_STATE_FLAG_ROOT_PREFIX_POINTER, + "name": "root_prefix_pointer" + }, + { + "bit": WPK_FORK_MODULE_STATE_FLAG_EXPLICIT_OWNERS, + "name": "explicit_owners" + }, + { + "bit": WPK_FORK_MODULE_STATE_FLAG_SPARSE_TABLES, + "name": "sparse_tables" + } + ]), + ); + module_state_descriptor.insert( + "known_flags".into(), + json!(WPK_FORK_MODULE_STATE_KNOWN_FLAGS), + ); + module_state_descriptor.insert( + "magic_bytes".into(), + json!(WPK_FORK_MODULE_STATE_FORMAT_MAGIC), + ); + module_state_descriptor.insert( + "required_flags".into(), + json!(WPK_FORK_MODULE_STATE_REQUIRED_FLAGS), + ); + module_state_descriptor.insert( + "root_pointer_word_offset".into(), + json!(WPK_FORK_MODULE_STATE_ROOT_POINTER_WORD_OFFSET), + ); + module_state_descriptor.insert( + "section".into(), + json!(WPK_FORK_MODULE_STATE_FORMAT_SECTION), + ); + module_state_descriptor.insert( + "version".into(), + json!(WPK_FORK_MODULE_STATE_FORMAT_VERSION), + ); + + let mut module_state_record: JsonMap = BTreeMap::new(); + module_state_record.insert( + "alignment".into(), + json!(WPK_FORK_MODULE_STATE_RECORD_ALIGNMENT), + ); + module_state_record.insert( + "header_size".into(), + json!(WPK_FORK_MODULE_STATE_RECORD_HEADER_SIZE), + ); + module_state_record.insert("kinds".into(), Value::Array(module_state_record_kinds)); + module_state_record.insert( + "magic_bytes".into(), + json!(WPK_FORK_MODULE_STATE_RECORD_MAGIC), + ); + module_state_record.insert( + "version".into(), + json!(WPK_FORK_MODULE_STATE_RECORD_VERSION), + ); + + let mut module_state_arena: JsonMap = BTreeMap::new(); + module_state_arena.insert( + "chunk_flags".into(), + json!([ + {"bit": WPK_FORK_MODULE_STATE_CHUNK_FLAG_ROOT, "name": "root"}, + {"bit": WPK_FORK_MODULE_STATE_CHUNK_FLAG_SEALED, "name": "sealed"} + ]), + ); + module_state_arena.insert( + "chunk_magic_bytes".into(), + json!(WPK_FORK_MODULE_STATE_CHUNK_MAGIC), + ); + module_state_arena.insert( + "known_chunk_flags".into(), + json!(WPK_FORK_MODULE_STATE_CHUNK_KNOWN_FLAGS), + ); + module_state_arena.insert( + "pointer_widths".into(), + Value::Array(module_state_pointer_widths), + ); + module_state_arena.insert( + "record".into(), + Value::Object(module_state_record.into_iter().collect()), + ); + module_state_arena.insert("version".into(), json!(WPK_FORK_MODULE_STATE_ARENA_VERSION)); + + let mut module_payload: JsonMap = BTreeMap::new(); + module_payload.insert( + "known_flags".into(), + json!(WPK_FORK_MODULE_STATE_MODULE_RECORD_KNOWN_FLAGS), + ); + module_payload.insert( + "payload_size".into(), + json!(WPK_FORK_MODULE_STATE_MODULE_RECORD_PAYLOAD_SIZE), + ); + module_payload.insert( + "template_id_size".into(), + json!(WPK_FORK_MODULE_STATE_MODULE_TEMPLATE_ID_SIZE), + ); + + let mut mutable_global_payload: JsonMap = BTreeMap::new(); + mutable_global_payload.insert( + "header_size".into(), + json!(WPK_FORK_MODULE_STATE_GLOBAL_HEADER_SIZE), + ); + mutable_global_payload.insert( + "value_types".into(), + json!([ + {"number": WPK_FORK_MODULE_STATE_GLOBAL_TYPE_I32, "name": "i32", "bytes": 4}, + {"number": WPK_FORK_MODULE_STATE_GLOBAL_TYPE_I64, "name": "i64", "bytes": 8}, + {"number": WPK_FORK_MODULE_STATE_GLOBAL_TYPE_F32, "name": "f32", "bytes": 4}, + {"number": WPK_FORK_MODULE_STATE_GLOBAL_TYPE_F64, "name": "f64", "bytes": 8}, + {"number": WPK_FORK_MODULE_STATE_GLOBAL_TYPE_V128, "name": "v128", "bytes": 16}, + {"number": WPK_FORK_MODULE_STATE_GLOBAL_TYPE_FUNCREF, "name": "funcref_recipe", "bytes": 4}, + {"number": WPK_FORK_MODULE_STATE_GLOBAL_TYPE_EXTERNREF, "name": "externref_recipe", "bytes": 4}, + {"number": WPK_FORK_MODULE_STATE_GLOBAL_TYPE_EXNREF, "name": "exnref_recipe", "bytes": 4}, + {"number": WPK_FORK_MODULE_STATE_GLOBAL_TYPE_ANYREF, "name": "anyref_recipe", "bytes": 4} + ]), + ); + + let mut table_payload: JsonMap = BTreeMap::new(); + table_payload.insert( + "baseline_fingerprint_size".into(), + json!(WPK_FORK_MODULE_STATE_TABLE_BASELINE_FINGERPRINT_SIZE), + ); + table_payload.insert( + "descriptor_payload_size".into(), + json!(WPK_FORK_MODULE_STATE_TABLE_DESCRIPTOR_PAYLOAD_SIZE), + ); + table_payload.insert( + "flags".into(), + json!([ + { + "bit": WPK_FORK_MODULE_STATE_TABLE_FLAG_SPARSE_OVERRIDES, + "name": "sparse_overrides" + } + ]), + ); + table_payload.insert( + "known_flags".into(), + json!(WPK_FORK_MODULE_STATE_TABLE_KNOWN_FLAGS), + ); + table_payload.insert( + "max_page_shift".into(), + json!(WPK_FORK_MODULE_STATE_MAX_TABLE_PAGE_SHIFT), + ); + table_payload.insert( + "min_page_shift".into(), + json!(WPK_FORK_MODULE_STATE_MIN_TABLE_PAGE_SHIFT), + ); + table_payload.insert( + "page_shift".into(), + json!(WPK_FORK_MODULE_STATE_TABLE_PAGE_SHIFT), + ); + table_payload.insert( + "page_header_size".into(), + json!(WPK_FORK_MODULE_STATE_TABLE_PAGE_HEADER_SIZE), + ); + table_payload.insert( + "run_header_size".into(), + json!(WPK_FORK_MODULE_STATE_TABLE_RUN_HEADER_SIZE), + ); + + let mut element_segments_payload: JsonMap = BTreeMap::new(); + element_segments_payload.insert( + "header_size".into(), + json!(WPK_FORK_MODULE_STATE_ELEMENT_SEGMENT_HEADER_SIZE), + ); + + let mut data_segments_payload: JsonMap = BTreeMap::new(); + data_segments_payload.insert( + "header_size".into(), + json!(WPK_FORK_MODULE_STATE_DATA_SEGMENT_HEADER_SIZE), + ); + + let mut replay_events_payload: JsonMap = BTreeMap::new(); + replay_events_payload.insert( + "entry_size".into(), + json!(WPK_FORK_MODULE_STATE_REPLAY_EVENT_SIZE), + ); + replay_events_payload.insert( + "magic".into(), + json!(WPK_FORK_MODULE_STATE_REPLAY_EVENTS_MAGIC), + ); + replay_events_payload.insert( + "header_size".into(), + json!(WPK_FORK_MODULE_STATE_REPLAY_EVENTS_HEADER_SIZE), + ); + replay_events_payload.insert( + "known_flags".into(), + json!(WPK_FORK_MODULE_STATE_REPLAY_EVENTS_KNOWN_FLAGS), + ); + replay_events_payload.insert( + "owner".into(), + json!(WPK_FORK_MODULE_STATE_REPLAY_EVENTS_OWNER), + ); + replay_events_payload.insert( + "version".into(), + json!(WPK_FORK_MODULE_STATE_REPLAY_EVENTS_VERSION), + ); + replay_events_payload.insert( + "segment_capacity".into(), + json!(WPK_FORK_MODULE_STATE_REPLAY_EVENT_SEGMENT_CAPACITY), + ); + replay_events_payload.insert( + "segment_header_size".into(), + json!(WPK_FORK_MODULE_STATE_REPLAY_EVENT_SEGMENT_HEADER_SIZE), + ); + replay_events_payload.insert( + "segment_known_flags".into(), + json!(WPK_FORK_MODULE_STATE_REPLAY_EVENT_SEGMENT_KNOWN_FLAGS), + ); + replay_events_payload.insert( + "segment_version".into(), + json!(WPK_FORK_MODULE_STATE_REPLAY_EVENT_SEGMENT_VERSION), + ); - let exports = WPK_FORK_REQUIRED_EXPORTS - .iter() - .map(|requirement| { - let mut item: JsonMap = BTreeMap::new(); - item.insert("kind".into(), json!("func")); - item.insert("name".into(), json!(requirement.name)); - item.insert("params".into(), value_types(requirement.params)); - item.insert("results".into(), value_types(requirement.results)); - Value::Object(item.into_iter().collect()) - }) - .collect(); + let mut reference_transaction_payload: JsonMap = BTreeMap::new(); + reference_transaction_payload.insert( + "known_flags".into(), + json!(WPK_FORK_REFERENCE_TRANSACTION_KNOWN_FLAGS), + ); + reference_transaction_payload.insert( + "magic".into(), + json!(WPK_FORK_REFERENCE_TRANSACTION_MAGIC), + ); + reference_transaction_payload.insert( + "manifest_size".into(), + json!(WPK_FORK_REFERENCE_TRANSACTION_MANIFEST_SIZE), + ); + reference_transaction_payload.insert( + "node_record_size".into(), + json!(WPK_FORK_REFERENCE_NODE_RECORD_SIZE), + ); + reference_transaction_payload.insert( + "owner".into(), + json!(WPK_FORK_REFERENCE_TRANSACTION_OWNER), + ); + reference_transaction_payload.insert( + "sealed_flag".into(), + json!(WPK_FORK_REFERENCE_TRANSACTION_FLAG_SEALED), + ); + reference_transaction_payload.insert( + "sections".into(), + json!([ + {"number": WPK_FORK_REFERENCE_SECTION_NODES, "name": "nodes"}, + {"number": WPK_FORK_REFERENCE_SECTION_EDGES, "name": "edges"}, + {"number": WPK_FORK_REFERENCE_SECTION_SCALARS, "name": "scalars"}, + {"number": WPK_FORK_REFERENCE_SECTION_VECTOR_INDEX, "name": "vector_index"}, + {"number": WPK_FORK_REFERENCE_SECTION_VECTOR_ENTRIES, "name": "vector_entries"} + ]), + ); + reference_transaction_payload.insert( + "segment_header_size".into(), + json!(WPK_FORK_REFERENCE_SEGMENT_HEADER_SIZE), + ); + reference_transaction_payload.insert( + "segment_known_flags".into(), + json!(WPK_FORK_REFERENCE_SEGMENT_KNOWN_FLAGS), + ); + reference_transaction_payload.insert( + "segment_magic".into(), + json!(WPK_FORK_REFERENCE_SEGMENT_MAGIC), + ); + reference_transaction_payload.insert( + "vector_index_size".into(), + json!(WPK_FORK_REFERENCE_VECTOR_INDEX_SIZE), + ); + reference_transaction_payload.insert( + "version".into(), + json!(WPK_FORK_REFERENCE_TRANSACTION_VERSION), + ); - let pointer_widths = WPK_FORK_LINKED_FRAME_POINTER_WIDTHS - .iter() - .map(|pointer_width| { - let mut item: JsonMap = BTreeMap::new(); - item.insert("bytes".into(), json!(pointer_width)); - item.insert( - "chunk_header_size".into(), - json!( - wpk_fork_linked_chunk_header_size(*pointer_width) - .expect("supported pointer width must have a chunk header") - ), - ); - item.insert( - "node_header_size".into(), - json!( - wpk_fork_linked_node_header_size(*pointer_width) - .expect("supported pointer width must have a node header") - ), - ); - Value::Object(item.into_iter().collect()) - }) - .collect(); + let mut imported_global_bindings_payload: JsonMap = BTreeMap::new(); + imported_global_bindings_payload.insert( + "binding_kinds".into(), + json!([ + {"number": WPK_FORK_IMPORTED_GLOBAL_BINDING_RAW_NUMBER, "name": "raw_number"}, + {"number": WPK_FORK_IMPORTED_GLOBAL_BINDING_RAW_BIGINT, "name": "raw_bigint"}, + {"number": WPK_FORK_IMPORTED_GLOBAL_BINDING_RAW_REFERENCE, "name": "raw_reference"}, + {"number": WPK_FORK_IMPORTED_GLOBAL_BINDING_ACTIVATION_GLOBAL, "name": "activation_global"}, + {"number": WPK_FORK_IMPORTED_GLOBAL_BINDING_BASE_IMPORT, "name": "base_import"} + ]), + ); + imported_global_bindings_payload.insert( + "entry_fields".into(), + json!([ + {"name": "consumer_activation", "offset": 0, "size": 4}, + {"name": "consumer_owner", "offset": 4, "size": 4}, + {"name": "source_activation", "offset": 8, "size": 4}, + {"name": "source_owner", "offset": 12, "size": 4}, + {"name": "reserved", "offset": 16, "size": 4}, + {"name": "recipe_id", "offset": 20, "size": 4}, + {"name": "raw_bits", "offset": 24, "size": 8}, + {"name": "binding_kind", "offset": 32, "size": 1}, + {"name": "import_flags", "offset": 33, "size": 1}, + {"name": "value_type", "offset": 34, "size": 1}, + {"name": "reserved", "offset": 35, "size": 5} + ]), + ); + imported_global_bindings_payload.insert( + "entry_size".into(), + json!(WPK_FORK_IMPORTED_GLOBAL_BINDINGS_ENTRY_SIZE), + ); + imported_global_bindings_payload.insert( + "header_size".into(), + json!(WPK_FORK_IMPORTED_GLOBAL_BINDINGS_HEADER_SIZE), + ); + imported_global_bindings_payload.insert( + "known_flags".into(), + json!(WPK_FORK_IMPORTED_GLOBAL_BINDINGS_KNOWN_FLAGS), + ); + imported_global_bindings_payload.insert( + "magic_bytes".into(), + json!(WPK_FORK_IMPORTED_GLOBAL_BINDINGS_MAGIC), + ); + imported_global_bindings_payload.insert( + "owner".into(), + json!(WPK_FORK_IMPORTED_GLOBAL_BINDINGS_OWNER), + ); + imported_global_bindings_payload.insert( + "version".into(), + json!(WPK_FORK_IMPORTED_GLOBAL_BINDINGS_VERSION), + ); - let mut descriptor: JsonMap = BTreeMap::new(); - descriptor.insert( - "alignment".into(), - json!(WPK_FORK_LINKED_FRAME_RECORD_ALIGNMENT), + let mut activation_continuations_payload: JsonMap = BTreeMap::new(); + activation_continuations_payload.insert( + "entry_fields".into(), + json!([ + {"name": "activation_id", "offset": 0, "size": 4}, + {"name": "flags", "offset": 4, "size": 4}, + {"name": "root", "offset": 8, "size": 8} + ]), ); - descriptor.insert( - "descriptor_size".into(), - json!(WPK_FORK_LINKED_FRAME_DESCRIPTOR_SIZE), + activation_continuations_payload.insert( + "entry_known_flags".into(), + json!(WPK_FORK_ACTIVATION_CONTINUATION_ENTRY_KNOWN_FLAGS), ); - descriptor.insert( - "flags".into(), + activation_continuations_payload.insert( + "entry_size".into(), + json!(WPK_FORK_ACTIVATION_CONTINUATION_ENTRY_SIZE), + ); + activation_continuations_payload.insert( + "header_size".into(), + json!(WPK_FORK_ACTIVATION_CONTINUATIONS_HEADER_SIZE), + ); + activation_continuations_payload.insert( + "known_flags".into(), + json!(WPK_FORK_ACTIVATION_CONTINUATIONS_KNOWN_FLAGS), + ); + activation_continuations_payload.insert( + "magic_bytes".into(), + json!(WPK_FORK_ACTIVATION_CONTINUATIONS_MAGIC), + ); + activation_continuations_payload.insert( + "owner".into(), + json!(WPK_FORK_ACTIVATION_CONTINUATIONS_OWNER), + ); + activation_continuations_payload.insert( + "version".into(), + json!(WPK_FORK_ACTIVATION_CONTINUATIONS_VERSION), + ); + + let mut imported_table_bindings_payload: JsonMap = BTreeMap::new(); + imported_table_bindings_payload.insert( + "binding_kinds".into(), json!([ { - "bit": WPK_FORK_LINKED_FRAME_FLAG_ABORT_UNWINDING, - "name": "abort_unwinding" + "number": WPK_FORK_IMPORTED_TABLE_BINDING_ACTIVATION_TABLE, + "name": "activation_table" }, { - "bit": WPK_FORK_LINKED_FRAME_FLAG_TRANSACTIONAL_NODES, - "name": "transactional_nodes" + "number": WPK_FORK_IMPORTED_TABLE_BINDING_BASE_IMPORT, + "name": "base_import" } ]), ); - descriptor.insert( + imported_table_bindings_payload.insert( + "entry_fields".into(), + json!([ + {"name": "consumer_activation", "offset": 0, "size": 4}, + {"name": "consumer_owner", "offset": 4, "size": 4}, + {"name": "source_activation", "offset": 8, "size": 4}, + {"name": "source_owner", "offset": 12, "size": 4}, + {"name": "reserved", "offset": 16, "size": 4}, + {"name": "binding_kind", "offset": 20, "size": 1}, + {"name": "reserved", "offset": 21, "size": 3} + ]), + ); + imported_table_bindings_payload.insert( + "entry_size".into(), + json!(WPK_FORK_IMPORTED_TABLE_BINDINGS_ENTRY_SIZE), + ); + imported_table_bindings_payload.insert( + "header_size".into(), + json!(WPK_FORK_IMPORTED_TABLE_BINDINGS_HEADER_SIZE), + ); + imported_table_bindings_payload.insert( + "known_flags".into(), + json!(WPK_FORK_IMPORTED_TABLE_BINDINGS_KNOWN_FLAGS), + ); + imported_table_bindings_payload.insert( "magic_bytes".into(), - json!(WPK_FORK_LINKED_FRAME_FORMAT_MAGIC), + json!(WPK_FORK_IMPORTED_TABLE_BINDINGS_MAGIC), ); - descriptor.insert("pointer_widths".into(), Value::Array(pointer_widths)); - descriptor.insert( - "required_flags".into(), - json!(WPK_FORK_LINKED_FRAME_REQUIRED_FLAGS), + imported_table_bindings_payload.insert( + "owner".into(), + json!(WPK_FORK_IMPORTED_TABLE_BINDINGS_OWNER), ); - descriptor.insert( + imported_table_bindings_payload.insert( + "version".into(), + json!(WPK_FORK_IMPORTED_TABLE_BINDINGS_VERSION), + ); + + let mut module_state_payloads: JsonMap = BTreeMap::new(); + module_state_payloads.insert( + "activation_continuations".into(), + Value::Object(activation_continuations_payload.into_iter().collect()), + ); + module_state_payloads.insert( + "data_segments".into(), + Value::Object(data_segments_payload.into_iter().collect()), + ); + module_state_payloads.insert( + "element_segments".into(), + Value::Object(element_segments_payload.into_iter().collect()), + ); + module_state_payloads.insert( + "imported_global_bindings".into(), + Value::Object(imported_global_bindings_payload.into_iter().collect()), + ); + module_state_payloads.insert( + "imported_table_bindings".into(), + Value::Object(imported_table_bindings_payload.into_iter().collect()), + ); + module_state_payloads.insert( + "module".into(), + Value::Object(module_payload.into_iter().collect()), + ); + module_state_payloads.insert( + "mutable_global".into(), + Value::Object(mutable_global_payload.into_iter().collect()), + ); + module_state_payloads.insert( + "replay_events".into(), + Value::Object(replay_events_payload.into_iter().collect()), + ); + module_state_payloads.insert( + "reference_transaction".into(), + Value::Object(reference_transaction_payload.into_iter().collect()), + ); + module_state_payloads.insert( + "table".into(), + Value::Object(table_payload.into_iter().collect()), + ); + + let mut module_state: JsonMap = BTreeMap::new(); + module_state.insert( + "arena".into(), + Value::Object(module_state_arena.into_iter().collect()), + ); + module_state.insert( + "descriptor".into(), + Value::Object(module_state_descriptor.into_iter().collect()), + ); + module_state.insert( + "record_payloads".into(), + Value::Object(module_state_payloads.into_iter().collect()), + ); + + let mut imported_globals: JsonMap = BTreeMap::new(); + imported_globals.insert( + "header_size".into(), + json!(WPK_FORK_IMPORTED_GLOBALS_HEADER_SIZE), + ); + imported_globals.insert( + "known_flags".into(), + json!(WPK_FORK_IMPORTED_GLOBAL_KNOWN_FLAGS), + ); + imported_globals.insert("magic_bytes".into(), json!(WPK_FORK_IMPORTED_GLOBALS_MAGIC)); + imported_globals.insert( + "mutable_flag".into(), + json!(WPK_FORK_IMPORTED_GLOBAL_FLAG_MUTABLE), + ); + imported_globals.insert( + "shared_flag".into(), + json!(WPK_FORK_IMPORTED_GLOBAL_FLAG_SHARED), + ); + imported_globals.insert( + "record_header_size".into(), + json!(WPK_FORK_IMPORTED_GLOBALS_RECORD_HEADER_SIZE), + ); + imported_globals.insert( + "record_fields".into(), + json!([ + {"name": "record_size", "offset": 0, "size": 4}, + {"name": "owner", "offset": 4, "size": 4}, + {"name": "value_type", "offset": 8, "size": 1}, + {"name": "flags", "offset": 9, "size": 1}, + {"name": "reserved", "offset": 10, "size": 2}, + {"name": "module_name_length", "offset": 12, "size": 4}, + {"name": "field_name_length", "offset": 16, "size": 4}, + {"name": "import_ordinal", "offset": 20, "size": 4} + ]), + ); + imported_globals.insert("section".into(), json!(WPK_FORK_IMPORTED_GLOBALS_SECTION)); + imported_globals.insert("version".into(), json!(WPK_FORK_IMPORTED_GLOBALS_VERSION)); + + let mut imported_tables: JsonMap = BTreeMap::new(); + imported_tables.insert( + "header_size".into(), + json!(WPK_FORK_IMPORTED_TABLES_HEADER_SIZE), + ); + imported_tables.insert( + "known_flags".into(), + json!(WPK_FORK_IMPORTED_TABLE_KNOWN_FLAGS), + ); + imported_tables.insert("magic_bytes".into(), json!(WPK_FORK_IMPORTED_TABLES_MAGIC)); + imported_tables.insert( + "record_header_size".into(), + json!(WPK_FORK_IMPORTED_TABLES_RECORD_HEADER_SIZE), + ); + imported_tables.insert( + "record_fields".into(), + json!([ + {"name": "record_size", "offset": 0, "size": 4}, + {"name": "owner", "offset": 4, "size": 4}, + {"name": "element_type", "offset": 8, "size": 1}, + {"name": "flags", "offset": 9, "size": 1}, + {"name": "reserved", "offset": 10, "size": 2}, + {"name": "module_name_length", "offset": 12, "size": 4}, + {"name": "field_name_length", "offset": 16, "size": 4}, + {"name": "import_ordinal", "offset": 20, "size": 4} + ]), + ); + imported_tables.insert("section".into(), json!(WPK_FORK_IMPORTED_TABLES_SECTION)); + imported_tables.insert( + "table64_flag".into(), + json!(WPK_FORK_IMPORTED_TABLE_FLAG_TABLE64), + ); + imported_tables.insert("version".into(), json!(WPK_FORK_IMPORTED_TABLES_VERSION)); + + let mut exception_codec: JsonMap = BTreeMap::new(); + exception_codec.insert( + "activation_import".into(), + json!({ + "module": WPK_FORK_EXCEPTION_CODEC_IMPORT_MODULE, + "name": WPK_FORK_EXCEPTION_IMPORT_ACTIVATION, + "type": "i32", + "mutable": false + }), + ); + exception_codec.insert( + "header_size".into(), + json!(WPK_FORK_EXCEPTION_CODEC_HEADER_SIZE), + ); + exception_codec.insert("section".into(), json!(WPK_FORK_EXCEPTION_CODEC_SECTION)); + exception_codec.insert( + "tag_record_size".into(), + json!(WPK_FORK_EXCEPTION_CODEC_TAG_RECORD_SIZE), + ); + exception_codec.insert("version".into(), json!(WPK_FORK_EXCEPTION_CODEC_VERSION)); + + let mut gc_codec: JsonMap = BTreeMap::new(); + gc_codec.insert( + "field_record".into(), + json!({ + "size": WPK_FORK_GC_CODEC_FIELD_RECORD_SIZE, + "fields": [ + {"name": "storage", "offset": 0, "size": 1}, + {"name": "flags", "offset": 1, "size": 1}, + {"name": "reserved", "offset": 2, "size": 2}, + {"name": "scalar_offset_or_none", "offset": 4, "size": 4}, + {"name": "reference_ordinal_or_none", "offset": 8, "size": 4} + ] + }), + ); + gc_codec.insert("header_size".into(), json!(WPK_FORK_GC_CODEC_HEADER_SIZE)); + gc_codec.insert( + "layout_record".into(), + json!({ + "size": WPK_FORK_GC_CODEC_LAYOUT_RECORD_SIZE, + "fields": [ + {"name": "layout_id", "offset": 0, "size": 4}, + {"name": "type_ordinal", "offset": 4, "size": 4}, + {"name": "kind", "offset": 8, "size": 1}, + {"name": "constructor", "offset": 9, "size": 1}, + {"name": "flags", "offset": 10, "size": 2}, + {"name": "snapshot_scalar_len_or_stride", "offset": 12, "size": 4}, + {"name": "field_start", "offset": 16, "size": 4}, + {"name": "field_count", "offset": 20, "size": 4}, + {"name": "super_type_ordinal_or_none", "offset": 24, "size": 4}, + {"name": "base_layout_id", "offset": 28, "size": 4}, + {"name": "auxiliary", "offset": 32, "size": 4}, + {"name": "provenance_scalar_len", "offset": 36, "size": 4}, + {"name": "provenance_ref_count", "offset": 40, "size": 4} + ] + }), + ); + gc_codec.insert("magic_bytes".into(), json!(WPK_FORK_GC_CODEC_MAGIC)); + gc_codec.insert("section".into(), json!(WPK_FORK_GC_CODEC_SECTION)); + gc_codec.insert( + "transit_table".into(), + json!({ + "module": shared::abi::WPK_FORK_REFERENCE_CODEC_IMPORT_MODULE, + "name": shared::abi::WPK_FORK_REFERENCE_IMPORT_GC_TRANSIT, + "table64": false, + "element": "anyref", + "minimum": 1, + "maximum": null + }), + ); + gc_codec.insert("version".into(), json!(WPK_FORK_GC_CODEC_VERSION)); + + let mut unwind_transport: JsonMap = BTreeMap::new(); + unwind_transport.insert( + "import".into(), + json!({ + "module": WPK_FORK_UNWIND_TAG_IMPORT_MODULE, + "name": WPK_FORK_UNWIND_TAG_IMPORT_NAME, + "kind": "tag" + }), + ); + unwind_transport.insert( + "payload_arity".into(), + json!(WPK_FORK_UNWIND_TRANSPORT_PAYLOAD_ARITY), + ); + unwind_transport.insert("section".into(), json!(WPK_FORK_UNWIND_TRANSPORT_SECTION)); + unwind_transport.insert("version".into(), json!(WPK_FORK_UNWIND_TRANSPORT_VERSION)); + + let mut static_root_catalog: JsonMap = BTreeMap::new(); + static_root_catalog.insert("export".into(), json!(WPK_FORK_STATIC_ROOT_CATALOG_EXPORT)); + static_root_catalog.insert( + "harvest_export".into(), + json!(WPK_FORK_STATIC_ROOT_HARVEST_EXPORT), + ); + static_root_catalog.insert( + "header_size".into(), + json!(WPK_FORK_STATIC_ROOT_CATALOG_HEADER_SIZE), + ); + static_root_catalog.insert( + "magic_bytes".into(), + json!(WPK_FORK_STATIC_ROOT_CATALOG_MAGIC), + ); + static_root_catalog.insert( "section".into(), - json!(WPK_FORK_LINKED_FRAME_FORMAT_SECTION), + json!(WPK_FORK_STATIC_ROOT_CATALOG_SECTION), ); - descriptor.insert( + static_root_catalog.insert( "version".into(), - json!(WPK_FORK_LINKED_FRAME_FORMAT_VERSION), + json!(WPK_FORK_STATIC_ROOT_CATALOG_VERSION), + ); + + let mut capabilities: JsonMap = BTreeMap::new(); + capabilities.insert("section".into(), json!(WPK_FORK_CAPABILITIES_SECTION)); + capabilities.insert("version".into(), json!(WPK_FORK_CAPABILITIES_VERSION)); + capabilities.insert("known_mask".into(), json!(WPK_FORK_CAP_KNOWN_MASK)); + capabilities.insert("required_flags".into(), json!(WPK_FORK_CAP_REQUIRED_FLAGS)); + capabilities.insert( + "flags".into(), + json!([ + {"bit": WPK_FORK_CAP_SIDE_ENTRY, "name": "side_entry"}, + {"bit": WPK_FORK_CAP_DYLINK_MAIN, "name": "dylink_main"}, + { + "bit": WPK_FORK_CAP_ACTIVATION_STATE_SAFE, + "name": "activation_state_safe" + } + ]), ); let mut fork: JsonMap = BTreeMap::new(); + fork.insert( + "capabilities".into(), + Value::Object(capabilities.into_iter().collect()), + ); + fork.insert( + "exception_codec".into(), + Value::Object(exception_codec.into_iter().collect()), + ); + fork.insert( + "gc_codec".into(), + Value::Object(gc_codec.into_iter().collect()), + ); + fork.insert( + "static_root_catalog".into(), + Value::Object(static_root_catalog.into_iter().collect()), + ); + fork.insert( + "unwind_transport".into(), + Value::Object(unwind_transport.into_iter().collect()), + ); fork.insert( "linked_frame_descriptor".into(), Value::Object(descriptor.into_iter().collect()), ); + fork.insert( + "imported_globals".into(), + Value::Object(imported_globals.into_iter().collect()), + ); + fork.insert( + "imported_tables".into(), + Value::Object(imported_tables.into_iter().collect()), + ); + fork.insert( + "module_state".into(), + Value::Object(module_state.into_iter().collect()), + ); fork.insert("required_exports".into(), Value::Array(exports)); fork.insert("required_imports".into(), Value::Array(imports)); @@ -2533,7 +4410,7 @@ mod tests { } #[test] - fn program_artifact_snapshot_captures_complete_abi42_fork_contract() { + fn program_artifact_snapshot_captures_complete_abi43_fork_contract() { let artifact = program_artifact(); let fork = &artifact["fork_instrumentation"]; let descriptor = &fork["linked_frame_descriptor"]; @@ -2545,6 +4422,38 @@ mod tests { assert_eq!(descriptor["version"], json!(1)); assert_eq!(descriptor["descriptor_size"], json!(24)); assert_eq!(descriptor["required_flags"], json!(3)); + assert_eq!( + fork["capabilities"], + json!({ + "section": "kandelo.wpk_fork.capabilities", + "version": 1, + "known_mask": 7, + "required_flags": 4, + "flags": [ + {"bit": 1, "name": "side_entry"}, + {"bit": 2, "name": "dylink_main"}, + {"bit": 4, "name": "activation_state_safe"} + ] + }) + ); + assert_eq!( + fork["exception_codec"], + json!({ + "activation_import": { + "module": "env", + "name": "__wpk_fork_module_activation", + "type": "i32", + "mutable": false + }, + "header_size": 8, + "section": "kandelo.wpk_fork.exception_codec", + "tag_record_size": 16, + "version": 1 + }) + ); + assert_eq!(fork["imported_globals"]["record_header_size"], json!(24)); + assert_eq!(fork["imported_globals"]["known_flags"], json!(3)); + assert_eq!(fork["imported_globals"]["shared_flag"], json!(2)); assert_eq!( descriptor["pointer_widths"], json!([ @@ -2552,9 +4461,60 @@ mod tests { {"bytes": 8, "chunk_header_size": 56, "node_header_size": 32} ]) ); + assert_eq!( + fork["module_state"]["descriptor"]["section"], + json!("kandelo.wpk_fork.module_state") + ); + let record_kinds = fork["module_state"]["arena"]["record"]["kinds"] + .as_array() + .unwrap(); + assert_eq!(record_kinds.len(), 13); + assert_eq!( + record_kinds[11], + json!({"name": "reference_recipe_segment", "number": 12}) + ); + assert_eq!( + record_kinds[12], + json!({"name": "replay_event_segment", "number": 13}) + ); + assert_eq!( + fork["module_state"]["record_payloads"]["mutable_global"]["header_size"], + json!(8) + ); + assert_eq!( + fork["module_state"]["record_payloads"]["replay_events"]["owner"], + json!(1) + ); + assert_eq!( + fork["module_state"]["record_payloads"]["imported_global_bindings"]["entry_size"], + json!(40) + ); + assert_eq!( + fork["module_state"]["record_payloads"]["imported_global_bindings"]["binding_kinds"] + .as_array() + .unwrap() + .len(), + 5 + ); + assert_eq!( + fork["module_state"]["record_payloads"]["activation_continuations"]["entry_size"], + json!(16) + ); + assert_eq!( + fork["module_state"]["record_payloads"]["imported_table_bindings"]["entry_size"], + json!(24) + ); + assert_eq!(fork["imported_tables"]["record_header_size"], json!(24)); + assert_eq!(fork["gc_codec"]["magic_bytes"], json!([75, 70, 71, 67])); + assert_eq!(fork["gc_codec"]["layout_record"]["size"], json!(44)); + assert_eq!(fork["gc_codec"]["field_record"]["size"], json!(12)); + assert_eq!( + fork["gc_codec"]["transit_table"]["element"], + json!("anyref") + ); let imports = fork["required_imports"].as_array().unwrap(); - assert_eq!(imports.len(), 3); + assert_eq!(imports.len(), 47); assert_eq!( imports[0], json!({ @@ -2565,9 +4525,75 @@ mod tests { "results": [] }) ); + assert!(imports.iter().any(|entry| { + entry["name"] == json!("__wpk_fork_module_state_record_reserve") + && entry["params"] == json!(["i32", "i32", "i32", "ptr"]) + && entry["results"] == json!(["ptr"]) + })); + assert!(!imports.iter().any(|entry| { + entry["name"] == json!("__wpk_fork_ref_encode_anyref") + || entry["name"] == json!("__wpk_fork_ref_decode_anyref") + })); + assert!(imports.iter().any(|entry| { + entry["name"] == json!("__wpk_fork_ref_exn_define") + && entry["params"] + == json!(["i32", "i32", "i32", "i32", "ptr", "i32", "ptr", "i32"]) + && entry["results"] == json!([]) + })); + assert!(imports.iter().any(|entry| { + entry["name"] == json!("__wpk_fork_ref_gc_define") + && entry["params"] + == json!(["i32", "i32", "i32", "i32", "i32", "ptr", "i32", "i32"]) + && entry["results"] == json!([]) + })); + assert!(imports.iter().any(|entry| { + entry["kind"] == json!("table") + && entry["name"] == json!("__wpk_fork_ref_gc_transit") + && entry["element"] == json!("anyref") + })); + assert!(imports.iter().any(|entry| { + entry["name"] == json!("__wpk_fork_ref_gc_provenance_begin") + && entry["params"] == json!(["i32", "i32", "i32", "i32", "i64", "i64", "i32"]) + && entry["results"] == json!(["i32"]) + })); + assert!(imports.iter().any(|entry| { + entry["name"] == json!("__wpk_fork_ref_gc_provenance_ref") + && entry["params"] == json!(["i32", "i32", "i32"]) + && entry["results"] == json!([]) + })); + assert!(imports.iter().any(|entry| { + entry["name"] == json!("__wpk_fork_ref_gc_provenance_end") + && entry["params"] == json!(["i32"]) + && entry["results"] == json!([]) + })); let exports = fork["required_exports"].as_array().unwrap(); - assert_eq!(exports.len(), 7); + assert_eq!(exports.len(), 28); + assert!(exports.iter().any(|entry| { + entry["name"] == json!("__wpk_fork_exception_materialize") + && entry["params"] == json!(["i32"]) + && entry["results"] == json!([]) + })); + assert!(exports.iter().any(|entry| { + entry["name"] == json!("__wpk_fork_ref_decode_exnref") + && entry["params"] == json!(["i32"]) + && entry["results"] == json!(["exnref"]) + })); + assert!(exports.iter().any(|entry| { + entry["name"] == json!("__wpk_fork_ref_gc_probe") + && entry["params"] == json!(["i32"]) + && entry["results"] == json!(["i64"]) + })); + assert!(exports.iter().any(|entry| { + entry["name"] == json!("__wpk_fork_ref_gc_publish_externref") + && entry["params"] == json!(["i32", "externref"]) + && entry["results"] == json!([]) + })); + assert!(exports.iter().any(|entry| { + entry["name"] == json!("__wpk_fork_static_root_harvest") + && entry["params"] == json!([]) + && entry["results"] == json!([]) + })); assert!(exports.iter().any(|entry| { entry["name"] == json!("wpk_fork_abort_begin") && entry["params"] == json!(["ptr"]) @@ -2578,14 +4604,45 @@ mod tests { && entry["params"] == json!([]) && entry["results"] == json!(["i32"]) })); + assert!(exports.iter().any(|entry| { + entry["name"] == json!("wpk_fork_module_state_finish_restore") + && entry["params"] == json!(["i32"]) + && entry["results"] == json!([]) + })); + assert!(exports.iter().any(|entry| { + entry["name"] == json!("wpk_fork_module_state_restore") + && entry["params"] == json!(["i32"]) + && entry["results"] == json!([]) + })); assert_eq!( custom_sections(), - json!(["kandelo.wpk_fork.linked_frames", "wasm-posix-abi"]) + json!([ + "kandelo.wpk_fork.capabilities", + "kandelo.wpk_fork.exception_codec", + "kandelo.wpk_fork.gc_codec", + "kandelo.wpk_fork.imported_globals", + "kandelo.wpk_fork.imported_tables", + "kandelo.wpk_fork.linked_frames", + "kandelo.wpk_fork.module_state", + "kandelo.wpk_fork.static_root_catalog", + "kandelo.wpk_fork.unwind_transport", + "wasm-posix-abi" + ]) ); let rendered = render_ts_module(); for expected in [ "export const WPK_FORK_LINKED_FRAME_DESCRIPTOR_SIZE = 24 as const;", + "export const WPK_FORK_MODULE_STATE_FORMAT_SECTION = \"kandelo.wpk_fork.module_state\" as const;", + "export const WPK_FORK_MODULE_STATE_RECORD_KIND_TABLE_PAGE = 5 as const;", + "export const WPK_FORK_MODULE_STATE_TABLE_DESCRIPTOR_PAYLOAD_SIZE = 56 as const;", + "export const WPK_FORK_EXPORT_MODULE_STATE_FINISH_RESTORE = \"wpk_fork_module_state_finish_restore\" as const;", + "export const WPK_FORK_CAP_ACTIVATION_STATE_SAFE = 4 as const;", + "export const WPK_FORK_EXCEPTION_CODEC_SECTION = \"kandelo.wpk_fork.exception_codec\" as const;", + "export const WPK_FORK_GC_CODEC_SECTION = \"kandelo.wpk_fork.gc_codec\" as const;", + "export const WPK_FORK_GC_CODEC_LAYOUT_RECORD_SIZE = 44 as const;", + "export const WPK_FORK_FRAME_IMPORT_COMMIT = \"__wpk_fork_frame_commit\" as const;", + "export const WPK_FORK_EXPORT_RESUME_START = \"wpk_fork_resume_start\" as const;", "name: \"__wpk_fork_frame_reserve\", params: [\"ptr\"], results: [\"ptr\"]", "name: \"wpk_fork_abort_end\", params: [], results: []", ] { @@ -2781,8 +4838,7 @@ mod tests { fn adding_optional_host_adapter_export_is_compatible() { let old = base_snapshot(); let mut new = old.clone(); - new["host_adapter"]["optional_kernel_exports"] = - json!(["kernel_get_process_exit_signal",]); + new["host_adapter"]["optional_kernel_exports"] = json!(["kernel_get_process_exit_signal",]); let report = classify_compat_change(&old, &new).unwrap(); assert!(report.breaking.is_empty(), "{report:?}"); diff --git a/tools/xtask/src/homebrew_sidecars.rs b/tools/xtask/src/homebrew_sidecars.rs index ead123def6..1e3387c4b6 100644 --- a/tools/xtask/src/homebrew_sidecars.rs +++ b/tools/xtask/src/homebrew_sidecars.rs @@ -541,10 +541,7 @@ impl Generator<'_> { link_outputs: &mut Vec<(String, Value)>, ) -> Result { require_relative_path(&package.formula_path, "formula_path")?; - require_sha256( - &package.formula_source_sha256, - "formula_source_sha256", - )?; + require_sha256(&package.formula_source_sha256, "formula_source_sha256")?; sha256_file(&self.options.tap_root.join(&package.formula_path))?; let formula_sidecar_path = format!("Kandelo/formula/{}.json", package.name); require_relative_path(&formula_sidecar_path, "formula sidecar path")?; @@ -556,12 +553,8 @@ impl Generator<'_> { let mut provenance_reports = 0; for bottle in &bottles { - let bottle_value = self.generate_bottle( - package, - bottle, - &formula_sidecar_path, - link_outputs, - )?; + let bottle_value = + self.generate_bottle(package, bottle, &formula_sidecar_path, link_outputs)?; if bottle_status(bottle) == "success" { link_manifests += 1; provenance_reports += 1; @@ -734,10 +727,7 @@ impl Generator<'_> { bottle, "archived_formula_sha256", )?; - require_sha256( - expected_archived_formula_sha, - "archived_formula_sha256", - )?; + require_sha256(expected_archived_formula_sha, "archived_formula_sha256")?; let payload_root = required_field(&bottle.payload_root, package, bottle, "payload_root")?; let build = bottle.build.as_ref().ok_or_else(|| { bottle_error(package, bottle, "success bottle requires build evidence") @@ -1178,7 +1168,11 @@ fn verify_bottle_payload( ) -> Result { require_relative_path(payload_root, "payload_root")?; let formula_receipt = format!(".brew/{}.rb", package.name); - if !bottle.receipts.iter().any(|receipt| receipt == &formula_receipt) { + if !bottle + .receipts + .iter() + .any(|receipt| receipt == &formula_receipt) + { return Err(bottle_error( package, bottle, @@ -1186,13 +1180,15 @@ fn verify_bottle_payload( )); } let (entries, archived_formula_sha) = - tar_gz_entries_and_formula_sha(bottle_path, payload_root, &formula_receipt).map_err(|e| { - bottle_error( - package, - bottle, - &format!("cannot inspect bottle payload: {e}"), - ) - })?; + tar_gz_entries_and_formula_sha(bottle_path, payload_root, &formula_receipt).map_err( + |e| { + bottle_error( + package, + bottle, + &format!("cannot inspect bottle payload: {e}"), + ) + }, + )?; for link in &bottle.links { require_relative_path(&link.source, "link source")?; @@ -1261,9 +1257,9 @@ fn tar_gz_entries_and_formula_sha( let mut hasher = Sha256::new(); let mut buffer = [0_u8; 64 * 1024]; loop { - let read = entry.read(&mut buffer).map_err(|e| { - format!("read {} formula receipt: {e}", path.display()) - })?; + let read = entry + .read(&mut buffer) + .map_err(|e| format!("read {} formula receipt: {e}", path.display()))?; if read == 0 { break; } @@ -1453,8 +1449,7 @@ mod tests { }); if status == "success" { bottle["bottle_file"] = json!(bottle_file); - bottle["archived_formula_sha256"] = - json!(sha256_bytes(FORMULA_TEXT.as_bytes())); + bottle["archived_formula_sha256"] = json!(sha256_bytes(FORMULA_TEXT.as_bytes())); bottle["url"] = json!(repository_bottle_url( "kandelo-dev/homebrew-tap-core", "hello", @@ -1539,10 +1534,7 @@ mod tests { let tap_root = dir.path().join("tap"); let input_dir = dir.path().join("inputs"); fs::create_dir_all(&input_dir).unwrap(); - write_text( - &tap_root.join("Formula/hello.rb"), - FORMULA_TEXT, - ); + write_text(&tap_root.join("Formula/hello.rb"), FORMULA_TEXT); let bottle_path = input_dir.join("hello.bottle.tar.gz"); write_bottle(&bottle_path, FORMULA_TEXT); let bottle_sha256 = sha256_file_and_len(&bottle_path).unwrap().0; @@ -1667,9 +1659,8 @@ mod tests { previous.run(None); let previous_metadata: Value = load_json(&previous.tap_root.join("Kandelo/metadata.json")).unwrap(); - let previous_built_from = previous_metadata["packages"][0]["bottles"][0] - ["built_from"] - .clone(); + let previous_built_from = + previous_metadata["packages"][0]["bottles"][0]["built_from"].clone(); let previous_formula_sha = previous_built_from["formula_sha256"] .as_str() .unwrap() @@ -1714,11 +1705,7 @@ mod tests { current.run(Some(&previous.tap_root.join("Kandelo/metadata.json"))); let metadata: Value = load_json(¤t.tap_root.join("Kandelo/metadata.json")).unwrap(); - write_formula_from_metadata( - ¤t.tap_root, - CURRENT_ARCHIVED_FORMULA_TEXT, - &metadata, - ); + write_formula_from_metadata(¤t.tap_root, CURRENT_ARCHIVED_FORMULA_TEXT, &metadata); let bottles = metadata["packages"][0]["bottles"].as_array().unwrap(); let arches: Vec<_> = bottles .iter() diff --git a/tools/xtask/src/homebrew_tier2_preflight.rs b/tools/xtask/src/homebrew_tier2_preflight.rs index b77342f39b..782bc8f71d 100644 --- a/tools/xtask/src/homebrew_tier2_preflight.rs +++ b/tools/xtask/src/homebrew_tier2_preflight.rs @@ -198,11 +198,10 @@ fn validate( .as_deref() .ok_or_else(|| "Tier-2 bridge plan is missing its support SHA-256".to_string())?; validate_sha256(support_sha256, "support SHA-256")?; - let support_runtime_sha256 = plan - .support_runtime_sha256 - .0 - .as_deref() - .ok_or_else(|| "Tier-2 bridge plan is missing its support runtime SHA-256".to_string())?; + let support_runtime_sha256 = + plan.support_runtime_sha256.0.as_deref().ok_or_else(|| { + "Tier-2 bridge plan is missing its support runtime SHA-256".to_string() + })?; validate_sha256(support_runtime_sha256, "support runtime SHA-256")?; let repo_root = exact_real_directory(repo_root, "repository root")?; @@ -1038,7 +1037,10 @@ index_url = "https://example.test/index.toml" serde_json::json!(["WASM_POSIX_INSTALL_LOCAL_MIRROR"]), "reserved variable", ), - (serde_json::json!(["PYTHON_CONFIGURE"]), "approved namespace"), + ( + serde_json::json!(["PYTHON_CONFIGURE"]), + "approved namespace", + ), (serde_json::json!(["PATH"]), "approved namespace"), ] { let fixture = Fixture::new(); diff --git a/tools/xtask/src/homebrew_validate.rs b/tools/xtask/src/homebrew_validate.rs index 502cfe6920..094e6fd97d 100644 --- a/tools/xtask/src/homebrew_validate.rs +++ b/tools/xtask/src/homebrew_validate.rs @@ -347,12 +347,7 @@ impl Validator<'_> { } } - fn validate_formula_file( - &mut self, - package_name: &str, - package: &Value, - metadata: &Value, - ) { + fn validate_formula_file(&mut self, package_name: &str, package: &Value, metadata: &Value) { let Some(formula_path_rel) = string_at(package, "/formula_path") else { return; }; @@ -666,14 +661,8 @@ impl Validator<'_> { "/repositories/kandelo_repository", "/built_from/kandelo_repository", ), - ( - "/repositories/kandelo_commit", - "/built_from/kandelo_commit", - ), - ( - "/repositories/tap_repository", - "/built_from/tap_repository", - ), + ("/repositories/kandelo_commit", "/built_from/kandelo_commit"), + ("/repositories/tap_repository", "/built_from/tap_repository"), ("/repositories/tap_commit", "/built_from/tap_commit"), ("/formula/sha256", "/built_from/formula_sha256"), ] { @@ -1472,12 +1461,7 @@ mod tests { ); let report = fixture.validate(); - assert!( - report - .errors - .join("\n") - .contains("Formula bottle tags") - ); + assert!(report.errors.join("\n").contains("Formula bottle tags")); } #[test] @@ -1632,9 +1616,7 @@ mod tests { let source_without_class_end = source.strip_suffix("end\n").unwrap(); write_text( &path, - &format!( - "{source_without_class_end} bottle {{ system \"false\" }}\nend\n" - ), + &format!("{source_without_class_end} bottle {{ system \"false\" }}\nend\n"), ); let report = fixture.validate(); diff --git a/tools/xtask/src/package_output_receipt.rs b/tools/xtask/src/package_output_receipt.rs index ef2ac3d32c..d67a0435fb 100644 --- a/tools/xtask/src/package_output_receipt.rs +++ b/tools/xtask/src/package_output_receipt.rs @@ -15,9 +15,9 @@ use sha2::{Digest, Sha256}; use crate::build_deps::{Registry, compute_cache_key_sha_for_package, resolve_relative_url}; use crate::index_toml::{EntryStatus, IndexToml}; -use crate::pkg_manifest::{BuildToml, DepsManifest, ManifestKind, TargetArch}; #[cfg(test)] use crate::pkg_manifest::write_cache_provenance; +use crate::pkg_manifest::{BuildToml, DepsManifest, ManifestKind, TargetArch}; use crate::remote_fetch; use crate::util::hex;